95 lines
3.6 KiB
C++
95 lines
3.6 KiB
C++
|
#pragma once
|
|||
|
|
|
|||
|
|
#include "limonp/Logging.hpp"
|
|||
|
|
#include "PreFilter.hpp"
|
|||
|
|
#include <cassert>
|
|||
|
|
|
|||
|
|
|
|||
|
|
namespace cppjieba {
|
|||
|
|
|
|||
|
|
const char* const SPECIAL_SEPARATORS = " \t\n\xEF\xBC\x8C\xE3\x80\x82";
|
|||
|
|
|
|||
|
|
using namespace limonp;
|
|||
|
|
|
|||
|
|
class SegmentBase {
|
|||
|
|
public:
|
|||
|
|
SegmentBase() {
|
|||
|
|
XCHECK(ResetSeparators(SPECIAL_SEPARATORS));
|
|||
|
|
}
|
|||
|
|
virtual ~SegmentBase() { }
|
|||
|
|
|
|||
|
|
virtual void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res, bool hmm,
|
|||
|
|
size_t max_word_len) const = 0;
|
|||
|
|
//添加基于sentence的cut方法，减少中间变量的存储与格式转换--jxx20210517
|
|||
|
|
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<string>& res, bool hmm,
|
|||
|
|
size_t max_word_len) const = 0;
|
|||
|
|
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, unordered_map<string, KeyWord>& res, bool hmm,
|
|||
|
|
size_t max_word_len) const = 0;
|
|||
|
|
//重写CutToStr函数，简化获取vector<string>& words的流程，降低内存占用--jxx20210517
|
|||
|
|
void CutToStr(const string& sentence, vector<string>& words, bool hmm = true,
|
|||
|
|
size_t max_word_len = MAX_WORD_LENGTH) const {
|
|||
|
|
PreFilter pre_filter(symbols_, sentence);
|
|||
|
|
words.clear();
|
|||
|
|
words.reserve(sentence.size() / 2);//todo 参考源码，参数待定
|
|||
|
|
RuneStrArray::const_iterator null_p;
|
|||
|
|
WordRange range(null_p, null_p);
|
|||
|
|
while (pre_filter.Next(range)) {
|
|||
|
|
CutWithSentence(sentence, range.left, range.right, words, hmm, max_word_len);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
void CutToStr(const string& sentence, WordRange range, vector<string>& words, bool hmm = true,
|
|||
|
|
size_t max_word_len = MAX_WORD_LENGTH) const {
|
|||
|
|
CutWithSentence(sentence, range.left, range.right, words, hmm, max_word_len);
|
|||
|
|
}
|
|||
|
|
void CutToStr(const string& sentence, WordRange range, unordered_map<string, KeyWord>& words, bool hmm = true,
|
|||
|
|
size_t max_word_len = MAX_WORD_LENGTH) const {
|
|||
|
|
CutWithSentence(sentence, range.left, range.right, words, hmm, max_word_len);
|
|||
|
|
}
|
|||
|
|
void CutToWord(const string& sentence, vector<Word>& words, bool hmm = true,
|
|||
|
|
size_t max_word_len = MAX_WORD_LENGTH) const {
|
|||
|
|
PreFilter pre_filter(symbols_, sentence);
|
|||
|
|
vector<WordRange> wrs;
|
|||
|
|
wrs.reserve(sentence.size() / 2);
|
|||
|
|
|
|||
|
|
while (pre_filter.HasNext()) {
|
|||
|
|
auto range = pre_filter.Next();
|
|||
|
|
Cut(range.left, range.right, wrs, hmm, max_word_len);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
words.clear();
|
|||
|
|
words.reserve(wrs.size());
|
|||
|
|
GetWordsFromWordRanges(sentence, wrs, words);
|
|||
|
|
wrs.clear();
|
|||
|
|
vector<WordRange>().swap(wrs);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
void CutRuneArray(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res,
|
|||
|
|
bool hmm = true, size_t max_word_len = MAX_WORD_LENGTH) const {
|
|||
|
|
Cut(begin, end, res, hmm, max_word_len);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
bool ResetSeparators(const string& s) {
|
|||
|
|
symbols_.clear();
|
|||
|
|
RuneStrArray runes;
|
|||
|
|
|
|||
|
|
if (!DecodeRunesInString(s, runes)) {
|
|||
|
|
XLOG(ERROR) << "decode " << s << " failed";
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for (size_t i = 0; i < runes.size(); i++) {
|
|||
|
|
if (!symbols_.insert(runes[i].rune).second) {
|
|||
|
|
XLOG(ERROR) << s.substr(runes[i].offset, runes[i].len) << " already exists";
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
protected:
|
|||
|
|
unordered_set<Rune> symbols_;
|
|||
|
|
}; // class SegmentBase
|
|||
|
|
|
|||
|
|
} // cppjieba
|
|||
|
|
|