openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Merge from main.

This commit is contained in:
iaom 2021-06-18 10:54:02 +08:00
parent c90b145b5a 7a24c09cdf
commit 4a0e751f42
35 changed files with 1324 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libchinese-segmentation/chinese-segmentation.cpp
View File

@ -58,7 +58,7 @@ QVector<SKeyWord> ChineseSegmentation::callSegement(std::string s) {
// str.squeeze();
const size_t topk = -1;
std::vector<cppjieba::KeywordExtractor::Word> keywordres;
std::vector<cppjieba::KeyWord> keywordres;
ChineseSegmentation::m_jieba->extractor.Extract(s, keywordres, topk);
std::string().swap(s);
QVector<SKeyWord> vecNeeds;
@ -72,16 +72,16 @@ QVector<SKeyWord> ChineseSegmentation::callSegement(std::string s) {
}
std::vector<cppjieba::KeywordExtractor::Word> ChineseSegmentation::callSegementStd(const std::string &str) {
std::vector<cppjieba::KeyWord> ChineseSegmentation::callSegementStd(const std::string &str) {
const size_t topk = -1;
std::vector<cppjieba::KeywordExtractor::Word> keywordres;
std::vector<cppjieba::KeyWord> keywordres;
ChineseSegmentation::m_jieba->extractor.Extract(str, keywordres, topk);
return keywordres;
}
void ChineseSegmentation::convert(std::vector<cppjieba::KeywordExtractor::Word> &keywordres, QVector<SKeyWord> &kw) {
void ChineseSegmentation::convert(std::vector<cppjieba::KeyWord> &keywordres, QVector<SKeyWord> &kw) {
for(auto i : keywordres) {
SKeyWord temp;
temp.word = i.word;

4
libchinese-segmentation/chinese-segmentation.h
View File

@ -50,8 +50,8 @@ public:
QVector<SKeyWord> callSegement(std::string s);
//新添加callSegementStd函数，修改返回值为std：：vector<cppjieba::KeywordExtractor::Word>并简化内部处理流程--jxx20210517
//修改函数入参形式为引用，去掉Qstring与std::string转换代码--jxx20210519
std::vector<cppjieba::KeywordExtractor::Word> callSegementStd(const std::string& str);
void convert(std::vector<cppjieba::KeywordExtractor::Word>& keywordres, QVector<SKeyWord>& kw);
std::vector<cppjieba::KeyWord> callSegementStd(const std::string& str);
void convert(std::vector<cppjieba::KeyWord>& keywordres, QVector<SKeyWord>& kw);
private:
static QMutex m_mutex;
cppjieba::Jieba *m_jieba;

3
libchinese-segmentation/cppjieba/FullSegment.hpp
View File

@ -47,7 +47,10 @@ public:
size_t) const override {
}
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, unordered_map<string, KeyWord>& res, bool hmm,
size_t) const override {
}
private:
const DictTrie* dictTrie_;
};

15
libchinese-segmentation/cppjieba/HMMSegment.hpp
View File

@ -21,7 +21,7 @@ public:
RuneStrArray::const_iterator right = begin;
while (right != end) {
if (right->rune < 0x80) {
if (right->rune < 0x80) { //asc码
if (left != right) {
InternalCut(left, right, res);
}
@ -29,13 +29,13 @@ public:
left = right;
do {
right = SequentialLetterRule(left, end);
right = SequentialLetterRule(left, end);//非英文字符则返回left，否则返回left后非英文字母的位置
if (right != left) {
break;
}
right = NumbersRule(left, end);
right = NumbersRule(left, end);//非数字则返回left，否则返回left后非数字的位置
if (right != left) {
break;
@ -61,7 +61,10 @@ public:
size_t) const override {
}
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, unordered_map<string, KeyWord>& res, bool hmm,
size_t) const override {
}
private:
// sequential letters rule
RuneStrArray::const_iterator SequentialLetterRule(RuneStrArray::const_iterator begin,
@ -135,8 +138,10 @@ private:
size_t now, old, stat;
double tmp, endE, endS;
vector<int> path(XYSize);
vector<double> weight(XYSize);
//vector<int> path(XYSize);
//vector<double> weight(XYSize);
int path[XYSize];
double weight[XYSize];
//start
for (size_t y = 0; y < Y; y++) {

4
libchinese-segmentation/cppjieba/Jieba.hpp
View File

@ -18,9 +18,9 @@ public:
model_(model_path),
mp_seg_(&dict_trie_),
hmm_seg_(&model_),
mix_seg_(&dict_trie_, &model_),
mix_seg_(&dict_trie_, &model_, stopWordPath),
full_seg_(&dict_trie_),
query_seg_(&dict_trie_, &model_),
query_seg_(&dict_trie_, &model_, stopWordPath),
extractor(&dict_trie_, &model_, idfPath, stopWordPath){ }
~Jieba() { }

68
libchinese-segmentation/cppjieba/KeywordExtractor.hpp
View File

@ -1,7 +1,6 @@
#pragma once
#include <cmath>
#include <set>
#include "MixSegment.hpp"
namespace cppjieba {
@ -12,25 +11,24 @@ using namespace std;
/*utf8*/
class KeywordExtractor {
public:
struct Word {
string word;
vector<size_t> offsets;
double weight;
}; // struct Word
// struct Word {
// string word;
// vector<size_t> offsets;
// double weight;
// }; // struct Word
KeywordExtractor(const DictTrie* dictTrie,
const HMMModel* model,
const string& idfPath,
const string& stopWordPath)
: segment_(dictTrie, model) {
: segment_(dictTrie, model, stopWordPath) {
LoadIdfDict(idfPath);
LoadStopWordDict(stopWordPath);
}
~KeywordExtractor() {
}
void Extract(const string& sentence, vector<string>& keywords, size_t topN) const {
vector<Word> topWords;
vector<KeyWord> topWords;
Extract(sentence, topWords, topN);
for (size_t i = 0; i < topWords.size(); i++) {
@ -39,7 +37,7 @@ public:
}
void Extract(const string& sentence, vector<pair<string, double> >& keywords, size_t topN) const {
vector<Word> topWords;
vector<KeyWord> topWords;
Extract(sentence, topWords, topN);
for (size_t i = 0; i < topWords.size(); i++) {
@ -47,34 +45,24 @@ public:
}
}
void Extract(const string& sentence, vector<Word>& keywords, size_t topN) const {
vector<string> words;
segment_.CutToStr(sentence, words);//将字符串string分解为words放入vector
void Extract(const string& sentence, vector<KeyWord>& keywords, size_t topN) const {
map<string, Word> wordmap;//插入字符串与Word的map，相同string统计词频叠加权重
size_t offset = 0;
for (size_t i = 0; i < words.size(); ++i) {
size_t t = offset;
offset += words[i].size();
if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
unordered_map<string, KeyWord> wordmap;//插入字符串与Word的map，相同string统计词频叠加权重
PreFilter pre_filter(symbols_, sentence);
RuneStrArray::const_iterator null_p;
WordRange range(null_p, null_p);
bool isNull(false);
while (pre_filter.Next(range, isNull)) {
if (isNull) {
continue;
}
wordmap[words[i]].offsets.push_back(t);
wordmap[words[i]].weight += 1.0;
}
if (offset != sentence.size()) {
XLOG(ERROR) << "words illegal";
return;
segment_.CutToStr(sentence, range, wordmap);
}
keywords.clear();
keywords.reserve(wordmap.size());
for (map<string, Word>::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) {
for (unordered_map<string, KeyWord>::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) {
unordered_map<string, double>::const_iterator cit = idfMap_.find(itr->first);//IDF词典查找
if (cit != idfMap_.end()) {
@ -129,22 +117,8 @@ private:
idfAverage_ = idfSum / lineno;
assert(idfAverage_ > 0.0);
}
void LoadStopWordDict(const string& filePath) {
ifstream ifs(filePath.c_str());
if(not ifs.is_open()){
return ;
}
XCHECK(ifs.is_open()) << "open " << filePath << " failed";
string line ;
while (getline(ifs, line)) {
stopWords_.insert(line);
}
assert(stopWords_.size());
}
static bool Compare(const Word& lhs, const Word& rhs) {
static bool Compare(const KeyWord& lhs, const KeyWord& rhs) {
return lhs.weight > rhs.weight;
}
@ -152,10 +126,10 @@ private:
unordered_map<string, double> idfMap_;
double idfAverage_;
unordered_set<string> stopWords_;
unordered_set<Rune> symbols_;
}; // class KeywordExtractor
inline ostream& operator << (ostream& os, const KeywordExtractor::Word& word) {
inline ostream& operator << (ostream& os, const KeyWord& word) {
return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight <<
"}";
}

6
libchinese-segmentation/cppjieba/MPSegment.hpp
View File

@ -32,7 +32,10 @@ public:
size_t) const override {
}
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, unordered_map<string, KeyWord>& res, bool hmm,
size_t) const override {
}
const DictTrie* GetDictTrie() const override {
return dictTrie_;
}
@ -46,13 +49,14 @@ public:
}
private:
void CalcDP(vector<DatDag>& dags) const {
double val(0);
for (auto rit = dags.rbegin(); rit != dags.rend(); rit++) {
rit->max_next = -1;
rit->max_weight = MIN_DOUBLE;
for (const auto & it : rit->nexts) {
const auto nextPos = it.first;
double val = dictTrie_->GetMinWeight();
val = dictTrie_->GetMinWeight();
if (nullptr != it.second) {
val = it.second->weight;

97
libchinese-segmentation/cppjieba/MixSegment.hpp
View File

@ -9,8 +9,11 @@
namespace cppjieba {
class MixSegment: public SegmentTagged {
public:
MixSegment(const DictTrie* dictTrie, const HMMModel* model)
MixSegment(const DictTrie* dictTrie,
const HMMModel* model,
const string& stopWordPath)
: mpSeg_(dictTrie), hmmSeg_(model) {
LoadStopWordDict(stopWordPath);
}
~MixSegment() {}
@ -81,16 +84,20 @@ public:
for (size_t i = 0; i < words.size(); i++) {
//if mp Get a word, it's ok, put it into result
if (words[i].left != words[i].right || (words[i].left == words[i].right &&
mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune))) {
if (words[i].left != words[i].right) {
res.push_back(GetStringFromRunes(s, words[i].left, words[i].right));
continue;
}
if (mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune)
|| i == (words.size() - 1)) {//i++后如果是最后一个字符则直接push_back
res.push_back(GetStringFromRunes(s, words[i].left, words[i].right));
continue;
}
// if mp Get a single one and it is not in userdict, collect it in sequence
size_t j = i;
size_t j = i + 1; //当前i字符为单独的字符并且不在用户字典里（i字符不是最后一个字符），直接判定j字符
while (j < words.size() && words[j].left == words[j].right &&
while (j < (words.size() - 1) && words[j].left == words[j].right &&
!mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) {
j++;
}
@ -113,6 +120,70 @@ public:
}
}
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, unordered_map<string, KeyWord>& res, bool hmm,
size_t) const override {
vector<WordRange> words;
assert(end >= begin);
words.reserve(end - begin);
mpSeg_.CutRuneArray(begin, end, words);
vector<WordRange> hmmRes;
hmmRes.reserve(end - begin);
for (size_t i = 0; i < words.size(); i++) {
string str = GetStringFromRunes(s, words[i].left, words[i].right);
if (stopWords_.find(str) != stopWords_.end()) {
continue;
}
if (words[i].left != words[i].right) {
res[str].offsets.push_back(words[i].left->offset);
res[str].weight += 1.0;
continue;
}
if (mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune)
|| i == (words.size() - 1)) {//i++后如果是最后一个字符则直接push_back
if (stopWords_.find(str) != stopWords_.end()) {
continue;
}
res[str].offsets.push_back(words[i].left->offset);
res[str].weight += 1.0;
continue;
}
// if mp Get a single one and it is not in userdict, collect it in sequence
size_t j = i + 1; //当前i字符为单独的字符并且不在用户字典里（i字符不是最后一个字符），直接判定j字符
while (j < (words.size() - 1) && words[j].left == words[j].right &&
!mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) {
j++;
}
// Cut the sequence with hmm
assert(j - 1 >= i);
// TODO
hmmSeg_.CutRuneArray(words[i].left, words[j - 1].left + 1, hmmRes);
//put hmm result to result
for (size_t k = 0; k < hmmRes.size(); k++) {
string hmmStr = GetStringFromRunes(s, hmmRes[k].left, hmmRes[k].right);
if (IsSingleWord(hmmStr) || stopWords_.find(hmmStr) != stopWords_.end()) {
continue;
}
res[hmmStr].offsets.push_back(hmmRes[k].left->offset);
res[hmmStr].weight += 1.0;
}
//clear tmp vars
hmmRes.clear();
//let i jump over this piece
i = j - 1;
}
}
const DictTrie* GetDictTrie() const override {
return mpSeg_.GetDictTrie();
}
@ -125,7 +196,23 @@ public:
return tagger_.LookupTag(str, *this);
}
void LoadStopWordDict(const string& filePath) {
ifstream ifs(filePath.c_str());
if(not ifs.is_open()){
return ;
}
XCHECK(ifs.is_open()) << "open " << filePath << " failed";
string line ;
while (getline(ifs, line)) {
stopWords_.insert(line);
}
assert(stopWords_.size());
}
private:
unordered_set<string> stopWords_;
MPSegment mpSeg_;
HMMSegment hmmSeg_;
PosTagger tagger_;

67
libchinese-segmentation/cppjieba/PreFilter.hpp
View File

@ -22,6 +22,73 @@ public:
bool HasNext() const {
return cursor_ != sentence_.end();
}
bool Next(WordRange& wordRange) {
if (cursor_ == sentence_.end()) {
return false;
}
wordRange.left = cursor_;
while (cursor_->rune == 0x20 && cursor_ != sentence_.end()) {
cursor_++;
}
if (cursor_ == sentence_.end()) {
wordRange.right = cursor_;
return true;
}
while (++cursor_ != sentence_.end()) {
if (cursor_->rune == 0x20) {
wordRange.right = cursor_;
return true;
}
}
wordRange.right = sentence_.end();
return true;
}
bool Next(WordRange& wordRange, bool& isNull) {
isNull = false;
if (cursor_ == sentence_.end()) {
return false;
}
wordRange.left = cursor_;
if (cursor_->rune == 0x20) {
while (cursor_ != sentence_.end()) {
if (cursor_->rune != 0x20) {
if (wordRange.left == cursor_) {
cursor_ ++;
}
wordRange.right = cursor_;
isNull = true;
return true;
}
cursor_ ++;
}
}
while (cursor_ != sentence_.end()) {
if (cursor_->rune == 0x20) {
if (wordRange.left == cursor_) {
cursor_ ++;
}
wordRange.right = cursor_;
return true;
}
cursor_ ++;
}
wordRange.right = sentence_.end();
return true;
}
WordRange Next() {
WordRange range(cursor_, cursor_);

9
libchinese-segmentation/cppjieba/QuerySegment.hpp
View File

@ -14,8 +14,10 @@
namespace cppjieba {
class QuerySegment: public SegmentBase {
public:
QuerySegment(const DictTrie* dictTrie, const HMMModel* model)
: mixSeg_(dictTrie, model), trie_(dictTrie) {
QuerySegment(const DictTrie* dictTrie,
const HMMModel* model,
const string& stopWordPath)
: mixSeg_(dictTrie, model, stopWordPath), trie_(dictTrie) {
}
~QuerySegment() {
}
@ -59,7 +61,10 @@ public:
size_t) const override {
}
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, unordered_map<string, KeyWord>& res, bool hmm,
size_t) const override {
}
private:
bool IsAllAscii(const RuneArray& s) const {
for (size_t i = 0; i < s.size(); i++) {

21
libchinese-segmentation/cppjieba/SegmentBase.hpp
View File

@ -23,23 +23,28 @@ public:
//添加基于sentence的cut方法，减少中间变量的存储与格式转换--jxx20210517
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<string>& res, bool hmm,
size_t max_word_len) const = 0;
virtual void CutWithSentence(const string& s, RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, unordered_map<string, KeyWord>& res, bool hmm,
size_t max_word_len) const = 0;
//重写CutToStr函数，简化获取vector<string>& words的流程，降低内存占用--jxx20210517
void CutToStr(const string& sentence, vector<string>& words, bool hmm = true,
size_t max_word_len = MAX_WORD_LENGTH) const {
/*
vector<Word> tmp;
CutToWord(sentence, tmp, hmm, max_word_len);
GetStringsFromWords(tmp, words);
*/
PreFilter pre_filter(symbols_, sentence);
words.clear();
words.reserve(sentence.size() / 2);//todo 参考源码，参数待定
while (pre_filter.HasNext()) {
auto range = pre_filter.Next();
RuneStrArray::const_iterator null_p;
WordRange range(null_p, null_p);
while (pre_filter.Next(range)) {
CutWithSentence(sentence, range.left, range.right, words, hmm, max_word_len);
}
}
void CutToStr(const string& sentence, WordRange range, vector<string>& words, bool hmm = true,
size_t max_word_len = MAX_WORD_LENGTH) const {
CutWithSentence(sentence, range.left, range.right, words, hmm, max_word_len);
}
void CutToStr(const string& sentence, WordRange range, unordered_map<string, KeyWord>& words, bool hmm = true,
size_t max_word_len = MAX_WORD_LENGTH) const {
CutWithSentence(sentence, range.left, range.right, words, hmm, max_word_len);
}
void CutToWord(const string& sentence, vector<Word>& words, bool hmm = true,
size_t max_word_len = MAX_WORD_LENGTH) const {
PreFilter pre_filter(symbols_, sentence);

23
libchinese-segmentation/cppjieba/Unicode.hpp
View File

@ -15,6 +15,12 @@ using std::vector;
typedef uint32_t Rune;
struct KeyWord {
string word;
vector<size_t> offsets;
double weight;
}; // struct Word
struct Word {
string word;
uint32_t offset;
@ -63,7 +69,7 @@ struct WordRange {
: left(l), right(r) {
}
size_t Length() const {
return right - left + 1;
return right - left;
}
bool IsAllAscii() const {
@ -113,11 +119,13 @@ inline bool DecodeRunesInString(const string& s, RuneStrArray& runes) {
uint32_t tmp;
uint32_t offset = 0;
runes.clear();
for(size_t i = 0; i < s.size();) {
if(!(s.data()[i] & 0x80)) { // 0xxxxxxx
uint32_t len(0);
for (size_t i = 0; i < s.size();) {
if (!(s.data()[i] & 0x80)) { // 0xxxxxxx
// 7bit, total 7bit
tmp = (uint8_t)(s.data()[i]) & 0x7f;
i++;
len = 1;
} else if ((uint8_t)s.data()[i] <= 0xdf && i + 1 < s.size()) { // 110xxxxxx
// 5bit, total 5bit
tmp = (uint8_t)(s.data()[i]) & 0x1f;
@ -126,6 +134,7 @@ inline bool DecodeRunesInString(const string& s, RuneStrArray& runes) {
tmp <<= 6;
tmp |= (uint8_t)(s.data()[i+1]) & 0x3f;
i += 2;
len = 2;
} else if((uint8_t)s.data()[i] <= 0xef && i + 2 < s.size()) { // 1110xxxxxx
// 4bit, total 4bit
tmp = (uint8_t)(s.data()[i]) & 0x0f;
@ -139,6 +148,7 @@ inline bool DecodeRunesInString(const string& s, RuneStrArray& runes) {
tmp |= (uint8_t)(s.data()[i+2]) & 0x3f;
i += 3;
len = 3;
} else if((uint8_t)s.data()[i] <= 0xf7 && i + 3 < s.size()) { // 11110xxxx
// 3bit, total 3bit
tmp = (uint8_t)(s.data()[i]) & 0x07;
@ -156,10 +166,10 @@ inline bool DecodeRunesInString(const string& s, RuneStrArray& runes) {
tmp |= (uint8_t)(s.data()[i+3]) & 0x3f;
i += 4;
len = 4;
} else {
return false;
}
uint32_t len = limonp::UnicodeToUtf8Bytes(tmp);
RuneInfo x(tmp, offset, len, i, 1);
runes.push_back(x);
offset += len;
@ -241,9 +251,8 @@ inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left,
inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
assert(right->offset >= left->offset);
uint32_t len = right->offset - left->offset + right->len;
uint32_t unicode_length = right->unicode_offset - left->unicode_offset + right->unicode_length;
return Word(s.substr(left->offset, len), left->offset, left->unicode_offset, unicode_length).word;
//uint32_t len = right->offset - left->offset + right->len;
return s.substr(left->offset, right->offset - left->offset + right->len);
}
inline void GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs, vector<Word>& words) {

19
libsearch/common.h
View File

@ -1,7 +1,22 @@
#ifndef COMMON_H
#define COMMON_H
#include <QMap>
#define UKUI_SEARCH_PIPE_PATH (QDir::homePath()+"/.config/org.ukui/ukui-search/ukuisearch").toLocal8Bit().constData()
#define HOME_PATH QDir::homePath()
static const QMap<QString, bool> targetFileTypeMap = {
std::map<QString, bool>::value_type("doc", true),
std::map<QString, bool>::value_type("docx", true),
std::map<QString, bool>::value_type("ppt", true),
std::map<QString, bool>::value_type("pptx", true),
std::map<QString, bool>::value_type("xls", true),
std::map<QString, bool>::value_type("xlsx", true),
std::map<QString, bool>::value_type("txt", true),
std::map<QString, bool>::value_type("dot", true),
std::map<QString, bool>::value_type("wps", true),
std::map<QString, bool>::value_type("pps", true),
std::map<QString, bool>::value_type("dps", true),
std::map<QString, bool>::value_type("et", true),
std::map<QString, bool>::value_type("pdf", true)
};
//TODO Put things that needed to be put here here.
#endif // COMMON_H

2
libsearch/file-utils.cpp
View File

@ -517,7 +517,7 @@ void FileUtils::getDocxTextContent(QString &path, QString &textcontent) {
QDomElement wr = wp.firstChildElement("w:r");
while(!wr.isNull()) {
QDomElement wt = wr.firstChildElement("w:t");
textcontent.append(wt.text().replace("\n", ""));
textcontent.append(wt.text().replace("\n", "")).replace("\r", " ");
if(textcontent.length() >= MAX_CONTENT_LENGTH / 3) {
file.close();
return;

24
libsearch/index/construct-document.cpp
View File

@ -34,8 +34,8 @@ ConstructDocumentForPath::ConstructDocumentForPath(QVector<QString> list) {
void ConstructDocumentForPath::run() {
// qDebug()<<"ConstructDocumentForPath";
if(!Zeeker::_doc_list_path)
Zeeker::_doc_list_path = new QList<Document>;
// if(!Zeeker::_doc_list_path)
// Zeeker::_doc_list_path = new QVector<Document>;
// qDebug()<<_doc_list_path->size();
QString index_text = m_list.at(0).toLower();
QString sourcePath = m_list.at(1);
@ -87,9 +87,9 @@ void ConstructDocumentForPath::run() {
}
// QMetaObject::invokeMethod(m_indexGenerator,"appendDocListPath",Q_ARG(Document,doc));
Zeeker::_mutex_doc_list_path.lock();
Zeeker::_doc_list_path->append(doc);
Zeeker::_mutex_doc_list_path.unlock();
IndexGenerator::_mutex_doc_list_path.lock();
IndexGenerator::_doc_list_path.append(doc);
IndexGenerator::_mutex_doc_list_path.unlock();
// qDebug()<<"ConstructDocumentForPath finish";
return;
}
@ -102,8 +102,8 @@ ConstructDocumentForContent::ConstructDocumentForContent(QString path) {
void ConstructDocumentForContent::run() {
// qDebug() << "ConstructDocumentForContent currentThreadId()" << QThread::currentThreadId();
// 构造文本索引的document
if(!Zeeker::_doc_list_content)
Zeeker::_doc_list_content = new QList<Document>;
// if(!Zeeker::_doc_list_content)
// Zeeker::_doc_list_content = new QVector<Document>;
QString content;
FileReader::getTextContent(m_path, content);
if(content.isEmpty())
@ -120,18 +120,18 @@ void ConstructDocumentForContent::run() {
content = content.replace("\t", " ").replace("\xEF\xBC\x8C", " ").replace("\xE3\x80\x82", " ");
// QVector<SKeyWord> term = ChineseSegmentation::getInstance()->callSegement(content.left(20480000));
//修改函数返回类型，修改入参为std::string引用--jxx20210519
std::vector<cppjieba::KeywordExtractor::Word> term = ChineseSegmentation::getInstance()->callSegementStd(content.left(20480000).toStdString());
std::vector<cppjieba::KeyWord> term = ChineseSegmentation::getInstance()->callSegementStd(content.left(20480000).toStdString());
for(size_t i = 0; i < term.size(); ++i) {
doc.addPosting(term.at(i).word, term.at(i).offsets, static_cast<int>(term.at(i).weight));
}
Zeeker::_mutex_doc_list_content.lock();
Zeeker::_doc_list_content->append(doc);
Zeeker::_mutex_doc_list_content.unlock();
IndexGenerator::_mutex_doc_list_content.lock();
IndexGenerator::_doc_list_content.append(doc);
IndexGenerator::_mutex_doc_list_content.unlock();
content.clear();
content.squeeze();
term.clear();
term.shrink_to_fit();
return;
}

3
libsearch/index/file-reader.cpp
View File

@ -31,9 +31,8 @@ void FileReader::getTextContent(QString path, QString &textContent) {
QFileInfo file(path);
QString strsfx = file.suffix();
if(name == "application/zip") {
if(strsfx.endsWith("docx")){
if(strsfx.endsWith("docx"))
FileUtils::getDocxTextContent(path, textContent);
}
if(strsfx.endsWith("pptx"))
FileUtils::getPptxTextContent(path, textContent);
if(strsfx.endsWith("xlsx"))

24
libsearch/index/first-index.cpp
View File

@ -49,7 +49,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
this->q_index->enqueue(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0"));
if((fileInfo.fileName().split(".", QString::SkipEmptyParts).length() > 1) && (true == targetFileTypeMap[fileInfo.fileName().split(".").last()])) {
//this->q_content_index->enqueue(fileInfo.absoluteFilePath());
if(fileInfo.fileName().split(".").last() == "docx"){
if (fileInfo.fileName().split(".").last() == "docx") {
QuaZip file(fileInfo.absoluteFilePath());
if(!file.open(QuaZip::mdUnzip))
return;
@ -57,10 +57,8 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
return;
QuaZipFile fileR(&file);
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//docx解压缩后的xml文件为实际需要解析文件大小
qDebug() << "文件路径:" <<fileInfo.absoluteFilePath();
qDebug() << "文件大小:" << fileR.usize();
file.close();
}else if(fileInfo.fileName().split(".").last() == "pptx"){
} else if (fileInfo.fileName().split(".").last() == "pptx") {
QuaZip file(fileInfo.absoluteFilePath());
if(!file.open(QuaZip::mdUnzip))
return;
@ -79,10 +77,8 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
}
}
file.close();
qDebug() << "文件路径:" <<fileInfo.absoluteFilePath();
qDebug() << "文件大小:" << fileSize;
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileSize));//pptx解压缩后的xml文件为实际需要解析文件大小
}else if(fileInfo.fileName().split(".").last() == "xlsx"){
} else if (fileInfo.fileName().split(".").last() == "xlsx") {
QuaZip file(fileInfo.absoluteFilePath());
if(!file.open(QuaZip::mdUnzip))
return;
@ -90,10 +86,8 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
return;
QuaZipFile fileR(&file);
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//xlsx解压缩后的xml文件为实际解析文件大小
qDebug() << "文件路径:" <<fileInfo.absoluteFilePath();
qDebug() << "文件大小:" << fileR.usize();
file.close();
}else{
} else {
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size()));
}
}
@ -225,9 +219,17 @@ void FirstIndex::run() {
// for (size_t i = 0; (i < this->u_send_length) && (!this->q_content_index->empty()); ++i){
qint64 fileSize = 0;
//修改一次处理的数据量，从30个文件改为文件总大小为50M以下，50M为暂定值--jxx20210519
for(size_t i = 0;/* (i < 30) && */(fileSize < 50*1024*1024) && (!this->q_content_index->empty()); ++i) {
for(size_t i = 0;/* (i < 30) && (fileSize < 52428800) && */(!this->q_content_index->empty()); ++i) {
QPair<QString,qint64> tempPair = this->q_content_index->dequeue();
fileSize += tempPair.second;
if (fileSize > 52428800 ) {
if (tmp2->size() == 0) {
tmp2->enqueue(tempPair.first);
break;
}
this->q_content_index->enqueue(tempPair);
break;
}
tmp2->enqueue(tempPair.first);
}
// qDebug() << ">>>>>>>>all fileSize:" << fileSize << "file num:" << tmp->size() << "<<<<<<<<<<<<<<<<<<<";

190
libsearch/index/index-generator.cpp
View File

@ -28,7 +28,6 @@
#include "file-utils.h"
#include "index-generator.h"
#include "chinese-segmentation.h"
#include "construct-document.h"
#include <QStandardPaths>
@ -41,10 +40,14 @@ using namespace Zeeker;
static IndexGenerator *global_instance = nullptr;
QMutex IndexGenerator::m_mutex;
QList<Document> *Zeeker::_doc_list_path;
QMutex Zeeker::_mutex_doc_list_path;
QList<Document> *Zeeker::_doc_list_content;
QMutex Zeeker::_mutex_doc_list_content;
//QVector<Document> *Zeeker::_doc_list_path;
//QMutex Zeeker::_mutex_doc_list_path;
//QVector<Document> *Zeeker::_doc_list_content;
//QMutex Zeeker::_mutex_doc_list_content;
QMutex IndexGenerator::_mutex_doc_list_path;
QMutex IndexGenerator::_mutex_doc_list_content;
QVector<Document> IndexGenerator::_doc_list_path = QVector<Document>();
QVector<Document> IndexGenerator::_doc_list_content = QVector<Document>();
IndexGenerator *IndexGenerator::getInstance(bool rebuild, QObject *parent) {
QMutexLocker locker(&m_mutex);
@ -63,45 +66,33 @@ bool IndexGenerator::setIndexdataPath() {
//文件名索引
bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) {
// FileUtils::_index_status |= 0x1;
// qDebug() << messageList->size();
HandlePathList(messageList);
if(_doc_list_path == NULL) {
// if(_doc_list_path == NULL) {
// return false;
// }
if(IndexGenerator::_doc_list_path.isEmpty()) {
return false;
}
qDebug() << "begin creatAllIndex";
// GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "0");
try {
// m_indexer = new Xapian::TermGenerator();
// m_indexer.set_database(*m_database_path);
//可以实现拼写纠正
// m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING);
// m_indexer.set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
// int count =0;
for(auto i : *_doc_list_path) {
for(auto i : IndexGenerator::_doc_list_path) {
insertIntoDatabase(i);
// if(++count > 8999){
// count = 0;
// m_database_path->commit();
// }
}
m_database_path->commit();
} catch(const Xapian::Error &e) {
qWarning() << "creatAllIndex fail!" << QString::fromStdString(e.get_description());
//need a record
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1");
// FileUtils::_index_status &= ~0x1;
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1");
assert(false);
}
// GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
qDebug() << "finish creatAllIndex";
// FileUtils::_index_status &= ~0x1;
_doc_list_path->clear();
delete _doc_list_path;
_doc_list_path = nullptr;
IndexGenerator::_doc_list_path.clear();
IndexGenerator::_doc_list_path.squeeze();
QVector<Document>().swap(IndexGenerator::_doc_list_path);
// delete _doc_list_path;
// _doc_list_path = nullptr;
return true;
}
//文件内容索引
@ -109,16 +100,19 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList) {
// FileUtils::_index_status |= 0x2;
HandlePathList(messageList);
qDebug() << "begin creatAllIndex for content";
if(_doc_list_content == NULL) {
// if(_doc_list_content == NULL) {
// return false;
// }
if(IndexGenerator::_doc_list_content.isEmpty()) {
return false;
}
int size = _doc_list_content->size();
int size = IndexGenerator::_doc_list_content.size();
qDebug() << "begin creatAllIndex for content" << size;
if(!size == 0) {
// GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "0");
try {
int count = 0;
for(auto i : *_doc_list_content) {
for(auto i : IndexGenerator::_doc_list_content) {
insertIntoContentDatabase(i);
if(++count > 999) {
count = 0;
@ -135,9 +129,11 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList) {
// GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2");
// FileUtils::_index_status &= ~0x2;
qDebug() << "finish creatAllIndex for content";
_doc_list_content->clear();
delete _doc_list_content;
_doc_list_content = nullptr;
IndexGenerator::_doc_list_content.clear();
IndexGenerator::_doc_list_content.squeeze();
QVector<Document>().swap(IndexGenerator::_doc_list_content);
// delete _doc_list_content;
// _doc_list_content = nullptr;
}
Q_EMIT this->transactionFinished();
return true;
@ -299,7 +295,7 @@ void IndexGenerator::HandlePathList(QQueue<QString> *messageList) {
return;
}
//deprecated
Document IndexGenerator::GenerateDocument(const QVector<QString> &list) {
Document doc;
// qDebug()<<QString::number(quintptr(QThread::currentThreadId()));
@ -344,7 +340,7 @@ Document IndexGenerator::GenerateDocument(const QVector<QString> &list) {
return doc;
}
//deprecated
Document IndexGenerator::GenerateContentDocument(const QString &path) {
// 构造文本索引的document
QString content;
@ -391,7 +387,7 @@ bool IndexGenerator::isIndexdataExist() {
}
//deprecated
QStringList IndexGenerator::IndexSearch(QString indexText) {
QStringList searchResult;
try {
@ -457,96 +453,60 @@ QStringList IndexGenerator::IndexSearch(QString indexText) {
return searchResult;
}
//void IndexGenerator::setSynonym()
//{
// try
// {
// m_database_path->add_synonym("a","A");
// m_database_path->add_synonym("b","B");
// m_database_path->add_synonym("c","C");
// m_database_path->add_synonym("d","D");
// m_database_path->add_synonym("e","A");
// m_database_path->add_synonym("f","F");
// m_database_path->add_synonym("g","G");
// m_database_path->add_synonym("h","H");
// m_database_path->add_synonym("i","I");
// m_database_path->add_synonym("j","J");
// m_database_path->add_synonym("k","K");
// m_database_path->add_synonym("l","L");
// m_database_path->add_synonym("m","M");
// m_database_path->add_synonym("n","N");
// m_database_path->add_synonym("o","O");
// m_database_path->add_synonym("p","P");
// m_database_path->add_synonym("q","Q");
// m_database_path->add_synonym("r","R");
// m_database_path->add_synonym("s","S");
// m_database_path->add_synonym("t","T");
// m_database_path->add_synonym("u","U");
// m_database_path->add_synonym("v","V");
// m_database_path->add_synonym("w","W");
// m_database_path->add_synonym("x","X");
// m_database_path->add_synonym("y","Y");
// m_database_path->add_synonym("z","Z");
// m_database_path->add_synonym("A","a");
// m_database_path->add_synonym("B","b");
// m_database_path->add_synonym("C","c");
// m_database_path->add_synonym("D","d");
// m_database_path->add_synonym("E","e");
// m_database_path->add_synonym("F","f");
// m_database_path->add_synonym("G","g");
// m_database_path->add_synonym("H","h");
// m_database_path->add_synonym("I","i");
// m_database_path->add_synonym("J","j");
// m_database_path->add_synonym("K","k");
// m_database_path->add_synonym("L","a");
// m_database_path->add_synonym("M","m");
// m_database_path->add_synonym("N","n");
// m_database_path->add_synonym("O","o");
// m_database_path->add_synonym("P","p");
// m_database_path->add_synonym("Q","q");
// m_database_path->add_synonym("R","r");
// m_database_path->add_synonym("S","s");
// m_database_path->add_synonym("T","t");
// m_database_path->add_synonym("U","u");
// m_database_path->add_synonym("V","v");
// m_database_path->add_synonym("W","w");
// m_database_path->add_synonym("X","x");
// m_database_path->add_synonym("Y","y");
// m_database_path->add_synonym("Z","z");
// m_database_path->commit();
// }
// catch(const Xapian::Error &e)
// {
// qWarning() <<QString::fromStdString(e.get_description());
// }
//}
bool IndexGenerator::deleteAllIndex(QStringList *pathlist) {
QStringList *list = pathlist;
if(list->isEmpty())
return true;
for(int i = 0; i < list->size(); i++) {
QString doc = list->at(i);
std::string uniqueterm = FileUtils::makeDocUterm(doc);
try {
try {
for(int i = 0; i < list->size(); i++) {
QString doc = list->at(i);
std::string uniqueterm = FileUtils::makeDocUterm(doc);
qDebug() << "--delete start--";
m_database_path->delete_document(uniqueterm);
m_database_content->delete_document(uniqueterm);
qDebug() << "delete path" << doc;
qDebug() << "delete md5" << QString::fromStdString(uniqueterm);
m_database_path->commit();
m_database_content->commit();
qDebug() << "--delete finish--";
// qDebug()<<"m_database_path->get_lastdocid()!!!"<<m_database_path->get_lastdocid();
// qDebug()<<"m_database_path->get_doccount()!!!"<<m_database_path->get_doccount();
} catch(const Xapian::Error &e) {
qWarning() << QString::fromStdString(e.get_description());
return false;
// qDebug()<<"m_database_path->get_lastdocid()!!!"<<m_database_path->get_lastdocid();
// qDebug()<<"m_database_path->get_doccount()!!!"<<m_database_path->get_doccount();
}
m_database_path->commit();
m_database_content->commit();
} catch(const Xapian::Error &e) {
qWarning() << QString::fromStdString(e.get_description());
return false;
}
Q_EMIT this->transactionFinished();
return true;
}
bool IndexGenerator::updateIndex(QVector<PendingFile> *pendingFiles)
{
QQueue<QVector<QString>> *fileIndexInfo = new QQueue<QVector<QString>>;
QQueue<QString> *fileContentIndexInfo = new QQueue<QString>;
QStringList *deleteList = new QStringList;
for(PendingFile file : *pendingFiles) {
if(file.shouldRemoveIndex()) {
deleteList->append(file.path());
continue;
}
fileIndexInfo->append(QVector<QString>() << file.path().section("/" , -1) << file.path() << QString(file.isDir() ? "1" : "0"));
if((!file.path().split(".").isEmpty()) && (true == targetFileTypeMap[file.path().section("/" , -1) .split(".").last()]))
fileContentIndexInfo->append(file.path());
}
if(!deleteList->isEmpty()) {
deleteAllIndex(deleteList);
}
if(!fileIndexInfo->isEmpty()) {
creatAllIndex(fileIndexInfo);
}
if(!fileContentIndexInfo->isEmpty()) {
creatAllIndex(fileContentIndexInfo);
}
delete fileIndexInfo;
delete fileContentIndexInfo;
return true;
}

19
libsearch/index/index-generator.h
View File

@ -29,18 +29,22 @@
#include <QMutex>
#include <QQueue>
//#include <QMetaObject>
#include "construct-document.h"
#include "index-status-recorder.h"
#include "document.h"
#include "file-reader.h"
#include "common.h"
#include "pending-file.h"
namespace Zeeker {
extern QList<Document> *_doc_list_path;
extern QMutex _mutex_doc_list_path;
extern QList<Document> *_doc_list_content;
extern QMutex _mutex_doc_list_content;
//extern QVector<Document> *_doc_list_path;
//extern QMutex _mutex_doc_list_path;
//extern QVector<Document> *_doc_list_content;
//extern QMutex _mutex_doc_list_content;
class IndexGenerator : public QObject {
friend class ConstructDocumentForPath;
friend class ConstructDocumentForContent;
Q_OBJECT
public:
static IndexGenerator *getInstance(bool rebuild = false, QObject *parent = nullptr);
@ -58,6 +62,7 @@ public Q_SLOTS:
bool creatAllIndex(QQueue<QVector<QString>> *messageList);
bool creatAllIndex(QQueue<QString> *messageList);
bool deleteAllIndex(QStringList *pathlist);
bool updateIndex(QVector<PendingFile> *pendingFiles);
private:
explicit IndexGenerator(bool rebuild = false, QObject *parent = nullptr);
@ -72,8 +77,10 @@ private:
void insertIntoDatabase(Document& doc);
void insertIntoContentDatabase(Document& doc);
// QList<Document> *m_doc_list_path; //for path index
// QList<Document> *m_doc_list_content; // for text content index
static QVector<Document> _doc_list_path;
static QMutex _mutex_doc_list_path;
static QVector<Document> _doc_list_content;
static QMutex _mutex_doc_list_content;
QMap<QString, QStringList> m_index_map;
QString m_index_data_path;
Xapian::WritableDatabase* m_database_path;

2
libsearch/index/index-status-recorder.cpp
View File

@ -13,8 +13,10 @@ IndexStatusRecorder *IndexStatusRecorder::getInstance()
void IndexStatusRecorder::setStatus(const QString &key, const QVariant &value)
{
m_mutex.lock();
m_status->setValue(key, value);
m_status->sync();
m_mutex.unlock();
}
const QVariant IndexStatusRecorder::getStatus(const QString &key)

3
libsearch/index/index-status-recorder.h
View File

@ -4,10 +4,12 @@
#include <QObject>
#include <QSettings>
#include <QDir>
#include <QMutex>
#define CONTENT_INDEX_DATABASE_STATE "content_index_database_state"
#define INDEX_DATABASE_STATE "index_database_state"
#define INOTIFY_NORMAL_EXIT "inotify_normal_exit"
#define INDEX_STATUS "/media/用户保险箱/.ukui-search/ukui-search-index-status.conf"
#define PENDING_FILE_QUEUE_FINISH "pending_file_queue_finish"
namespace Zeeker {
//fixme: we need a better way to record index status.
class IndexStatusRecorder : public QObject
@ -21,6 +23,7 @@ public:
private:
explicit IndexStatusRecorder(QObject *parent = nullptr);
QSettings *m_status;
QMutex m_mutex;
};
}

6
libsearch/index/index.pri
View File

@ -8,6 +8,9 @@ HEADERS += \
$$PWD/index-generator.h \
$$PWD/index-status-recorder.h \
$$PWD/inotify-index.h \
$$PWD/inotify-watch.h \
$$PWD/pending-file-queue.h \
$$PWD/pending-file.h \
$$PWD/search-manager.h \
$$PWD/searchmethodmanager.h \
$$PWD/traverse_bfs.h \
@ -21,6 +24,9 @@ SOURCES += \
$$PWD/index-generator.cpp \
$$PWD/index-status-recorder.cpp \
$$PWD/inotify-index.cpp \
$$PWD/inotify-watch.cpp \
$$PWD/pending-file-queue.cpp \
$$PWD/pending-file.cpp \
$$PWD/search-manager.cpp \
$$PWD/searchmethodmanager.cpp \
$$PWD/traverse_bfs.cpp \

448
libsearch/index/inotify-watch.cpp Normal file
View File

@ -0,0 +1,448 @@
#include "inotify-watch.h"
#include <sys/ioctl.h>
#include <malloc.h>
using namespace Zeeker;
static InotifyWatch* global_instance_InotifyWatch = nullptr;
Zeeker::InotifyWatch *Zeeker::InotifyWatch::getInstance(const QStringList &pathList)
{
if(!global_instance_InotifyWatch) {
global_instance_InotifyWatch = new InotifyWatch(pathList);
}
return global_instance_InotifyWatch;
}
Zeeker::InotifyWatch::InotifyWatch(const QStringList &pathList): Traverse_BFS(pathList)
{
qDebug() << "setInotifyMaxUserWatches start";
UkuiSearchQDBus usQDBus;
usQDBus.setInotifyMaxUserWatches();
qDebug() << "setInotifyMaxUserWatches end";
m_sharedMemory = new QSharedMemory("ukui-search-shared-map", this);
}
InotifyWatch::~InotifyWatch()
{
if(m_notifier)
delete m_notifier;
m_notifier = nullptr;
}
bool InotifyWatch::addWatch(const QString &path)
{
int ret = inotify_add_watch(m_inotifyFd, path.toStdString().c_str(), (IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_DELETE | IN_MODIFY));
if(ret == -1) {
qWarning() << "AddWatch error:" << path;
return false;
}
currentPath[ret] = path;
// qDebug() << "Watch: " << path << "ret: " << ret;
return true;
}
bool InotifyWatch::removeWatch(const QString &path, bool removeFromDatabase)
{
inotify_rm_watch(m_inotifyFd, currentPath.key(path));
if(removeFromDatabase) {
for(QMap<int, QString>::Iterator i = currentPath.begin(); i != currentPath.end();) {
// qDebug() << i.value();
// if(i.value().length() > path.length()) {
if(i.value().startsWith(path)) {
qDebug() << "remove path: " << i.value();
inotify_rm_watch(m_inotifyFd, currentPath.key(path));
PendingFile f(i.value());
f.setDeleted();
f.setIsDir();
PendingFileQueue::getInstance()->enqueue(f);
currentPath.erase(i++);
} else {
i++;
}
}
} else {
for(QMap<int, QString>::Iterator i = currentPath.begin(); i != currentPath.end();) {
// qDebug() << i.value();
if(i.value().length() > path.length()) {
if(i.value().startsWith(path)) {
qDebug() << "remove path: " << i.value();
inotify_rm_watch(m_inotifyFd, currentPath.key(path));
currentPath.erase(i++);
} else {
i++;
}
} else {
i++;
}
}
}
currentPath.remove(currentPath.key(path));
return true;
}
void InotifyWatch::DoSomething(const QFileInfo &info)
{
qDebug() << info.fileName() << "-------" << info.absoluteFilePath();
if(info.isDir() && (!info.isSymLink())) {
this->addWatch(info.absoluteFilePath());
}
PendingFile f(info.absoluteFilePath());
if(info.isDir()) {
f.setIsDir();
}
PendingFileQueue::getInstance()->enqueue(f);
}
void InotifyWatch::firstTraverse()
{
QQueue<QString> bfs;
for(QString path : this->m_pathList) {
this->addWatch(path);
bfs.enqueue(path);
QFileInfoList list;
QDir dir;
dir.setFilter(QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot);
dir.setSorting(QDir::DirsFirst);
while(!bfs.empty()) {
dir.setPath(bfs.dequeue());
list = dir.entryInfoList();
for(auto i : list) {
if(i.isDir() && (!(i.isSymLink()))) {
this->addWatch(i.absoluteFilePath());
bfs.enqueue(i.absoluteFilePath());
}
}
}
}
}
void InotifyWatch::stopWatch()
{
// if(this->isRunning()) {
// this->quit();
// if(m_notifier)
// delete m_notifier;
// m_notifier = nullptr;
// removeWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false);
// }
// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3");
}
void InotifyWatch::run()
{
m_inotifyFd = inotify_init();
if (m_inotifyFd > 0) {
qDebug()<<"Inotify init success!";
} else {
Q_ASSERT_X(0, "InotifyWatch", "Failed to initialize inotify");
}
// this->addWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation));
// this->setPath(QStandardPaths::writableLocation(QStandardPaths::HomeLocation));
this->firstTraverse();
int fifo_fd;
char buffer[2];
memset(buffer, 0, sizeof(buffer));
fifo_fd = open(UKUI_SEARCH_PIPE_PATH, O_RDWR);
if(fifo_fd == -1) {
qWarning() << "Open fifo error\n";
assert(false);
}
int retval = read(fifo_fd, buffer, sizeof(buffer));
if(retval == -1) {
qWarning() << "read error\n";
assert(false);
}
qDebug("Read fifo[%s]", buffer);
qDebug("Read data ok");
close(fifo_fd);
if(buffer[0] & 0x1) {
qDebug("Data confirmed\n");
}
unlink(UKUI_SEARCH_PIPE_PATH);
while(FileUtils::SearchMethod::INDEXSEARCH == FileUtils::searchMethod) {
fd_set fds;
FD_ZERO(&fds);
FD_SET(m_inotifyFd, &fds);
int rc;
rc = select(m_inotifyFd + 1, &fds, NULL, NULL, NULL);
if(rc > 0) {
int avail;
if (ioctl(m_inotifyFd, FIONREAD, &avail) == EINVAL) {
qWarning() << "Did not receive an entire inotify event.";
return;
}
char* buf = (char*)malloc(avail);
memset(buf, 0x00, avail);
const ssize_t len = read(m_inotifyFd, buf, avail);
if(len != avail) {
qWarning()<<"read event error";
// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
}
int i = 0;
while (i < len) {
const struct inotify_event* event = (struct inotify_event*)&buf[i];
if(event->name[0] != '.') {
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
break;
}
i += sizeof(struct inotify_event) + event->len;
}
if(i < len ) {
qDebug() << "fork";
slotEvent(buf, len);
free(buf);
}
} else if(rc < 0) {
// error
qWarning() << "select result < 0, error!";
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
}
}
if(FileUtils::SearchMethod::DIRECTSEARCH == FileUtils::searchMethod) {
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3");
removeWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false);
}
// fcntl(m_inotifyFd, F_SETFD, FD_CLOEXEC);
// m_notifier = new QSocketNotifier(m_inotifyFd, QSocketNotifier::Read);
// connect(m_notifier, &QSocketNotifier::activated, this, &InotifyWatch::slotEvent, Qt::DirectConnection);
// exec();
}
void InotifyWatch::slotEvent(char *buf, ssize_t len)
{
// eventProcess(socket);
++FileUtils::_index_status;
if(FileUtils::SearchMethod::INDEXSEARCH == FileUtils::searchMethod) {
pid_t pid;
pid = fork();
if(pid == 0) {
prctl(PR_SET_PDEATHSIG, SIGTERM);
prctl(PR_SET_NAME, "inotify-index");
this->eventProcess(buf, len);
fd_set read_fds;
int rc;
timeval* read_timeout = (timeval*)malloc(sizeof(timeval));
read_timeout->tv_sec = 40;
read_timeout->tv_usec = 0;
for(;;) {
FD_ZERO(&read_fds);
FD_SET(m_inotifyFd, &read_fds);
rc = select(m_inotifyFd + 1, &read_fds, NULL, NULL, read_timeout);
if(rc < 0) {
// error
qWarning() << "fork select result < 0, error!";
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
} else if(rc == 0) {
qDebug() << "select timeout!";
::free(read_timeout);
QBuffer buffer;
QDataStream out(&buffer);
if (m_sharedMemory->isAttached()) {
m_sharedMemory->detach();
}
buffer.open(QBuffer::ReadWrite);
out << currentPath;
int size = buffer.size();
if (!m_sharedMemory->create(size)) {
qDebug() << "Create sharedMemory Error: " << m_sharedMemory->errorString();
} else {
m_sharedMemory->lock();
char *to = static_cast<char *>(m_sharedMemory->data());
const char *from = buffer.data().constData();
memcpy(to, from, qMin(size, m_sharedMemory->size()));
m_sharedMemory->unlock();
}
// GlobalSettings::getInstance()->forceSync();
PendingFileQueue::getInstance()->forceFinish();
PendingFileQueue::getInstance()->~PendingFileQueue();
::_exit(0);
} else {
// qDebug() << "Select remain:" <<read_timeout->tv_sec;
this->eventProcess(m_inotifyFd);
// qDebug() << "Select remain:" <<read_timeout->tv_sec;
}
}
} else if(pid > 0) {
waitpid(pid, NULL, 0);
if (!m_sharedMemory->attach()) {
qDebug() << "SharedMemory attach Error: " << m_sharedMemory->errorString();
} else {
QBuffer buffer;
QDataStream in(&buffer);
QMap<int, QString> pathMap;
m_sharedMemory->lock();
buffer.setData(static_cast<const char *>(m_sharedMemory->constData()), m_sharedMemory->size());
buffer.open(QBuffer::ReadWrite);
in >> pathMap;
m_sharedMemory->unlock();
m_sharedMemory->detach();
currentPath = pathMap;
}
--FileUtils::_index_status;
} else {
assert(false);
}
}
}
char * InotifyWatch::filter()
{
int avail;
if (ioctl(m_inotifyFd, FIONREAD, &avail) == EINVAL) {
qWarning() << "Did not receive an entire inotify event.";
return NULL;
}
char* buffer = (char*)malloc(avail);
memset(buffer, 0x00, avail);
const int len = read(m_inotifyFd, buffer, avail);
if(len != avail) {
qWarning()<<"read event error";
// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
}
int i = 0;
while (i < len) {
const struct inotify_event* event = (struct inotify_event*)&buffer[i];
if(event->name[0] == '.') {
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
i += sizeof(struct inotify_event) + event->len;
return buffer;
}
}
free(buffer);
return NULL;
}
void InotifyWatch::eventProcess(int socket)
{
// qDebug()<< "Enter eventProcess!";
int avail;
if (ioctl(socket, FIONREAD, &avail) == EINVAL) {
qWarning() << "Did not receive an entire inotify event.";
return;
}
char* buffer = (char*)malloc(avail);
memset(buffer, 0x00, avail);
const ssize_t len = read(socket, buffer, avail);
if(len != avail) {
qWarning()<<"read event error";
// IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
}
int i = 0;
while (i < len) {
const struct inotify_event* event = (struct inotify_event*)&buffer[i];
if(event->name[0] != '.') {
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
break;
}
i += sizeof(struct inotify_event) + event->len;
}
if(i >= len) {
qDebug() << "There is nothing to do!";
return;
}
eventProcess(buffer, len);
free(buffer);
}
void InotifyWatch::eventProcess(const char *buffer, ssize_t len)
{
// qDebug()<< "Begin eventProcess! len:" << len;
char * p = const_cast<char*>(buffer);
while (p < buffer + len) {
const struct inotify_event* event = reinterpret_cast<inotify_event *>(p);
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
if(event->name[0] != '.') {
QString path = currentPath[event->wd] + '/' + event->name;
//Create top dir first, traverse it last.
if(event->mask & IN_CREATE) {
// qDebug() << "IN_CREATE";
PendingFile f(path);
if(event->mask & IN_ISDIR) {
f.setIsDir();
}
PendingFileQueue::getInstance(this)->enqueue(f);
if(event->mask & IN_ISDIR) {
if(!QFileInfo(path).isSymLink()){
addWatch(path);
setPath(QStringList(path));
Traverse();
}
}
goto next;
}
if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)) {
qDebug() << "IN_DELETE or IN_MOVED_FROM";
if(event->mask & IN_ISDIR) {
removeWatch(path);
} else {
PendingFile f(path);
f.setDeleted();
PendingFileQueue::getInstance()->enqueue(f);
}
p += sizeof(struct inotify_event) + event->len;
continue;
}
if(event->mask & IN_MODIFY) {
// qDebug() << "IN_MODIFY";
if(!(event->mask & IN_ISDIR)) {
PendingFileQueue::getInstance()->enqueue(PendingFile(path));
}
goto next;
}
if(event->mask & IN_MOVED_TO) {
qDebug() << "IN_MOVED_TO";
if(event->mask & IN_ISDIR) {
removeWatch(path);
PendingFile f(path);
f.setIsDir();
PendingFileQueue::getInstance()->enqueue(f);
if(!QFileInfo(path).isSymLink()){
addWatch(path);
setPath(QStringList(path));
Traverse();
}
} else {
//Enqueue a deleted file to merge.
PendingFile f(path);
f.setDeleted();
PendingFileQueue::getInstance()->enqueue(f);
//Enqueue a new one.
PendingFileQueue::getInstance()->enqueue(PendingFile(path));
}
goto next;
}
}
next:
p += sizeof(struct inotify_event) + event->len;
}
// qDebug()<< "Finish eventProcess!";
}

52
libsearch/index/inotify-watch.h Normal file
View File

@ -0,0 +1,52 @@
#ifndef INOTIFYWATCH_H
#define INOTIFYWATCH_H
#include <QThread>
#include <unistd.h>
#include <sys/inotify.h>
#include <QSocketNotifier>
#include <QDataStream>
#include <QSharedMemory>
#include "traverse_bfs.h"
#include "ukui-search-qdbus.h"
#include "index-status-recorder.h"
#include "file-utils.h"
#include "first-index.h"
#include "pending-file-queue.h"
#include "common.h"
namespace Zeeker {
class InotifyWatch : public QThread, public Traverse_BFS
{
Q_OBJECT
public:
static InotifyWatch* getInstance(const QStringList &pathList);
bool addWatch(const QString &path);
bool removeWatch(const QString &path, bool removeFromDatabase = true);
virtual void DoSomething(const QFileInfo &info) final;
void firstTraverse();
void stopWatch();
protected:
void run() override;
private Q_SLOTS:
void slotEvent(char *buf, ssize_t len);
private:
explicit InotifyWatch(const QStringList &pathList);
~InotifyWatch();
char * filter();
void eventProcess(int socket);
void eventProcess(const char *buffer, ssize_t len);
int m_inotifyFd;
QSocketNotifier* m_notifier = nullptr;
QSharedMemory *m_sharedMemory = nullptr;
QMap<int, QString> currentPath;
QMutex m_mutex;
};
}
#endif // INOTIFYWATCH_H

165
libsearch/index/pending-file-queue.cpp Normal file
View File

@ -0,0 +1,165 @@
/*
* Copyright (C) 2021, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#include "pending-file-queue.h"
#include <malloc.h>
using namespace Zeeker;
static PendingFileQueue *global_instance_pending_file_queue = nullptr;
PendingFileQueue::PendingFileQueue(QObject *parent) : QThread(parent)
{
this->start();
m_cacheTimer = new QTimer;
m_minProcessTimer = new QTimer;
m_cacheTimer->setInterval(10*1000);
m_cacheTimer->setSingleShot(true);
m_minProcessTimer->setInterval(500);
m_minProcessTimer->setSingleShot(true);
m_cacheTimer->moveToThread(this);
m_minProcessTimer->moveToThread(this);
// connect(this, &PendingFileQueue::cacheTimerStart, m_cacheTimer, f, Qt::DirectConnection);
// connect(this, &PendingFileQueue::minProcessTimerStart, m_minProcessTimer, f,Qt::DirectConnection);
connect(this, SIGNAL(cacheTimerStart()), m_cacheTimer, SLOT(start()));
connect(this, SIGNAL(minProcessTimerStart()), m_minProcessTimer, SLOT(start()));
connect(this, &PendingFileQueue::timerStop, m_cacheTimer, &QTimer::stop);
connect(this, &PendingFileQueue::timerStop, m_minProcessTimer, &QTimer::stop);
connect(m_cacheTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection);
connect(m_minProcessTimer, &QTimer::timeout, this, &PendingFileQueue::processCache, Qt::DirectConnection);
}
PendingFileQueue *PendingFileQueue::getInstance(QObject *parent)
{
if (!global_instance_pending_file_queue) {
global_instance_pending_file_queue = new PendingFileQueue(parent);
}
return global_instance_pending_file_queue;
}
PendingFileQueue::~PendingFileQueue()
{
if(m_cacheTimer) {
delete m_cacheTimer;
m_cacheTimer = nullptr;
}
if(m_minProcessTimer) {
delete m_minProcessTimer;
m_minProcessTimer = nullptr;
}
IndexGenerator::getInstance()->~IndexGenerator();
}
void PendingFileQueue::forceFinish()
{
QThread::msleep(600);
Q_EMIT timerStop();
this->quit();
this->wait();
}
void PendingFileQueue::enqueue(const PendingFile &file)
{
// qDebug() << "enqueuq file: " << file.path();
m_mutex.lock();
m_enqueuetimes++;
if(m_cache.isEmpty()) {
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "0");
}
// Remove all indexs of files under a dir which is to about be deleted,but keep delete signals.
// Because our datebase need to delete those indexs one by one.
if(file.shouldRemoveIndex() && file.isDir()) {
const auto keepFile = [&file](const PendingFile& pending) {
return (!pending.path().startsWith(file.path()) || pending.shouldRemoveIndex());
};
const auto end = m_cache.end();
const auto droppedFilesBegin = std::stable_partition(m_cache.begin(), end, keepFile);
m_cache.erase(droppedFilesBegin, end);
}
if(file.shouldRemoveIndex()) {
m_cache.removeOne(file);
}
int i = m_cache.indexOf(file);
if (i == -1) {
// qDebug() << "insert file" << file.path() << file.shouldRemoveIndex();
m_cache << file;
} else {
// qDebug() << "merge file" << file.path() << file.shouldRemoveIndex();
m_cache[i].merge(file);
}
if(!m_cacheTimer->isActive()) {
// qDebug()<<"m_cacheTimer-----start!!";
// m_cacheTimer->start();
Q_EMIT cacheTimerStart();
}
Q_EMIT minProcessTimerStart();
// m_minProcessTimer->start();
// qDebug()<<"m_minProcessTimer-----start!!";
m_mutex.unlock();
// qDebug() << "Current cache-------------";
// for(PendingFile i : m_cache) {
// qDebug() << "|" << i.path();
// qDebug() << "|" <<i.shouldRemoveIndex();
// }
// qDebug() << "Current cache-------------";
// qDebug()<<"enqueuq file finish!!"<<file.path();
}
void PendingFileQueue::run()
{
exec();
}
void PendingFileQueue::processCache()
{
qDebug()<< "Begin processCache!" ;
m_mutex.lock();
qDebug() << "Events:" << m_enqueuetimes;
m_enqueuetimes = 0;
m_cache.swap(m_pendingFiles);
// m_pendingFiles = m_cache;
// m_cache.clear();
// m_cache.squeeze();
m_mutex.unlock();
qDebug() << "Current process-------------";
for(PendingFile i : m_pendingFiles) {
qDebug() << "|" << i.path();
qDebug() << "|" <<i.shouldRemoveIndex();
}
qDebug() << "Current process-------------";
if(m_pendingFiles.isEmpty()) {
qDebug()<< "Empty, finish processCache!";
return;
}
IndexGenerator::getInstance()->updateIndex(&m_pendingFiles);
m_mutex.lock();
if(m_cache.isEmpty()) {
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2");
}
m_mutex.unlock();
m_pendingFiles.clear();
m_pendingFiles.squeeze();
malloc_trim(0);
qDebug()<< "Finish processCache!";
return;
}

67
libsearch/index/pending-file-queue.h Normal file
View File

@ -0,0 +1,67 @@
/*
* Copyright (C) 2021, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#ifndef PENDINGFILEQUEUE_H
#define PENDINGFILEQUEUE_H
#include <QObject>
#include <QVector>
#include <QTimer>
#include <QThread>
#include <QMutex>
#include "pending-file.h"
#include "index-generator.h"
namespace Zeeker {
class PendingFileQueue : public QThread
{
Q_OBJECT
public:
static PendingFileQueue *getInstance(QObject *parent = nullptr);
~PendingFileQueue();
//This method will block until current cache has been processed.
//Do not do enqueue operation in other thread while this method is running.
void forceFinish();
void enqueue(const PendingFile& file);
QTimer *m_cacheTimer = nullptr;
QTimer *m_minProcessTimer = nullptr;
protected:
void run() override;
Q_SIGNALS:
void cacheTimerStart();
void minProcessTimerStart();
void timerStop();
private:
void processCache();
explicit PendingFileQueue(QObject *parent = nullptr);
QVector<PendingFile> m_cache;
QVector<PendingFile> m_pendingFiles;
QMutex m_mutex;
QMutex m_timeoutMutex;
QThread *m_timerThread = nullptr;
bool m_timeout = false;
int m_enqueuetimes = 0;
};
}
#endif // PENDINGFILEQUEUE_H

81
libsearch/index/pending-file.cpp Normal file
View File

@ -0,0 +1,81 @@
/*
* Copyright (C) 2021, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#include "pending-file.h"
using namespace Zeeker;
PendingFile::PendingFile(const QString &path)
: m_path(path)
, m_deleted(false)
, m_modified(false)
, m_isDir(false)
{
}
QString PendingFile::path() const
{
return m_path;
}
void PendingFile::setPath(const QString& path)
{
if (path.endsWith(QLatin1Char('/'))) {
m_path = path.mid(0, m_path.length() - 1);
return;
}
m_path = path;
}
//bool PendingFile::isNewFile() const
//{
// return m_created;
//}
//bool PendingFile::shouldIndexContents() const
//{
// if (m_created || m_modified) {
// return true;
// }
// return false;
//}
bool PendingFile::isDir() const
{
return m_isDir;
}
bool PendingFile::shouldRemoveIndex() const
{
return m_deleted;
}
void PendingFile::merge(const PendingFile& file)
{
// m_created |= file.m_created;
m_modified = file.m_modified;
m_deleted = file.m_deleted;
}
void PendingFile::printFlags() const
{
// qDebug() << "Created:" << m_created;
qDebug() << "Deleted:" << m_deleted;
qDebug() << "Modified:" << m_modified;
qDebug() << "Is dir:" << m_isDir;
}

65
libsearch/index/pending-file.h Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright (C) 2021, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#ifndef PENDINGFILE_H
#define PENDINGFILE_H
#include <QString>
#include <QDebug>
namespace Zeeker {
/**
* Represents a file/folder which needs to be indexed.
*/
class PendingFile
{
public:
explicit PendingFile(const QString& path = QString());
QString path() const;
void setPath(const QString& path);
void setIsDir(){ m_isDir = true; }
void setModified() { m_modified = true; }
// void setCreated() { m_created = true; }
void setDeleted() { m_deleted = true; }
bool shouldRemoveIndex() const;
// bool shouldIndexContents() const;
bool isDir() const;
bool operator == (const PendingFile& rhs) const {
return (m_path == rhs.m_path);
}
/**
* Takes a PendingFile \p file and merges its flags into
* the current PendingFile
*/
void merge(const PendingFile& file);
private:
QString m_path;
// bool m_created : 1;
bool m_deleted : 1;
bool m_modified : 1;
bool m_isDir : 1;
void printFlags() const;
};
}
#endif // PENDINGFILE_H

29
libsearch/index/searchmethodmanager.cpp
View File

@ -1,30 +1,39 @@
#include "searchmethodmanager.h"
using namespace Zeeker;
SearchMethodManager::SearchMethodManager()
{
QStringList pathList;
pathList.append("/media/用户保险箱");
pathList.append("/media/邮件保险箱");
pathList.append("/media/公共保险箱");
pathList.append("/media/备份保险箱");
m_iw = InotifyWatch::getInstance(pathList);
}
void SearchMethodManager::searchMethod(FileUtils::SearchMethod sm) {
qWarning() << "searchMethod start: " << static_cast<int>(sm);
if(FileUtils::SearchMethod::INDEXSEARCH == sm || FileUtils::SearchMethod::DIRECTSEARCH == sm) {
FileUtils::searchMethod = sm;
} else {
printf("enum class error!!!\n");
qWarning("enum class error!!!\n");
}
if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::_index_status) {
qWarning() << "start first index";
// m_fi = FirstIndex("/home/zhangzihao/Desktop");
m_fi.start();
qWarning() << "start inotify index";
// InotifyIndex ii("/home");
// ii.start();
QStringList pathList;
pathList.append("/media/用户保险箱");
pathList.append("/media/邮件保险箱");
pathList.append("/media/公共保险箱");
pathList.append("/media/备份保险箱");
this->m_ii = InotifyIndex::getInstance(pathList);
if(!this->m_ii->isRunning()) {
this->m_ii->start();
// this->m_ii = InotifyIndex::getInstance("/home");
// if(!this->m_ii->isRunning()) {
// this->m_ii->start();
// }
if(!this->m_iw->isRunning()) {
this->m_iw->start();
}
qDebug() << "Search method has been set to INDEXSEARCH";
}
if(FileUtils::SearchMethod::DIRECTSEARCH == sm) {
m_iw->stopWatch();
}
qWarning() << "searchMethod end: " << static_cast<int>(FileUtils::searchMethod);
}

8
libsearch/index/searchmethodmanager.h
View File

@ -2,15 +2,17 @@
#define SEARCHMETHODMANAGER_H
#include "first-index.h"
#include "inotify-index.h"
//#include "inotify-index.h"
#include "inotify-watch.h"
namespace Zeeker {
class SearchMethodManager {
public:
SearchMethodManager() = default;
SearchMethodManager();
void searchMethod(FileUtils::SearchMethod sm);
private:
FirstIndex m_fi;
InotifyIndex* m_ii;
// InotifyIndex* m_ii;
InotifyWatch *m_iw = nullptr;
};
}

11
src/content-widget.cpp
View File

@ -60,8 +60,10 @@ ContentWidget::~ContentWidget() {
void ContentWidget::initUI() {
this->setFixedHeight(486);
QPalette pal = palette();
pal.setColor(QPalette::Base, QColor(0, 0, 0, 0));
pal.setColor(QPalette::Window, QColor(0, 0, 0, 0)); //使用此palette的窗口背景将为透明
QPalette scroll_bar_pal = palette();
// pal.setColor(QPalette::Base, QColor(0, 0, 0, 0));
// pal.setColor(QPalette::Window, QColor(0, 0, 0, 0)); //使用此palette的窗口背景将为透明
// scroll_bar_pal.setColor(QPalette::Base, QColor(0, 0, 0, 0));
m_homePage = new QWidget(this);
m_homePageLyt = new QVBoxLayout(m_homePage);
m_homePageLyt->setSpacing(0);
@ -106,6 +108,8 @@ void ContentWidget::initUI() {
m_resultDetailArea->setFrameShape(QFrame::NoFrame);
m_resultListArea->setPalette(pal);
m_resultDetailArea->setPalette(pal);
// m_resultListArea->verticalScrollBar()->setPalette(scroll_bar_pal);
// m_resultDetailArea->verticalScrollBar()->setPalette(scroll_bar_pal);
this->addWidget(m_homePage);
this->addWidget(m_resultPage);
@ -387,7 +391,7 @@ void ContentWidget::initHomePage() {
itemWidget->setLayout(layout);
for(int j = 0; j < lists.at(i).count(); j++) {
HomePageItem * item = new HomePageItem(itemWidget, i, lists.at(i).at(j));
item->setFixedSize(300, 48);
item->setFixedSize(312, 48);
layout->addWidget(item, j / 2, j % 2);
}
if(lists.at(i).length() == 1) {
@ -421,6 +425,7 @@ void ContentWidget::initHomePage() {
}
itemWidgetLyt->setSpacing(6);
titleLabel->setFixedHeight(24);
titleLabel->setContentsMargins(6,0,0,0);
itemWidgetLyt->addWidget(titleLabel);
itemWidgetLyt->addWidget(itemWidget);
m_homePageLyt->addWidget(listWidget);

1
src/content-widget.h
View File

@ -25,6 +25,7 @@
#include <QStackedWidget>
#include <QScrollArea>
#include <QGridLayout>
#include <QScrollBar>
#include "control/search-detail-view.h"
#include "home-page-item.h"
#include "show-more-label.h"

43
src/control/search-detail-view.cpp
View File

@ -313,25 +313,26 @@ void SearchDetailView::setupWidget(const int& type, const QString& path) {
m_pathLabel_1->show();
m_pathLabel_2->show();
// m_pathLabel_2->setText(path);
QString showPath = path;
QFontMetrics fontMetrics = m_pathLabel_2->fontMetrics();
if(fontMetrics.width(path) > m_pathLabel_2->width() - 10) {
//路径长度超过230,手动添加换行符以实现折叠
int lastIndex = 0;
for(int i = lastIndex; i < path.length(); i++) {
if(fontMetrics.width(path.mid(lastIndex, i - lastIndex)) == m_pathLabel_2->width() - 10) {
lastIndex = i;
showPath.insert(i, '\n');
} else if(fontMetrics.width(path.mid(lastIndex, i - lastIndex)) > m_pathLabel_2->width() - 10) {
lastIndex = i;
showPath.insert(i - 1, '\n');
} else {
continue;
}
}
}
m_pathLabel_2->setText(showPath);
// QString showPath = path;
// QFontMetrics fontMetrics = m_pathLabel_2->fontMetrics();
// if(fontMetrics.width(path) > m_pathLabel_2->width() - 10) {
// //路径长度超过230,手动添加换行符以实现折叠
// int lastIndex = 0;
// for(int i = lastIndex; i < path.length(); i++) {
// if(fontMetrics.width(path.mid(lastIndex, i - lastIndex)) == m_pathLabel_2->width() - 10) {
// lastIndex = i;
// showPath.insert(i, '\n');
// } else if(fontMetrics.width(path.mid(lastIndex, i - lastIndex)) > m_pathLabel_2->width() - 10) {
// lastIndex = i;
// showPath.insert(i - 1, '\n');
// } else {
// continue;
// }
// }
// }
// m_pathLabel_2->setText(showPath);
m_pathLabel_2->setText(m_pathLabel_2->fontMetrics().elidedText(path, Qt::ElideRight, m_pathLabel_2->width()));
m_pathLabel_2->setToolTip(path);
m_timeLabel_1->show();
m_timeLabel_2->show();
QFileInfo fileInfo(path);
@ -465,7 +466,7 @@ void SearchDetailView::initUI() {
m_layout = new QVBoxLayout(this);
this->setLayout(m_layout);
m_layout->setContentsMargins(16, 60, 16, 24);
this->setFixedWidth(378);
this->setFixedWidth(368);
//没有网络的时候的提示信息
m_noNetFrame = new QFrame(this);
@ -526,12 +527,14 @@ void SearchDetailView::initUI() {
m_pathLabel_2 = new QLabel(m_pathFrame);
m_pathLabel_1->setText(tr("Path"));
m_pathLabel_2->setFixedWidth(240);
m_pathLabel_2->setAlignment(Qt::AlignRight);
// m_pathLabel_2->setWordWrap(true);
m_pathLyt->addWidget(m_pathLabel_1);
m_pathLyt->addStretch();
m_pathLyt->addWidget(m_pathLabel_2);
m_timeLabel_1 = new QLabel(m_timeFrame);
m_timeLabel_2 = new QLabel(m_timeFrame);
m_timeLabel_2->setAlignment(Qt::AlignRight);
m_timeLabel_1->setText(tr("Last time modified"));
m_timeLyt->addWidget(m_timeLabel_1);
m_timeLyt->addStretch();

7
src/input-box.cpp
View File

@ -89,7 +89,7 @@ SearchBarHLayout::~SearchBarHLayout() {
* @brief 初始化ui
*/
void SearchBarHLayout::initUI() {
m_queryLineEdit = new SearchLineEdit();
m_queryLineEdit = new SearchLineEdit(this->parentWidget());
m_queryLineEdit->installEventFilter(this);
m_queryLineEdit->setTextMargins(30, 1, 0, 1);
this->setContentsMargins(0, 0, 0, 0);
@ -150,7 +150,8 @@ void SearchBarHLayout::effectiveSearchRecord() {
}
void SearchBarHLayout::focusIn() {
m_queryLineEdit->setFocus();
if (!m_queryLineEdit->hasFocus())
m_queryLineEdit->setFocus(Qt::MouseFocusReason);
}
void SearchBarHLayout::focusOut() {
@ -203,7 +204,7 @@ bool SearchBarHLayout::eventFilter(QObject *watched, QEvent *event) {
/**
* @brief UKuiSearchLineEdit 全局搜索的输入框
*/
SearchLineEdit::SearchLineEdit() {
SearchLineEdit::SearchLineEdit(QWidget *parent) : QLineEdit(parent) {
this->setFocusPolicy(Qt::ClickFocus);
this->setAttribute(Qt::WA_InputMethodEnabled);
// this->installEventFilter(this);

2
src/input-box.h
View File

@ -100,7 +100,7 @@ class SearchLineEdit : public QLineEdit {
　　　*/
Q_CLASSINFO("D-Bus Interface", "org.ukui.search.inputbox")
public:
SearchLineEdit();
SearchLineEdit(QWidget *parent = nullptr);
void record();
~SearchLineEdit();