openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

增加文件索引状态查询功能；优化文建索引启动校验逻辑。

This commit is contained in:
iaom 2022-03-17 15:40:55 +08:00
parent 7126068efb
commit baaac7a4dc
11 changed files with 285 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
frontend/control/settings-widget.cpp
View File

@ -277,7 +277,7 @@ void SettingsWidget::refreshIndexState() {
} else { } else {
this->setIndexState(false); this->setIndexState(false);
} }
m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::_max_index_count))); m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::maxIndexCount)));
m_timer = new QTimer; m_timer = new QTimer;
connect(m_timer, &QTimer::timeout, this, [ = ]() { connect(m_timer, &QTimer::timeout, this, [ = ]() {
qDebug() << "FileUtils::indexStatus: " << FileUtils::indexStatus; qDebug() << "FileUtils::indexStatus: " << FileUtils::indexStatus;
@ -286,7 +286,7 @@ void SettingsWidget::refreshIndexState() {
} else { } else {
this->setIndexState(false); this->setIndexState(false);
} }
m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::_max_index_count))); m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::maxIndexCount)));
}); });
m_timer->start(0.5 * 1000); m_timer->start(0.5 * 1000);
} }

60
libsearch/common.h
View File

@ -1,38 +1,44 @@
#ifndef COMMON_H #pragma once
#define COMMON_H
#include <QMap> #include <QMap>
#include <QString> #include <QString>
#include <QDir> #include <QDir>
#define UKUI_SEARCH_PIPE_PATH (QDir::homePath()+"/.config/org.ukui/ukui-search/ukuisearch").toLocal8Bit().constData()
#define FILE_SEARCH_VALUE "0" static const int LABEL_MAX_WIDTH = 300;
#define DIR_SEARCH_VALUE "1"
#define LABEL_MAX_WIDTH 300 static const QString HOME_PATH = QDir::homePath();
#define HOME_PATH QDir::homePath() static const QString INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/index_data");
#define INDEX_SEM "ukui-search-index-sem" static const QString CONTENT_INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/content_index_data");
static const QString OCR_INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/ocr_index_data");
static const QString FILE_SEARCH_VALUE = QStringLiteral("0");
static const QString DIR_SEARCH_VALUE = QStringLiteral("1");
static const QString INDEX_SEM = QStringLiteral("ukui-search-index-sem");
static const QStringList allAppPath = { static const QStringList allAppPath = {
QDir::homePath()+"/.local/share/applications/", {HOME_PATH + "/.local/share/applications/"},
"/usr/share/applications/" {"/usr/share/applications/"}
}; };
static const QMap<QString, bool> targetFileTypeMap = { static const QMap<QString, bool> targetFileTypeMap = {
std::map<QString, bool>::value_type("doc", true), {"doc", true},
std::map<QString, bool>::value_type("docx", true), {"docx", true},
std::map<QString, bool>::value_type("ppt", true), {"ppt", true},
std::map<QString, bool>::value_type("pptx", true), {"pptx", true},
std::map<QString, bool>::value_type("xls", true), {"xls", true},
std::map<QString, bool>::value_type("xlsx", true), {"xlsx", true},
std::map<QString, bool>::value_type("txt", true), {"txt", true},
std::map<QString, bool>::value_type("dot", true), {"dot", true},
std::map<QString, bool>::value_type("wps", true), {"wps", true},
std::map<QString, bool>::value_type("pps", true), {"pps", true},
std::map<QString, bool>::value_type("dps", true), {"dps", true},
std::map<QString, bool>::value_type("et", true), {"et", true},
std::map<QString, bool>::value_type("pdf", true) {"pdf", true}
}; };
static const QMap<QString, bool> targetPhotographTypeMap = { static const QMap<QString, bool> targetPhotographTypeMap = {
std::map<QString, bool>::value_type("png", true), {"png", true},
std::map<QString, bool>::value_type("jpg", true), {"jpg", true},
std::map<QString, bool>::value_type("jpeg", true)//TODO 待完善,后续改为配置文件 {"jpeg", true} // TODO 待完善,后续改为配置文件
}; };
//TODO Put things that needed to be put here here. //TODO Put things that needed to be put here here.
#endif // COMMON_H

3
libsearch/file-utils.cpp
View File

@ -30,8 +30,7 @@
#include "pinyinmanager.h" #include "pinyinmanager.h"
using namespace UkuiSearch; using namespace UkuiSearch;
size_t FileUtils::_max_index_count = 0; size_t FileUtils::maxIndexCount = 0;
size_t FileUtils::_current_index_count = 0;
unsigned short FileUtils::indexStatus = 0; unsigned short FileUtils::indexStatus = 0;
FileUtils::SearchMethod FileUtils::searchMethod = FileUtils::SearchMethod::DIRECTSEARCH; FileUtils::SearchMethod FileUtils::searchMethod = FileUtils::SearchMethod::DIRECTSEARCH;
QMap<QString, QStringList> FileUtils::map_chinese2pinyin = QMap<QString, QStringList>(); QMap<QString, QStringList> FileUtils::map_chinese2pinyin = QMap<QString, QStringList>();

3
libsearch/file-utils.h
View File

@ -97,8 +97,7 @@ public:
static QIcon iconFromTheme(const QString &name, const QIcon &iconDefault); static QIcon iconFromTheme(const QString &name, const QIcon &iconDefault);
static bool isOpenXMLFileEncrypted(QString &path); static bool isOpenXMLFileEncrypted(QString &path);
static bool isEncrypedOrUnreadable(QString path); static bool isEncrypedOrUnreadable(QString path);
static size_t _max_index_count; static size_t maxIndexCount;
static size_t _current_index_count; //this one has been Abandoned,do not use it.
static unsigned short indexStatus; static unsigned short indexStatus;
enum class SearchMethod { DIRECTSEARCH = 0, INDEXSEARCH = 1}; enum class SearchMethod { DIRECTSEARCH = 0, INDEXSEARCH = 1};

190
libsearch/index/first-index.cpp
View File

@ -42,24 +42,21 @@ FirstIndex *FirstIndex::getInstance()
FirstIndex::~FirstIndex() { FirstIndex::~FirstIndex() {
qDebug() << "~FirstIndex"; qDebug() << "~FirstIndex";
if(this->q_index) if(this->m_indexData)
delete this->q_index; delete this->m_indexData;
this->q_index = nullptr; this->m_indexData = nullptr;
if(this->q_content_index) if(this->m_contentIndexData)
delete this->q_content_index; delete this->m_contentIndexData;
this->q_content_index = nullptr; this->m_contentIndexData = nullptr;
if(this->m_ocr_index) if(this->m_ocrIndexData)
delete this->m_ocr_index; delete this->m_ocrIndexData;
this->m_ocr_index = nullptr; this->m_ocrIndexData = nullptr;
if(this->p_indexGenerator)
delete this->p_indexGenerator;
this->p_indexGenerator = nullptr;
qDebug() << "~FirstIndex end"; qDebug() << "~FirstIndex end";
} }
void FirstIndex::DoSomething(const QFileInfo& fileInfo) { void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
// qDebug() << "there are some shit here"<<fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0"); // qDebug() << "there are some shit here"<<fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0");
this->q_index->enqueue(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0")); this->m_indexData->enqueue(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0"));
if (fileInfo.fileName().split(".", QString::SkipEmptyParts).length() < 2) if (fileInfo.fileName().split(".", QString::SkipEmptyParts).length() < 2)
return; return;
if (true == targetFileTypeMap[fileInfo.fileName().split(".").last()] if (true == targetFileTypeMap[fileInfo.fileName().split(".").last()]
@ -71,7 +68,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
if(!file.setCurrentFile("word/document.xml", QuaZip::csSensitive)) if(!file.setCurrentFile("word/document.xml", QuaZip::csSensitive))
return; return;
QuaZipFile fileR(&file); QuaZipFile fileR(&file);
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//docx解压缩后的xml文件为实际需要解析文件大小 this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//docx解压缩后的xml文件为实际需要解析文件大小
file.close(); file.close();
} else if (fileInfo.fileName().split(".").last() == "pptx") { } else if (fileInfo.fileName().split(".").last() == "pptx") {
QuaZip file(fileInfo.absoluteFilePath()); QuaZip file(fileInfo.absoluteFilePath());
@ -92,7 +89,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
} }
} }
file.close(); file.close();
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileSize));//pptx解压缩后的xml文件为实际需要解析文件大小 this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileSize));//pptx解压缩后的xml文件为实际需要解析文件大小
} else if (fileInfo.fileName().split(".").last() == "xlsx") { } else if (fileInfo.fileName().split(".").last() == "xlsx") {
QuaZip file(fileInfo.absoluteFilePath()); QuaZip file(fileInfo.absoluteFilePath());
if(!file.open(QuaZip::mdUnzip)) if(!file.open(QuaZip::mdUnzip))
@ -100,14 +97,14 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
if(!file.setCurrentFile("xl/sharedStrings.xml", QuaZip::csSensitive)) if(!file.setCurrentFile("xl/sharedStrings.xml", QuaZip::csSensitive))
return; return;
QuaZipFile fileR(&file); QuaZipFile fileR(&file);
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//xlsx解压缩后的xml文件为实际解析文件大小 this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//xlsx解压缩后的xml文件为实际解析文件大小
file.close(); file.close();
} else { } else {
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size())); this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size()));
} }
} else if (true == targetPhotographTypeMap[fileInfo.fileName().split(".").last()]) { } /*else if (true == targetPhotographTypeMap[fileInfo.fileName().split(".").last()]) {
this->m_ocr_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size())); this->m_ocrIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size()));
} }*/
} }
void FirstIndex::run() { void FirstIndex::run() {
@ -115,26 +112,28 @@ void FirstIndex::run() {
// Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running. // Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running.
QString indexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(INDEX_DATABASE_STATE).toString(); QString indexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(INDEX_DATABASE_STATE).toString();
QString contentIndexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE).toString(); QString contentIndexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE).toString();
// QString ocrIndexDatabaseStatus = IndexStatusRecorder::getInstance()->getStatus(OCR_DATABASE_STATE).toString();
QString inotifyIndexStatus = IndexStatusRecorder::getInstance()->getStatus(INOTIFY_NORMAL_EXIT).toString(); QString inotifyIndexStatus = IndexStatusRecorder::getInstance()->getStatus(INOTIFY_NORMAL_EXIT).toString();
qDebug() << "indexDataBaseStatus: " << indexDataBaseStatus; qInfo() << "indexDataBaseStatus: " << indexDataBaseStatus;
qDebug() << "contentIndexDataBaseStatus: " << contentIndexDataBaseStatus; qInfo() << "contentIndexDataBaseStatus: " << contentIndexDataBaseStatus;
qDebug() << "inotifyIndexStatus: " << inotifyIndexStatus; // qInfo() << "ocrIndexDatabaseStatus: " << ocrIndexDatabaseStatus;
qInfo() << "inotifyIndexStatus: " << inotifyIndexStatus;
if(indexDataBaseStatus == "") { m_allDatadaseStatus = inotifyIndexStatus == "2" ? true : false;
this->bool_dataBaseExist = false; m_indexDatabaseStatus = indexDataBaseStatus == "2" ? true : false;
} else { m_contentIndexDatabaseStatus = contentIndexDataBaseStatus == "2" ? true : false;
this->bool_dataBaseExist = true; // m_ocrIndexDatabaseStatus = ocrIndexDatabaseStatus == "2" ? true : false;
}
if(indexDataBaseStatus != "2" || contentIndexDataBaseStatus != "2" || inotifyIndexStatus != "2") { if(m_allDatadaseStatus && m_indexDatabaseStatus && m_contentIndexDatabaseStatus /*&& m_ocrIndexDatabaseStatus*/) {
this->bool_dataBaseStatusOK = false; m_semaphore.release(1);
} else { return;
this->bool_dataBaseStatusOK = true;
} }
this->q_index = new QQueue<QVector<QString>>();
this->q_content_index = new QQueue<QPair<QString,qint64>>(); this->m_indexData = new QQueue<QVector<QString>>();
this->m_ocr_index = new QQueue<QPair<QString,qint64>>(); this->m_contentIndexData = new QQueue<QPair<QString,qint64>>();
// this->m_ocrIndexData = new QQueue<QPair<QString,qint64>>();
++FileUtils::indexStatus; ++FileUtils::indexStatus;
pid_t pid; pid_t pid;
@ -142,23 +141,12 @@ void FirstIndex::run() {
if(pid == 0) { if(pid == 0) {
prctl(PR_SET_PDEATHSIG, SIGTERM); prctl(PR_SET_PDEATHSIG, SIGTERM);
prctl(PR_SET_NAME, "first-index"); prctl(PR_SET_NAME, "first-index");
if(this->bool_dataBaseExist) {
if(this->bool_dataBaseStatusOK) {
::_exit(0);
} else {
//if the parameter is false, index won't be rebuild
//if it is true, index will be rebuild
p_indexGenerator = IndexGenerator::getInstance(true, this);
}
} else {
// p_indexGenerator = IndexGenerator::getInstance(false,this);
p_indexGenerator = IndexGenerator::getInstance(true, this);
}
//TODO Fix these weird code.
QSemaphore sem(5); QSemaphore sem(5);
QMutex mutex1, mutex2, mutex3; QMutex mutex1, mutex2, mutex3;
mutex1.lock(); mutex1.lock();
mutex2.lock(); mutex2.lock();
// mutex3.lock();
qInfo() << "index dir" << DirWatcher::getDirWatcher()->currentindexableDir(); qInfo() << "index dir" << DirWatcher::getDirWatcher()->currentindexableDir();
qInfo() << "index block dir" << DirWatcher::getDirWatcher()->currentBlackListOfIndex(); qInfo() << "index block dir" << DirWatcher::getDirWatcher()->currentBlackListOfIndex();
@ -166,59 +154,89 @@ void FirstIndex::run() {
setBlockPath(DirWatcher::getDirWatcher()->currentBlackListOfIndex()); setBlockPath(DirWatcher::getDirWatcher()->currentBlackListOfIndex());
this->Traverse(); this->Traverse();
FileUtils::_max_index_count = this->q_index->length(); FileUtils::maxIndexCount = this->m_indexData->length();
qDebug() << "max_index_count:" << FileUtils::_max_index_count; qDebug() << "max_index_count:" << FileUtils::maxIndexCount;
QtConcurrent::run(&m_pool, [&]() { QtConcurrent::run(&m_pool, [&]() {
sem.acquire(2); sem.acquire(2);
mutex1.unlock(); mutex1.unlock();
qDebug() << "index start;"; if(m_allDatadaseStatus && m_indexDatabaseStatus) {
sem.release(2);
return;
}
qDebug() << "index start;" << m_indexData->size();
IndexGenerator::getInstance()->rebuildIndexDatabase();
QQueue<QVector<QString>>* tmp1 = new QQueue<QVector<QString>>(); QQueue<QVector<QString>>* tmp1 = new QQueue<QVector<QString>>();
while(!this->q_index->empty()) { bool sucess = true;
for(size_t i = 0; (i < 8192) && (!this->q_index->empty()); ++i) { while(!this->m_indexData->empty()) {
tmp1->enqueue(this->q_index->dequeue()); for(size_t i = 0; (i < 8192) && (!this->m_indexData->empty()); ++i) {
tmp1->enqueue(this->m_indexData->dequeue());
}
if(!IndexGenerator::getInstance()->creatAllIndex(tmp1)) {
sucess = false;
break;
} }
this->p_indexGenerator->creatAllIndex(tmp1);
tmp1->clear(); tmp1->clear();
} }
delete tmp1; delete tmp1;
qDebug() << "index end;"; qDebug() << "index end;";
if(sucess) {
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "2");
}
sem.release(2); sem.release(2);
}); });
QtConcurrent::run(&m_pool,[&]() { QtConcurrent::run(&m_pool,[&]() {
sem.acquire(2); sem.acquire(2);
mutex2.unlock(); mutex2.unlock();
if(m_allDatadaseStatus && m_contentIndexDatabaseStatus) {
sem.release(2);
return;
}
qDebug() << "content index start:" << m_contentIndexData->size();
IndexGenerator::getInstance()->rebuildContentIndexDatabase();
QQueue<QString>* tmp2 = new QQueue<QString>(); QQueue<QString>* tmp2 = new QQueue<QString>();
qDebug() << "q_content_index:" << q_content_index->size(); bool sucess = true;
while(!this->q_content_index->empty()) { while(!this->m_contentIndexData->empty()) {
qint64 fileSize = 0; qint64 fileSize = 0;
//修改一次处理的数据量，从30个文件改为文件总大小为50M以下，50M为暂定值--jxx20210519 //修改一次处理的数据量，从30个文件改为文件总大小为50M以下，50M为暂定值--jxx20210519
for(size_t i = 0;/* (i < 30) && (fileSize < 52428800) && */(!this->q_content_index->empty()); ++i) { for(size_t i = 0;/* (i < 30) && (fileSize < 52428800) && */(!this->m_contentIndexData->empty()); ++i) {
QPair<QString,qint64> tempPair = this->q_content_index->dequeue(); QPair<QString,qint64> tempPair = this->m_contentIndexData->dequeue();
fileSize += tempPair.second; fileSize += tempPair.second;
if (fileSize > 52428800 ) { if (fileSize > 52428800 ) {
if (tmp2->size() == 0) { if (tmp2->size() == 0) {
tmp2->enqueue(tempPair.first); tmp2->enqueue(tempPair.first);
break; break;
} }
this->q_content_index->enqueue(tempPair); this->m_contentIndexData->enqueue(tempPair);
break; break;
} }
tmp2->enqueue(tempPair.first); tmp2->enqueue(tempPair.first);
} }
// qDebug() << ">>>>>>>>all fileSize:" << fileSize << "file num:" << tmp->size() << "<<<<<<<<<<<<<<<<<<<"; // qDebug() << ">>>>>>>>all fileSize:" << fileSize << "file num:" << tmp->size() << "<<<<<<<<<<<<<<<<<<<";
this->p_indexGenerator->creatAllIndex(tmp2); if(!IndexGenerator::getInstance()->creatAllIndex(tmp2)) {
sucess = false;
break;
}
tmp2->clear(); tmp2->clear();
} }
delete tmp2; delete tmp2;
qDebug() << "content index end;"; qDebug() << "content index end;";
if(sucess) {
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "2");
}
sem.release(2); sem.release(2);
}); });
// OCR功能暂时屏蔽 // OCR功能暂时屏蔽
// QtConcurrent::run(&m_pool,[&]() { // QtConcurrent::run(&m_pool,[&]() {
// sem.acquire(5); // sem.acquire(5);
// mutex3.unlock(); // mutex3.unlock();
// QQueue<QString>* tmpOcr = new QQueue<QString>(); // QQueue<QString>* tmpOcr = new QQueue<QString>();
// qDebug() << "m_ocr_index:" << m_ocr_index->size(); // qDebug() << "m_ocr_index:" << m_ocr_index->size();
// if(m_allDatadaseStatus && m_contentIndexDatabaseStatus) {
// sem.release(2);
// return;
// }
// IndexGenerator::getInstance()->rebuildOcrIndexDatabase();
// bool sucess = true;
// while(!this->m_ocr_index->empty()) { // while(!this->m_ocr_index->empty()) {
// qint64 fileSize = 0; // qint64 fileSize = 0;
// //一次处理的数据量文件总大小为50M以下，50M为暂定值 // //一次处理的数据量文件总大小为50M以下，50M为暂定值
@ -235,36 +253,36 @@ void FirstIndex::run() {
// } // }
// tmpOcr->enqueue(tempPair.first); // tmpOcr->enqueue(tempPair.first);
// } // }
// this->p_indexGenerator->creatOcrIndex(tmpOcr); // if(!IndexGenerator::getInstance()->creatAllIndex(tmpOcr)) {
// sucess = false;
// break;
// }
// tmpOcr->clear(); // tmpOcr->clear();
// } // }
// delete tmpOcr; // delete tmpOcr;
// qDebug() << "OCR index end;"; // qDebug() << "OCR index end;";
// if(sucess) {
// IndexStatusRecorder::getInstance()->setStatus(OCR_DATABASE_STATE, "2");
// }
// sem.release(5); // sem.release(5);
// }); // });
mutex1.lock(); mutex1.lock();
mutex2.lock(); mutex2.lock();
// mutex3.lock(); // mutex3.lock();
sem.acquire(5); sem.acquire(5);
mutex1.unlock(); mutex1.unlock();
mutex2.unlock(); mutex2.unlock();
// mutex3.unlock(); // mutex3.unlock();
if(this->q_index) if(this->m_indexData)
delete this->q_index; delete this->m_indexData;
this->q_index = nullptr; this->m_indexData = nullptr;
if(this->q_content_index) if(this->m_contentIndexData)
delete this->q_content_index; delete this->m_contentIndexData;
this->q_content_index = nullptr; this->m_contentIndexData = nullptr;
if(this->m_ocr_index) if(this->m_ocrIndexData)
delete this->m_ocr_index; delete this->m_ocrIndexData;
this->m_ocr_index = nullptr; this->m_ocrIndexData = nullptr;
if(p_indexGenerator)
delete p_indexGenerator;
p_indexGenerator = nullptr;
// GlobalSettings::getInstance()->forceSync();
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "2");
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "2");
::_exit(0); ::_exit(0);
} else if(pid < 0) { } else if(pid < 0) {
qWarning() << "First Index fork error!!"; qWarning() << "First Index fork error!!";
@ -275,11 +293,11 @@ void FirstIndex::run() {
m_semaphore.release(1); m_semaphore.release(1);
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2"); IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2");
// int retval1 = write(fifo_fd, buffer, strlen(buffer)); // int retval1 = write(fifo_fd, buffer, strlen(buffer));
// if(retval1 == -1) { // if(retval1 == -1) {
// qWarning("write error\n"); // qWarning("write error\n");
// } // }
// qDebug("write data ok!\n"); // qDebug("write data ok!\n");
QTime t2 = QTime::currentTime(); QTime t2 = QTime::currentTime();
qWarning() << t1; qWarning() << t1;
qWarning() << t2; qWarning() << t2;

16
libsearch/index/first-index.h
View File

@ -47,24 +47,28 @@ public:
static FirstIndex* getInstance(); static FirstIndex* getInstance();
~FirstIndex(); ~FirstIndex();
virtual void DoSomething(const QFileInfo &) final; virtual void DoSomething(const QFileInfo &) final;
protected: protected:
void run() override; void run() override;
private: private:
FirstIndex(); FirstIndex();
FirstIndex(const FirstIndex&) = delete; FirstIndex(const FirstIndex&) = delete;
void operator=(const FirstIndex&) = delete; void operator=(const FirstIndex&) = delete;
static FirstIndex *m_instance; static FirstIndex *m_instance;
bool bool_dataBaseStatusOK = false;
bool bool_dataBaseExist = false; bool m_indexDatabaseStatus = false;
IndexGenerator* p_indexGenerator = nullptr; bool m_contentIndexDatabaseStatus = false;
bool m_ocrIndexDatabaseStatus = false;
bool m_allDatadaseStatus = false;
QThreadPool m_pool; QThreadPool m_pool;
QQueue<QVector<QString>>* q_index; QQueue<QVector<QString>>* m_indexData = nullptr;
// QQueue<QString>* q_content_index; // QQueue<QString>* q_content_index;
//修改QQueue存储数据为QPair<QString,qint64>，增加存储文件大小数据便于处理时统计--jxx20210519 //修改QQueue存储数据为QPair<QString,qint64>，增加存储文件大小数据便于处理时统计--jxx20210519
QQueue<QPair<QString,qint64>>* q_content_index; QQueue<QPair<QString,qint64>>* m_contentIndexData = nullptr;
//新增ocr队列存储ocr可识别处理的图片信息及大小； //新增ocr队列存储ocr可识别处理的图片信息及大小；
QQueue<QPair<QString,qint64>>* m_ocr_index; QQueue<QPair<QString,qint64>>* m_ocrIndexData = nullptr;
//xapian will auto commit per 10,000 changes, donnot change it!!! //xapian will auto commit per 10,000 changes, donnot change it!!!
const size_t u_send_length = 8192; const size_t u_send_length = 8192;
QSystemSemaphore m_semaphore; QSystemSemaphore m_semaphore;

113
libsearch/index/index-generator.cpp
View File

@ -25,15 +25,11 @@
#include <QFuture> #include <QFuture>
#include <QThreadPool> #include <QThreadPool>
#include <QFile> #include <QFile>
#include <QStandardPaths>
#include <malloc.h>
#include "file-utils.h" #include "file-utils.h"
#include "index-generator.h" #include "index-generator.h"
#include "chinese-segmentation.h" #include "chinese-segmentation.h"
#include <QStandardPaths>
#include <malloc.h>
#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/index_data").toStdString()
#define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/content_index_data").toStdString()
#define OCR_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/ocr_index_data").toStdString()
using namespace UkuiSearch; using namespace UkuiSearch;
@ -51,21 +47,14 @@ QVector<Document> IndexGenerator::g_docListForContent = QVector<Document>();
QVector<Document> IndexGenerator::g_docListForOcr = QVector<Document>(); QVector<Document> IndexGenerator::g_docListForOcr = QVector<Document>();
IndexGenerator *IndexGenerator::getInstance(bool rebuild, QObject *parent) { IndexGenerator *IndexGenerator::getInstance() {
QMutexLocker locker(&m_mutex); QMutexLocker locker(&m_mutex);
if(!global_instance) { if(!global_instance) {
qDebug() << "IndexGenerator================="; global_instance = new IndexGenerator();
global_instance = new IndexGenerator(rebuild, parent);
} }
qDebug() << "global_instance" << global_instance;
qDebug() << "QThread::currentThreadId()" << QThread::currentThreadId();
return global_instance; return global_instance;
} }
bool IndexGenerator::setIndexdataPath() {
return true;
}
//文件名索引 //文件名索引
bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) { bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) {
HandlePathList(messageList); HandlePathList(messageList);
@ -86,7 +75,7 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) {
qWarning() << "creatAllIndex fail!" << QString::fromStdString(e.get_description()); qWarning() << "creatAllIndex fail!" << QString::fromStdString(e.get_description());
//need a record //need a record
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1"); IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1");
assert(false); return false;
} }
qDebug() << "finish creatAllIndex"; qDebug() << "finish creatAllIndex";
IndexGenerator::g_docListForPath.clear(); IndexGenerator::g_docListForPath.clear();
@ -124,7 +113,7 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList) {
} catch(const Xapian::Error &e) { } catch(const Xapian::Error &e) {
qWarning() << "creat content Index fail!" << QString::fromStdString(e.get_description()); qWarning() << "creat content Index fail!" << QString::fromStdString(e.get_description());
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "1"); IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "1");
assert(false); return false;
} }
qDebug() << "finish creatAllIndex for content"; qDebug() << "finish creatAllIndex for content";
@ -163,8 +152,8 @@ bool IndexGenerator::creatOcrIndex(QQueue<QString> *messageList)
m_database_ocr->commit(); m_database_ocr->commit();
} catch(const Xapian::Error &e) { } catch(const Xapian::Error &e) {
qWarning() << "creat ocr Index fail!" << QString::fromStdString(e.get_description()); qWarning() << "creat ocr Index fail!" << QString::fromStdString(e.get_description());
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "1"); IndexStatusRecorder::getInstance()->setStatus(OCR_DATABASE_STATE, "1");
assert(false); return false;
} }
qDebug() << "finish creatAllIndex for ocr"; qDebug() << "finish creatAllIndex for ocr";
@ -176,26 +165,30 @@ bool IndexGenerator::creatOcrIndex(QQueue<QString> *messageList)
return true; return true;
} }
IndexGenerator::IndexGenerator(bool rebuild, QObject *parent) : QObject(parent) { IndexGenerator::IndexGenerator(QObject *parent) : QObject(parent)
QDir database(QString::fromStdString(INDEX_PATH)); {
QDir database(INDEX_PATH);
if(database.exists()) { if(!database.exists()) {
if(rebuild) qDebug() << "create index path" << INDEX_PATH<< database.mkpath(INDEX_PATH);
qDebug() << "remove" << database.removeRecursively();
} else {
qDebug() << "create index path" << database.mkpath(QString::fromStdString(INDEX_PATH));
} }
database.setPath(QString::fromStdString(CONTENT_INDEX_PATH)); database.setPath(CONTENT_INDEX_PATH);
if(database.exists()) { if(!database.exists()) {
if(rebuild) qDebug() << "create content index path" << CONTENT_INDEX_PATH << database.mkpath(CONTENT_INDEX_PATH);
qDebug() << "remove" << database.removeRecursively(); }
} else { database.setPath(OCR_INDEX_PATH);
qDebug() << "create content index path" << database.mkpath(QString::fromStdString(CONTENT_INDEX_PATH)); if(!database.exists()) {
qDebug() << "create ocr index path" << OCR_INDEX_PATH << database.mkpath(OCR_INDEX_PATH);
} }
m_database_path = new Xapian::WritableDatabase(INDEX_PATH, Xapian::DB_CREATE_OR_OPEN); try {
m_database_content = new Xapian::WritableDatabase(CONTENT_INDEX_PATH, Xapian::DB_CREATE_OR_OPEN); m_database_path = new Xapian::WritableDatabase(INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN);
m_database_ocr = new Xapian::WritableDatabase(OCR_INDEX_PATH, Xapian::DB_CREATE_OR_OPEN); m_database_content = new Xapian::WritableDatabase(CONTENT_INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN);
m_database_ocr = new Xapian::WritableDatabase(OCR_INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN);
} catch(const Xapian::Error &e) {
qWarning() << "creat Index fail!" << QString::fromStdString(e.get_description());
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
}
} }
IndexGenerator::~IndexGenerator() { IndexGenerator::~IndexGenerator() {
@ -237,6 +230,45 @@ IndexGenerator::~IndexGenerator() {
qDebug() << "~IndexGenerator end"; qDebug() << "~IndexGenerator end";
} }
void IndexGenerator::rebuildIndexDatabase(const QString &path)
{
QDir database(path);
if(database.exists()) {
qDebug() << "remove" << path << database.removeRecursively();
} else {
qDebug() << "create index path" << path << database.mkpath(path);
}
if(m_database_path)
m_database_path->~WritableDatabase();
m_database_path = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN);
}
void IndexGenerator::rebuildContentIndexDatabase(const QString &path)
{
QDir database(path);
if(database.exists()) {
qDebug() << "remove" << path << database.removeRecursively();
} else {
qDebug() << "create content index path" << path << database.mkpath(path);
}
if(m_database_content)
m_database_content->~WritableDatabase();
m_database_content = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN);
}
void IndexGenerator::rebuildOcrIndexDatabase(const QString &path)
{
QDir database(path);
if(database.exists()) {
qDebug() << "remove" << path << database.removeRecursively();
} else {
qDebug() << "create ocr index path" << path << database.mkpath(path);
}
if(m_database_ocr)
m_database_ocr->~WritableDatabase();
m_database_ocr = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN);
}
void IndexGenerator::insertIntoDatabase(Document& doc) { void IndexGenerator::insertIntoDatabase(Document& doc) {
// qDebug()<< "--index start--"; // qDebug()<< "--index start--";
Xapian::Document document = doc.getXapianDocument(); Xapian::Document document = doc.getXapianDocument();
@ -417,20 +449,13 @@ Document IndexGenerator::GenerateContentDocument(const QString &path) {
return doc; return doc;
} }
bool IndexGenerator::isIndexdataExist() {
// Xapian::Database db(m_index_data_path->toStdString());
return true;
}
//deprecated //deprecated
QStringList IndexGenerator::IndexSearch(QString indexText) { QStringList IndexGenerator::IndexSearch(QString indexText) {
QStringList searchResult; QStringList searchResult;
try { try {
qDebug() << "--search start--"; qDebug() << "--search start--";
Xapian::Database db(INDEX_PATH); Xapian::Database db(INDEX_PATH.toStdString());
Xapian::Enquire enquire(db); Xapian::Enquire enquire(db);
Xapian::QueryParser qp; Xapian::QueryParser qp;
qp.set_default_op(Xapian::Query::OP_PHRASE); qp.set_default_op(Xapian::Query::OP_PHRASE);

17
libsearch/index/index-generator.h
View File

@ -35,6 +35,7 @@
#include "file-reader.h" #include "file-reader.h"
#include "common.h" #include "common.h"
#include "pending-file.h" #include "pending-file.h"
#include "common.h"
namespace UkuiSearch { namespace UkuiSearch {
//extern QVector<Document> *_doc_list_path; //extern QVector<Document> *_doc_list_path;
@ -48,10 +49,12 @@ class IndexGenerator : public QObject {
friend class ConstructDocumentForOcr; friend class ConstructDocumentForOcr;
Q_OBJECT Q_OBJECT
public: public:
static IndexGenerator *getInstance(bool rebuild = false, QObject *parent = nullptr); static IndexGenerator *getInstance();
~IndexGenerator(); ~IndexGenerator();
bool setIndexdataPath();
bool isIndexdataExist(); void rebuildIndexDatabase(const QString &path = INDEX_PATH);
void rebuildContentIndexDatabase(const QString &path = CONTENT_INDEX_PATH);
void rebuildOcrIndexDatabase(const QString &path = OCR_INDEX_PATH);
// Q_INVOKABLE void appendDocListPath(Document doc); // Q_INVOKABLE void appendDocListPath(Document doc);
//for search test //for search test
static QStringList IndexSearch(QString indexText); static QStringList IndexSearch(QString indexText);
@ -69,7 +72,7 @@ public Q_SLOTS:
bool updateIndex(QVector<PendingFile> *pendingFiles); bool updateIndex(QVector<PendingFile> *pendingFiles);
private: private:
explicit IndexGenerator(bool rebuild = false, QObject *parent = nullptr); explicit IndexGenerator(QObject *parent = nullptr);
static QMutex m_mutex; static QMutex m_mutex;
//For file name index //For file name index
void HandlePathList(QQueue<QVector<QString> > *messageList); void HandlePathList(QQueue<QVector<QString> > *messageList);
@ -91,9 +94,9 @@ private:
static QMutex g_mutexDocListForOcr; static QMutex g_mutexDocListForOcr;
QMap<QString, QStringList> m_index_map; QMap<QString, QStringList> m_index_map;
QString m_index_data_path; QString m_index_data_path;
Xapian::WritableDatabase* m_database_path; Xapian::WritableDatabase* m_database_path = nullptr;
Xapian::WritableDatabase* m_database_content; Xapian::WritableDatabase* m_database_content = nullptr;
Xapian::WritableDatabase* m_database_ocr; Xapian::WritableDatabase* m_database_ocr = nullptr;
std::string m_docstr; std::string m_docstr;
std::string m_index_text_str; std::string m_index_text_str;
Xapian::TermGenerator m_indexer; Xapian::TermGenerator m_indexer;

40
libsearch/index/index-status-recorder.cpp
View File

@ -1,14 +1,15 @@
#include "index-status-recorder.h" #include "index-status-recorder.h"
#include <mutex>
using namespace UkuiSearch; using namespace UkuiSearch;
static IndexStatusRecorder *global_instance_indexStatusRecorder = nullptr; IndexStatusRecorder *IndexStatusRecorder::m_instance = nullptr;
std::once_flag g_IndexStatusRecorderInstanceFlag;
IndexStatusRecorder *IndexStatusRecorder::getInstance() IndexStatusRecorder *IndexStatusRecorder::getInstance()
{ {
if(!global_instance_indexStatusRecorder) { std::call_once(g_IndexStatusRecorderInstanceFlag, [] () {
global_instance_indexStatusRecorder = new IndexStatusRecorder; m_instance = new IndexStatusRecorder;
} });
return global_instance_indexStatusRecorder; return m_instance;
} }
void IndexStatusRecorder::setStatus(const QString &key, const QVariant &value) void IndexStatusRecorder::setStatus(const QString &key, const QVariant &value)
@ -24,6 +25,33 @@ const QVariant IndexStatusRecorder::getStatus(const QString &key)
return m_status->value(key); return m_status->value(key);
} }
bool IndexStatusRecorder::indexDatabaseEnable()
{
m_mutex.lock();
m_status->sync();
m_mutex.unlock();
return m_status->value(INDEX_DATABASE_STATE, QVariant(false)).toBool();
}
bool IndexStatusRecorder::contentIndexDatabaseEnable()
{
m_mutex.lock();
m_status->sync();
m_mutex.unlock();
return m_status->value(CONTENT_INDEX_DATABASE_STATE, QVariant(false)).toBool();
}
bool IndexStatusRecorder::ocrDatabaseEnable()
{
m_mutex.lock();
m_status->sync();
m_mutex.unlock();
return m_status->value(OCR_DATABASE_STATE, QVariant(false)).toBool();
}
IndexStatusRecorder::IndexStatusRecorder(QObject *parent) : QObject(parent) IndexStatusRecorder::IndexStatusRecorder(QObject *parent) : QObject(parent)
{ {
m_status = new QSettings(INDEX_STATUS, QSettings::IniFormat, this); m_status = new QSettings(INDEX_STATUS, QSettings::IniFormat, this);

7
libsearch/index/index-status-recorder.h
View File

@ -7,7 +7,8 @@
#include <QMutex> #include <QMutex>
#define CONTENT_INDEX_DATABASE_STATE "content_index_database_state" #define CONTENT_INDEX_DATABASE_STATE "content_index_database_state"
#define INDEX_DATABASE_STATE "index_database_state" #define INDEX_DATABASE_STATE "index_database_state"
#define INOTIFY_NORMAL_EXIT "inotify_normal_exit" #define OCR_DATABASE_STATE "ocr_database_state"
#define INOTIFY_NORMAL_EXIT "inotify_normal_exit" // 1 - 出错；2 - 正常；3-关闭索引； 0-有信号正在处理
#define PENDING_FILE_QUEUE_FINISH "pending_file_queue_finish" #define PENDING_FILE_QUEUE_FINISH "pending_file_queue_finish"
#define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf" #define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf"
namespace UkuiSearch { namespace UkuiSearch {
@ -19,9 +20,13 @@ public:
static IndexStatusRecorder *getInstance(); static IndexStatusRecorder *getInstance();
void setStatus(const QString& key, const QVariant& value); void setStatus(const QString& key, const QVariant& value);
const QVariant getStatus(const QString& key); const QVariant getStatus(const QString& key);
bool indexDatabaseEnable();
bool contentIndexDatabaseEnable();
bool ocrDatabaseEnable();
private: private:
explicit IndexStatusRecorder(QObject *parent = nullptr); explicit IndexStatusRecorder(QObject *parent = nullptr);
static IndexStatusRecorder *m_instance;
QSettings *m_status; QSettings *m_status;
QMutex m_mutex; QMutex m_mutex;
}; };

37
libsearch/index/search-method-manager.cpp
View File

@ -3,7 +3,6 @@ using namespace UkuiSearch;
static SearchMethodManager* global_instance = nullptr; static SearchMethodManager* global_instance = nullptr;
SearchMethodManager::SearchMethodManager() : m_semaphore(INDEX_SEM, 1, QSystemSemaphore::AccessMode::Create) SearchMethodManager::SearchMethodManager() : m_semaphore(INDEX_SEM, 1, QSystemSemaphore::AccessMode::Create)
{ {
qDebug() << m_semaphore.errorString();
m_fi = FirstIndex::getInstance(); m_fi = FirstIndex::getInstance();
m_iw = InotifyWatch::getInstance(); m_iw = InotifyWatch::getInstance();
} }
@ -25,30 +24,24 @@ void SearchMethodManager::searchMethod(FileUtils::SearchMethod sm) {
} }
if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::indexStatus) { if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::indexStatus) {
// Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running. // // Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running.
QDir fifoDir = QDir(QDir::homePath() + "/.config/org.ukui/ukui-search"); // QDir fifoDir = QDir(QDir::homePath() + "/.config/org.ukui/ukui-search");
if(!fifoDir.exists()) // if(!fifoDir.exists())
qDebug() << "create fifo path" << fifoDir.mkpath(fifoDir.absolutePath()); // qDebug() << "create fifo path" << fifoDir.mkpath(fifoDir.absolutePath());
unlink(UKUI_SEARCH_PIPE_PATH); // unlink(UKUI_SEARCH_PIPE_PATH);
int retval = mkfifo(UKUI_SEARCH_PIPE_PATH, 0777); // int retval = mkfifo(UKUI_SEARCH_PIPE_PATH, 0777);
if(retval == -1) { // if(retval == -1) {
qCritical() << "creat fifo error!!"; // qCritical() << "creat fifo error!!";
syslog(LOG_ERR, "creat fifo error!!\n"); // syslog(LOG_ERR, "creat fifo error!!\n");
assert(false); // assert(false);
return; // return;
} // }
qDebug() << "create fifo success\n"; // qDebug() << "create fifo success\n";
qWarning() << "start first index"; qDebug() << "start first index";
m_semaphore.acquire(); m_semaphore.acquire();
m_fi->start(); m_fi->start();
qWarning() << "start inotify index"; qDebug() << "start inotify index";
// InotifyIndex ii("/home");
// ii.start();
// this->m_ii = InotifyIndex::getInstance("/home");
// if(!this->m_ii->isRunning()) {
// this->m_ii->start();
// }
if(!this->m_iw->isRunning()) { if(!this->m_iw->isRunning()) {
this->m_iw->start(); this->m_iw->start();
} }