From baaac7a4dc52994682c972f7eb1e5db3c227cce6 Mon Sep 17 00:00:00 2001 From: iaom Date: Thu, 17 Mar 2022 15:40:55 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=87=E4=BB=B6=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E7=8A=B6=E6=80=81=E6=9F=A5=E8=AF=A2=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=EF=BC=9B=E4=BC=98=E5=8C=96=E6=96=87=E5=BB=BA=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E6=A0=A1=E9=AA=8C=E9=80=BB=E8=BE=91=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/control/settings-widget.cpp | 4 +- libsearch/common.h | 60 ++++--- libsearch/file-utils.cpp | 3 +- libsearch/file-utils.h | 3 +- libsearch/index/first-index.cpp | 190 ++++++++++++---------- libsearch/index/first-index.h | 16 +- libsearch/index/index-generator.cpp | 113 ++++++++----- libsearch/index/index-generator.h | 17 +- libsearch/index/index-status-recorder.cpp | 40 ++++- libsearch/index/index-status-recorder.h | 7 +- libsearch/index/search-method-manager.cpp | 37 ++--- 11 files changed, 285 insertions(+), 205 deletions(-) diff --git a/frontend/control/settings-widget.cpp b/frontend/control/settings-widget.cpp index 1873acb..39abded 100644 --- a/frontend/control/settings-widget.cpp +++ b/frontend/control/settings-widget.cpp @@ -277,7 +277,7 @@ void SettingsWidget::refreshIndexState() { } else { this->setIndexState(false); } - m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::_max_index_count))); + m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::maxIndexCount))); m_timer = new QTimer; connect(m_timer, &QTimer::timeout, this, [ = ]() { qDebug() << "FileUtils::indexStatus: " << FileUtils::indexStatus; @@ -286,7 +286,7 @@ void SettingsWidget::refreshIndexState() { } else { this->setIndexState(false); } - m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::_max_index_count))); + m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::maxIndexCount))); }); m_timer->start(0.5 * 1000); } diff --git a/libsearch/common.h b/libsearch/common.h index 487a0dd..3edf4cd 100644 --- a/libsearch/common.h +++ b/libsearch/common.h @@ -1,38 +1,44 @@ -#ifndef COMMON_H -#define COMMON_H +#pragma once + #include #include #include -#define UKUI_SEARCH_PIPE_PATH (QDir::homePath()+"/.config/org.ukui/ukui-search/ukuisearch").toLocal8Bit().constData() -#define FILE_SEARCH_VALUE "0" -#define DIR_SEARCH_VALUE "1" -#define LABEL_MAX_WIDTH 300 -#define HOME_PATH QDir::homePath() -#define INDEX_SEM "ukui-search-index-sem" + +static const int LABEL_MAX_WIDTH = 300; + +static const QString HOME_PATH = QDir::homePath(); +static const QString INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/index_data"); +static const QString CONTENT_INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/content_index_data"); +static const QString OCR_INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/ocr_index_data"); +static const QString FILE_SEARCH_VALUE = QStringLiteral("0"); +static const QString DIR_SEARCH_VALUE = QStringLiteral("1"); +static const QString INDEX_SEM = QStringLiteral("ukui-search-index-sem"); + static const QStringList allAppPath = { - QDir::homePath()+"/.local/share/applications/", - "/usr/share/applications/" + {HOME_PATH + "/.local/share/applications/"}, + {"/usr/share/applications/"} }; + static const QMap targetFileTypeMap = { - std::map::value_type("doc", true), - std::map::value_type("docx", true), - std::map::value_type("ppt", true), - std::map::value_type("pptx", true), - std::map::value_type("xls", true), - std::map::value_type("xlsx", true), - std::map::value_type("txt", true), - std::map::value_type("dot", true), - std::map::value_type("wps", true), - std::map::value_type("pps", true), - std::map::value_type("dps", true), - std::map::value_type("et", true), - std::map::value_type("pdf", true) + {"doc", true}, + {"docx", true}, + {"ppt", true}, + {"pptx", true}, + {"xls", true}, + {"xlsx", true}, + {"txt", true}, + {"dot", true}, + {"wps", true}, + {"pps", true}, + {"dps", true}, + {"et", true}, + {"pdf", true} }; static const QMap targetPhotographTypeMap = { - std::map::value_type("png", true), - std::map::value_type("jpg", true), - std::map::value_type("jpeg", true)//TODO 待完善,后续改为配置文件 + {"png", true}, + {"jpg", true}, + {"jpeg", true} // TODO 待完善,后续改为配置文件 }; //TODO Put things that needed to be put here here. -#endif // COMMON_H + diff --git a/libsearch/file-utils.cpp b/libsearch/file-utils.cpp index 6a359b0..e646758 100644 --- a/libsearch/file-utils.cpp +++ b/libsearch/file-utils.cpp @@ -30,8 +30,7 @@ #include "pinyinmanager.h" using namespace UkuiSearch; -size_t FileUtils::_max_index_count = 0; -size_t FileUtils::_current_index_count = 0; +size_t FileUtils::maxIndexCount = 0; unsigned short FileUtils::indexStatus = 0; FileUtils::SearchMethod FileUtils::searchMethod = FileUtils::SearchMethod::DIRECTSEARCH; QMap FileUtils::map_chinese2pinyin = QMap(); diff --git a/libsearch/file-utils.h b/libsearch/file-utils.h index f3b8654..4397ef1 100644 --- a/libsearch/file-utils.h +++ b/libsearch/file-utils.h @@ -97,8 +97,7 @@ public: static QIcon iconFromTheme(const QString &name, const QIcon &iconDefault); static bool isOpenXMLFileEncrypted(QString &path); static bool isEncrypedOrUnreadable(QString path); - static size_t _max_index_count; - static size_t _current_index_count; //this one has been Abandoned,do not use it. + static size_t maxIndexCount; static unsigned short indexStatus; enum class SearchMethod { DIRECTSEARCH = 0, INDEXSEARCH = 1}; diff --git a/libsearch/index/first-index.cpp b/libsearch/index/first-index.cpp index 9e26048..07d896d 100644 --- a/libsearch/index/first-index.cpp +++ b/libsearch/index/first-index.cpp @@ -42,24 +42,21 @@ FirstIndex *FirstIndex::getInstance() FirstIndex::~FirstIndex() { qDebug() << "~FirstIndex"; - if(this->q_index) - delete this->q_index; - this->q_index = nullptr; - if(this->q_content_index) - delete this->q_content_index; - this->q_content_index = nullptr; - if(this->m_ocr_index) - delete this->m_ocr_index; - this->m_ocr_index = nullptr; - if(this->p_indexGenerator) - delete this->p_indexGenerator; - this->p_indexGenerator = nullptr; + if(this->m_indexData) + delete this->m_indexData; + this->m_indexData = nullptr; + if(this->m_contentIndexData) + delete this->m_contentIndexData; + this->m_contentIndexData = nullptr; + if(this->m_ocrIndexData) + delete this->m_ocrIndexData; + this->m_ocrIndexData = nullptr; qDebug() << "~FirstIndex end"; } void FirstIndex::DoSomething(const QFileInfo& fileInfo) { -// qDebug() << "there are some shit here"<q_index->enqueue(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0")); + // qDebug() << "there are some shit here"<m_indexData->enqueue(QVector() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0")); if (fileInfo.fileName().split(".", QString::SkipEmptyParts).length() < 2) return; if (true == targetFileTypeMap[fileInfo.fileName().split(".").last()] @@ -71,7 +68,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) { if(!file.setCurrentFile("word/document.xml", QuaZip::csSensitive)) return; QuaZipFile fileR(&file); - this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//docx解压缩后的xml文件为实际需要解析文件大小 + this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//docx解压缩后的xml文件为实际需要解析文件大小 file.close(); } else if (fileInfo.fileName().split(".").last() == "pptx") { QuaZip file(fileInfo.absoluteFilePath()); @@ -92,7 +89,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) { } } file.close(); - this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileSize));//pptx解压缩后的xml文件为实际需要解析文件大小 + this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileSize));//pptx解压缩后的xml文件为实际需要解析文件大小 } else if (fileInfo.fileName().split(".").last() == "xlsx") { QuaZip file(fileInfo.absoluteFilePath()); if(!file.open(QuaZip::mdUnzip)) @@ -100,14 +97,14 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) { if(!file.setCurrentFile("xl/sharedStrings.xml", QuaZip::csSensitive)) return; QuaZipFile fileR(&file); - this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//xlsx解压缩后的xml文件为实际解析文件大小 + this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//xlsx解压缩后的xml文件为实际解析文件大小 file.close(); } else { - this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size())); + this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size())); } - } else if (true == targetPhotographTypeMap[fileInfo.fileName().split(".").last()]) { - this->m_ocr_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size())); - } + } /*else if (true == targetPhotographTypeMap[fileInfo.fileName().split(".").last()]) { + this->m_ocrIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size())); + }*/ } void FirstIndex::run() { @@ -115,26 +112,28 @@ void FirstIndex::run() { // Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running. QString indexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(INDEX_DATABASE_STATE).toString(); QString contentIndexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE).toString(); +// QString ocrIndexDatabaseStatus = IndexStatusRecorder::getInstance()->getStatus(OCR_DATABASE_STATE).toString(); QString inotifyIndexStatus = IndexStatusRecorder::getInstance()->getStatus(INOTIFY_NORMAL_EXIT).toString(); - qDebug() << "indexDataBaseStatus: " << indexDataBaseStatus; - qDebug() << "contentIndexDataBaseStatus: " << contentIndexDataBaseStatus; - qDebug() << "inotifyIndexStatus: " << inotifyIndexStatus; + qInfo() << "indexDataBaseStatus: " << indexDataBaseStatus; + qInfo() << "contentIndexDataBaseStatus: " << contentIndexDataBaseStatus; +// qInfo() << "ocrIndexDatabaseStatus: " << ocrIndexDatabaseStatus; + qInfo() << "inotifyIndexStatus: " << inotifyIndexStatus; - if(indexDataBaseStatus == "") { - this->bool_dataBaseExist = false; - } else { - this->bool_dataBaseExist = true; - } - if(indexDataBaseStatus != "2" || contentIndexDataBaseStatus != "2" || inotifyIndexStatus != "2") { - this->bool_dataBaseStatusOK = false; - } else { - this->bool_dataBaseStatusOK = true; + m_allDatadaseStatus = inotifyIndexStatus == "2" ? true : false; + m_indexDatabaseStatus = indexDataBaseStatus == "2" ? true : false; + m_contentIndexDatabaseStatus = contentIndexDataBaseStatus == "2" ? true : false; +// m_ocrIndexDatabaseStatus = ocrIndexDatabaseStatus == "2" ? true : false; + + if(m_allDatadaseStatus && m_indexDatabaseStatus && m_contentIndexDatabaseStatus /*&& m_ocrIndexDatabaseStatus*/) { + m_semaphore.release(1); + return; } - this->q_index = new QQueue>(); - this->q_content_index = new QQueue>(); - this->m_ocr_index = new QQueue>(); + + this->m_indexData = new QQueue>(); + this->m_contentIndexData = new QQueue>(); +// this->m_ocrIndexData = new QQueue>(); ++FileUtils::indexStatus; pid_t pid; @@ -142,23 +141,12 @@ void FirstIndex::run() { if(pid == 0) { prctl(PR_SET_PDEATHSIG, SIGTERM); prctl(PR_SET_NAME, "first-index"); - if(this->bool_dataBaseExist) { - if(this->bool_dataBaseStatusOK) { - ::_exit(0); - } else { - //if the parameter is false, index won't be rebuild - //if it is true, index will be rebuild - p_indexGenerator = IndexGenerator::getInstance(true, this); - } - } else { -// p_indexGenerator = IndexGenerator::getInstance(false,this); - p_indexGenerator = IndexGenerator::getInstance(true, this); - } - //TODO Fix these weird code. + QSemaphore sem(5); QMutex mutex1, mutex2, mutex3; mutex1.lock(); mutex2.lock(); + // mutex3.lock(); qInfo() << "index dir" << DirWatcher::getDirWatcher()->currentindexableDir(); qInfo() << "index block dir" << DirWatcher::getDirWatcher()->currentBlackListOfIndex(); @@ -166,59 +154,89 @@ void FirstIndex::run() { setBlockPath(DirWatcher::getDirWatcher()->currentBlackListOfIndex()); this->Traverse(); - FileUtils::_max_index_count = this->q_index->length(); - qDebug() << "max_index_count:" << FileUtils::_max_index_count; + FileUtils::maxIndexCount = this->m_indexData->length(); + qDebug() << "max_index_count:" << FileUtils::maxIndexCount; QtConcurrent::run(&m_pool, [&]() { sem.acquire(2); mutex1.unlock(); - qDebug() << "index start;"; + if(m_allDatadaseStatus && m_indexDatabaseStatus) { + sem.release(2); + return; + } + qDebug() << "index start;" << m_indexData->size(); + IndexGenerator::getInstance()->rebuildIndexDatabase(); QQueue>* tmp1 = new QQueue>(); - while(!this->q_index->empty()) { - for(size_t i = 0; (i < 8192) && (!this->q_index->empty()); ++i) { - tmp1->enqueue(this->q_index->dequeue()); + bool sucess = true; + while(!this->m_indexData->empty()) { + for(size_t i = 0; (i < 8192) && (!this->m_indexData->empty()); ++i) { + tmp1->enqueue(this->m_indexData->dequeue()); + } + if(!IndexGenerator::getInstance()->creatAllIndex(tmp1)) { + sucess = false; + break; } - this->p_indexGenerator->creatAllIndex(tmp1); tmp1->clear(); } delete tmp1; qDebug() << "index end;"; + if(sucess) { + IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "2"); + } sem.release(2); }); QtConcurrent::run(&m_pool,[&]() { sem.acquire(2); mutex2.unlock(); + if(m_allDatadaseStatus && m_contentIndexDatabaseStatus) { + sem.release(2); + return; + } + qDebug() << "content index start:" << m_contentIndexData->size(); + IndexGenerator::getInstance()->rebuildContentIndexDatabase(); QQueue* tmp2 = new QQueue(); - qDebug() << "q_content_index:" << q_content_index->size(); - while(!this->q_content_index->empty()) { + bool sucess = true; + while(!this->m_contentIndexData->empty()) { qint64 fileSize = 0; //修改一次处理的数据量，从30个文件改为文件总大小为50M以下，50M为暂定值--jxx20210519 - for(size_t i = 0;/* (i < 30) && (fileSize < 52428800) && */(!this->q_content_index->empty()); ++i) { - QPair tempPair = this->q_content_index->dequeue(); + for(size_t i = 0;/* (i < 30) && (fileSize < 52428800) && */(!this->m_contentIndexData->empty()); ++i) { + QPair tempPair = this->m_contentIndexData->dequeue(); fileSize += tempPair.second; if (fileSize > 52428800 ) { if (tmp2->size() == 0) { tmp2->enqueue(tempPair.first); break; } - this->q_content_index->enqueue(tempPair); + this->m_contentIndexData->enqueue(tempPair); break; } tmp2->enqueue(tempPair.first); } // qDebug() << ">>>>>>>>all fileSize:" << fileSize << "file num:" << tmp->size() << "<<<<<<<<<<<<<<<<<<<"; - this->p_indexGenerator->creatAllIndex(tmp2); + if(!IndexGenerator::getInstance()->creatAllIndex(tmp2)) { + sucess = false; + break; + } tmp2->clear(); } delete tmp2; qDebug() << "content index end;"; + if(sucess) { + IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "2"); + } sem.release(2); }); -// OCR功能暂时屏蔽 + // OCR功能暂时屏蔽 // QtConcurrent::run(&m_pool,[&]() { // sem.acquire(5); // mutex3.unlock(); // QQueue* tmpOcr = new QQueue(); // qDebug() << "m_ocr_index:" << m_ocr_index->size(); +// if(m_allDatadaseStatus && m_contentIndexDatabaseStatus) { +// sem.release(2); +// return; +// } +// IndexGenerator::getInstance()->rebuildOcrIndexDatabase(); +// bool sucess = true; // while(!this->m_ocr_index->empty()) { // qint64 fileSize = 0; // //一次处理的数据量文件总大小为50M以下，50M为暂定值 @@ -235,36 +253,36 @@ void FirstIndex::run() { // } // tmpOcr->enqueue(tempPair.first); // } -// this->p_indexGenerator->creatOcrIndex(tmpOcr); +// if(!IndexGenerator::getInstance()->creatAllIndex(tmpOcr)) { +// sucess = false; +// break; +// } // tmpOcr->clear(); // } // delete tmpOcr; // qDebug() << "OCR index end;"; +// if(sucess) { +// IndexStatusRecorder::getInstance()->setStatus(OCR_DATABASE_STATE, "2"); +// } // sem.release(5); // }); mutex1.lock(); mutex2.lock(); -// mutex3.lock(); + // mutex3.lock(); sem.acquire(5); mutex1.unlock(); mutex2.unlock(); -// mutex3.unlock(); + // mutex3.unlock(); - if(this->q_index) - delete this->q_index; - this->q_index = nullptr; - if(this->q_content_index) - delete this->q_content_index; - this->q_content_index = nullptr; - if(this->m_ocr_index) - delete this->m_ocr_index; - this->m_ocr_index = nullptr; - if(p_indexGenerator) - delete p_indexGenerator; - p_indexGenerator = nullptr; -// GlobalSettings::getInstance()->forceSync(); - IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "2"); - IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "2"); + if(this->m_indexData) + delete this->m_indexData; + this->m_indexData = nullptr; + if(this->m_contentIndexData) + delete this->m_contentIndexData; + this->m_contentIndexData = nullptr; + if(this->m_ocrIndexData) + delete this->m_ocrIndexData; + this->m_ocrIndexData = nullptr; ::_exit(0); } else if(pid < 0) { qWarning() << "First Index fork error!!"; @@ -275,11 +293,11 @@ void FirstIndex::run() { m_semaphore.release(1); IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2"); -// int retval1 = write(fifo_fd, buffer, strlen(buffer)); -// if(retval1 == -1) { -// qWarning("write error\n"); -// } -// qDebug("write data ok!\n"); + // int retval1 = write(fifo_fd, buffer, strlen(buffer)); + // if(retval1 == -1) { + // qWarning("write error\n"); + // } + // qDebug("write data ok!\n"); QTime t2 = QTime::currentTime(); qWarning() << t1; qWarning() << t2; diff --git a/libsearch/index/first-index.h b/libsearch/index/first-index.h index a85a88c..dd15fd7 100644 --- a/libsearch/index/first-index.h +++ b/libsearch/index/first-index.h @@ -47,24 +47,28 @@ public: static FirstIndex* getInstance(); ~FirstIndex(); virtual void DoSomething(const QFileInfo &) final; + protected: void run() override; private: FirstIndex(); FirstIndex(const FirstIndex&) = delete; void operator=(const FirstIndex&) = delete; + static FirstIndex *m_instance; - bool bool_dataBaseStatusOK = false; - bool bool_dataBaseExist = false; - IndexGenerator* p_indexGenerator = nullptr; + + bool m_indexDatabaseStatus = false; + bool m_contentIndexDatabaseStatus = false; + bool m_ocrIndexDatabaseStatus = false; + bool m_allDatadaseStatus = false; QThreadPool m_pool; - QQueue>* q_index; + QQueue>* m_indexData = nullptr; // QQueue* q_content_index; //修改QQueue存储数据为QPair，增加存储文件大小数据便于处理时统计--jxx20210519 - QQueue>* q_content_index; + QQueue>* m_contentIndexData = nullptr; //新增ocr队列存储ocr可识别处理的图片信息及大小； - QQueue>* m_ocr_index; + QQueue>* m_ocrIndexData = nullptr; //xapian will auto commit per 10,000 changes, donnot change it!!! const size_t u_send_length = 8192; QSystemSemaphore m_semaphore; diff --git a/libsearch/index/index-generator.cpp b/libsearch/index/index-generator.cpp index ea7f7d8..e158a05 100644 --- a/libsearch/index/index-generator.cpp +++ b/libsearch/index/index-generator.cpp @@ -25,15 +25,11 @@ #include #include #include +#include +#include #include "file-utils.h" #include "index-generator.h" #include "chinese-segmentation.h" -#include -#include - -#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/index_data").toStdString() -#define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/content_index_data").toStdString() -#define OCR_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/ocr_index_data").toStdString() using namespace UkuiSearch; @@ -51,21 +47,14 @@ QVector IndexGenerator::g_docListForContent = QVector(); QVector IndexGenerator::g_docListForOcr = QVector(); -IndexGenerator *IndexGenerator::getInstance(bool rebuild, QObject *parent) { +IndexGenerator *IndexGenerator::getInstance() { QMutexLocker locker(&m_mutex); if(!global_instance) { - qDebug() << "IndexGenerator================="; - global_instance = new IndexGenerator(rebuild, parent); + global_instance = new IndexGenerator(); } - qDebug() << "global_instance" << global_instance; - qDebug() << "QThread::currentThreadId()" << QThread::currentThreadId(); return global_instance; } -bool IndexGenerator::setIndexdataPath() { - return true; -} - //文件名索引 bool IndexGenerator::creatAllIndex(QQueue > *messageList) { HandlePathList(messageList); @@ -86,7 +75,7 @@ bool IndexGenerator::creatAllIndex(QQueue > *messageList) { qWarning() << "creatAllIndex fail!" << QString::fromStdString(e.get_description()); //need a record IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1"); - assert(false); + return false; } qDebug() << "finish creatAllIndex"; IndexGenerator::g_docListForPath.clear(); @@ -124,7 +113,7 @@ bool IndexGenerator::creatAllIndex(QQueue *messageList) { } catch(const Xapian::Error &e) { qWarning() << "creat content Index fail!" << QString::fromStdString(e.get_description()); IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "1"); - assert(false); + return false; } qDebug() << "finish creatAllIndex for content"; @@ -163,8 +152,8 @@ bool IndexGenerator::creatOcrIndex(QQueue *messageList) m_database_ocr->commit(); } catch(const Xapian::Error &e) { qWarning() << "creat ocr Index fail!" << QString::fromStdString(e.get_description()); - IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "1"); - assert(false); + IndexStatusRecorder::getInstance()->setStatus(OCR_DATABASE_STATE, "1"); + return false; } qDebug() << "finish creatAllIndex for ocr"; @@ -176,26 +165,30 @@ bool IndexGenerator::creatOcrIndex(QQueue *messageList) return true; } -IndexGenerator::IndexGenerator(bool rebuild, QObject *parent) : QObject(parent) { - QDir database(QString::fromStdString(INDEX_PATH)); - - if(database.exists()) { - if(rebuild) - qDebug() << "remove" << database.removeRecursively(); - } else { - qDebug() << "create index path" << database.mkpath(QString::fromStdString(INDEX_PATH)); +IndexGenerator::IndexGenerator(QObject *parent) : QObject(parent) +{ + QDir database(INDEX_PATH); + if(!database.exists()) { + qDebug() << "create index path" << INDEX_PATH<< database.mkpath(INDEX_PATH); } - database.setPath(QString::fromStdString(CONTENT_INDEX_PATH)); - if(database.exists()) { - if(rebuild) - qDebug() << "remove" << database.removeRecursively(); - } else { - qDebug() << "create content index path" << database.mkpath(QString::fromStdString(CONTENT_INDEX_PATH)); + database.setPath(CONTENT_INDEX_PATH); + if(!database.exists()) { + qDebug() << "create content index path" << CONTENT_INDEX_PATH << database.mkpath(CONTENT_INDEX_PATH); + } + database.setPath(OCR_INDEX_PATH); + if(!database.exists()) { + qDebug() << "create ocr index path" << OCR_INDEX_PATH << database.mkpath(OCR_INDEX_PATH); } - m_database_path = new Xapian::WritableDatabase(INDEX_PATH, Xapian::DB_CREATE_OR_OPEN); - m_database_content = new Xapian::WritableDatabase(CONTENT_INDEX_PATH, Xapian::DB_CREATE_OR_OPEN); - m_database_ocr = new Xapian::WritableDatabase(OCR_INDEX_PATH, Xapian::DB_CREATE_OR_OPEN); + try { + m_database_path = new Xapian::WritableDatabase(INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN); + m_database_content = new Xapian::WritableDatabase(CONTENT_INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN); + m_database_ocr = new Xapian::WritableDatabase(OCR_INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN); + } catch(const Xapian::Error &e) { + qWarning() << "creat Index fail!" << QString::fromStdString(e.get_description()); + IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1"); + assert(false); + } } IndexGenerator::~IndexGenerator() { @@ -237,6 +230,45 @@ IndexGenerator::~IndexGenerator() { qDebug() << "~IndexGenerator end"; } +void IndexGenerator::rebuildIndexDatabase(const QString &path) +{ + QDir database(path); + if(database.exists()) { + qDebug() << "remove" << path << database.removeRecursively(); + } else { + qDebug() << "create index path" << path << database.mkpath(path); + } + if(m_database_path) + m_database_path->~WritableDatabase(); + m_database_path = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN); +} + +void IndexGenerator::rebuildContentIndexDatabase(const QString &path) +{ + QDir database(path); + if(database.exists()) { + qDebug() << "remove" << path << database.removeRecursively(); + } else { + qDebug() << "create content index path" << path << database.mkpath(path); + } + if(m_database_content) + m_database_content->~WritableDatabase(); + m_database_content = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN); +} + +void IndexGenerator::rebuildOcrIndexDatabase(const QString &path) +{ + QDir database(path); + if(database.exists()) { + qDebug() << "remove" << path << database.removeRecursively(); + } else { + qDebug() << "create ocr index path" << path << database.mkpath(path); + } + if(m_database_ocr) + m_database_ocr->~WritableDatabase(); + m_database_ocr = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN); +} + void IndexGenerator::insertIntoDatabase(Document& doc) { // qDebug()<< "--index start--"; Xapian::Document document = doc.getXapianDocument(); @@ -417,20 +449,13 @@ Document IndexGenerator::GenerateContentDocument(const QString &path) { return doc; } -bool IndexGenerator::isIndexdataExist() { - -// Xapian::Database db(m_index_data_path->toStdString()); - return true; - - -} //deprecated QStringList IndexGenerator::IndexSearch(QString indexText) { QStringList searchResult; try { qDebug() << "--search start--"; - Xapian::Database db(INDEX_PATH); + Xapian::Database db(INDEX_PATH.toStdString()); Xapian::Enquire enquire(db); Xapian::QueryParser qp; qp.set_default_op(Xapian::Query::OP_PHRASE); diff --git a/libsearch/index/index-generator.h b/libsearch/index/index-generator.h index d7ecdce..0b4c236 100644 --- a/libsearch/index/index-generator.h +++ b/libsearch/index/index-generator.h @@ -35,6 +35,7 @@ #include "file-reader.h" #include "common.h" #include "pending-file.h" +#include "common.h" namespace UkuiSearch { //extern QVector *_doc_list_path; @@ -48,10 +49,12 @@ class IndexGenerator : public QObject { friend class ConstructDocumentForOcr; Q_OBJECT public: - static IndexGenerator *getInstance(bool rebuild = false, QObject *parent = nullptr); + static IndexGenerator *getInstance(); ~IndexGenerator(); - bool setIndexdataPath(); - bool isIndexdataExist(); + + void rebuildIndexDatabase(const QString &path = INDEX_PATH); + void rebuildContentIndexDatabase(const QString &path = CONTENT_INDEX_PATH); + void rebuildOcrIndexDatabase(const QString &path = OCR_INDEX_PATH); // Q_INVOKABLE void appendDocListPath(Document doc); //for search test static QStringList IndexSearch(QString indexText); @@ -69,7 +72,7 @@ public Q_SLOTS: bool updateIndex(QVector *pendingFiles); private: - explicit IndexGenerator(bool rebuild = false, QObject *parent = nullptr); + explicit IndexGenerator(QObject *parent = nullptr); static QMutex m_mutex; //For file name index void HandlePathList(QQueue > *messageList); @@ -91,9 +94,9 @@ private: static QMutex g_mutexDocListForOcr; QMap m_index_map; QString m_index_data_path; - Xapian::WritableDatabase* m_database_path; - Xapian::WritableDatabase* m_database_content; - Xapian::WritableDatabase* m_database_ocr; + Xapian::WritableDatabase* m_database_path = nullptr; + Xapian::WritableDatabase* m_database_content = nullptr; + Xapian::WritableDatabase* m_database_ocr = nullptr; std::string m_docstr; std::string m_index_text_str; Xapian::TermGenerator m_indexer; diff --git a/libsearch/index/index-status-recorder.cpp b/libsearch/index/index-status-recorder.cpp index 2ac7f3c..fa295ab 100644 --- a/libsearch/index/index-status-recorder.cpp +++ b/libsearch/index/index-status-recorder.cpp @@ -1,14 +1,15 @@ #include "index-status-recorder.h" +#include using namespace UkuiSearch; -static IndexStatusRecorder *global_instance_indexStatusRecorder = nullptr; - +IndexStatusRecorder *IndexStatusRecorder::m_instance = nullptr; +std::once_flag g_IndexStatusRecorderInstanceFlag; IndexStatusRecorder *IndexStatusRecorder::getInstance() { - if(!global_instance_indexStatusRecorder) { - global_instance_indexStatusRecorder = new IndexStatusRecorder; - } - return global_instance_indexStatusRecorder; + std::call_once(g_IndexStatusRecorderInstanceFlag, [] () { + m_instance = new IndexStatusRecorder; + }); + return m_instance; } void IndexStatusRecorder::setStatus(const QString &key, const QVariant &value) @@ -24,6 +25,33 @@ const QVariant IndexStatusRecorder::getStatus(const QString &key) return m_status->value(key); } +bool IndexStatusRecorder::indexDatabaseEnable() +{ + m_mutex.lock(); + m_status->sync(); + m_mutex.unlock(); + return m_status->value(INDEX_DATABASE_STATE, QVariant(false)).toBool(); + +} + +bool IndexStatusRecorder::contentIndexDatabaseEnable() +{ + m_mutex.lock(); + m_status->sync(); + m_mutex.unlock(); + return m_status->value(CONTENT_INDEX_DATABASE_STATE, QVariant(false)).toBool(); + +} + +bool IndexStatusRecorder::ocrDatabaseEnable() +{ + m_mutex.lock(); + m_status->sync(); + m_mutex.unlock(); + return m_status->value(OCR_DATABASE_STATE, QVariant(false)).toBool(); + +} + IndexStatusRecorder::IndexStatusRecorder(QObject *parent) : QObject(parent) { m_status = new QSettings(INDEX_STATUS, QSettings::IniFormat, this); diff --git a/libsearch/index/index-status-recorder.h b/libsearch/index/index-status-recorder.h index 280950f..7a61a1b 100644 --- a/libsearch/index/index-status-recorder.h +++ b/libsearch/index/index-status-recorder.h @@ -7,7 +7,8 @@ #include #define CONTENT_INDEX_DATABASE_STATE "content_index_database_state" #define INDEX_DATABASE_STATE "index_database_state" -#define INOTIFY_NORMAL_EXIT "inotify_normal_exit" +#define OCR_DATABASE_STATE "ocr_database_state" +#define INOTIFY_NORMAL_EXIT "inotify_normal_exit" // 1 - 出错；2 - 正常；3-关闭索引； 0-有信号正在处理 #define PENDING_FILE_QUEUE_FINISH "pending_file_queue_finish" #define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf" namespace UkuiSearch { @@ -19,9 +20,13 @@ public: static IndexStatusRecorder *getInstance(); void setStatus(const QString& key, const QVariant& value); const QVariant getStatus(const QString& key); + bool indexDatabaseEnable(); + bool contentIndexDatabaseEnable(); + bool ocrDatabaseEnable(); private: explicit IndexStatusRecorder(QObject *parent = nullptr); + static IndexStatusRecorder *m_instance; QSettings *m_status; QMutex m_mutex; }; diff --git a/libsearch/index/search-method-manager.cpp b/libsearch/index/search-method-manager.cpp index a4920da..e045b62 100644 --- a/libsearch/index/search-method-manager.cpp +++ b/libsearch/index/search-method-manager.cpp @@ -3,7 +3,6 @@ using namespace UkuiSearch; static SearchMethodManager* global_instance = nullptr; SearchMethodManager::SearchMethodManager() : m_semaphore(INDEX_SEM, 1, QSystemSemaphore::AccessMode::Create) { - qDebug() << m_semaphore.errorString(); m_fi = FirstIndex::getInstance(); m_iw = InotifyWatch::getInstance(); } @@ -25,30 +24,24 @@ void SearchMethodManager::searchMethod(FileUtils::SearchMethod sm) { } if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::indexStatus) { - // Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running. - QDir fifoDir = QDir(QDir::homePath() + "/.config/org.ukui/ukui-search"); - if(!fifoDir.exists()) - qDebug() << "create fifo path" << fifoDir.mkpath(fifoDir.absolutePath()); +// // Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running. +// QDir fifoDir = QDir(QDir::homePath() + "/.config/org.ukui/ukui-search"); +// if(!fifoDir.exists()) +// qDebug() << "create fifo path" << fifoDir.mkpath(fifoDir.absolutePath()); - unlink(UKUI_SEARCH_PIPE_PATH); - int retval = mkfifo(UKUI_SEARCH_PIPE_PATH, 0777); - if(retval == -1) { - qCritical() << "creat fifo error!!"; - syslog(LOG_ERR, "creat fifo error!!\n"); - assert(false); - return; - } - qDebug() << "create fifo success\n"; - qWarning() << "start first index"; +// unlink(UKUI_SEARCH_PIPE_PATH); +// int retval = mkfifo(UKUI_SEARCH_PIPE_PATH, 0777); +// if(retval == -1) { +// qCritical() << "creat fifo error!!"; +// syslog(LOG_ERR, "creat fifo error!!\n"); +// assert(false); +// return; +// } +// qDebug() << "create fifo success\n"; + qDebug() << "start first index"; m_semaphore.acquire(); m_fi->start(); - qWarning() << "start inotify index"; -// InotifyIndex ii("/home"); -// ii.start(); -// this->m_ii = InotifyIndex::getInstance("/home"); -// if(!this->m_ii->isRunning()) { -// this->m_ii->start(); -// } + qDebug() << "start inotify index"; if(!this->m_iw->isRunning()) { this->m_iw->start(); }