openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

增加文件索引状态查询功能；优化文建索引启动校验逻辑。

This commit is contained in:
iaom 2022-03-17 15:40:55 +08:00
parent 7126068efb
commit baaac7a4dc
11 changed files with 285 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
frontend/control/settings-widget.cpp
View File

@ -277,7 +277,7 @@ void SettingsWidget::refreshIndexState() {
} else {
this->setIndexState(false);
}
m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::_max_index_count)));
m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::maxIndexCount)));
m_timer = new QTimer;
connect(m_timer, &QTimer::timeout, this, [ = ]() {
qDebug() << "FileUtils::indexStatus: " << FileUtils::indexStatus;
@ -286,7 +286,7 @@ void SettingsWidget::refreshIndexState() {
} else {
this->setIndexState(false);
}
m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::_max_index_count)));
m_indexNumLabel->setText(QString("%1/%2").arg(QString::number(SearchManager::getCurrentIndexCount())).arg(QString::number(FileUtils::maxIndexCount)));
});
m_timer->start(0.5 * 1000);
}

60
libsearch/common.h
View File

@ -1,38 +1,44 @@
#ifndef COMMON_H
#define COMMON_H
#pragma once
#include <QMap>
#include <QString>
#include <QDir>
#define UKUI_SEARCH_PIPE_PATH (QDir::homePath()+"/.config/org.ukui/ukui-search/ukuisearch").toLocal8Bit().constData()
#define FILE_SEARCH_VALUE "0"
#define DIR_SEARCH_VALUE "1"
#define LABEL_MAX_WIDTH 300
#define HOME_PATH QDir::homePath()
#define INDEX_SEM "ukui-search-index-sem"
static const int LABEL_MAX_WIDTH = 300;
static const QString HOME_PATH = QDir::homePath();
static const QString INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/index_data");
static const QString CONTENT_INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/content_index_data");
static const QString OCR_INDEX_PATH = HOME_PATH + QStringLiteral("/.config/org.ukui/ukui-search/ocr_index_data");
static const QString FILE_SEARCH_VALUE = QStringLiteral("0");
static const QString DIR_SEARCH_VALUE = QStringLiteral("1");
static const QString INDEX_SEM = QStringLiteral("ukui-search-index-sem");
static const QStringList allAppPath = {
QDir::homePath()+"/.local/share/applications/",
"/usr/share/applications/"
{HOME_PATH + "/.local/share/applications/"},
{"/usr/share/applications/"}
};
static const QMap<QString, bool> targetFileTypeMap = {
std::map<QString, bool>::value_type("doc", true),
std::map<QString, bool>::value_type("docx", true),
std::map<QString, bool>::value_type("ppt", true),
std::map<QString, bool>::value_type("pptx", true),
std::map<QString, bool>::value_type("xls", true),
std::map<QString, bool>::value_type("xlsx", true),
std::map<QString, bool>::value_type("txt", true),
std::map<QString, bool>::value_type("dot", true),
std::map<QString, bool>::value_type("wps", true),
std::map<QString, bool>::value_type("pps", true),
std::map<QString, bool>::value_type("dps", true),
std::map<QString, bool>::value_type("et", true),
std::map<QString, bool>::value_type("pdf", true)
{"doc", true},
{"docx", true},
{"ppt", true},
{"pptx", true},
{"xls", true},
{"xlsx", true},
{"txt", true},
{"dot", true},
{"wps", true},
{"pps", true},
{"dps", true},
{"et", true},
{"pdf", true}
};
static const QMap<QString, bool> targetPhotographTypeMap = {
std::map<QString, bool>::value_type("png", true),
std::map<QString, bool>::value_type("jpg", true),
std::map<QString, bool>::value_type("jpeg", true)//TODO 待完善,后续改为配置文件
{"png", true},
{"jpg", true},
{"jpeg", true} // TODO 待完善,后续改为配置文件
};
//TODO Put things that needed to be put here here.
#endif // COMMON_H

3
libsearch/file-utils.cpp
View File

@ -30,8 +30,7 @@
#include "pinyinmanager.h"
using namespace UkuiSearch;
size_t FileUtils::_max_index_count = 0;
size_t FileUtils::_current_index_count = 0;
size_t FileUtils::maxIndexCount = 0;
unsigned short FileUtils::indexStatus = 0;
FileUtils::SearchMethod FileUtils::searchMethod = FileUtils::SearchMethod::DIRECTSEARCH;
QMap<QString, QStringList> FileUtils::map_chinese2pinyin = QMap<QString, QStringList>();

3
libsearch/file-utils.h
View File

@ -97,8 +97,7 @@ public:
static QIcon iconFromTheme(const QString &name, const QIcon &iconDefault);
static bool isOpenXMLFileEncrypted(QString &path);
static bool isEncrypedOrUnreadable(QString path);
static size_t _max_index_count;
static size_t _current_index_count; //this one has been Abandoned,do not use it.
static size_t maxIndexCount;
static unsigned short indexStatus;
enum class SearchMethod { DIRECTSEARCH = 0, INDEXSEARCH = 1};

190
libsearch/index/first-index.cpp
View File

@ -42,24 +42,21 @@ FirstIndex *FirstIndex::getInstance()
FirstIndex::~FirstIndex() {
qDebug() << "~FirstIndex";
if(this->q_index)
delete this->q_index;
this->q_index = nullptr;
if(this->q_content_index)
delete this->q_content_index;
this->q_content_index = nullptr;
if(this->m_ocr_index)
delete this->m_ocr_index;
this->m_ocr_index = nullptr;
if(this->p_indexGenerator)
delete this->p_indexGenerator;
this->p_indexGenerator = nullptr;
if(this->m_indexData)
delete this->m_indexData;
this->m_indexData = nullptr;
if(this->m_contentIndexData)
delete this->m_contentIndexData;
this->m_contentIndexData = nullptr;
if(this->m_ocrIndexData)
delete this->m_ocrIndexData;
this->m_ocrIndexData = nullptr;
qDebug() << "~FirstIndex end";
}
void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
// qDebug() << "there are some shit here"<<fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0");
this->q_index->enqueue(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0"));
// qDebug() << "there are some shit here"<<fileInfo.fileName() << fileInfo.absoluteFilePath() << QString(fileInfo.isDir() ? "1" : "0");
this->m_indexData->enqueue(QVector<QString>() << fileInfo.fileName() << fileInfo.absoluteFilePath() << QString((fileInfo.isDir() && (!fileInfo.isSymLink())) ? "1" : "0"));
if (fileInfo.fileName().split(".", QString::SkipEmptyParts).length() < 2)
return;
if (true == targetFileTypeMap[fileInfo.fileName().split(".").last()]
@ -71,7 +68,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
if(!file.setCurrentFile("word/document.xml", QuaZip::csSensitive))
return;
QuaZipFile fileR(&file);
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//docx解压缩后的xml文件为实际需要解析文件大小
this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//docx解压缩后的xml文件为实际需要解析文件大小
file.close();
} else if (fileInfo.fileName().split(".").last() == "pptx") {
QuaZip file(fileInfo.absoluteFilePath());
@ -92,7 +89,7 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
}
}
file.close();
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileSize));//pptx解压缩后的xml文件为实际需要解析文件大小
this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileSize));//pptx解压缩后的xml文件为实际需要解析文件大小
} else if (fileInfo.fileName().split(".").last() == "xlsx") {
QuaZip file(fileInfo.absoluteFilePath());
if(!file.open(QuaZip::mdUnzip))
@ -100,14 +97,14 @@ void FirstIndex::DoSomething(const QFileInfo& fileInfo) {
if(!file.setCurrentFile("xl/sharedStrings.xml", QuaZip::csSensitive))
return;
QuaZipFile fileR(&file);
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//xlsx解压缩后的xml文件为实际解析文件大小
this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileR.usize()));//xlsx解压缩后的xml文件为实际解析文件大小
file.close();
} else {
this->q_content_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size()));
this->m_contentIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size()));
}
} else if (true == targetPhotographTypeMap[fileInfo.fileName().split(".").last()]) {
this->m_ocr_index->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size()));
}
} /*else if (true == targetPhotographTypeMap[fileInfo.fileName().split(".").last()]) {
this->m_ocrIndexData->enqueue(qMakePair(fileInfo.absoluteFilePath(),fileInfo.size()));
}*/
}
void FirstIndex::run() {
@ -115,26 +112,28 @@ void FirstIndex::run() {
// Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running.
QString indexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(INDEX_DATABASE_STATE).toString();
QString contentIndexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE).toString();
// QString ocrIndexDatabaseStatus = IndexStatusRecorder::getInstance()->getStatus(OCR_DATABASE_STATE).toString();
QString inotifyIndexStatus = IndexStatusRecorder::getInstance()->getStatus(INOTIFY_NORMAL_EXIT).toString();
qDebug() << "indexDataBaseStatus: " << indexDataBaseStatus;
qDebug() << "contentIndexDataBaseStatus: " << contentIndexDataBaseStatus;
qDebug() << "inotifyIndexStatus: " << inotifyIndexStatus;
qInfo() << "indexDataBaseStatus: " << indexDataBaseStatus;
qInfo() << "contentIndexDataBaseStatus: " << contentIndexDataBaseStatus;
// qInfo() << "ocrIndexDatabaseStatus: " << ocrIndexDatabaseStatus;
qInfo() << "inotifyIndexStatus: " << inotifyIndexStatus;
if(indexDataBaseStatus == "") {
this->bool_dataBaseExist = false;
} else {
this->bool_dataBaseExist = true;
}
if(indexDataBaseStatus != "2" || contentIndexDataBaseStatus != "2" || inotifyIndexStatus != "2") {
this->bool_dataBaseStatusOK = false;
} else {
this->bool_dataBaseStatusOK = true;
m_allDatadaseStatus = inotifyIndexStatus == "2" ? true : false;
m_indexDatabaseStatus = indexDataBaseStatus == "2" ? true : false;
m_contentIndexDatabaseStatus = contentIndexDataBaseStatus == "2" ? true : false;
// m_ocrIndexDatabaseStatus = ocrIndexDatabaseStatus == "2" ? true : false;
if(m_allDatadaseStatus && m_indexDatabaseStatus && m_contentIndexDatabaseStatus /*&& m_ocrIndexDatabaseStatus*/) {
m_semaphore.release(1);
return;
}
this->q_index = new QQueue<QVector<QString>>();
this->q_content_index = new QQueue<QPair<QString,qint64>>();
this->m_ocr_index = new QQueue<QPair<QString,qint64>>();
this->m_indexData = new QQueue<QVector<QString>>();
this->m_contentIndexData = new QQueue<QPair<QString,qint64>>();
// this->m_ocrIndexData = new QQueue<QPair<QString,qint64>>();
++FileUtils::indexStatus;
pid_t pid;
@ -142,23 +141,12 @@ void FirstIndex::run() {
if(pid == 0) {
prctl(PR_SET_PDEATHSIG, SIGTERM);
prctl(PR_SET_NAME, "first-index");
if(this->bool_dataBaseExist) {
if(this->bool_dataBaseStatusOK) {
::_exit(0);
} else {
//if the parameter is false, index won't be rebuild
//if it is true, index will be rebuild
p_indexGenerator = IndexGenerator::getInstance(true, this);
}
} else {
// p_indexGenerator = IndexGenerator::getInstance(false,this);
p_indexGenerator = IndexGenerator::getInstance(true, this);
}
//TODO Fix these weird code.
QSemaphore sem(5);
QMutex mutex1, mutex2, mutex3;
mutex1.lock();
mutex2.lock();
// mutex3.lock();
qInfo() << "index dir" << DirWatcher::getDirWatcher()->currentindexableDir();
qInfo() << "index block dir" << DirWatcher::getDirWatcher()->currentBlackListOfIndex();
@ -166,59 +154,89 @@ void FirstIndex::run() {
setBlockPath(DirWatcher::getDirWatcher()->currentBlackListOfIndex());
this->Traverse();
FileUtils::_max_index_count = this->q_index->length();
qDebug() << "max_index_count:" << FileUtils::_max_index_count;
FileUtils::maxIndexCount = this->m_indexData->length();
qDebug() << "max_index_count:" << FileUtils::maxIndexCount;
QtConcurrent::run(&m_pool, [&]() {
sem.acquire(2);
mutex1.unlock();
qDebug() << "index start;";
if(m_allDatadaseStatus && m_indexDatabaseStatus) {
sem.release(2);
return;
}
qDebug() << "index start;" << m_indexData->size();
IndexGenerator::getInstance()->rebuildIndexDatabase();
QQueue<QVector<QString>>* tmp1 = new QQueue<QVector<QString>>();
while(!this->q_index->empty()) {
for(size_t i = 0; (i < 8192) && (!this->q_index->empty()); ++i) {
tmp1->enqueue(this->q_index->dequeue());
bool sucess = true;
while(!this->m_indexData->empty()) {
for(size_t i = 0; (i < 8192) && (!this->m_indexData->empty()); ++i) {
tmp1->enqueue(this->m_indexData->dequeue());
}
if(!IndexGenerator::getInstance()->creatAllIndex(tmp1)) {
sucess = false;
break;
}
this->p_indexGenerator->creatAllIndex(tmp1);
tmp1->clear();
}
delete tmp1;
qDebug() << "index end;";
if(sucess) {
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "2");
}
sem.release(2);
});
QtConcurrent::run(&m_pool,[&]() {
sem.acquire(2);
mutex2.unlock();
if(m_allDatadaseStatus && m_contentIndexDatabaseStatus) {
sem.release(2);
return;
}
qDebug() << "content index start:" << m_contentIndexData->size();
IndexGenerator::getInstance()->rebuildContentIndexDatabase();
QQueue<QString>* tmp2 = new QQueue<QString>();
qDebug() << "q_content_index:" << q_content_index->size();
while(!this->q_content_index->empty()) {
bool sucess = true;
while(!this->m_contentIndexData->empty()) {
qint64 fileSize = 0;
//修改一次处理的数据量，从30个文件改为文件总大小为50M以下，50M为暂定值--jxx20210519
for(size_t i = 0;/* (i < 30) && (fileSize < 52428800) && */(!this->q_content_index->empty()); ++i) {
QPair<QString,qint64> tempPair = this->q_content_index->dequeue();
for(size_t i = 0;/* (i < 30) && (fileSize < 52428800) && */(!this->m_contentIndexData->empty()); ++i) {
QPair<QString,qint64> tempPair = this->m_contentIndexData->dequeue();
fileSize += tempPair.second;
if (fileSize > 52428800 ) {
if (tmp2->size() == 0) {
tmp2->enqueue(tempPair.first);
break;
}
this->q_content_index->enqueue(tempPair);
this->m_contentIndexData->enqueue(tempPair);
break;
}
tmp2->enqueue(tempPair.first);
}
// qDebug() << ">>>>>>>>all fileSize:" << fileSize << "file num:" << tmp->size() << "<<<<<<<<<<<<<<<<<<<";
this->p_indexGenerator->creatAllIndex(tmp2);
if(!IndexGenerator::getInstance()->creatAllIndex(tmp2)) {
sucess = false;
break;
}
tmp2->clear();
}
delete tmp2;
qDebug() << "content index end;";
if(sucess) {
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "2");
}
sem.release(2);
});
// OCR功能暂时屏蔽
// OCR功能暂时屏蔽
// QtConcurrent::run(&m_pool,[&]() {
// sem.acquire(5);
// mutex3.unlock();
// QQueue<QString>* tmpOcr = new QQueue<QString>();
// qDebug() << "m_ocr_index:" << m_ocr_index->size();
// if(m_allDatadaseStatus && m_contentIndexDatabaseStatus) {
// sem.release(2);
// return;
// }
// IndexGenerator::getInstance()->rebuildOcrIndexDatabase();
// bool sucess = true;
// while(!this->m_ocr_index->empty()) {
// qint64 fileSize = 0;
// //一次处理的数据量文件总大小为50M以下，50M为暂定值
@ -235,36 +253,36 @@ void FirstIndex::run() {
// }
// tmpOcr->enqueue(tempPair.first);
// }
// this->p_indexGenerator->creatOcrIndex(tmpOcr);
// if(!IndexGenerator::getInstance()->creatAllIndex(tmpOcr)) {
// sucess = false;
// break;
// }
// tmpOcr->clear();
// }
// delete tmpOcr;
// qDebug() << "OCR index end;";
// if(sucess) {
// IndexStatusRecorder::getInstance()->setStatus(OCR_DATABASE_STATE, "2");
// }
// sem.release(5);
// });
mutex1.lock();
mutex2.lock();
// mutex3.lock();
// mutex3.lock();
sem.acquire(5);
mutex1.unlock();
mutex2.unlock();
// mutex3.unlock();
// mutex3.unlock();
if(this->q_index)
delete this->q_index;
this->q_index = nullptr;
if(this->q_content_index)
delete this->q_content_index;
this->q_content_index = nullptr;
if(this->m_ocr_index)
delete this->m_ocr_index;
this->m_ocr_index = nullptr;
if(p_indexGenerator)
delete p_indexGenerator;
p_indexGenerator = nullptr;
// GlobalSettings::getInstance()->forceSync();
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "2");
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "2");
if(this->m_indexData)
delete this->m_indexData;
this->m_indexData = nullptr;
if(this->m_contentIndexData)
delete this->m_contentIndexData;
this->m_contentIndexData = nullptr;
if(this->m_ocrIndexData)
delete this->m_ocrIndexData;
this->m_ocrIndexData = nullptr;
::_exit(0);
} else if(pid < 0) {
qWarning() << "First Index fork error!!";
@ -275,11 +293,11 @@ void FirstIndex::run() {
m_semaphore.release(1);
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2");
// int retval1 = write(fifo_fd, buffer, strlen(buffer));
// if(retval1 == -1) {
// qWarning("write error\n");
// }
// qDebug("write data ok!\n");
// int retval1 = write(fifo_fd, buffer, strlen(buffer));
// if(retval1 == -1) {
// qWarning("write error\n");
// }
// qDebug("write data ok!\n");
QTime t2 = QTime::currentTime();
qWarning() << t1;
qWarning() << t2;

16
libsearch/index/first-index.h
View File

@ -47,24 +47,28 @@ public:
static FirstIndex* getInstance();
~FirstIndex();
virtual void DoSomething(const QFileInfo &) final;
protected:
void run() override;
private:
FirstIndex();
FirstIndex(const FirstIndex&) = delete;
void operator=(const FirstIndex&) = delete;
static FirstIndex *m_instance;
bool bool_dataBaseStatusOK = false;
bool bool_dataBaseExist = false;
IndexGenerator* p_indexGenerator = nullptr;
bool m_indexDatabaseStatus = false;
bool m_contentIndexDatabaseStatus = false;
bool m_ocrIndexDatabaseStatus = false;
bool m_allDatadaseStatus = false;
QThreadPool m_pool;
QQueue<QVector<QString>>* q_index;
QQueue<QVector<QString>>* m_indexData = nullptr;
// QQueue<QString>* q_content_index;
//修改QQueue存储数据为QPair<QString,qint64>，增加存储文件大小数据便于处理时统计--jxx20210519
QQueue<QPair<QString,qint64>>* q_content_index;
QQueue<QPair<QString,qint64>>* m_contentIndexData = nullptr;
//新增ocr队列存储ocr可识别处理的图片信息及大小；
QQueue<QPair<QString,qint64>>* m_ocr_index;
QQueue<QPair<QString,qint64>>* m_ocrIndexData = nullptr;
//xapian will auto commit per 10,000 changes, donnot change it!!!
const size_t u_send_length = 8192;
QSystemSemaphore m_semaphore;

113
libsearch/index/index-generator.cpp
View File

@ -25,15 +25,11 @@
#include <QFuture>
#include <QThreadPool>
#include <QFile>
#include <QStandardPaths>
#include <malloc.h>
#include "file-utils.h"
#include "index-generator.h"
#include "chinese-segmentation.h"
#include <QStandardPaths>
#include <malloc.h>
#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/index_data").toStdString()
#define CONTENT_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/content_index_data").toStdString()
#define OCR_INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/ukui-search/ocr_index_data").toStdString()
using namespace UkuiSearch;
@ -51,21 +47,14 @@ QVector<Document> IndexGenerator::g_docListForContent = QVector<Document>();
QVector<Document> IndexGenerator::g_docListForOcr = QVector<Document>();
IndexGenerator *IndexGenerator::getInstance(bool rebuild, QObject *parent) {
IndexGenerator *IndexGenerator::getInstance() {
QMutexLocker locker(&m_mutex);
if(!global_instance) {
qDebug() << "IndexGenerator=================";
global_instance = new IndexGenerator(rebuild, parent);
global_instance = new IndexGenerator();
}
qDebug() << "global_instance" << global_instance;
qDebug() << "QThread::currentThreadId()" << QThread::currentThreadId();
return global_instance;
}
bool IndexGenerator::setIndexdataPath() {
return true;
}
//文件名索引
bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) {
HandlePathList(messageList);
@ -86,7 +75,7 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) {
qWarning() << "creatAllIndex fail!" << QString::fromStdString(e.get_description());
//need a record
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1");
assert(false);
return false;
}
qDebug() << "finish creatAllIndex";
IndexGenerator::g_docListForPath.clear();
@ -124,7 +113,7 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList) {
} catch(const Xapian::Error &e) {
qWarning() << "creat content Index fail!" << QString::fromStdString(e.get_description());
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "1");
assert(false);
return false;
}
qDebug() << "finish creatAllIndex for content";
@ -163,8 +152,8 @@ bool IndexGenerator::creatOcrIndex(QQueue<QString> *messageList)
m_database_ocr->commit();
} catch(const Xapian::Error &e) {
qWarning() << "creat ocr Index fail!" << QString::fromStdString(e.get_description());
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "1");
assert(false);
IndexStatusRecorder::getInstance()->setStatus(OCR_DATABASE_STATE, "1");
return false;
}
qDebug() << "finish creatAllIndex for ocr";
@ -176,26 +165,30 @@ bool IndexGenerator::creatOcrIndex(QQueue<QString> *messageList)
return true;
}
IndexGenerator::IndexGenerator(bool rebuild, QObject *parent) : QObject(parent) {
QDir database(QString::fromStdString(INDEX_PATH));
if(database.exists()) {
if(rebuild)
qDebug() << "remove" << database.removeRecursively();
} else {
qDebug() << "create index path" << database.mkpath(QString::fromStdString(INDEX_PATH));
IndexGenerator::IndexGenerator(QObject *parent) : QObject(parent)
{
QDir database(INDEX_PATH);
if(!database.exists()) {
qDebug() << "create index path" << INDEX_PATH<< database.mkpath(INDEX_PATH);
}
database.setPath(QString::fromStdString(CONTENT_INDEX_PATH));
if(database.exists()) {
if(rebuild)
qDebug() << "remove" << database.removeRecursively();
} else {
qDebug() << "create content index path" << database.mkpath(QString::fromStdString(CONTENT_INDEX_PATH));
database.setPath(CONTENT_INDEX_PATH);
if(!database.exists()) {
qDebug() << "create content index path" << CONTENT_INDEX_PATH << database.mkpath(CONTENT_INDEX_PATH);
}
database.setPath(OCR_INDEX_PATH);
if(!database.exists()) {
qDebug() << "create ocr index path" << OCR_INDEX_PATH << database.mkpath(OCR_INDEX_PATH);
}
m_database_path = new Xapian::WritableDatabase(INDEX_PATH, Xapian::DB_CREATE_OR_OPEN);
m_database_content = new Xapian::WritableDatabase(CONTENT_INDEX_PATH, Xapian::DB_CREATE_OR_OPEN);
m_database_ocr = new Xapian::WritableDatabase(OCR_INDEX_PATH, Xapian::DB_CREATE_OR_OPEN);
try {
m_database_path = new Xapian::WritableDatabase(INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN);
m_database_content = new Xapian::WritableDatabase(CONTENT_INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN);
m_database_ocr = new Xapian::WritableDatabase(OCR_INDEX_PATH.toStdString(), Xapian::DB_CREATE_OR_OPEN);
} catch(const Xapian::Error &e) {
qWarning() << "creat Index fail!" << QString::fromStdString(e.get_description());
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
}
}
IndexGenerator::~IndexGenerator() {
@ -237,6 +230,45 @@ IndexGenerator::~IndexGenerator() {
qDebug() << "~IndexGenerator end";
}
void IndexGenerator::rebuildIndexDatabase(const QString &path)
{
QDir database(path);
if(database.exists()) {
qDebug() << "remove" << path << database.removeRecursively();
} else {
qDebug() << "create index path" << path << database.mkpath(path);
}
if(m_database_path)
m_database_path->~WritableDatabase();
m_database_path = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN);
}
void IndexGenerator::rebuildContentIndexDatabase(const QString &path)
{
QDir database(path);
if(database.exists()) {
qDebug() << "remove" << path << database.removeRecursively();
} else {
qDebug() << "create content index path" << path << database.mkpath(path);
}
if(m_database_content)
m_database_content->~WritableDatabase();
m_database_content = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN);
}
void IndexGenerator::rebuildOcrIndexDatabase(const QString &path)
{
QDir database(path);
if(database.exists()) {
qDebug() << "remove" << path << database.removeRecursively();
} else {
qDebug() << "create ocr index path" << path << database.mkpath(path);
}
if(m_database_ocr)
m_database_ocr->~WritableDatabase();
m_database_ocr = new Xapian::WritableDatabase(path.toStdString(), Xapian::DB_CREATE_OR_OPEN);
}
void IndexGenerator::insertIntoDatabase(Document& doc) {
// qDebug()<< "--index start--";
Xapian::Document document = doc.getXapianDocument();
@ -417,20 +449,13 @@ Document IndexGenerator::GenerateContentDocument(const QString &path) {
return doc;
}
bool IndexGenerator::isIndexdataExist() {
// Xapian::Database db(m_index_data_path->toStdString());
return true;
}
//deprecated
QStringList IndexGenerator::IndexSearch(QString indexText) {
QStringList searchResult;
try {
qDebug() << "--search start--";
Xapian::Database db(INDEX_PATH);
Xapian::Database db(INDEX_PATH.toStdString());
Xapian::Enquire enquire(db);
Xapian::QueryParser qp;
qp.set_default_op(Xapian::Query::OP_PHRASE);

17
libsearch/index/index-generator.h
View File

@ -35,6 +35,7 @@
#include "file-reader.h"
#include "common.h"
#include "pending-file.h"
#include "common.h"
namespace UkuiSearch {
//extern QVector<Document> *_doc_list_path;
@ -48,10 +49,12 @@ class IndexGenerator : public QObject {
friend class ConstructDocumentForOcr;
Q_OBJECT
public:
static IndexGenerator *getInstance(bool rebuild = false, QObject *parent = nullptr);
static IndexGenerator *getInstance();
~IndexGenerator();
bool setIndexdataPath();
bool isIndexdataExist();
void rebuildIndexDatabase(const QString &path = INDEX_PATH);
void rebuildContentIndexDatabase(const QString &path = CONTENT_INDEX_PATH);
void rebuildOcrIndexDatabase(const QString &path = OCR_INDEX_PATH);
// Q_INVOKABLE void appendDocListPath(Document doc);
//for search test
static QStringList IndexSearch(QString indexText);
@ -69,7 +72,7 @@ public Q_SLOTS:
bool updateIndex(QVector<PendingFile> *pendingFiles);
private:
explicit IndexGenerator(bool rebuild = false, QObject *parent = nullptr);
explicit IndexGenerator(QObject *parent = nullptr);
static QMutex m_mutex;
//For file name index
void HandlePathList(QQueue<QVector<QString> > *messageList);
@ -91,9 +94,9 @@ private:
static QMutex g_mutexDocListForOcr;
QMap<QString, QStringList> m_index_map;
QString m_index_data_path;
Xapian::WritableDatabase* m_database_path;
Xapian::WritableDatabase* m_database_content;
Xapian::WritableDatabase* m_database_ocr;
Xapian::WritableDatabase* m_database_path = nullptr;
Xapian::WritableDatabase* m_database_content = nullptr;
Xapian::WritableDatabase* m_database_ocr = nullptr;
std::string m_docstr;
std::string m_index_text_str;
Xapian::TermGenerator m_indexer;

40
libsearch/index/index-status-recorder.cpp
View File

@ -1,14 +1,15 @@
#include "index-status-recorder.h"
#include <mutex>
using namespace UkuiSearch;
static IndexStatusRecorder *global_instance_indexStatusRecorder = nullptr;
IndexStatusRecorder *IndexStatusRecorder::m_instance = nullptr;
std::once_flag g_IndexStatusRecorderInstanceFlag;
IndexStatusRecorder *IndexStatusRecorder::getInstance()
{
if(!global_instance_indexStatusRecorder) {
global_instance_indexStatusRecorder = new IndexStatusRecorder;
}
return global_instance_indexStatusRecorder;
std::call_once(g_IndexStatusRecorderInstanceFlag, [] () {
m_instance = new IndexStatusRecorder;
});
return m_instance;
}
void IndexStatusRecorder::setStatus(const QString &key, const QVariant &value)
@ -24,6 +25,33 @@ const QVariant IndexStatusRecorder::getStatus(const QString &key)
return m_status->value(key);
}
bool IndexStatusRecorder::indexDatabaseEnable()
{
m_mutex.lock();
m_status->sync();
m_mutex.unlock();
return m_status->value(INDEX_DATABASE_STATE, QVariant(false)).toBool();
}
bool IndexStatusRecorder::contentIndexDatabaseEnable()
{
m_mutex.lock();
m_status->sync();
m_mutex.unlock();
return m_status->value(CONTENT_INDEX_DATABASE_STATE, QVariant(false)).toBool();
}
bool IndexStatusRecorder::ocrDatabaseEnable()
{
m_mutex.lock();
m_status->sync();
m_mutex.unlock();
return m_status->value(OCR_DATABASE_STATE, QVariant(false)).toBool();
}
IndexStatusRecorder::IndexStatusRecorder(QObject *parent) : QObject(parent)
{
m_status = new QSettings(INDEX_STATUS, QSettings::IniFormat, this);

7
libsearch/index/index-status-recorder.h
View File

@ -7,7 +7,8 @@
#include <QMutex>
#define CONTENT_INDEX_DATABASE_STATE "content_index_database_state"
#define INDEX_DATABASE_STATE "index_database_state"
#define INOTIFY_NORMAL_EXIT "inotify_normal_exit"
#define OCR_DATABASE_STATE "ocr_database_state"
#define INOTIFY_NORMAL_EXIT "inotify_normal_exit" // 1 - 出错；2 - 正常；3-关闭索引； 0-有信号正在处理
#define PENDING_FILE_QUEUE_FINISH "pending_file_queue_finish"
#define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf"
namespace UkuiSearch {
@ -19,9 +20,13 @@ public:
static IndexStatusRecorder *getInstance();
void setStatus(const QString& key, const QVariant& value);
const QVariant getStatus(const QString& key);
bool indexDatabaseEnable();
bool contentIndexDatabaseEnable();
bool ocrDatabaseEnable();
private:
explicit IndexStatusRecorder(QObject *parent = nullptr);
static IndexStatusRecorder *m_instance;
QSettings *m_status;
QMutex m_mutex;
};

37
libsearch/index/search-method-manager.cpp
View File

@ -3,7 +3,6 @@ using namespace UkuiSearch;
static SearchMethodManager* global_instance = nullptr;
SearchMethodManager::SearchMethodManager() : m_semaphore(INDEX_SEM, 1, QSystemSemaphore::AccessMode::Create)
{
qDebug() << m_semaphore.errorString();
m_fi = FirstIndex::getInstance();
m_iw = InotifyWatch::getInstance();
}
@ -25,30 +24,24 @@ void SearchMethodManager::searchMethod(FileUtils::SearchMethod sm) {
}
if(FileUtils::SearchMethod::INDEXSEARCH == sm && 0 == FileUtils::indexStatus) {
// Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running.
QDir fifoDir = QDir(QDir::homePath() + "/.config/org.ukui/ukui-search");
if(!fifoDir.exists())
qDebug() << "create fifo path" << fifoDir.mkpath(fifoDir.absolutePath());
// // Create a fifo at ~/.config/org.ukui/ukui-search, the fifo is used to control the order of child processes' running.
// QDir fifoDir = QDir(QDir::homePath() + "/.config/org.ukui/ukui-search");
// if(!fifoDir.exists())
// qDebug() << "create fifo path" << fifoDir.mkpath(fifoDir.absolutePath());
unlink(UKUI_SEARCH_PIPE_PATH);
int retval = mkfifo(UKUI_SEARCH_PIPE_PATH, 0777);
if(retval == -1) {
qCritical() << "creat fifo error!!";
syslog(LOG_ERR, "creat fifo error!!\n");
assert(false);
return;
}
qDebug() << "create fifo success\n";
qWarning() << "start first index";
// unlink(UKUI_SEARCH_PIPE_PATH);
// int retval = mkfifo(UKUI_SEARCH_PIPE_PATH, 0777);
// if(retval == -1) {
// qCritical() << "creat fifo error!!";
// syslog(LOG_ERR, "creat fifo error!!\n");
// assert(false);
// return;
// }
// qDebug() << "create fifo success\n";
qDebug() << "start first index";
m_semaphore.acquire();
m_fi->start();
qWarning() << "start inotify index";
// InotifyIndex ii("/home");
// ii.start();
// this->m_ii = InotifyIndex::getInstance("/home");
// if(!this->m_ii->isRunning()) {
// this->m_ii->start();
// }
qDebug() << "start inotify index";
if(!this->m_iw->isRunning()) {
this->m_iw->start();
}