p90463782
/
ukui-search
forked from openkylin/ukui-search
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #231 from iaom/0512-dev 

Optimized index status record mechanism.
This commit is contained in:
fensite 2021-05-13 13:49:19 +08:00 committed by GitHub
parent 865ac7d5d0 1ad0542fca
commit cc9b5dde11
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 123 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libchinese-segmentation/libchinese-segmentation.pro
View File

@ -40,8 +40,10 @@ unix {
!isEmpty(target.path): INSTALLS += target
header.path = /usr/include/chinese-seg/
header.files += *.h cppjieba/*
INSTALLS += header header_cppjieba
header.files += *.h
headercppjieba.path = /usr/include/chinese-seg/cppjieba/
headercppjieba.files = cppjieba/*
INSTALLS += header headercppjieba
#DISTFILES += \
# jiaba/jieba.pri

7
libsearch/common.h Normal file
View File

@ -0,0 +1,7 @@
#ifndef COMMON_H
#define COMMON_H
#define UKUI_SEARCH_PIPE_PATH (QDir::homePath()+"/.config/org.ukui/ukui-search/ukuisearch").toLocal8Bit().constData()
//TODO Put things that needed to be put here here.
#endif // COMMON_H

2
libsearch/file-utils.h
View File

@ -56,8 +56,6 @@
//#define FINISH_CREATING_INDEX 2
#define MAX_CONTENT_LENGTH 20480000
#define UKUI_SEARCH_PIPE_PATH (QDir::homePath()+"/.config/org.ukui/ukui-search/ukuisearch").toLocal8Bit().constData()
namespace Zeeker {
class LIBSEARCH_EXPORT FileUtils {
public:

15
libsearch/global-settings.cpp
View File

@ -247,22 +247,15 @@ QStringList GlobalSettings::getSearchRecord() {
// return values;
//}
//here should be override
//MouseZhangZh
//this method is designed for main process settings only!!
void GlobalSettings::setValue(const QString &key, const QVariant &value) {
// qDebug()<<"setvalue========"<<key<<":"<<value;
m_cache.insert(key, value);
// m_settings->sync();
QtConcurrent::run([ = ]() {
// qDebug()<<m_settings->status();
// if (m_mutex.tryLock(1000)) {
// m_mutex.lock();
if (m_mutex.tryLock(1000)) {
m_settings->setValue(key, value);
// qDebug()<<"setvalue========finish!!!"<<key<<":"<<value;
m_settings->sync();
// qDebug()<<"setvalue========sync!!!"<<key<<":"<<value;
// m_mutex.unlock();
// }
m_mutex.unlock();
}
});
}

7
libsearch/global-settings.h
View File

@ -42,10 +42,7 @@
#define STYLE_NAME_KEY "styleName"
#define FONT_SIZE_KEY "systemFontSize"
#define ICON_THEME_KEY "iconThemeName"
#define INDEX_DATABASE_STATE "index_database_state"
#define CONTENT_INDEX_DATABASE_STATE "content_index_database_state"
#define INDEX_GENERATOR_NORMAL_EXIT "index_generator_normal_exit"
#define INOTIFY_NORMAL_EXIT "inotify_normal_exit"
#define ENABLE_CREATE_INDEX_ASK_DIALOG "enable_create_index_ask_dialog"
#define WEB_ENGINE "web_engine"
#define PATH_EMPTY 1;
@ -107,6 +104,8 @@ private:
QStringList m_history;
QMutex m_mutex;
// size_t test = 0;
};

11
libsearch/index/first-index.cpp
View File

@ -68,9 +68,9 @@ void FirstIndex::run() {
}
qDebug() << "create fifo success\n";
QString indexDataBaseStatus = GlobalSettings::getInstance()->getValue(INDEX_DATABASE_STATE).toString();
QString contentIndexDataBaseStatus = GlobalSettings::getInstance()->getValue(CONTENT_INDEX_DATABASE_STATE).toString();
QString inotifyIndexStatus = GlobalSettings::getInstance()->getValue(INOTIFY_NORMAL_EXIT).toString();
QString indexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(INDEX_DATABASE_STATE).toString();
QString contentIndexDataBaseStatus = IndexStatusRecorder::getInstance()->getStatus(CONTENT_INDEX_DATABASE_STATE).toString();
QString inotifyIndexStatus = IndexStatusRecorder::getInstance()->getStatus(INOTIFY_NORMAL_EXIT).toString();
qDebug() << "indexDataBaseStatus: " << indexDataBaseStatus;
qDebug() << "contentIndexDataBaseStatus: " << contentIndexDataBaseStatus;
@ -196,6 +196,8 @@ void FirstIndex::run() {
delete p_indexGenerator;
p_indexGenerator = nullptr;
// GlobalSettings::getInstance()->forceSync();
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "2");
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "2");
::_exit(0);
} else if(pid < 0) {
qWarning() << "First Index fork error!!";
@ -204,8 +206,7 @@ void FirstIndex::run() {
--FileUtils::_index_status;
}
GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "2");
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2");
int retval1 = write(fifo_fd, buffer, strlen(buffer));
if(retval1 == -1) {
qWarning("write error\n");

4
libsearch/index/first-index.h
View File

@ -37,11 +37,11 @@
#include <syslog.h>
//#include <QtConcurrent>
#include "traverse_bfs.h"
#include "global-settings.h"
#include "index-status-recorder.h"
#include "index-generator.h"
#include "inotify-index.h"
#include "file-utils.h"
#include "common.h"
namespace Zeeker {
class FirstIndex : public QThread, public Traverse_BFS {
public:

13
libsearch/index/index-generator.cpp
View File

@ -27,7 +27,6 @@
#include <QFile>
#include "file-utils.h"
#include "index-generator.h"
#include "global-settings.h"
#include "chinese-segmentation.h"
#include "construct-document.h"
#include <QStandardPaths>
@ -69,7 +68,7 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) {
return false;
}
qDebug() << "begin creatAllIndex";
GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "0");
// GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "0");
try {
// m_indexer = new Xapian::TermGenerator();
// m_indexer.set_database(*m_database_path);
@ -91,11 +90,11 @@ bool IndexGenerator::creatAllIndex(QQueue<QVector<QString> > *messageList) {
} catch(const Xapian::Error &e) {
qWarning() << "creatAllIndex fail!" << QString::fromStdString(e.get_description());
//need a record
GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "1");
IndexStatusRecorder::getInstance()->setStatus(INDEX_DATABASE_STATE, "1");
// FileUtils::_index_status &= ~0x1;
assert(false);
}
GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
// GlobalSettings::getInstance()->setValue(INDEX_DATABASE_STATE, "2");
qDebug() << "finish creatAllIndex";
// FileUtils::_index_status &= ~0x1;
_doc_list_path->clear();
@ -114,7 +113,7 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList) {
int size = _doc_list_content->size();
qDebug() << "begin creatAllIndex for content" << size;
if(!size == 0) {
GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "0");
// GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "0");
try {
int count = 0;
for(auto i : *_doc_list_content) {
@ -127,11 +126,11 @@ bool IndexGenerator::creatAllIndex(QQueue<QString> *messageList) {
m_database_content->commit();
} catch(const Xapian::Error &e) {
qWarning() << "creat content Index fail!" << QString::fromStdString(e.get_description());
GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "1");
IndexStatusRecorder::getInstance()->setStatus(CONTENT_INDEX_DATABASE_STATE, "1");
// FileUtils::_index_status &= ~0x2;
assert(false);
}
GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2");
// GlobalSettings::getInstance()->setValue(CONTENT_INDEX_DATABASE_STATE, "2");
// FileUtils::_index_status &= ~0x2;
qDebug() << "finish creatAllIndex for content";
_doc_list_content->clear();

2
libsearch/index/index-generator.h
View File

@ -29,8 +29,10 @@
#include <QMutex>
#include <QQueue>
//#include <QMetaObject>
#include "index-status-recorder.h"
#include "document.h"
#include "file-reader.h"
#include "common.h"
namespace Zeeker {
extern QList<Document> *_doc_list_path;

28
libsearch/index/index-status-recorder.cpp Normal file
View File

@ -0,0 +1,28 @@
#include "index-status-recorder.h"
using namespace Zeeker;
static IndexStatusRecorder *global_instance_indexStatusRecorder = nullptr;
IndexStatusRecorder *IndexStatusRecorder::getInstance()
{
if(!global_instance_indexStatusRecorder) {
global_instance_indexStatusRecorder = new IndexStatusRecorder;
}
return global_instance_indexStatusRecorder;
}
void IndexStatusRecorder::setStatus(const QString &key, const QVariant &value)
{
m_status->setValue(key, value);
m_status->sync();
}
const QVariant IndexStatusRecorder::getStatus(const QString &key)
{
return m_status->value(key);
}
IndexStatusRecorder::IndexStatusRecorder(QObject *parent) : QObject(parent)
{
m_status = new QSettings(INDEX_STATUS, QSettings::IniFormat, this);
}

27
libsearch/index/index-status-recorder.h Normal file
View File

@ -0,0 +1,27 @@
#ifndef INDEXSTATUSRECORDER_H
#define INDEXSTATUSRECORDER_H
#include <QObject>
#include <QSettings>
#include <QDir>
#define CONTENT_INDEX_DATABASE_STATE "content_index_database_state"
#define INDEX_DATABASE_STATE "index_database_state"
#define INOTIFY_NORMAL_EXIT "inotify_normal_exit"
#define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf"
namespace Zeeker {
//fixme: we need a better way to record index status.
class IndexStatusRecorder : public QObject
{
Q_OBJECT
public:
static IndexStatusRecorder *getInstance();
void setStatus(const QString& key, const QVariant& value);
const QVariant getStatus(const QString& key);
private:
explicit IndexStatusRecorder(QObject *parent = nullptr);
QSettings *m_status;
};
}
#endif // INDEXSTATUSRECORDER_H

2
libsearch/index/index.pri
View File

@ -6,6 +6,7 @@ HEADERS += \
$$PWD/file-reader.h \
$$PWD/first-index.h \
$$PWD/index-generator.h \
$$PWD/index-status-recorder.h \
$$PWD/inotify-index.h \
$$PWD/search-manager.h \
$$PWD/searchmethodmanager.h \
@ -18,6 +19,7 @@ SOURCES += \
$$PWD/file-reader.cpp \
$$PWD/first-index.cpp \
$$PWD/index-generator.cpp \
$$PWD/index-status-recorder.cpp \
$$PWD/inotify-index.cpp \
$$PWD/search-manager.cpp \
$$PWD/searchmethodmanager.cpp \

41
libsearch/index/inotify-index.cpp
View File

@ -180,12 +180,14 @@ void InotifyIndex::eventProcess(const char* buf, ssize_t tmp) {
ssize_t numRead = 0;
numRead = tmp;
char * p = const_cast<char*>(buf);
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "0");
for(; p < buf + numRead;) {
struct inotify_event * event = reinterpret_cast<inotify_event *>(p);
if(event->name[0] != '.') {
qDebug() << "Read Event event->wd: " << event->wd;
qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
if(event->name[0] != '.') {
qDebug() << QString(currentPath[event->wd] + '/' + event->name);
// switch (event->mask) {
if(event->mask & IN_CREATE) {
@ -235,7 +237,7 @@ void InotifyIndex::eventProcess(const char* buf, ssize_t tmp) {
next:
p += sizeof(struct inotify_event) + event->len;
}
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "2");
delete indexQueue;
indexQueue = nullptr;
delete contentIndexQueue;
@ -283,19 +285,19 @@ void InotifyIndex::run() {
if(numRead == -1) {
printf("\033[1;31;40mread event error\033[0m\n");
GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "1");
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
fflush(stdout);
assert(false);
}
//TODO: Merge multiple signals.
char * tmp = const_cast<char*>(buf);
for(; tmp < buf + numRead;) {
struct inotify_event * event = reinterpret_cast<inotify_event *>(tmp);
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
if(event->name[0] != '.') {
GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "0");
// qDebug() << "Read Event: " << currentPath[event->wd] << QString(event->name) << event->cookie << event->wd << event->mask;
// qDebug("mask:0x%x,",event->mask);
break;
}
tmp += sizeof(struct inotify_event) + event->len;
@ -318,7 +320,6 @@ void InotifyIndex::run() {
qDebug() << "read";
}
eventProcess(buf, numRead);
GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "2");
fd_set read_fds;
int rc;
@ -334,7 +335,7 @@ void InotifyIndex::run() {
if(rc < 0) {
// error
qWarning() << "select result < 0, error!";
GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "1");
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "1");
assert(false);
} else if(rc == 0) {
qDebug() << "select timeout!";
@ -343,7 +344,6 @@ void InotifyIndex::run() {
// GlobalSettings::getInstance()->forceSync();
::_exit(0);
} else {
GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "0");
memset(buf, 0x00, BUF_LEN);
numRead = read(m_fd, buf, BUF_LEN);
if(numRead == -1) {
@ -351,9 +351,23 @@ void InotifyIndex::run() {
fflush(stdout);
assert(false);
}
//TODO: Merge multiple signals.
char * tmp = const_cast<char*>(buf);
for(; tmp < buf + numRead; ) {
struct inotify_event * event = reinterpret_cast<inotify_event *>(tmp);
if(event->name[0] != '.') {
break;
}
tmp += sizeof(struct inotify_event) + event->len;
}
if(tmp >= buf + numRead) {
continue;
}
qDebug() << "Read " << numRead << " bytes from inotify fd";
this->eventProcess(buf, numRead);
GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "2");
}
}
} else if(pid > 0) {
@ -366,10 +380,7 @@ void InotifyIndex::run() {
}
if(FileUtils::SearchMethod::DIRECTSEARCH == FileUtils::searchMethod) {
GlobalSettings::getInstance()->setValue(INOTIFY_NORMAL_EXIT, "3");
IndexStatusRecorder::getInstance()->setStatus(INOTIFY_NORMAL_EXIT, "3");
RemoveWatch(QStandardPaths::writableLocation(QStandardPaths::HomeLocation), false);
}
}

3
libsearch/index/inotify-index.h
View File

@ -27,9 +27,10 @@
#include "index-generator.h"
#include "traverse_bfs.h"
#include "ukui-search-qdbus.h"
#include "global-settings.h"
#include "index-status-recorder.h"
#include "file-utils.h"
#include "first-index.h"
#include "common.h"
#define BUF_LEN 1024000

1
libsearch/index/searchmethodmanager.h
View File

@ -3,7 +3,6 @@
#include "first-index.h"
#include "inotify-index.h"
namespace Zeeker {
class SearchMethodManager {
public:

1
libsearch/libsearch.pro
View File

@ -42,6 +42,7 @@ SOURCES += \
libsearch.cpp
HEADERS += \
common.h \
file-utils.h \
global-settings.h \
gobject-template.h \