openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

feat(file-index):增加AI索引功能

This commit is contained in:
iaom 2024-04-17 17:43:50 +08:00
parent 600c543ea7
commit 1b920b4f5a
26 changed files with 777 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
data/org.ukui.search.data.gschema.xml
View File

@ -30,5 +30,10 @@
<summary>content index enable ocr</summary>
<description>Enable or disable OCR in content index.</description>
</key>
<key name="ai-index-enable" type="b">
<default>true</default>
<summary>ai index enable</summary>
<description>Enable or disable AI index service.</description>
</key>
</schema>
</schemalist>

1
frontend/mainwindow.cpp
View File

@ -53,6 +53,7 @@
#define MAIN_SETTINGS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search.conf"
#define ENABLE_CREATE_INDEX_ASK_DIALOG "enable_create_index_ask_dialog"
const static QString FILE_INDEX_ENABLE_KEY = "fileIndexEnable";
static const QByteArray UKUI_SEARCH_SCHEMAS = QByteArrayLiteral("org.ukui.search.settings");
using namespace UkuiSearch;
extern void qt_blurImage(QImage &blurImage, qreal radius, bool quality, int transposed);

1
frontend/mainwindow.h
View File

@ -47,7 +47,6 @@
#include <QTimer>
#include "search-app-widget-plugin/search.h"
#include "libsearch.h"
#include "create-index-ask-dialog.h"
#include "search-line-edit.h"
#include "search-result-page.h"

5
libsearch/CMakeLists.txt
View File

@ -29,7 +29,8 @@ set(LIBUKUI_SEARCH_PC_PKGS
kysdk-qtwidgets
uchardet
kysdk-systime
kysdk-datacollect)
kysdk-datacollect
kylin-ai-base)
foreach(PC_LIB IN ITEMS ${LIBUKUI_SEARCH_PC_PKGS})
pkg_check_modules(${PC_LIB} REQUIRED IMPORTED_TARGET ${PC_LIB})
@ -58,6 +59,7 @@ set(LIBUKUI_SEARCH_SRC
gobject-template.cpp gobject-template.h
index/basic-indexer.cpp index/basic-indexer.h
index/batch-indexer.cpp index/batch-indexer.h
index/ai-indexer.cpp index/ai-indexer.h
index/compatible-define.h
index/database.cpp index/database.h
index/document.cpp index/document.h
@ -77,7 +79,6 @@ set(LIBUKUI_SEARCH_SRC
index/file-extraction-result.cpp
index/file-extraction-result.h
index/index-monitor.cpp
libsearch.cpp libsearch.h
libsearch_global.h
log-utils.cpp log-utils.h
notesearch/note-search-plugin.cpp notesearch/note-search-plugin.h

11
libsearch/common.h
View File

@ -25,8 +25,6 @@
namespace UkuiSearch {
#define CONTENT_DATABASE_PATH_SLOT 1
#define CONTENT_DATABASE_SUFFIX_SLOT 2
static const int LABEL_MAX_WIDTH = 320;
static const QString HOME_PATH = QDir::homePath();
@ -36,10 +34,8 @@ static const QString OCR_CONTENT_INDEX_PATH = HOME_PATH + QStringLiteral("/.conf
static const QString FILE_SEARCH_VALUE = QStringLiteral("0");
static const QString DIR_SEARCH_VALUE = QStringLiteral("1");
static const QString INDEX_SEM = QStringLiteral("ukui-search-index-sem");
static const int OCR_MIN_SIZE = 200;
static const QByteArray UKUI_SEARCH_SCHEMAS = QByteArrayLiteral("org.ukui.search.settings");
static const QString SEARCH_METHOD_KEY = QStringLiteral("fileIndexEnable");
/**
* changelog 1.0.1 修复部分中文字符在term中被截断的问题
*/
@ -71,11 +67,12 @@ enum class DataBaseType {
/**
* @brief The IndexType enum
* Index type, notice:OCR index is part of Content index.
* Index type
*/
enum class IndexType {
Basic,
Contents,
OCR
OCR,
Ai
};
}

137
libsearch/index/ai-indexer.cpp Normal file
View File

@ -0,0 +1,137 @@
/*
* Copyright (C) 2024, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: iaom <zhangpengfei@kylinos.cn>
*
*/
#include "ai-indexer.h"
#include <unistd.h>
#include <QJsonDocument>
#include <QJsonValue>
#include <QJsonArray>
#include <QMutex>
#include "kylin-ai/ai-base/datamanagement.h"
namespace UkuiSearch {
class AiIndexerPrivate
{
public:
QMutex m_mutex;
bool m_sessionValid = false;
DataManagementSession m_session = nullptr;
};
AiIndexer::AiIndexer(QObject *parent) : QObject(parent), d(new AiIndexerPrivate)
{
d->m_mutex.lock();
}
AiIndexer::~AiIndexer()
{
destroySession();
d->m_mutex.unlock();
if(d) {
delete d;
d = nullptr;
}
}
bool AiIndexer::creatSession()
{
DataManagementResult result = data_management_create_session(&d->m_session, DataManagementType::SYS_SEARCH, getuid());
qDebug() << "===creatSession===" <<result;
if(result == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
d->m_sessionValid = true;
return true;
}
return false;
}
bool AiIndexer::destroySession()
{
d->m_sessionValid = false;
return data_management_destroy_session(d->m_session) == DataManagementResult::DATA_MANAGEMENT_SUCCESS;
}
bool AiIndexer::checkFileSupported(const QString &filePath, QString &result)
{
if(!d->m_sessionValid) {
return false;
}
char *originalResult;
if(data_management_check_file_format(d->m_session, filePath.toLocal8Bit().data(), &originalResult) == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
QJsonObject jo = QJsonDocument::fromJson(QByteArray(originalResult)).object();
data_management_free_check_file_format_result(d->m_session, originalResult);
if(jo.value(QStringLiteral("supported")).toBool()) {
result = jo.value(QStringLiteral("format")).toString();
return true;
} else {
return false;
}
};
return false;
}
bool AiIndexer::addTextFileIndex(const QJsonArray &object)
{
if(!d->m_sessionValid) {
return false;
}
return data_management_add_text_files(d->m_session, QJsonDocument(object).toJson().data()) == DataManagementResult::DATA_MANAGEMENT_SUCCESS;
}
bool AiIndexer::addImageFileIndex(const QJsonArray &object)
{
if(!d->m_sessionValid) {
return false;
}
return data_management_add_image_files(d->m_session, QJsonDocument(object).toJson().data()) == DataManagementResult::DATA_MANAGEMENT_SUCCESS;
}
bool AiIndexer::deleteFileIndex(const QStringList &files)
{
if(!d->m_sessionValid) {
return false;
}
QJsonArray tmpArray;
for(const QString &file : files) {
QJsonObject oneFile;
oneFile.insert(QStringLiteral("filepath"), file);
tmpArray.append(oneFile);
}
return data_management_delete_files(d->m_session, QJsonDocument(tmpArray).toJson().data()) == DataManagementResult::DATA_MANAGEMENT_SUCCESS;
}
bool AiIndexer::getAllIndexedFiles(QJsonObject &object)
{
if(!d->m_sessionValid) {
return false;
}
char *originalResult;
DataManagementResult result = data_management_get_all_fileinfos(d->m_session, &originalResult);
if(result == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
QJsonArray array = QJsonDocument::fromJson(QByteArray(originalResult)).array();
for(const QJsonValueRef value : array) {
object.insert(value.toObject().value(QStringLiteral("filepath")).toString(), value.toObject().value(QStringLiteral("timestamp")).toInt());
}
return true;
}
return false;
}
} // UkuiSearch

48
libsearch/index/ai-indexer.h Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (C) 2024, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: iaom <zhangpengfei@kylinos.cn>
*
*/
#ifndef UKUI_SEARCH_AI_INDEXER_H
#define UKUI_SEARCH_AI_INDEXER_H
#include <QObject>
#include <QJsonObject>
namespace UkuiSearch {
class AiIndexerPrivate;
class AiIndexer : public QObject
{
Q_OBJECT
public:
explicit AiIndexer(QObject *parent = nullptr);
~AiIndexer();
bool creatSession();
bool checkFileSupported(const QString &filePath, QString &result);
bool addTextFileIndex(const QJsonArray &object);
bool addImageFileIndex(const QJsonArray &object);
bool deleteFileIndex(const QStringList &files);
bool getAllIndexedFiles(QJsonObject &object);
private:
bool destroySession();
AiIndexerPrivate *d = nullptr;
};
} // UkuiSearch
#endif //UKUI_SEARCH_AI_INDEXER_H

132
libsearch/index/batch-indexer.cpp
View File

@ -25,6 +25,7 @@
#include <QQueue>
#include <QDateTime>
#include <QMetaEnum>
#include <QJsonArray>
#include "file-utils.h"
#include "basic-indexer.h"
@ -32,15 +33,20 @@
#include "file-content-indexer.h"
#include "writable-database.h"
#include "compatible-define.h"
#include "ai-indexer.h"
using namespace UkuiSearch;
BatchIndexer::BatchIndexer(const QStringList &folders, const QStringList &blackList,
QAtomicInt& indexStop, QAtomicInt &contentIndexStop, QAtomicInt &contentIndexOcrStop,
QAtomicInt& indexStop,
QAtomicInt &contentIndexStop,
QAtomicInt &contentIndexOcrStop,
QAtomicInt& aiIndexStop,
WorkMode mode, Targets target)
: m_folders(folders),
m_blackList(blackList),
m_indexStop(&indexStop),
m_contentIndexStop(&contentIndexStop),
m_contentIndexOcrStop(&contentIndexOcrStop),
m_aiIndexStop(&aiIndexStop),
m_mode(mode),
m_target(target)
{
@ -52,7 +58,8 @@ void BatchIndexer::run()
timer.start();
if((!m_target.testFlag(Target::Basic) || m_indexStop->LOAD)
&& (!m_target.testFlag(Target::Content) || m_contentIndexStop->LOAD)
&& (!m_target.testFlag(Target::Ocr) || m_contentIndexOcrStop->LOAD)) {
&& (!m_target.testFlag(Target::Ocr) || m_contentIndexOcrStop->LOAD)
&& (!m_target.testFlag(Target::Ai) || m_aiIndexStop->LOAD)) {
Q_EMIT done(m_mode, m_target);
return;
}
@ -74,6 +81,11 @@ void BatchIndexer::run()
ocrIndex();
Q_EMIT ocrContentIndexDone(m_mode);
}
if(m_target & Target::Ai) {
Q_EMIT aiIndexStart(m_mode);
aiIndex();
Q_EMIT aiIndexDone(m_mode);
}
m_cache.clear();
malloc_trim(0);
@ -378,3 +390,119 @@ void BatchIndexer::ocrIndex()
qDebug() << "Ocr content index finished,";
}
void BatchIndexer::aiIndex()
{
qDebug() << "Begin ai index";
if (m_aiIndexStop->LOAD) {
qDebug() << "Index stopped, abort ai index.";
return;
}
AiIndexer indexer;
if(!indexer.creatSession()) {
qWarning() << "Creat session failed, fail to run ai index!";
return;
}
if (m_mode == WorkMode::Rebuild) {
//暂不支持重建
return;
}
QJsonObject filesNeedAiIndex;
QJsonObject imagesNeedAiIndex;
QFileInfo info;
if (m_mode == WorkMode::Add) {
for (const QString &path: m_cache) {
info.setFile(path);
if(!info.isFile()) {
continue;
}
QString type;
if (FileIndexerConfig::getInstance()->aiIndexFileTarget()[info.suffix()]) {
if(indexer.checkFileSupported(path, type)) { //这个检查可能会读文件，不要每个文件都执行一遍
filesNeedAiIndex.insert(path, type);
}
}
if (FileIndexerConfig::getInstance()->aiIndexImageTarget()[info.suffix()]) {
if(indexer.checkFileSupported(path, type)) {
imagesNeedAiIndex.insert(path, type);
}
}
}
} else {
QJsonObject indexTimes; //校验完整性
if(indexer.getAllIndexedFiles(indexTimes)) {
qDebug() << indexTimes.size() << "documents recorded";
for (const QString &path: m_cache) {
info.setFile(path);
if(!info.isFile()) {
continue;
}
QString type;
if (FileIndexerConfig::getInstance()->aiIndexFileTarget()[info.suffix()]) {
if(indexTimes.value(path).toInt() != info.lastModified().toSecsSinceEpoch()) {
if(indexer.checkFileSupported(path, type)) {
filesNeedAiIndex.insert(path, type);
indexTimes.remove(path);
}
} else {
indexTimes.remove(path);
}
} else if (FileIndexerConfig::getInstance()->aiIndexImageTarget()[info.suffix()]) {
if(indexTimes.value(path).toInt() != info.lastModified().toSecsSinceEpoch()) {
if(indexer.checkFileSupported(path, type)) {
imagesNeedAiIndex.insert(path, type);
indexTimes.remove(path);
}
} else {
indexTimes.remove(path);
}
}
}
if(!indexTimes.isEmpty()) {
qDebug() << indexTimes.size() << "documents need remove";
indexer.deleteFileIndex(indexTimes.keys());
}
}
}
uint allSize = filesNeedAiIndex.size() + imagesNeedAiIndex.size();
qDebug() << allSize << "files need ai index.";
Q_EMIT progress(IndexType::Ai, allSize, 0);
uint finishNum = 0;
uint batchSize = 0;
QJsonArray tmpArray;
QJsonObject::const_iterator iter;
auto creatIndex = [&](bool(AiIndexer::*creat)(const QJsonArray &), const QJsonObject &files){
for(iter = files.begin(); iter != files.end(); ++iter) {
if(m_aiIndexStop->LOAD) {
qDebug() << "Index stopped, interrupt ai index.";
return;
}
QJsonObject oneFile;
oneFile.insert(QStringLiteral("filepath"), iter.key());
oneFile.insert(QStringLiteral("fileformat"), iter.value());
tmpArray.append(oneFile);
++batchSize;
++finishNum;
if(batchSize >= 10) {
(indexer.*creat)(tmpArray);
qDebug() << finishNum << "of" << allSize <<"finished.";
Q_EMIT progress(IndexType::Ai, allSize, finishNum);
batchSize = 0;
tmpArray = QJsonArray();
}
}
if(!tmpArray.isEmpty()) {
(indexer.*creat)(tmpArray);
qDebug() << finishNum << "of" << allSize <<"finished.";
Q_EMIT progress(IndexType::Ai, allSize, finishNum);
tmpArray = QJsonArray();
}
};
creatIndex(&AiIndexer::addImageFileIndex, imagesNeedAiIndex);
creatIndex(&AiIndexer::addTextFileIndex, filesNeedAiIndex);
}

8
libsearch/index/batch-indexer.h
View File

@ -54,7 +54,8 @@ public:
Basic = 1u << 0,
Content = 1u << 1,
Ocr = 1u << 2,
All = Basic | Content | Ocr
Ai = 1u << 3,
All = Basic | Content | Ocr | Ai
};
Q_ENUM(Target)
Q_DECLARE_FLAGS(Targets, Target)
@ -65,6 +66,7 @@ public:
QAtomicInt& indexStop,
QAtomicInt& contentIndexStop,
QAtomicInt& contentIndexOcrStop,
QAtomicInt& aiIndexStop,
WorkMode mode = WorkMode::Update,
Targets target = Target::All);
void run() override;
@ -74,9 +76,11 @@ Q_SIGNALS:
void basicIndexStart(WorkMode);
void contentIndexStart(WorkMode);
void ocrContentIndexStart(WorkMode);
void aiIndexStart(WorkMode);
void basicIndexDone(WorkMode);
void contentIndexDone(WorkMode);
void ocrContentIndexDone(WorkMode);
void aiIndexDone(WorkMode);
void done(WorkMode, Targets);
private:
@ -84,12 +88,14 @@ private:
void basicIndex();
void contentIndex();
void ocrIndex();
void aiIndex();
QStringList m_folders;
QStringList m_blackList;
QAtomicInt *m_indexStop = nullptr;
QAtomicInt *m_contentIndexStop = nullptr;
QAtomicInt *m_contentIndexOcrStop = nullptr;
QAtomicInt *m_aiIndexStop = nullptr;
WorkMode m_mode;
Targets m_target;
QStringList m_cache;

106
libsearch/index/file-indexer-config.cpp
View File

@ -27,17 +27,27 @@
static const QString INDEX_SETTINGS = QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-service.conf";
static const QString CONTENT_INDEX_TARGET_TYPE = "contentIndexTarget";
static const QString OCR_CONTENT_INDEX_TARGET_TYPE = "ocrContentIndexTarget";
static const QString S_AI_FILE_TYPE = "sAiFileType";
static const QString S_AI_IMAGE_TYPE = "sAiImageType";
/**
* changelog: 1.2 增加AI索引开关
* changelog: 1.1 增加ocr开关
*/
static const QString CONFIG_VERSION = QStringLiteral("1.1");
static const QString GSETTINGS_VERSION = QStringLiteral("1.2");
/**
* changelog: 1.0 init
*/
static const QString SETTINGS_VERSION = QStringLiteral("1.2");
static const QByteArray UKUI_SEARCH_SCHEMAS = QByteArrayLiteral("org.ukui.search.settings");
static const QString FILE_INDEX_ENABLE_KEY = QStringLiteral("fileIndexEnable");
static const QString CONTENT_INDEX_ENABLE_KEY = QStringLiteral("contentIndexEnable");
static const QString CONTENT_FUZZY_SEARCH_KEY = QStringLiteral("contentFuzzySearch");
static const QString CONTENT_INDEX_ENABLE_OCR_KEY = QStringLiteral("contentIndexEnableOcr");
static const QString META_DATA_INDEX_ENABLE_KEY = QStringLiteral("metaDataIndexEnable");
static const QString CONFIG_VERSION_KEY = QStringLiteral("version");
static const QString AI_INDEX_ENABLE_KEY = QStringLiteral("aiIndexEnable");
static const QString GSETTINGS_VERSION_KEY = QStringLiteral("version");
static const QString SETTINGS_VERSION_KEY = QStringLiteral("version");
static std::once_flag flag;
static UkuiSearch::FileIndexerConfig *globalInstance = nullptr;
@ -61,8 +71,8 @@ FileIndexerConfig::FileIndexerConfig(QObject *parent)
if(QGSettings::isSchemaInstalled(id)) {
m_gsettings = new QGSettings(id, QByteArray(), this);
//保留旧版本配置
if(m_gsettings->keys().contains(CONFIG_VERSION_KEY)) {
QString oldVersion = m_gsettings->get(CONFIG_VERSION_KEY).toString();
if(m_gsettings->keys().contains(GSETTINGS_VERSION_KEY)) {
QString oldVersion = m_gsettings->get(GSETTINGS_VERSION_KEY).toString();
if(oldVersion == "0.0") {
bool fileIndexEnable = false;
if(m_gsettings->keys().contains(FILE_INDEX_ENABLE_KEY)) {
@ -76,7 +86,7 @@ FileIndexerConfig::FileIndexerConfig(QObject *parent)
m_gsettings->set(CONTENT_INDEX_ENABLE_OCR_KEY, true);
}
}
m_gsettings->set(CONFIG_VERSION_KEY, CONFIG_VERSION);
m_gsettings->set(GSETTINGS_VERSION_KEY, GSETTINGS_VERSION);
} else if (oldVersion == "1.0") {
bool contentIndex = false;
if(m_gsettings->keys().contains(CONTENT_INDEX_ENABLE_KEY)) {
@ -87,23 +97,33 @@ FileIndexerConfig::FileIndexerConfig(QObject *parent)
m_gsettings->set(CONTENT_INDEX_ENABLE_OCR_KEY, true);
}
}
m_gsettings->set(CONFIG_VERSION_KEY, CONFIG_VERSION);
m_gsettings->set(GSETTINGS_VERSION_KEY, GSETTINGS_VERSION);
}
}
connect(m_gsettings, &QGSettings::changed, this, [ = ](const QString &key) {
if(key == FILE_INDEX_ENABLE_KEY) {
Q_EMIT this->fileIndexEnableStatusChanged(m_gsettings->get(FILE_INDEX_ENABLE_KEY).toBool());
Q_EMIT fileIndexEnableStatusChanged(m_gsettings->get(FILE_INDEX_ENABLE_KEY).toBool());
} else if (key == CONTENT_INDEX_ENABLE_KEY) {
Q_EMIT this->contentIndexEnableStatusChanged(m_gsettings->get(CONTENT_INDEX_ENABLE_KEY).toBool());
Q_EMIT contentIndexEnableStatusChanged(m_gsettings->get(CONTENT_INDEX_ENABLE_KEY).toBool());
} else if (key == CONTENT_INDEX_ENABLE_OCR_KEY) {
Q_EMIT this->contentIndexEnableOcrStatusChanged(m_gsettings->get(CONTENT_INDEX_ENABLE_OCR_KEY).toBool());
Q_EMIT contentIndexEnableOcrStatusChanged(m_gsettings->get(CONTENT_INDEX_ENABLE_OCR_KEY).toBool());
} else if (key == AI_INDEX_ENABLE_KEY) {
Q_EMIT aiIndexEnableStatusChanged(m_gsettings->get(AI_INDEX_ENABLE_KEY).toBool());
}
});
} else {
qWarning() << UKUI_SEARCH_SCHEMAS << " is not found!";
}
m_settings = new QSettings(INDEX_SETTINGS, QSettings::IniFormat, this);
sync();
bool useDefaultSettings = false;
if(m_settings->value(SETTINGS_VERSION_KEY).toString() != SETTINGS_VERSION) {
qDebug() << "File index Settings version changed, using default file types, current version:"
<< SETTINGS_VERSION << "previous version:"
<< m_settings->value(SETTINGS_VERSION_KEY).toString();
useDefaultSettings = true;
m_settings->setValue(SETTINGS_VERSION_KEY, SETTINGS_VERSION);
}
syncIndexFileTypes(useDefaultSettings);
}
FileIndexerConfig::~FileIndexerConfig()
@ -188,6 +208,22 @@ bool FileIndexerConfig::isMetaDataIndexEnable()
return m_settings->value(META_DATA_INDEX_ENABLE_KEY, true).toBool();
}
bool FileIndexerConfig::isAiIndexEnable()
{
QMutexLocker locker(&m_mutex);
if(m_gsettings) {
if(m_gsettings->keys().contains(AI_INDEX_ENABLE_KEY)) {
return m_gsettings->get(AI_INDEX_ENABLE_KEY).toBool();
} else {
qWarning() << "FileIndexerConfig: Can not find key:" << AI_INDEX_ENABLE_KEY << "in" << UKUI_SEARCH_SCHEMAS;
return false;
}
} else {
qWarning() << "FileIndexerConfig:" << UKUI_SEARCH_SCHEMAS << " is not found!";
return false;
}
}
QMap<QString, bool> FileIndexerConfig::contentIndexTarget()
{
QMutexLocker locker(&m_mutex);
@ -200,12 +236,26 @@ QMap<QString, bool> FileIndexerConfig::ocrContentIndexTarget()
return m_targetPhotographTypeMap;
}
void FileIndexerConfig::sync()
QMap<QString, bool> FileIndexerConfig::aiIndexFileTarget()
{
QMutexLocker locker(&m_mutex);
return m_sAiFileTypeMap;
}
QMap<QString, bool> FileIndexerConfig::aiIndexImageTarget()
{
QMutexLocker locker(&m_mutex);
return m_sAiImageTypeMap;
}
void FileIndexerConfig::syncIndexFileTypes(bool useDefaultSettings)
{
QMutexLocker locker(&m_mutex);
m_settings->sync();
//文本内容
m_settings->beginGroup(CONTENT_INDEX_TARGET_TYPE);
if(m_settings->allKeys().isEmpty()) {
if(m_settings->allKeys().isEmpty() || useDefaultSettings) {
m_settings->remove("");
for(const QString &type : m_targetFileTypeMap.keys()) {
m_settings->setValue(type, m_targetFileTypeMap[type]);
}
@ -216,8 +266,10 @@ void FileIndexerConfig::sync()
}
}
m_settings->endGroup();
//图片ocr
m_settings->beginGroup(OCR_CONTENT_INDEX_TARGET_TYPE);
if(m_settings->allKeys().isEmpty()) {
if(m_settings->allKeys().isEmpty() || useDefaultSettings) {
m_settings->remove("");
for(const QString &type : m_targetPhotographTypeMap.keys()) {
m_settings->setValue(type, m_targetPhotographTypeMap[type]);
}
@ -228,4 +280,32 @@ void FileIndexerConfig::sync()
}
}
m_settings->endGroup();
//“AI文本内容”
m_settings->beginGroup(S_AI_FILE_TYPE);
if(m_settings->allKeys().isEmpty() || useDefaultSettings) {
m_settings->remove("");
for(const QString &type : m_sAiFileTypeMap.keys()) {
m_settings->setValue(type, m_sAiFileTypeMap[type]);
}
} else {
m_sAiFileTypeMap.clear();
for(const QString &type : m_settings->allKeys()) {
m_sAiFileTypeMap.insert(type, m_settings->value(type).toBool());
}
}
m_settings->endGroup();
//“AI图片”
m_settings->beginGroup(S_AI_IMAGE_TYPE);
if(m_settings->allKeys().isEmpty() || useDefaultSettings) {
m_settings->remove("");
for(const QString &type : m_sAiImageTypeMap.keys()) {
m_settings->setValue(type, m_sAiImageTypeMap[type]);
}
} else {
m_sAiImageTypeMap.clear();
for(const QString &type : m_settings->allKeys()) {
m_sAiImageTypeMap.insert(type, m_settings->value(type).toBool());
}
}
m_settings->endGroup();
}

25
libsearch/index/file-indexer-config.h
View File

@ -67,9 +67,16 @@ public:
* @return 是否激活元数据索引
*/
bool isMetaDataIndexEnable();
/**
* 是否激活AI索引
* @return
*/
bool isAiIndexEnable();
QMap<QString, bool> contentIndexTarget();
QMap<QString, bool> ocrContentIndexTarget();
void sync();
QMap<QString, bool> aiIndexFileTarget();
QMap<QString, bool> aiIndexImageTarget();
void syncIndexFileTypes(bool useDefaultSettings = false);
Q_SIGNALS:
/**
@ -97,6 +104,10 @@ Q_SIGNALS:
* ocr
*/
void contentIndexEnableOcrStatusChanged(bool);
/**
* AI索引
*/
void aiIndexEnableStatusChanged(bool);
private:
explicit FileIndexerConfig(QObject *parent = nullptr);
@ -143,6 +154,18 @@ private:
{"jpeg", true}
};
QMap<QString, bool> m_sAiFileTypeMap = {
{"docx", true},
{"pptx", true},
{"txt", true},
{"pdf", true},
};
QMap<QString, bool> m_sAiImageTypeMap = {
{"png", true},
{"jpg", true},
{"jpeg", true}
};
DirWatcher *m_dirWatcher = nullptr;
QGSettings *m_gsettings = nullptr;
QSettings *m_settings = nullptr;

31
libsearch/index/index-monitor.cpp
View File

@ -76,21 +76,27 @@ IndexMonitor::IndexMonitor(QObject *parent) : QObject(parent), d(new IndexMonito
connect(d->m_monitorReplica, &MonitorReplica::basicIndexSizeChanged, this, &IndexMonitor::basicIndexSizeChanged);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexSizeChanged, this, &IndexMonitor::contentIndexSizeChanged);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexSizeChanged, this, &IndexMonitor::ocrContentIndexSizeChanged);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexSizeChanged, this, &IndexMonitor::aiIndexSizeChanged);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexProgressChanged, this, &IndexMonitor::basicIndexProgressChanged);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexProgressChanged, this, &IndexMonitor::contentIndexProgressChanged);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexProgressChanged, this, &IndexMonitor::ocrContentIndexProgressChanged);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexProgressChanged, this, &IndexMonitor::aiIndexProgressChanged);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexDocNumChanged, this, &IndexMonitor::basicIndexDocNumChanged);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexDocNumChanged, this, &IndexMonitor::contentIndexDocNumChanged);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexDocNumChanged, this, &IndexMonitor::ocrContentIndexDocNumChanged);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexDocNumChanged, this, &IndexMonitor::aiIndexDocNumChanged);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexStart, this, &IndexMonitor::basicIndexStart);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexStart, this, &IndexMonitor::contentIndexStart);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexStart, this, &IndexMonitor::ocrContentIndexStart);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexStart, this, &IndexMonitor::aiIndexStart);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexDone, this, &IndexMonitor::basicIndexDone);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexDone, this, &IndexMonitor::contentIndexDone);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexDone, this, &IndexMonitor::ocrContentIndexDone);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexDone, this, &IndexMonitor::aiIndexDone);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexUpdatingChanged, this, &IndexMonitor::basicIndexUpdatingChanged);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexUpdatingChanged, this, &IndexMonitor::contentIndexUpdatingChanged);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexUpdatingChanged, this, &IndexMonitor::ocrContentIndexUpdatingChanged);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexUpdatingChanged, this, &IndexMonitor::aiIndexUpdatingChanged);
connect(d->m_monitorReplica, &QRemoteObjectReplica::initialized, this, &IndexMonitor::serviceReady);
connect(d->m_monitorReplica, &QRemoteObjectReplica::stateChanged, this, [&](QRemoteObjectReplica::State state, QRemoteObjectReplica::State oldState){
@ -133,6 +139,11 @@ uint IndexMonitor::ocrContentIndexSize() const
return d->m_monitorReplica->ocrContentIndexSize();
}
uint IndexMonitor::aiIndexSize() const
{
return d->m_monitorReplica->aiIndexSize();
}
uint IndexMonitor::basicIndexProgress() const
{
return d->m_monitorReplica->basicIndexProgress();
@ -143,16 +154,21 @@ uint IndexMonitor::contentIndexProgress() const
return d->m_monitorReplica->contentIndexProgress();
}
uint IndexMonitor::basicIndexDocNum() const
{
return d->m_monitorReplica->basicIndexDocNum();
}
uint IndexMonitor::ocrContentIndexProgress() const
{
return d->m_monitorReplica->ocrContentIndexProgress();
}
uint IndexMonitor::aiIndexProgress() const
{
return d->m_monitorReplica->aiIndexProgress();
}
uint IndexMonitor::basicIndexDocNum() const
{
return d->m_monitorReplica->basicIndexDocNum();
}
uint IndexMonitor::contentIndexDocNum() const
{
return d->m_monitorReplica->contentIndexDocNum();
@ -177,4 +193,9 @@ bool IndexMonitor::ocrContentIndexUpdating() const
{
return d->m_monitorReplica->ocrContentIndexUpdating();
}
bool IndexMonitor::aiIndexUpdating() const
{
return d->m_monitorReplica->aiIndexUpdating();
}
} // UkuiSearch

35
libsearch/index/index-monitor.h
View File

@ -35,12 +35,18 @@ class IndexMonitor : public QObject
Q_PROPERTY(uint basicIndexSize READ basicIndexSize NOTIFY basicIndexSizeChanged)
Q_PROPERTY(uint contentIndexSize READ contentIndexSize NOTIFY contentIndexSizeChanged)
Q_PROPERTY(uint ocrContentIndexSize READ ocrContentIndexSize NOTIFY ocrContentIndexSizeChanged)
Q_PROPERTY(uint aiIndexSize READ aiIndexSize NOTIFY aiIndexSizeChanged)
Q_PROPERTY(uint basicIndexProgress READ basicIndexProgress NOTIFY basicIndexProgressChanged)
Q_PROPERTY(uint contentIndexProgress READ contentIndexProgress NOTIFY contentIndexProgressChanged)
Q_PROPERTY(uint ocrContentIndexProgress READ ocrContentIndexProgress NOTIFY ocrContentIndexProgressChanged)
Q_PROPERTY(uint aiIndexProgress READ aiIndexProgress NOTIFY aiIndexProgressChanged)
Q_PROPERTY(uint basicIndexDocNum READ basicIndexDocNum NOTIFY basicIndexDocNumChanged)
Q_PROPERTY(uint contentIndexDocNum READ contentIndexDocNum NOTIFY contentIndexDocNumChanged)
Q_PROPERTY(uint ocrContentIndexDocNum READ ocrContentIndexDocNum NOTIFY ocrContentIndexDocNumChanged)
Q_PROPERTY(uint basicIndexUpdating READ basicIndexUpdating NOTIFY basicIndexUpdatingChanged)
Q_PROPERTY(uint contentIndexUpdating READ contentIndexUpdating NOTIFY contentIndexUpdatingChanged)
Q_PROPERTY(uint ocrContentIndexUpdating READ ocrContentIndexUpdating NOTIFY ocrContentIndexUpdatingChanged)
Q_PROPERTY(uint aiIndexUpdating READ aiIndexUpdating NOTIFY aiIndexUpdatingChanged)
public:
static IndexMonitor *self();
@ -71,6 +77,11 @@ public:
* @return 当前需要处理的OCR索引数量
*/
uint ocrContentIndexSize() const;
/**
* @brief aiIndexSize
* @return 当前需要处理的ai索引数量
*/
uint aiIndexSize() const;
/**
* @brief basicIndexProgress
* @return 基础索引进度
@ -86,6 +97,11 @@ public:
* @return ocr索引进度
*/
uint ocrContentIndexProgress() const;
/**
* @brief aiIndexProgress
* @return ai索引进度
*/
uint aiIndexProgress() const;
/**
* @brief basicIndexDocNum
* @return 基础索引完成的总文档数
@ -105,39 +121,50 @@ public:
* basicIndexUpdating
* @return 基础索引是否正在执行增量更新
*/
virtual bool basicIndexUpdating() const;
bool basicIndexUpdating() const;
/**
* basicIndexUpdating
* @return 文本内容索引是否正在执行增量更新
*/
virtual bool contentIndexUpdating() const;
bool contentIndexUpdating() const;
/**
* basicIndexUpdating
* @return ocr索引是否正在执行增量更新
*/
virtual bool ocrContentIndexUpdating() const;
bool ocrContentIndexUpdating() const;
/**
* aiIndexUpdating
* @return ai索引是否正在执行增量更新
*/
bool aiIndexUpdating() const;
Q_SIGNALS:
void currentIndexPathsChanged(QStringList currentIndexPaths);
void indexStateChanged(QString indexState);
void basicIndexSizeChanged(uint basicIndexSize);
void contentIndexSizeChanged(uint contentIndexSize);
void ocrContentIndexSizeChanged(uint ocrIndexSize);
void ocrContentIndexSizeChanged(uint ocrContentIndexSize);
void aiIndexSizeChanged(uint aiIndexSize);
void basicIndexProgressChanged(uint basicIndexProgress);
void contentIndexProgressChanged(uint contentIndexProgress);
void ocrContentIndexProgressChanged(uint ocrIndexProgress);
void aiIndexProgressChanged(uint aiIndexProgress);
void basicIndexDocNumChanged(uint basicIndexDocNum);
void contentIndexDocNumChanged(uint contentIndexDocNum);
void ocrContentIndexDocNumChanged(uint ocrContentIndexDocNum);
void aiIndexDocNumChanged(uint ocrContentIndexDocNum);
void basicIndexStart();
void contentIndexStart();
void ocrContentIndexStart();
void aiIndexStart();
void basicIndexDone(bool success);
void contentIndexDone(bool success);
void ocrContentIndexDone(bool success);
void aiIndexDone(bool success);
void basicIndexUpdatingChanged(bool basicIndexUpdating);
void contentIndexUpdatingChanged(bool contentIndexUpdating);
void ocrContentIndexUpdatingChanged(bool ocrContentIndexUpdating);
void aiIndexUpdatingChanged(bool ocrContentIndexUpdating);
void serviceReady();
void serviceOffline();

74
libsearch/index/index-scheduler.cpp
View File

@ -22,6 +22,7 @@
#include <QMetaEnum>
#include "index-updater.h"
#include "compatible-define.h"
#include "ai-indexer.h"
using namespace UkuiSearch;
IndexScheduler::IndexScheduler(QObject *parent) :
@ -31,7 +32,8 @@ IndexScheduler::IndexScheduler(QObject *parent) :
m_state(Startup),
m_indexStop(0),
m_contentIndexStop(0),
m_ocrContentIndexStop(0)
m_ocrContentIndexStop(0),
m_aiIndexStop(0)
{
qRegisterMetaType<IndexerState>("IndexerState");
qRegisterMetaType<BatchIndexer::WorkMode>("BatchIndexer::WorkMode");
@ -43,6 +45,7 @@ IndexScheduler::IndexScheduler(QObject *parent) :
connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable);
connect(m_config, &FileIndexerConfig::contentIndexEnableStatusChanged, this, &IndexScheduler::contentIndexEnable);
connect(m_config, &FileIndexerConfig::contentIndexEnableOcrStatusChanged, this, &IndexScheduler::ocrContentIndexEnable);
connect(m_config, &FileIndexerConfig::aiIndexEnableStatusChanged, this, &IndexScheduler::aiIndexEnable);
connect(m_config, &FileIndexerConfig::appendIndexDir, this, &IndexScheduler::addNewPath);
connect(m_config, &FileIndexerConfig::removeIndexDir, this, &IndexScheduler::removeIndex);
@ -63,6 +66,11 @@ IndexScheduler::IndexScheduler(QObject *parent) :
} else {
m_ocrContentIndexStop.fetchAndStoreRelaxed(1);
}
if(m_config->isAiIndexEnable()) {
targets |= BatchIndexer::Target::Ai;
} else {
m_aiIndexStop.fetchAndStoreRelaxed(1);
}
start(targets);
}
@ -119,7 +127,12 @@ void IndexScheduler::stop(BatchIndexer::Targets target)
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Off);
qDebug() << "File ocr content index has been stopped.";
}
if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD) {
if(target & BatchIndexer::Target::Ai) {
m_aiIndexStop.fetchAndStoreRelaxed(1);
m_statusRecorder->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Off);
qDebug() << "Ai index has been stopped.";
}
if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD && m_aiIndexStop.LOAD) {
m_fileWatcher.removeWatch();
m_watchInstalled = false;
m_threadPool.clear();
@ -139,7 +152,7 @@ IndexScheduler::IndexerState IndexScheduler::getIndexState()
void IndexScheduler::forceUpdate(BatchIndexer::Targets target)
{
m_config->sync();
m_config->syncIndexFileTypes();
stop(target);
if(target & BatchIndexer::Basic) {
fileIndexEnable(true);
@ -150,6 +163,9 @@ void IndexScheduler::forceUpdate(BatchIndexer::Targets target)
if(target & BatchIndexer::Ocr) {
ocrContentIndexEnable(true);
}
if(target & BatchIndexer::Ai) {
aiIndexEnable(true);
}
}
void IndexScheduler::start(BatchIndexer::Targets target)
@ -166,6 +182,9 @@ void IndexScheduler::start(BatchIndexer::Targets target)
if(target & BatchIndexer::Ocr) {
m_ocrContentIndexStop.fetchAndStoreRelaxed(0);
}
if(target & BatchIndexer::Ai) {
m_aiIndexStop.fetchAndStoreRelaxed(0);
}
if(target == BatchIndexer::None) {
return;
}
@ -189,6 +208,10 @@ void IndexScheduler::start(BatchIndexer::Targets target)
startTarget |= BatchIndexer::Target::Ocr;
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
}
if(target & BatchIndexer::Ai && !(rebuiltTarget & BatchIndexer::Target::Ai)) {
startTarget |= BatchIndexer::Target::Ai;
m_statusRecorder->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
}
startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, startTarget);
//启动监听
@ -239,19 +262,26 @@ void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList&
if(target & BatchIndexer::Ocr) {
m_ocrContentIndexPendingWorkCount++;
}
if(target & BatchIndexer::Ai) {
m_aiIndexPendingWorkCount++;
}
m_state = Running;
Q_EMIT stateChange(m_state);
auto *indexer = new BatchIndexer(folders, blackList, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop, mode, target);
auto *indexer = new BatchIndexer(folders, blackList,
m_indexStop, m_contentIndexStop, m_ocrContentIndexStop, m_aiIndexStop,
mode, target);
connect(indexer, &BatchIndexer::done, this, &IndexScheduler::batchIndexerFinished, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::progress, this, &IndexScheduler::process, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::basicIndexDone, this, &IndexScheduler::onBasicIndexDone, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::contentIndexDone, this, &IndexScheduler::onContentIndexDone, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::ocrContentIndexDone, this, &IndexScheduler::onOcrContentIndexDone, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::aiIndexDone, this, &IndexScheduler::onAiIndexDone, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::basicIndexStart, this, &IndexScheduler::basicIndexStart, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::contentIndexStart, this, &IndexScheduler::contentIndexStart, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::ocrContentIndexStart, this, &IndexScheduler::ocrContentIndexStart, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::aiIndexStart, this, &IndexScheduler::aiIndexStart, Qt::QueuedConnection);
m_threadPool.start(indexer);
}
}
@ -295,6 +325,19 @@ void IndexScheduler::ocrContentIndexEnable(bool enable)
}
}
void IndexScheduler::aiIndexEnable(bool enable)
{
if(enable) {
if(m_aiIndexPendingWorkCount == 0) {
start(BatchIndexer::Ai);
} else {
m_aiIndexStartWaiting = true;
}
} else {
stop(BatchIndexer::Ai);
}
}
void IndexScheduler::updateIndex(const QVector<PendingFile> &files)
{
qDebug() << "updateIndex=====";
@ -303,7 +346,7 @@ void IndexScheduler::updateIndex(const QVector<PendingFile> &files)
m_state = Running;
Q_EMIT stateChange(m_state);
auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop);
auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop, m_aiIndexStop);
connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection);
m_threadPoolForRTUpdate.start(updateJob);
}
@ -333,6 +376,7 @@ bool IndexScheduler::isIdle()
return m_basicIndexPendingWorkCount == 0
&& m_contentIndexPendingWorkCount == 0
&& m_ocrContentIndexPendingWorkCount == 0
&& m_aiIndexPendingWorkCount == 0
&& m_updatePendingWorkCount == 0
&& m_addNewPathPendingWorkCount == 0;
}
@ -387,6 +431,22 @@ void IndexScheduler::onOcrContentIndexDone(BatchIndexer::WorkMode mode)
Q_EMIT ocrContentIndexDone(success);
checkIfStartsWaiting();
}
void IndexScheduler::onAiIndexDone(BatchIndexer::WorkMode mode)
{
Q_UNUSED(mode)
m_aiIndexPendingWorkCount--;
bool success = false;
if(m_statusRecorder->getStatus(AI_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Initializing ||
m_statusRecorder->getStatus(AI_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Updating) {
m_statusRecorder->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
success = true;
} else if (m_statusRecorder->getStatus(AI_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::RealTimeUpdating ) {
success = true;
}
Q_EMIT aiIndexDone(success);
checkIfStartsWaiting();
}
void IndexScheduler::checkIfStartsWaiting()
{
@ -403,6 +463,10 @@ void IndexScheduler::checkIfStartsWaiting()
targets |= BatchIndexer::Ocr;
m_ocrContentIndexStartWaiting = false;
}
if(m_aiIndexStartWaiting && !m_aiIndexPendingWorkCount) {
targets |= BatchIndexer::Ai;
m_aiIndexStartWaiting = false;
}
if(targets != BatchIndexer::None) {
start(targets);
}

7
libsearch/index/index-scheduler.h
View File

@ -58,9 +58,11 @@ Q_SIGNALS:
void basicIndexStart();
void contentIndexStart();
void ocrContentIndexStart();
void aiIndexStart();
void basicIndexDone(bool success);
void contentIndexDone(bool success);
void ocrContentIndexDone(bool success);
void aiIndexDone(bool success);
void done();
private Q_SLOTS:
@ -80,6 +82,7 @@ private Q_SLOTS:
void fileIndexEnable(bool enable);
void contentIndexEnable(bool enable);
void ocrContentIndexEnable(bool enable);
void aiIndexEnable(bool enable);
void updateIndex(const QVector<PendingFile>& files);
void batchIndexerFinished(BatchIndexer::WorkMode mode, BatchIndexer::Targets targets);
void updateFinished();
@ -87,6 +90,7 @@ private Q_SLOTS:
void onBasicIndexDone(BatchIndexer::WorkMode mode);
void onContentIndexDone(BatchIndexer::WorkMode mode);
void onOcrContentIndexDone(BatchIndexer::WorkMode mode);
void onAiIndexDone(BatchIndexer::WorkMode mode);
void checkIfStartsWaiting();
void installWatches();
@ -105,6 +109,7 @@ private:
QAtomicInt m_indexStop;
QAtomicInt m_contentIndexStop;
QAtomicInt m_ocrContentIndexStop;
QAtomicInt m_aiIndexStop;
QThreadPool m_threadPool;
QThreadPool m_threadPoolForRTUpdate; //用于实时更新索引的线程池
@ -112,6 +117,7 @@ private:
quint64 m_basicIndexPendingWorkCount = 0;
quint64 m_contentIndexPendingWorkCount = 0;
quint64 m_ocrContentIndexPendingWorkCount= 0;
quint64 m_aiIndexPendingWorkCount= 0;
quint64 m_updatePendingWorkCount = 0;
quint64 m_addNewPathPendingWorkCount = 0;
@ -119,6 +125,7 @@ private:
bool m_basicIndexStartWaiting = false;
bool m_contentIndexStartWaiting = false;
bool m_ocrContentIndexStartWaiting = false;
bool m_aiIndexStartWaiting = false;
//监听是否已安装
bool m_watchInstalled = false;

1
libsearch/index/index-status-recorder.h
View File

@ -27,6 +27,7 @@
#define CONTENT_INDEX_DATABASE_STATE_KEY "content_index_database_state"
#define OCR_CONTENT_INDEX_DATABASE_STATE_KEY "ocr_content_index_database_state"
#define INDEX_DATABASE_STATE_KEY "index_database_state"
#define AI_INDEX_DATABASE_STATE_KEY "ai_index_database_state"
#define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf"
#define INDEX_DATABASE_VERSION_KEY "index_database_version"
#define CONTENT_DATABASE_VERSION_KEY "content_database_version"

68
libsearch/index/index-updater.cpp
View File

@ -19,6 +19,7 @@
*/
#include "index-updater.h"
#include <malloc.h>
#include <QJsonArray>
#include "writable-database.h"
#include "basic-indexer.h"
#include "file-indexer-config.h"
@ -27,12 +28,18 @@
#include "file-utils.h"
#include "compatible-define.h"
#include "index-status-recorder.h"
#include "ai-indexer.h"
using namespace UkuiSearch;
IndexUpdater::IndexUpdater(const QVector<PendingFile>& files, QAtomicInt& indexstop, QAtomicInt& contentIndexstop, QAtomicInt& contentIndexOcrStop)
IndexUpdater::IndexUpdater(const QVector<PendingFile>& files,
QAtomicInt& indexstop,
QAtomicInt& contentIndexstop,
QAtomicInt& contentIndexOcrStop,
QAtomicInt& aiIndexStop)
: m_cache(files),
m_indexStop(&indexstop),
m_contentIndexStop(&contentIndexstop),
m_contentIndexOcrStop(&contentIndexOcrStop)
m_contentIndexOcrStop(&contentIndexOcrStop),
m_aiIndexStop(&aiIndexStop)
{
}
void IndexUpdater::updateIndex()
@ -83,6 +90,10 @@ void IndexUpdater::run()
if(IndexStatusRecorder::getInstance()->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) {
IndexStatusRecorder::getInstance()->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
}
updateAiIndex();
if(IndexStatusRecorder::getInstance()->getStatus(AI_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) {
IndexStatusRecorder::getInstance()->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
}
m_cache.clear();
m_cache.shrink_to_fit();
malloc_trim(0);
@ -192,3 +203,56 @@ void IndexUpdater::updateOcrContentIndex()
qDebug() << "===finish update ocr content index===";
}
}
void IndexUpdater::updateAiIndex()
{
if(FileIndexerConfig::getInstance()->isAiIndexEnable() && !m_aiIndexStop->LOAD) {
IndexStatusRecorder::getInstance()->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating);
AiIndexer indexer;
if(!indexer.creatSession()) {
qWarning() << "Creat session failed, fail to update ai index";
return;
}
qDebug() << "===update ai index===";
for(PendingFile file : m_cache) {
if(m_aiIndexStop->LOAD) {
qDebug() << "Ai index update interrupted";
return;
}
auto update = [&](bool(AiIndexer::*creat)(const QJsonArray &)){
QString type;
if(!indexer.checkFileSupported(file.path(), type)) {
if(file.isModified() || file.isMoveTo()) {
qDebug() << "| remove:" <<file.path();
indexer.deleteFileIndex({file.path()});
}
return;
}
qDebug() << "| index:" <<file.path();
if(indexer.addTextFileIndex(QJsonArray{
QJsonObject{
{QStringLiteral("filepath"), file.path()},
{QStringLiteral("filetype"), type}
}}))
{
} else {
qWarning() << "Ai index failed for" << file.path();
}
};
QString suffix = QFileInfo(file.path()).suffix();
if(file.shouldRemoveIndex()) {
if(file.isDir() || (FileIndexerConfig::getInstance()->aiIndexFileTarget()[suffix] || FileIndexerConfig::getInstance()->aiIndexImageTarget()[suffix])) {
qDebug() << "| remove:" <<file.path();
indexer.deleteFileIndex({file.path()});
}
} else if(FileIndexerConfig::getInstance()->aiIndexFileTarget()[suffix] && !file.isDir()) {
update(&AiIndexer::addTextFileIndex);
} else if (FileIndexerConfig::getInstance()->aiIndexImageTarget()[suffix] && !file.isDir()) {
update(&AiIndexer::addImageFileIndex);
}
}
qDebug() << "===finish update ai content index===";
}
}

8
libsearch/index/index-updater.h
View File

@ -31,7 +31,11 @@ class IndexUpdater : public QObject, public QRunnable
{
Q_OBJECT
public:
explicit IndexUpdater(const QVector<PendingFile>& files, QAtomicInt& indexstop, QAtomicInt& contentIndexstop, QAtomicInt& contentIndexOcrStop);
explicit IndexUpdater(const QVector<PendingFile>& files,
QAtomicInt& indexstop,
QAtomicInt& contentIndexstop,
QAtomicInt& contentIndexOcrStop,
QAtomicInt& aiIndexStop);
void run() override;
Q_SIGNALS:
@ -41,11 +45,13 @@ private:
void updateIndex();
void updateContentIndex();
void updateOcrContentIndex();
void updateAiIndex();
QVector<PendingFile> m_cache;
QAtomicInt *m_contentIndexStop = nullptr;
QAtomicInt *m_indexStop = nullptr;
QAtomicInt *m_contentIndexOcrStop = nullptr;
QAtomicInt *m_aiIndexStop = nullptr;
};
}
#endif // INDEXUPDATER_H

6
libsearch/index/monitor.rep
View File

@ -6,19 +6,25 @@ class Monitor
PROP(uint basicIndexSize READONLY);
PROP(uint contentIndexSize READONLY);
PROP(uint ocrContentIndexSize READONLY);
PROP(uint aiIndexSize READONLY);
PROP(uint basicIndexProgress READONLY);
PROP(uint contentIndexProgress READONLY);
PROP(uint ocrContentIndexProgress READONLY);
PROP(uint aiIndexProgress READONLY);
PROP(uint basicIndexDocNum READONLY);
PROP(uint contentIndexDocNum READONLY);
PROP(uint ocrContentIndexDocNum READONLY);
PROP(uint aiIndexDocNum READONLY);
PROP(bool basicIndexUpdating READONLY);
PROP(bool contentIndexUpdating READONLY);
PROP(bool ocrContentIndexUpdating READONLY);
PROP(bool aiIndexUpdating READONLY);
SIGNAL(basicIndexStart());
SIGNAL(contentIndexStart());
SIGNAL(ocrContentIndexStart());
SIGNAL(aiIndexStart());
SIGNAL(basicIndexDone(bool success));
SIGNAL(contentIndexDone(bool success));
SIGNAL(ocrContentIndexDone(bool success));
SIGNAL(aiIndexDone(bool success));
};

23
libsearch/libsearch.cpp
View File

@ -1,23 +0,0 @@
/*
* Copyright (C) 2020, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#include "libsearch.h"
using namespace UkuiSearch;
// Encapsulate advanced interfaces here.

44
libsearch/libsearch.h
View File

@ -1,44 +0,0 @@
/*
* Copyright (C) 2020, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#ifndef LIBSEARCH_H
#define LIBSEARCH_H
#include "libsearch_global.h"
#include "file-utils.h"
#include "global-settings.h"
#include "plugininterface/search-plugin-iface.h"
#include "plugininterface/data-queue.h"
#include "index/ukui-search-qdbus.h"
#include "index/search-manager.h"
namespace UkuiSearch {
//class LIBSEARCH_EXPORT GlobalSearch {
//public:
// static QStringList fileSearch(QString keyword, int begin = 0, int num = -1);
//private:
// GlobalSearch();
//};
}
#endif // LIBSEARCH_H

59
ukui-search-service/monitor.cpp
View File

@ -44,6 +44,11 @@ Monitor::Monitor(IndexScheduler *scheduler, QObject *parent) : MonitorSource(par
m_ocrContentIndexUpdating = false;
Q_EMIT ocrContentIndexUpdatingChanged(m_ocrContentIndexUpdating);
});
connect(scheduler, &IndexScheduler::aiIndexDone, this, [&](bool success){
Q_EMIT aiIndexDone(success);
m_aiIndexUpdating = false;
Q_EMIT aiIndexUpdatingChanged(m_aiIndexUpdating);
});
connect(scheduler, &IndexScheduler::basicIndexStart, this, [&](){
Q_EMIT basicIndexStart();
m_basicIndexUpdating = true;
@ -59,6 +64,11 @@ Monitor::Monitor(IndexScheduler *scheduler, QObject *parent) : MonitorSource(par
m_ocrContentIndexUpdating = true;
Q_EMIT ocrContentIndexUpdatingChanged(m_ocrContentIndexUpdating);
});
connect(scheduler, &IndexScheduler::aiIndexStart, this, [&](){
Q_EMIT aiIndexStart();
m_aiIndexUpdating = true;
Q_EMIT aiIndexUpdatingChanged(m_aiIndexUpdating);
});
connect(FileIndexerConfig::getInstance(), &FileIndexerConfig::appendIndexDir, this, [&](){
m_currentIndexPaths = FileIndexerConfig::getInstance()->currentIndexableDir();
Q_EMIT currentIndexPathsChanged(m_currentIndexPaths);
@ -92,7 +102,12 @@ uint Monitor::contentIndexSize() const
uint Monitor::ocrContentIndexSize() const
{
return m_ocrIndexSize;
return m_ocrContentIndexSize;
}
uint Monitor::aiIndexSize() const
{
return m_aiIndexSize;
}
uint Monitor::basicIndexProgress() const
@ -107,9 +122,15 @@ uint Monitor::contentIndexProgress() const
uint Monitor::ocrContentIndexProgress() const
{
return m_ocrIndexProgress;
return m_ocrContentIndexProgress;
}
uint Monitor::aiIndexProgress() const
{
return m_aiIndexProgress;
}
uint Monitor::basicIndexDocNum() const
{
return m_basicIndexDocNum;
@ -125,6 +146,11 @@ uint Monitor::ocrContentIndexDocNum() const
return m_ocrContentIndexDocNum;
}
uint Monitor::aiIndexDocNum() const
{
return m_aiIndexDocNum;
}
void Monitor::onIndexStateChanged(IndexScheduler::IndexerState state)
{
if(state == IndexScheduler::IndexerState::Idle || state == IndexScheduler::IndexerState::Stop) {
@ -175,13 +201,13 @@ void Monitor::processUpdate(IndexType type, uint all, uint finished)
break;
}
case IndexType::OCR: {
if(m_ocrIndexSize != all) {
m_ocrIndexSize = all;
Q_EMIT ocrContentIndexSizeChanged(m_ocrIndexSize);
if(m_ocrContentIndexSize != all) {
m_ocrContentIndexSize = all;
Q_EMIT ocrContentIndexSizeChanged(m_ocrContentIndexSize);
}
if(m_ocrIndexProgress != finished) {
m_ocrIndexProgress = finished;
Q_EMIT ocrContentIndexProgressChanged(m_ocrIndexProgress);
if(m_ocrContentIndexProgress != finished) {
m_ocrContentIndexProgress = finished;
Q_EMIT ocrContentIndexProgressChanged(m_ocrContentIndexProgress);
}
uint count = m_contentDatabase.getIndexDocCount();
if(m_ocrContentIndexDocNum != count) {
@ -190,6 +216,18 @@ void Monitor::processUpdate(IndexType type, uint all, uint finished)
}
break;
}
case IndexType::Ai: {
if(m_aiIndexSize != all) {
m_aiIndexSize = all;
Q_EMIT aiIndexSizeChanged(m_aiIndexSize);
}
if(m_aiIndexProgress != finished) {
m_aiIndexProgress = finished;
Q_EMIT aiIndexProgressChanged(m_aiIndexProgress);
}
//AI索引暂不支持文档数量信号
break;
}
default:
break;
}
@ -209,3 +247,8 @@ bool Monitor::ocrContentIndexUpdating() const
{
return m_ocrContentIndexUpdating;
}
bool Monitor::aiIndexUpdating() const
{
return m_aiIndexUpdating;
}

35
ukui-search-service/monitor.h
View File

@ -60,6 +60,11 @@ public:
* @return 当前需要处理的OCR索引数量
*/
uint ocrContentIndexSize() const override;
/**
* @brief aiIndexSize
* @return 当前需要处理的AI索引数量
*/
uint aiIndexSize() const override;
/**
* @brief basicIndexProgress
* @return 基础索引进度
@ -75,6 +80,11 @@ public:
* @return ocr索引进度
*/
uint ocrContentIndexProgress() const override;
/**
* @brief aiIndexProgress
* @return ai索引进度
*/
uint aiIndexProgress() const override;
/**
* @brief basicIndexDocNum
* @return 基础索引完成的总文档数
@ -90,21 +100,31 @@ public:
* @return ocr容索引完成的总文档数
*/
uint ocrContentIndexDocNum() const override;
/**
* @brief aiIndexDocNum
* @return ai索引完成的总文档数
*/
uint aiIndexDocNum() const override;
/**
* basicIndexUpdating
* @return 基础索引是否正在执行增量更新
*/
virtual bool basicIndexUpdating() const override;
bool basicIndexUpdating() const override;
/**
* basicIndexUpdating
* @return 文本内容索引是否正在执行增量更新
*/
virtual bool contentIndexUpdating() const override;
bool contentIndexUpdating() const override;
/**
* basicIndexUpdating
* @return ocr索引是否正在执行增量更新
*/
virtual bool ocrContentIndexUpdating() const override;
bool ocrContentIndexUpdating() const override;
/**
* aiIndexUpdating
* @return ai索引是否正在执行增量更新
*/
bool aiIndexUpdating() const override;
private Q_SLOTS:
void onIndexStateChanged(IndexScheduler::IndexerState);
@ -118,16 +138,21 @@ private:
Database m_ocrContentDatabase;
uint m_basicIndexSize = 0;
uint m_contentIndexSize = 0;
uint m_ocrIndexSize = 0;
uint m_ocrContentIndexSize = 0;
uint m_aiIndexSize = 0;
uint m_basicIndexProgress = 0;
uint m_contentIndexProgress = 0;
uint m_ocrIndexProgress = 0;
uint m_ocrContentIndexProgress = 0;
uint m_aiIndexProgress = 0;
uint m_basicIndexDocNum = 0;
uint m_contentIndexDocNum = 0;
uint m_ocrContentIndexDocNum = 0;
uint m_aiIndexDocNum = 0;
bool m_basicIndexUpdating = false;
bool m_contentIndexUpdating = false;
bool m_ocrContentIndexUpdating = false;
bool m_aiIndexUpdating = false;
};
}
#endif // MONITOR_H

24
ukui-search-service/qml/IndexMonitor.qml
View File

@ -21,7 +21,7 @@ import QtQuick 2.0
import QtQuick.Layouts 1.12
Rectangle {
height: 180
height: 250
width: 320
Column {
visible: true
@ -43,6 +43,11 @@ Rectangle {
keyText: qsTr("OCR content index document number")
valueText: monitor.ocrContentIndexDocNum
}
StatusKeyValue {
id: aiIndexDocumentNum
keyText: qsTr("ai index document number")
valueText: monitor.aiIndexDocNum
}
StatusKeyValue {
id: indexState
@ -79,14 +84,25 @@ Rectangle {
value: monitor.ocrContentIndexProgress
visible: true
}
IndexProgressBar {
id: aiIndexProgress
name: "Ai progress"
width: parent.width;
from: 0
to: monitor.aiIndexSize
value: monitor.aiIndexProgress
visible: true
}
}
Component.onCompleted: {
monitor.basicIndexDone.connect(onBasicIndexDone);
monitor.contentIndexDone.connect(onContentIndexDone);
monitor.ocrContentIndexDone.connect(onOcrContentIndexDone);
monitor.aiIndexDone.connect(onAiIndexDone);
monitor.basicIndexStart.connect(onBasicIndexStart);
monitor.contentIndexStart.connect(onContentIndexStart);
monitor.ocrContentIndexStart.connect(onOcrContentIndexStart);
monitor.aiIndexStart.connect(onAiIndexStart);
}
function onBasicIndexStart() {
basicIndexProgress.state = "Running"
@ -97,6 +113,9 @@ Rectangle {
function onOcrContentIndexStart() {
ocrContentIndexProgress.state = "Running"
}
function onAiIndexStart() {
aiIndexProgress.state = "Running"
}
function onBasicIndexDone() {
basicIndexProgress.state = "Done"
}
@ -106,4 +125,7 @@ Rectangle {
function onOcrContentIndexDone() {
ocrContentIndexProgress.state = "Done"
}
function onAiIndexDone() {
aiIndexProgress.state = "Done"
}
}

2
ukui-search-service/ukui-search-service.cpp
View File

@ -104,6 +104,8 @@ void UkuiSearchService::parseCmd(const QString& msg, bool isPrimary)
targets = BatchIndexer::Content;
} else if (target == "ocr") {
targets = BatchIndexer::Ocr;
} else if (target == "ai") {
target = BatchIndexer::Ai;
}
m_indexScheduler->forceUpdate(targets);
}

4
ukuisearch-systemdbus/sysdbusregister.cpp
View File

@ -151,7 +151,7 @@ int SysdbusRegister::AddInotifyMaxUserInstance(int addNum)
// autoSettings->setValue("autologin-user", username);
// autoSettings->endGroup();
// autoSettings->sync();
// autoSettings->syncIndexFileTypes();
//}
//QString SysdbusRegister::getSuspendThenHibernate() {
@ -160,7 +160,7 @@ int SysdbusRegister::AddInotifyMaxUserInstance(int addNum)
// QString time = mHibernateSet->value("HibernateDelaySec").toString();
// mHibernateSet->endGroup();
// mHibernateSet->sync();
// mHibernateSet->syncIndexFileTypes();
// return time;
//}