feat(file-index):增加AI索引功能

This commit is contained in:
iaom 2024-04-17 17:43:50 +08:00
parent f9d727de1c
commit ede828a430
26 changed files with 777 additions and 131 deletions

View File

@ -30,5 +30,10 @@
<summary>content index enable ocr</summary> <summary>content index enable ocr</summary>
<description>Enable or disable OCR in content index.</description> <description>Enable or disable OCR in content index.</description>
</key> </key>
<key name="ai-index-enable" type="b">
<default>true</default>
<summary>ai index enable</summary>
<description>Enable or disable AI index service.</description>
</key>
</schema> </schema>
</schemalist> </schemalist>

View File

@ -53,6 +53,7 @@
#define MAIN_SETTINGS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search.conf" #define MAIN_SETTINGS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search.conf"
#define ENABLE_CREATE_INDEX_ASK_DIALOG "enable_create_index_ask_dialog" #define ENABLE_CREATE_INDEX_ASK_DIALOG "enable_create_index_ask_dialog"
const static QString FILE_INDEX_ENABLE_KEY = "fileIndexEnable"; const static QString FILE_INDEX_ENABLE_KEY = "fileIndexEnable";
static const QByteArray UKUI_SEARCH_SCHEMAS = QByteArrayLiteral("org.ukui.search.settings");
using namespace UkuiSearch; using namespace UkuiSearch;
extern void qt_blurImage(QImage &blurImage, qreal radius, bool quality, int transposed); extern void qt_blurImage(QImage &blurImage, qreal radius, bool quality, int transposed);

View File

@ -47,7 +47,6 @@
#include <QTimer> #include <QTimer>
#include "search-app-widget-plugin/search.h" #include "search-app-widget-plugin/search.h"
#include "libsearch.h"
#include "create-index-ask-dialog.h" #include "create-index-ask-dialog.h"
#include "search-line-edit.h" #include "search-line-edit.h"
#include "search-result-page.h" #include "search-result-page.h"

View File

@ -29,7 +29,8 @@ set(LIBUKUI_SEARCH_PC_PKGS
kysdk-qtwidgets kysdk-qtwidgets
uchardet uchardet
kysdk-systime kysdk-systime
kysdk-datacollect) kysdk-datacollect
kylin-ai-base)
foreach(PC_LIB IN ITEMS ${LIBUKUI_SEARCH_PC_PKGS}) foreach(PC_LIB IN ITEMS ${LIBUKUI_SEARCH_PC_PKGS})
pkg_check_modules(${PC_LIB} REQUIRED IMPORTED_TARGET ${PC_LIB}) pkg_check_modules(${PC_LIB} REQUIRED IMPORTED_TARGET ${PC_LIB})
@ -58,6 +59,7 @@ set(LIBUKUI_SEARCH_SRC
gobject-template.cpp gobject-template.h gobject-template.cpp gobject-template.h
index/basic-indexer.cpp index/basic-indexer.h index/basic-indexer.cpp index/basic-indexer.h
index/batch-indexer.cpp index/batch-indexer.h index/batch-indexer.cpp index/batch-indexer.h
index/ai-indexer.cpp index/ai-indexer.h
index/compatible-define.h index/compatible-define.h
index/database.cpp index/database.h index/database.cpp index/database.h
index/document.cpp index/document.h index/document.cpp index/document.h
@ -77,7 +79,6 @@ set(LIBUKUI_SEARCH_SRC
index/file-extraction-result.cpp index/file-extraction-result.cpp
index/file-extraction-result.h index/file-extraction-result.h
index/index-monitor.cpp index/index-monitor.cpp
libsearch.cpp libsearch.h
libsearch_global.h libsearch_global.h
log-utils.cpp log-utils.h log-utils.cpp log-utils.h
notesearch/note-search-plugin.cpp notesearch/note-search-plugin.h notesearch/note-search-plugin.cpp notesearch/note-search-plugin.h

View File

@ -25,8 +25,6 @@
namespace UkuiSearch { namespace UkuiSearch {
#define CONTENT_DATABASE_PATH_SLOT 1 #define CONTENT_DATABASE_PATH_SLOT 1
#define CONTENT_DATABASE_SUFFIX_SLOT 2
static const int LABEL_MAX_WIDTH = 320; static const int LABEL_MAX_WIDTH = 320;
static const QString HOME_PATH = QDir::homePath(); static const QString HOME_PATH = QDir::homePath();
@ -36,10 +34,8 @@ static const QString OCR_CONTENT_INDEX_PATH = HOME_PATH + QStringLiteral("/.conf
static const QString FILE_SEARCH_VALUE = QStringLiteral("0"); static const QString FILE_SEARCH_VALUE = QStringLiteral("0");
static const QString DIR_SEARCH_VALUE = QStringLiteral("1"); static const QString DIR_SEARCH_VALUE = QStringLiteral("1");
static const QString INDEX_SEM = QStringLiteral("ukui-search-index-sem");
static const int OCR_MIN_SIZE = 200; static const int OCR_MIN_SIZE = 200;
static const QByteArray UKUI_SEARCH_SCHEMAS = QByteArrayLiteral("org.ukui.search.settings");
static const QString SEARCH_METHOD_KEY = QStringLiteral("fileIndexEnable");
/** /**
* changelog 1.0.1 term中被截断的问题 * changelog 1.0.1 term中被截断的问题
*/ */
@ -71,11 +67,12 @@ enum class DataBaseType {
/** /**
* @brief The IndexType enum * @brief The IndexType enum
* Index type, notice:OCR index is part of Content index. * Index type
*/ */
enum class IndexType { enum class IndexType {
Basic, Basic,
Contents, Contents,
OCR OCR,
Ai
}; };
} }

View File

@ -0,0 +1,137 @@
/*
* Copyright (C) 2024, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: iaom <zhangpengfei@kylinos.cn>
*
*/
#include "ai-indexer.h"
#include <unistd.h>
#include <QJsonDocument>
#include <QJsonValue>
#include <QJsonArray>
#include <QMutex>
#include "kylin-ai/ai-base/datamanagement.h"
namespace UkuiSearch {
class AiIndexerPrivate
{
public:
QMutex m_mutex;
bool m_sessionValid = false;
DataManagementSession m_session = nullptr;
};
AiIndexer::AiIndexer(QObject *parent) : QObject(parent), d(new AiIndexerPrivate)
{
d->m_mutex.lock();
}
AiIndexer::~AiIndexer()
{
destroySession();
d->m_mutex.unlock();
if(d) {
delete d;
d = nullptr;
}
}
bool AiIndexer::creatSession()
{
DataManagementResult result = data_management_create_session(&d->m_session, DataManagementType::SYS_SEARCH, getuid());
qDebug() << "===creatSession===" <<result;
if(result == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
d->m_sessionValid = true;
return true;
}
return false;
}
bool AiIndexer::destroySession()
{
d->m_sessionValid = false;
return data_management_destroy_session(d->m_session) == DataManagementResult::DATA_MANAGEMENT_SUCCESS;
}
bool AiIndexer::checkFileSupported(const QString &filePath, QString &result)
{
if(!d->m_sessionValid) {
return false;
}
char *originalResult;
if(data_management_check_file_format(d->m_session, filePath.toLocal8Bit().data(), &originalResult) == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
QJsonObject jo = QJsonDocument::fromJson(QByteArray(originalResult)).object();
data_management_free_check_file_format_result(d->m_session, originalResult);
if(jo.value(QStringLiteral("supported")).toBool()) {
result = jo.value(QStringLiteral("format")).toString();
return true;
} else {
return false;
}
};
return false;
}
bool AiIndexer::addTextFileIndex(const QJsonArray &object)
{
if(!d->m_sessionValid) {
return false;
}
return data_management_add_text_files(d->m_session, QJsonDocument(object).toJson().data()) == DataManagementResult::DATA_MANAGEMENT_SUCCESS;
}
bool AiIndexer::addImageFileIndex(const QJsonArray &object)
{
if(!d->m_sessionValid) {
return false;
}
return data_management_add_image_files(d->m_session, QJsonDocument(object).toJson().data()) == DataManagementResult::DATA_MANAGEMENT_SUCCESS;
}
bool AiIndexer::deleteFileIndex(const QStringList &files)
{
if(!d->m_sessionValid) {
return false;
}
QJsonArray tmpArray;
for(const QString &file : files) {
QJsonObject oneFile;
oneFile.insert(QStringLiteral("filepath"), file);
tmpArray.append(oneFile);
}
return data_management_delete_files(d->m_session, QJsonDocument(tmpArray).toJson().data()) == DataManagementResult::DATA_MANAGEMENT_SUCCESS;
}
bool AiIndexer::getAllIndexedFiles(QJsonObject &object)
{
if(!d->m_sessionValid) {
return false;
}
char *originalResult;
DataManagementResult result = data_management_get_all_fileinfos(d->m_session, &originalResult);
if(result == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
QJsonArray array = QJsonDocument::fromJson(QByteArray(originalResult)).array();
for(const QJsonValueRef value : array) {
object.insert(value.toObject().value(QStringLiteral("filepath")).toString(), value.toObject().value(QStringLiteral("timestamp")).toInt());
}
return true;
}
return false;
}
} // UkuiSearch

View File

@ -0,0 +1,48 @@
/*
* Copyright (C) 2024, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: iaom <zhangpengfei@kylinos.cn>
*
*/
#ifndef UKUI_SEARCH_AI_INDEXER_H
#define UKUI_SEARCH_AI_INDEXER_H
#include <QObject>
#include <QJsonObject>
namespace UkuiSearch {
class AiIndexerPrivate;
class AiIndexer : public QObject
{
Q_OBJECT
public:
explicit AiIndexer(QObject *parent = nullptr);
~AiIndexer();
bool creatSession();
bool checkFileSupported(const QString &filePath, QString &result);
bool addTextFileIndex(const QJsonArray &object);
bool addImageFileIndex(const QJsonArray &object);
bool deleteFileIndex(const QStringList &files);
bool getAllIndexedFiles(QJsonObject &object);
private:
bool destroySession();
AiIndexerPrivate *d = nullptr;
};
} // UkuiSearch
#endif //UKUI_SEARCH_AI_INDEXER_H

View File

@ -25,6 +25,7 @@
#include <QQueue> #include <QQueue>
#include <QDateTime> #include <QDateTime>
#include <QMetaEnum> #include <QMetaEnum>
#include <QJsonArray>
#include "file-utils.h" #include "file-utils.h"
#include "basic-indexer.h" #include "basic-indexer.h"
@ -32,15 +33,20 @@
#include "file-content-indexer.h" #include "file-content-indexer.h"
#include "writable-database.h" #include "writable-database.h"
#include "compatible-define.h" #include "compatible-define.h"
#include "ai-indexer.h"
using namespace UkuiSearch; using namespace UkuiSearch;
BatchIndexer::BatchIndexer(const QStringList &folders, const QStringList &blackList, BatchIndexer::BatchIndexer(const QStringList &folders, const QStringList &blackList,
QAtomicInt& indexStop, QAtomicInt &contentIndexStop, QAtomicInt &contentIndexOcrStop, QAtomicInt& indexStop,
QAtomicInt &contentIndexStop,
QAtomicInt &contentIndexOcrStop,
QAtomicInt& aiIndexStop,
WorkMode mode, Targets target) WorkMode mode, Targets target)
: m_folders(folders), : m_folders(folders),
m_blackList(blackList), m_blackList(blackList),
m_indexStop(&indexStop), m_indexStop(&indexStop),
m_contentIndexStop(&contentIndexStop), m_contentIndexStop(&contentIndexStop),
m_contentIndexOcrStop(&contentIndexOcrStop), m_contentIndexOcrStop(&contentIndexOcrStop),
m_aiIndexStop(&aiIndexStop),
m_mode(mode), m_mode(mode),
m_target(target) m_target(target)
{ {
@ -52,7 +58,8 @@ void BatchIndexer::run()
timer.start(); timer.start();
if((!m_target.testFlag(Target::Basic) || m_indexStop->LOAD) if((!m_target.testFlag(Target::Basic) || m_indexStop->LOAD)
&& (!m_target.testFlag(Target::Content) || m_contentIndexStop->LOAD) && (!m_target.testFlag(Target::Content) || m_contentIndexStop->LOAD)
&& (!m_target.testFlag(Target::Ocr) || m_contentIndexOcrStop->LOAD)) { && (!m_target.testFlag(Target::Ocr) || m_contentIndexOcrStop->LOAD)
&& (!m_target.testFlag(Target::Ai) || m_aiIndexStop->LOAD)) {
Q_EMIT done(m_mode, m_target); Q_EMIT done(m_mode, m_target);
return; return;
} }
@ -74,6 +81,11 @@ void BatchIndexer::run()
ocrIndex(); ocrIndex();
Q_EMIT ocrContentIndexDone(m_mode); Q_EMIT ocrContentIndexDone(m_mode);
} }
if(m_target & Target::Ai) {
Q_EMIT aiIndexStart(m_mode);
aiIndex();
Q_EMIT aiIndexDone(m_mode);
}
m_cache.clear(); m_cache.clear();
malloc_trim(0); malloc_trim(0);
@ -378,3 +390,119 @@ void BatchIndexer::ocrIndex()
qDebug() << "Ocr content index finished,"; qDebug() << "Ocr content index finished,";
} }
void BatchIndexer::aiIndex()
{
qDebug() << "Begin ai index";
if (m_aiIndexStop->LOAD) {
qDebug() << "Index stopped, abort ai index.";
return;
}
AiIndexer indexer;
if(!indexer.creatSession()) {
qWarning() << "Creat session failed, fail to run ai index!";
return;
}
if (m_mode == WorkMode::Rebuild) {
//暂不支持重建
return;
}
QJsonObject filesNeedAiIndex;
QJsonObject imagesNeedAiIndex;
QFileInfo info;
if (m_mode == WorkMode::Add) {
for (const QString &path: m_cache) {
info.setFile(path);
if(!info.isFile()) {
continue;
}
QString type;
if (FileIndexerConfig::getInstance()->aiIndexFileTarget()[info.suffix()]) {
if(indexer.checkFileSupported(path, type)) { //这个检查可能会读文件,不要每个文件都执行一遍
filesNeedAiIndex.insert(path, type);
}
}
if (FileIndexerConfig::getInstance()->aiIndexImageTarget()[info.suffix()]) {
if(indexer.checkFileSupported(path, type)) {
imagesNeedAiIndex.insert(path, type);
}
}
}
} else {
QJsonObject indexTimes; //校验完整性
if(indexer.getAllIndexedFiles(indexTimes)) {
qDebug() << indexTimes.size() << "documents recorded";
for (const QString &path: m_cache) {
info.setFile(path);
if(!info.isFile()) {
continue;
}
QString type;
if (FileIndexerConfig::getInstance()->aiIndexFileTarget()[info.suffix()]) {
if(indexTimes.value(path).toInt() != info.lastModified().toSecsSinceEpoch()) {
if(indexer.checkFileSupported(path, type)) {
filesNeedAiIndex.insert(path, type);
indexTimes.remove(path);
}
} else {
indexTimes.remove(path);
}
} else if (FileIndexerConfig::getInstance()->aiIndexImageTarget()[info.suffix()]) {
if(indexTimes.value(path).toInt() != info.lastModified().toSecsSinceEpoch()) {
if(indexer.checkFileSupported(path, type)) {
imagesNeedAiIndex.insert(path, type);
indexTimes.remove(path);
}
} else {
indexTimes.remove(path);
}
}
}
if(!indexTimes.isEmpty()) {
qDebug() << indexTimes.size() << "documents need remove";
indexer.deleteFileIndex(indexTimes.keys());
}
}
}
uint allSize = filesNeedAiIndex.size() + imagesNeedAiIndex.size();
qDebug() << allSize << "files need ai index.";
Q_EMIT progress(IndexType::Ai, allSize, 0);
uint finishNum = 0;
uint batchSize = 0;
QJsonArray tmpArray;
QJsonObject::const_iterator iter;
auto creatIndex = [&](bool(AiIndexer::*creat)(const QJsonArray &), const QJsonObject &files){
for(iter = files.begin(); iter != files.end(); ++iter) {
if(m_aiIndexStop->LOAD) {
qDebug() << "Index stopped, interrupt ai index.";
return;
}
QJsonObject oneFile;
oneFile.insert(QStringLiteral("filepath"), iter.key());
oneFile.insert(QStringLiteral("fileformat"), iter.value());
tmpArray.append(oneFile);
++batchSize;
++finishNum;
if(batchSize >= 10) {
(indexer.*creat)(tmpArray);
qDebug() << finishNum << "of" << allSize <<"finished.";
Q_EMIT progress(IndexType::Ai, allSize, finishNum);
batchSize = 0;
tmpArray = QJsonArray();
}
}
if(!tmpArray.isEmpty()) {
(indexer.*creat)(tmpArray);
qDebug() << finishNum << "of" << allSize <<"finished.";
Q_EMIT progress(IndexType::Ai, allSize, finishNum);
tmpArray = QJsonArray();
}
};
creatIndex(&AiIndexer::addImageFileIndex, imagesNeedAiIndex);
creatIndex(&AiIndexer::addTextFileIndex, filesNeedAiIndex);
}

View File

@ -54,7 +54,8 @@ public:
Basic = 1u << 0, Basic = 1u << 0,
Content = 1u << 1, Content = 1u << 1,
Ocr = 1u << 2, Ocr = 1u << 2,
All = Basic | Content | Ocr Ai = 1u << 3,
All = Basic | Content | Ocr | Ai
}; };
Q_ENUM(Target) Q_ENUM(Target)
Q_DECLARE_FLAGS(Targets, Target) Q_DECLARE_FLAGS(Targets, Target)
@ -65,6 +66,7 @@ public:
QAtomicInt& indexStop, QAtomicInt& indexStop,
QAtomicInt& contentIndexStop, QAtomicInt& contentIndexStop,
QAtomicInt& contentIndexOcrStop, QAtomicInt& contentIndexOcrStop,
QAtomicInt& aiIndexStop,
WorkMode mode = WorkMode::Update, WorkMode mode = WorkMode::Update,
Targets target = Target::All); Targets target = Target::All);
void run() override; void run() override;
@ -74,9 +76,11 @@ Q_SIGNALS:
void basicIndexStart(WorkMode); void basicIndexStart(WorkMode);
void contentIndexStart(WorkMode); void contentIndexStart(WorkMode);
void ocrContentIndexStart(WorkMode); void ocrContentIndexStart(WorkMode);
void aiIndexStart(WorkMode);
void basicIndexDone(WorkMode); void basicIndexDone(WorkMode);
void contentIndexDone(WorkMode); void contentIndexDone(WorkMode);
void ocrContentIndexDone(WorkMode); void ocrContentIndexDone(WorkMode);
void aiIndexDone(WorkMode);
void done(WorkMode, Targets); void done(WorkMode, Targets);
private: private:
@ -84,12 +88,14 @@ private:
void basicIndex(); void basicIndex();
void contentIndex(); void contentIndex();
void ocrIndex(); void ocrIndex();
void aiIndex();
QStringList m_folders; QStringList m_folders;
QStringList m_blackList; QStringList m_blackList;
QAtomicInt *m_indexStop = nullptr; QAtomicInt *m_indexStop = nullptr;
QAtomicInt *m_contentIndexStop = nullptr; QAtomicInt *m_contentIndexStop = nullptr;
QAtomicInt *m_contentIndexOcrStop = nullptr; QAtomicInt *m_contentIndexOcrStop = nullptr;
QAtomicInt *m_aiIndexStop = nullptr;
WorkMode m_mode; WorkMode m_mode;
Targets m_target; Targets m_target;
QStringList m_cache; QStringList m_cache;

View File

@ -27,17 +27,27 @@
static const QString INDEX_SETTINGS = QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-service.conf"; static const QString INDEX_SETTINGS = QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-service.conf";
static const QString CONTENT_INDEX_TARGET_TYPE = "contentIndexTarget"; static const QString CONTENT_INDEX_TARGET_TYPE = "contentIndexTarget";
static const QString OCR_CONTENT_INDEX_TARGET_TYPE = "ocrContentIndexTarget"; static const QString OCR_CONTENT_INDEX_TARGET_TYPE = "ocrContentIndexTarget";
static const QString S_AI_FILE_TYPE = "sAiFileType";
static const QString S_AI_IMAGE_TYPE = "sAiImageType";
/** /**
* changelog: 1.2 AI索引开关
* changelog: 1.1 ocr开关 * changelog: 1.1 ocr开关
*/ */
static const QString CONFIG_VERSION = QStringLiteral("1.1"); static const QString GSETTINGS_VERSION = QStringLiteral("1.2");
/**
* changelog: 1.0 init
*/
static const QString SETTINGS_VERSION = QStringLiteral("1.2");
static const QByteArray UKUI_SEARCH_SCHEMAS = QByteArrayLiteral("org.ukui.search.settings"); static const QByteArray UKUI_SEARCH_SCHEMAS = QByteArrayLiteral("org.ukui.search.settings");
static const QString FILE_INDEX_ENABLE_KEY = QStringLiteral("fileIndexEnable"); static const QString FILE_INDEX_ENABLE_KEY = QStringLiteral("fileIndexEnable");
static const QString CONTENT_INDEX_ENABLE_KEY = QStringLiteral("contentIndexEnable"); static const QString CONTENT_INDEX_ENABLE_KEY = QStringLiteral("contentIndexEnable");
static const QString CONTENT_FUZZY_SEARCH_KEY = QStringLiteral("contentFuzzySearch"); static const QString CONTENT_FUZZY_SEARCH_KEY = QStringLiteral("contentFuzzySearch");
static const QString CONTENT_INDEX_ENABLE_OCR_KEY = QStringLiteral("contentIndexEnableOcr"); static const QString CONTENT_INDEX_ENABLE_OCR_KEY = QStringLiteral("contentIndexEnableOcr");
static const QString META_DATA_INDEX_ENABLE_KEY = QStringLiteral("metaDataIndexEnable"); static const QString META_DATA_INDEX_ENABLE_KEY = QStringLiteral("metaDataIndexEnable");
static const QString CONFIG_VERSION_KEY = QStringLiteral("version"); static const QString AI_INDEX_ENABLE_KEY = QStringLiteral("aiIndexEnable");
static const QString GSETTINGS_VERSION_KEY = QStringLiteral("version");
static const QString SETTINGS_VERSION_KEY = QStringLiteral("version");
static std::once_flag flag; static std::once_flag flag;
static UkuiSearch::FileIndexerConfig *globalInstance = nullptr; static UkuiSearch::FileIndexerConfig *globalInstance = nullptr;
@ -61,8 +71,8 @@ FileIndexerConfig::FileIndexerConfig(QObject *parent)
if(QGSettings::isSchemaInstalled(id)) { if(QGSettings::isSchemaInstalled(id)) {
m_gsettings = new QGSettings(id, QByteArray(), this); m_gsettings = new QGSettings(id, QByteArray(), this);
//保留旧版本配置 //保留旧版本配置
if(m_gsettings->keys().contains(CONFIG_VERSION_KEY)) { if(m_gsettings->keys().contains(GSETTINGS_VERSION_KEY)) {
QString oldVersion = m_gsettings->get(CONFIG_VERSION_KEY).toString(); QString oldVersion = m_gsettings->get(GSETTINGS_VERSION_KEY).toString();
if(oldVersion == "0.0") { if(oldVersion == "0.0") {
bool fileIndexEnable = false; bool fileIndexEnable = false;
if(m_gsettings->keys().contains(FILE_INDEX_ENABLE_KEY)) { if(m_gsettings->keys().contains(FILE_INDEX_ENABLE_KEY)) {
@ -76,7 +86,7 @@ FileIndexerConfig::FileIndexerConfig(QObject *parent)
m_gsettings->set(CONTENT_INDEX_ENABLE_OCR_KEY, true); m_gsettings->set(CONTENT_INDEX_ENABLE_OCR_KEY, true);
} }
} }
m_gsettings->set(CONFIG_VERSION_KEY, CONFIG_VERSION); m_gsettings->set(GSETTINGS_VERSION_KEY, GSETTINGS_VERSION);
} else if (oldVersion == "1.0") { } else if (oldVersion == "1.0") {
bool contentIndex = false; bool contentIndex = false;
if(m_gsettings->keys().contains(CONTENT_INDEX_ENABLE_KEY)) { if(m_gsettings->keys().contains(CONTENT_INDEX_ENABLE_KEY)) {
@ -87,23 +97,33 @@ FileIndexerConfig::FileIndexerConfig(QObject *parent)
m_gsettings->set(CONTENT_INDEX_ENABLE_OCR_KEY, true); m_gsettings->set(CONTENT_INDEX_ENABLE_OCR_KEY, true);
} }
} }
m_gsettings->set(CONFIG_VERSION_KEY, CONFIG_VERSION); m_gsettings->set(GSETTINGS_VERSION_KEY, GSETTINGS_VERSION);
} }
} }
connect(m_gsettings, &QGSettings::changed, this, [ = ](const QString &key) { connect(m_gsettings, &QGSettings::changed, this, [ = ](const QString &key) {
if(key == FILE_INDEX_ENABLE_KEY) { if(key == FILE_INDEX_ENABLE_KEY) {
Q_EMIT this->fileIndexEnableStatusChanged(m_gsettings->get(FILE_INDEX_ENABLE_KEY).toBool()); Q_EMIT fileIndexEnableStatusChanged(m_gsettings->get(FILE_INDEX_ENABLE_KEY).toBool());
} else if(key == CONTENT_INDEX_ENABLE_KEY) { } else if (key == CONTENT_INDEX_ENABLE_KEY) {
Q_EMIT this->contentIndexEnableStatusChanged(m_gsettings->get(CONTENT_INDEX_ENABLE_KEY).toBool()); Q_EMIT contentIndexEnableStatusChanged(m_gsettings->get(CONTENT_INDEX_ENABLE_KEY).toBool());
} else if(key == CONTENT_INDEX_ENABLE_OCR_KEY) { } else if (key == CONTENT_INDEX_ENABLE_OCR_KEY) {
Q_EMIT this->contentIndexEnableOcrStatusChanged(m_gsettings->get(CONTENT_INDEX_ENABLE_OCR_KEY).toBool()); Q_EMIT contentIndexEnableOcrStatusChanged(m_gsettings->get(CONTENT_INDEX_ENABLE_OCR_KEY).toBool());
} else if (key == AI_INDEX_ENABLE_KEY) {
Q_EMIT aiIndexEnableStatusChanged(m_gsettings->get(AI_INDEX_ENABLE_KEY).toBool());
} }
}); });
} else { } else {
qWarning() << UKUI_SEARCH_SCHEMAS << " is not found!"; qWarning() << UKUI_SEARCH_SCHEMAS << " is not found!";
} }
m_settings = new QSettings(INDEX_SETTINGS, QSettings::IniFormat, this); m_settings = new QSettings(INDEX_SETTINGS, QSettings::IniFormat, this);
sync(); bool useDefaultSettings = false;
if(m_settings->value(SETTINGS_VERSION_KEY).toString() != SETTINGS_VERSION) {
qDebug() << "File index Settings version changed, using default file types, current version:"
<< SETTINGS_VERSION << "previous version:"
<< m_settings->value(SETTINGS_VERSION_KEY).toString();
useDefaultSettings = true;
m_settings->setValue(SETTINGS_VERSION_KEY, SETTINGS_VERSION);
}
syncIndexFileTypes(useDefaultSettings);
} }
FileIndexerConfig::~FileIndexerConfig() FileIndexerConfig::~FileIndexerConfig()
@ -188,6 +208,22 @@ bool FileIndexerConfig::isMetaDataIndexEnable()
return m_settings->value(META_DATA_INDEX_ENABLE_KEY, true).toBool(); return m_settings->value(META_DATA_INDEX_ENABLE_KEY, true).toBool();
} }
bool FileIndexerConfig::isAiIndexEnable()
{
QMutexLocker locker(&m_mutex);
if(m_gsettings) {
if(m_gsettings->keys().contains(AI_INDEX_ENABLE_KEY)) {
return m_gsettings->get(AI_INDEX_ENABLE_KEY).toBool();
} else {
qWarning() << "FileIndexerConfig: Can not find key:" << AI_INDEX_ENABLE_KEY << "in" << UKUI_SEARCH_SCHEMAS;
return false;
}
} else {
qWarning() << "FileIndexerConfig:" << UKUI_SEARCH_SCHEMAS << " is not found!";
return false;
}
}
QMap<QString, bool> FileIndexerConfig::contentIndexTarget() QMap<QString, bool> FileIndexerConfig::contentIndexTarget()
{ {
QMutexLocker locker(&m_mutex); QMutexLocker locker(&m_mutex);
@ -200,12 +236,26 @@ QMap<QString, bool> FileIndexerConfig::ocrContentIndexTarget()
return m_targetPhotographTypeMap; return m_targetPhotographTypeMap;
} }
void FileIndexerConfig::sync() QMap<QString, bool> FileIndexerConfig::aiIndexFileTarget()
{
QMutexLocker locker(&m_mutex);
return m_sAiFileTypeMap;
}
QMap<QString, bool> FileIndexerConfig::aiIndexImageTarget()
{
QMutexLocker locker(&m_mutex);
return m_sAiImageTypeMap;
}
void FileIndexerConfig::syncIndexFileTypes(bool useDefaultSettings)
{ {
QMutexLocker locker(&m_mutex); QMutexLocker locker(&m_mutex);
m_settings->sync(); m_settings->sync();
//文本内容
m_settings->beginGroup(CONTENT_INDEX_TARGET_TYPE); m_settings->beginGroup(CONTENT_INDEX_TARGET_TYPE);
if(m_settings->allKeys().isEmpty()) { if(m_settings->allKeys().isEmpty() || useDefaultSettings) {
m_settings->remove("");
for(const QString &type : m_targetFileTypeMap.keys()) { for(const QString &type : m_targetFileTypeMap.keys()) {
m_settings->setValue(type, m_targetFileTypeMap[type]); m_settings->setValue(type, m_targetFileTypeMap[type]);
} }
@ -216,8 +266,10 @@ void FileIndexerConfig::sync()
} }
} }
m_settings->endGroup(); m_settings->endGroup();
//图片ocr
m_settings->beginGroup(OCR_CONTENT_INDEX_TARGET_TYPE); m_settings->beginGroup(OCR_CONTENT_INDEX_TARGET_TYPE);
if(m_settings->allKeys().isEmpty()) { if(m_settings->allKeys().isEmpty() || useDefaultSettings) {
m_settings->remove("");
for(const QString &type : m_targetPhotographTypeMap.keys()) { for(const QString &type : m_targetPhotographTypeMap.keys()) {
m_settings->setValue(type, m_targetPhotographTypeMap[type]); m_settings->setValue(type, m_targetPhotographTypeMap[type]);
} }
@ -228,4 +280,32 @@ void FileIndexerConfig::sync()
} }
} }
m_settings->endGroup(); m_settings->endGroup();
} //“AI文本内容”
m_settings->beginGroup(S_AI_FILE_TYPE);
if(m_settings->allKeys().isEmpty() || useDefaultSettings) {
m_settings->remove("");
for(const QString &type : m_sAiFileTypeMap.keys()) {
m_settings->setValue(type, m_sAiFileTypeMap[type]);
}
} else {
m_sAiFileTypeMap.clear();
for(const QString &type : m_settings->allKeys()) {
m_sAiFileTypeMap.insert(type, m_settings->value(type).toBool());
}
}
m_settings->endGroup();
//“AI图片”
m_settings->beginGroup(S_AI_IMAGE_TYPE);
if(m_settings->allKeys().isEmpty() || useDefaultSettings) {
m_settings->remove("");
for(const QString &type : m_sAiImageTypeMap.keys()) {
m_settings->setValue(type, m_sAiImageTypeMap[type]);
}
} else {
m_sAiImageTypeMap.clear();
for(const QString &type : m_settings->allKeys()) {
m_sAiImageTypeMap.insert(type, m_settings->value(type).toBool());
}
}
m_settings->endGroup();
}

View File

@ -67,9 +67,16 @@ public:
* @return * @return
*/ */
bool isMetaDataIndexEnable(); bool isMetaDataIndexEnable();
/**
* AI索引
* @return
*/
bool isAiIndexEnable();
QMap<QString, bool> contentIndexTarget(); QMap<QString, bool> contentIndexTarget();
QMap<QString, bool> ocrContentIndexTarget(); QMap<QString, bool> ocrContentIndexTarget();
void sync(); QMap<QString, bool> aiIndexFileTarget();
QMap<QString, bool> aiIndexImageTarget();
void syncIndexFileTypes(bool useDefaultSettings = false);
Q_SIGNALS: Q_SIGNALS:
/** /**
@ -97,6 +104,10 @@ Q_SIGNALS:
* ocr * ocr
*/ */
void contentIndexEnableOcrStatusChanged(bool); void contentIndexEnableOcrStatusChanged(bool);
/**
* AI索引
*/
void aiIndexEnableStatusChanged(bool);
private: private:
explicit FileIndexerConfig(QObject *parent = nullptr); explicit FileIndexerConfig(QObject *parent = nullptr);
@ -143,6 +154,18 @@ private:
{"jpeg", true} {"jpeg", true}
}; };
QMap<QString, bool> m_sAiFileTypeMap = {
{"docx", true},
{"pptx", true},
{"txt", true},
{"pdf", true},
};
QMap<QString, bool> m_sAiImageTypeMap = {
{"png", true},
{"jpg", true},
{"jpeg", true}
};
DirWatcher *m_dirWatcher = nullptr; DirWatcher *m_dirWatcher = nullptr;
QGSettings *m_gsettings = nullptr; QGSettings *m_gsettings = nullptr;
QSettings *m_settings = nullptr; QSettings *m_settings = nullptr;

View File

@ -76,21 +76,27 @@ IndexMonitor::IndexMonitor(QObject *parent) : QObject(parent), d(new IndexMonito
connect(d->m_monitorReplica, &MonitorReplica::basicIndexSizeChanged, this, &IndexMonitor::basicIndexSizeChanged); connect(d->m_monitorReplica, &MonitorReplica::basicIndexSizeChanged, this, &IndexMonitor::basicIndexSizeChanged);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexSizeChanged, this, &IndexMonitor::contentIndexSizeChanged); connect(d->m_monitorReplica, &MonitorReplica::contentIndexSizeChanged, this, &IndexMonitor::contentIndexSizeChanged);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexSizeChanged, this, &IndexMonitor::ocrContentIndexSizeChanged); connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexSizeChanged, this, &IndexMonitor::ocrContentIndexSizeChanged);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexSizeChanged, this, &IndexMonitor::aiIndexSizeChanged);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexProgressChanged, this, &IndexMonitor::basicIndexProgressChanged); connect(d->m_monitorReplica, &MonitorReplica::basicIndexProgressChanged, this, &IndexMonitor::basicIndexProgressChanged);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexProgressChanged, this, &IndexMonitor::contentIndexProgressChanged); connect(d->m_monitorReplica, &MonitorReplica::contentIndexProgressChanged, this, &IndexMonitor::contentIndexProgressChanged);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexProgressChanged, this, &IndexMonitor::ocrContentIndexProgressChanged); connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexProgressChanged, this, &IndexMonitor::ocrContentIndexProgressChanged);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexProgressChanged, this, &IndexMonitor::aiIndexProgressChanged);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexDocNumChanged, this, &IndexMonitor::basicIndexDocNumChanged); connect(d->m_monitorReplica, &MonitorReplica::basicIndexDocNumChanged, this, &IndexMonitor::basicIndexDocNumChanged);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexDocNumChanged, this, &IndexMonitor::contentIndexDocNumChanged); connect(d->m_monitorReplica, &MonitorReplica::contentIndexDocNumChanged, this, &IndexMonitor::contentIndexDocNumChanged);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexDocNumChanged, this, &IndexMonitor::ocrContentIndexDocNumChanged); connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexDocNumChanged, this, &IndexMonitor::ocrContentIndexDocNumChanged);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexDocNumChanged, this, &IndexMonitor::aiIndexDocNumChanged);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexStart, this, &IndexMonitor::basicIndexStart); connect(d->m_monitorReplica, &MonitorReplica::basicIndexStart, this, &IndexMonitor::basicIndexStart);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexStart, this, &IndexMonitor::contentIndexStart); connect(d->m_monitorReplica, &MonitorReplica::contentIndexStart, this, &IndexMonitor::contentIndexStart);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexStart, this, &IndexMonitor::ocrContentIndexStart); connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexStart, this, &IndexMonitor::ocrContentIndexStart);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexStart, this, &IndexMonitor::aiIndexStart);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexDone, this, &IndexMonitor::basicIndexDone); connect(d->m_monitorReplica, &MonitorReplica::basicIndexDone, this, &IndexMonitor::basicIndexDone);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexDone, this, &IndexMonitor::contentIndexDone); connect(d->m_monitorReplica, &MonitorReplica::contentIndexDone, this, &IndexMonitor::contentIndexDone);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexDone, this, &IndexMonitor::ocrContentIndexDone); connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexDone, this, &IndexMonitor::ocrContentIndexDone);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexDone, this, &IndexMonitor::aiIndexDone);
connect(d->m_monitorReplica, &MonitorReplica::basicIndexUpdatingChanged, this, &IndexMonitor::basicIndexUpdatingChanged); connect(d->m_monitorReplica, &MonitorReplica::basicIndexUpdatingChanged, this, &IndexMonitor::basicIndexUpdatingChanged);
connect(d->m_monitorReplica, &MonitorReplica::contentIndexUpdatingChanged, this, &IndexMonitor::contentIndexUpdatingChanged); connect(d->m_monitorReplica, &MonitorReplica::contentIndexUpdatingChanged, this, &IndexMonitor::contentIndexUpdatingChanged);
connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexUpdatingChanged, this, &IndexMonitor::ocrContentIndexUpdatingChanged); connect(d->m_monitorReplica, &MonitorReplica::ocrContentIndexUpdatingChanged, this, &IndexMonitor::ocrContentIndexUpdatingChanged);
connect(d->m_monitorReplica, &MonitorReplica::aiIndexUpdatingChanged, this, &IndexMonitor::aiIndexUpdatingChanged);
connect(d->m_monitorReplica, &QRemoteObjectReplica::initialized, this, &IndexMonitor::serviceReady); connect(d->m_monitorReplica, &QRemoteObjectReplica::initialized, this, &IndexMonitor::serviceReady);
connect(d->m_monitorReplica, &QRemoteObjectReplica::stateChanged, this, [&](QRemoteObjectReplica::State state, QRemoteObjectReplica::State oldState){ connect(d->m_monitorReplica, &QRemoteObjectReplica::stateChanged, this, [&](QRemoteObjectReplica::State state, QRemoteObjectReplica::State oldState){
@ -133,6 +139,11 @@ uint IndexMonitor::ocrContentIndexSize() const
return d->m_monitorReplica->ocrContentIndexSize(); return d->m_monitorReplica->ocrContentIndexSize();
} }
uint IndexMonitor::aiIndexSize() const
{
return d->m_monitorReplica->aiIndexSize();
}
uint IndexMonitor::basicIndexProgress() const uint IndexMonitor::basicIndexProgress() const
{ {
return d->m_monitorReplica->basicIndexProgress(); return d->m_monitorReplica->basicIndexProgress();
@ -143,16 +154,21 @@ uint IndexMonitor::contentIndexProgress() const
return d->m_monitorReplica->contentIndexProgress(); return d->m_monitorReplica->contentIndexProgress();
} }
uint IndexMonitor::basicIndexDocNum() const
{
return d->m_monitorReplica->basicIndexDocNum();
}
uint IndexMonitor::ocrContentIndexProgress() const uint IndexMonitor::ocrContentIndexProgress() const
{ {
return d->m_monitorReplica->ocrContentIndexProgress(); return d->m_monitorReplica->ocrContentIndexProgress();
} }
uint IndexMonitor::aiIndexProgress() const
{
return d->m_monitorReplica->aiIndexProgress();
}
uint IndexMonitor::basicIndexDocNum() const
{
return d->m_monitorReplica->basicIndexDocNum();
}
uint IndexMonitor::contentIndexDocNum() const uint IndexMonitor::contentIndexDocNum() const
{ {
return d->m_monitorReplica->contentIndexDocNum(); return d->m_monitorReplica->contentIndexDocNum();
@ -177,4 +193,9 @@ bool IndexMonitor::ocrContentIndexUpdating() const
{ {
return d->m_monitorReplica->ocrContentIndexUpdating(); return d->m_monitorReplica->ocrContentIndexUpdating();
} }
bool IndexMonitor::aiIndexUpdating() const
{
return d->m_monitorReplica->aiIndexUpdating();
}
} // UkuiSearch } // UkuiSearch

View File

@ -35,12 +35,18 @@ class IndexMonitor : public QObject
Q_PROPERTY(uint basicIndexSize READ basicIndexSize NOTIFY basicIndexSizeChanged) Q_PROPERTY(uint basicIndexSize READ basicIndexSize NOTIFY basicIndexSizeChanged)
Q_PROPERTY(uint contentIndexSize READ contentIndexSize NOTIFY contentIndexSizeChanged) Q_PROPERTY(uint contentIndexSize READ contentIndexSize NOTIFY contentIndexSizeChanged)
Q_PROPERTY(uint ocrContentIndexSize READ ocrContentIndexSize NOTIFY ocrContentIndexSizeChanged) Q_PROPERTY(uint ocrContentIndexSize READ ocrContentIndexSize NOTIFY ocrContentIndexSizeChanged)
Q_PROPERTY(uint aiIndexSize READ aiIndexSize NOTIFY aiIndexSizeChanged)
Q_PROPERTY(uint basicIndexProgress READ basicIndexProgress NOTIFY basicIndexProgressChanged) Q_PROPERTY(uint basicIndexProgress READ basicIndexProgress NOTIFY basicIndexProgressChanged)
Q_PROPERTY(uint contentIndexProgress READ contentIndexProgress NOTIFY contentIndexProgressChanged) Q_PROPERTY(uint contentIndexProgress READ contentIndexProgress NOTIFY contentIndexProgressChanged)
Q_PROPERTY(uint ocrContentIndexProgress READ ocrContentIndexProgress NOTIFY ocrContentIndexProgressChanged) Q_PROPERTY(uint ocrContentIndexProgress READ ocrContentIndexProgress NOTIFY ocrContentIndexProgressChanged)
Q_PROPERTY(uint aiIndexProgress READ aiIndexProgress NOTIFY aiIndexProgressChanged)
Q_PROPERTY(uint basicIndexDocNum READ basicIndexDocNum NOTIFY basicIndexDocNumChanged) Q_PROPERTY(uint basicIndexDocNum READ basicIndexDocNum NOTIFY basicIndexDocNumChanged)
Q_PROPERTY(uint contentIndexDocNum READ contentIndexDocNum NOTIFY contentIndexDocNumChanged) Q_PROPERTY(uint contentIndexDocNum READ contentIndexDocNum NOTIFY contentIndexDocNumChanged)
Q_PROPERTY(uint ocrContentIndexDocNum READ ocrContentIndexDocNum NOTIFY ocrContentIndexDocNumChanged) Q_PROPERTY(uint ocrContentIndexDocNum READ ocrContentIndexDocNum NOTIFY ocrContentIndexDocNumChanged)
Q_PROPERTY(uint basicIndexUpdating READ basicIndexUpdating NOTIFY basicIndexUpdatingChanged)
Q_PROPERTY(uint contentIndexUpdating READ contentIndexUpdating NOTIFY contentIndexUpdatingChanged)
Q_PROPERTY(uint ocrContentIndexUpdating READ ocrContentIndexUpdating NOTIFY ocrContentIndexUpdatingChanged)
Q_PROPERTY(uint aiIndexUpdating READ aiIndexUpdating NOTIFY aiIndexUpdatingChanged)
public: public:
static IndexMonitor *self(); static IndexMonitor *self();
@ -71,6 +77,11 @@ public:
* @return OCR索引数量 * @return OCR索引数量
*/ */
uint ocrContentIndexSize() const; uint ocrContentIndexSize() const;
/**
* @brief aiIndexSize
* @return ai索引数量
*/
uint aiIndexSize() const;
/** /**
* @brief basicIndexProgress * @brief basicIndexProgress
* @return * @return
@ -86,6 +97,11 @@ public:
* @return ocr索引进度 * @return ocr索引进度
*/ */
uint ocrContentIndexProgress() const; uint ocrContentIndexProgress() const;
/**
* @brief aiIndexProgress
* @return ai索引进度
*/
uint aiIndexProgress() const;
/** /**
* @brief basicIndexDocNum * @brief basicIndexDocNum
* @return * @return
@ -105,39 +121,50 @@ public:
* basicIndexUpdating * basicIndexUpdating
* @return * @return
*/ */
virtual bool basicIndexUpdating() const; bool basicIndexUpdating() const;
/** /**
* basicIndexUpdating * basicIndexUpdating
* @return * @return
*/ */
virtual bool contentIndexUpdating() const; bool contentIndexUpdating() const;
/** /**
* basicIndexUpdating * basicIndexUpdating
* @return ocr索引是否正在执行增量更新 * @return ocr索引是否正在执行增量更新
*/ */
virtual bool ocrContentIndexUpdating() const; bool ocrContentIndexUpdating() const;
/**
* aiIndexUpdating
* @return ai索引是否正在执行增量更新
*/
bool aiIndexUpdating() const;
Q_SIGNALS: Q_SIGNALS:
void currentIndexPathsChanged(QStringList currentIndexPaths); void currentIndexPathsChanged(QStringList currentIndexPaths);
void indexStateChanged(QString indexState); void indexStateChanged(QString indexState);
void basicIndexSizeChanged(uint basicIndexSize); void basicIndexSizeChanged(uint basicIndexSize);
void contentIndexSizeChanged(uint contentIndexSize); void contentIndexSizeChanged(uint contentIndexSize);
void ocrContentIndexSizeChanged(uint ocrIndexSize); void ocrContentIndexSizeChanged(uint ocrContentIndexSize);
void aiIndexSizeChanged(uint aiIndexSize);
void basicIndexProgressChanged(uint basicIndexProgress); void basicIndexProgressChanged(uint basicIndexProgress);
void contentIndexProgressChanged(uint contentIndexProgress); void contentIndexProgressChanged(uint contentIndexProgress);
void ocrContentIndexProgressChanged(uint ocrIndexProgress); void ocrContentIndexProgressChanged(uint ocrIndexProgress);
void aiIndexProgressChanged(uint aiIndexProgress);
void basicIndexDocNumChanged(uint basicIndexDocNum); void basicIndexDocNumChanged(uint basicIndexDocNum);
void contentIndexDocNumChanged(uint contentIndexDocNum); void contentIndexDocNumChanged(uint contentIndexDocNum);
void ocrContentIndexDocNumChanged(uint ocrContentIndexDocNum); void ocrContentIndexDocNumChanged(uint ocrContentIndexDocNum);
void aiIndexDocNumChanged(uint ocrContentIndexDocNum);
void basicIndexStart(); void basicIndexStart();
void contentIndexStart(); void contentIndexStart();
void ocrContentIndexStart(); void ocrContentIndexStart();
void aiIndexStart();
void basicIndexDone(bool success); void basicIndexDone(bool success);
void contentIndexDone(bool success); void contentIndexDone(bool success);
void ocrContentIndexDone(bool success); void ocrContentIndexDone(bool success);
void aiIndexDone(bool success);
void basicIndexUpdatingChanged(bool basicIndexUpdating); void basicIndexUpdatingChanged(bool basicIndexUpdating);
void contentIndexUpdatingChanged(bool contentIndexUpdating); void contentIndexUpdatingChanged(bool contentIndexUpdating);
void ocrContentIndexUpdatingChanged(bool ocrContentIndexUpdating); void ocrContentIndexUpdatingChanged(bool ocrContentIndexUpdating);
void aiIndexUpdatingChanged(bool ocrContentIndexUpdating);
void serviceReady(); void serviceReady();
void serviceOffline(); void serviceOffline();

View File

@ -22,6 +22,7 @@
#include <QMetaEnum> #include <QMetaEnum>
#include "index-updater.h" #include "index-updater.h"
#include "compatible-define.h" #include "compatible-define.h"
#include "ai-indexer.h"
using namespace UkuiSearch; using namespace UkuiSearch;
IndexScheduler::IndexScheduler(QObject *parent) : IndexScheduler::IndexScheduler(QObject *parent) :
@ -31,7 +32,8 @@ IndexScheduler::IndexScheduler(QObject *parent) :
m_state(Startup), m_state(Startup),
m_indexStop(0), m_indexStop(0),
m_contentIndexStop(0), m_contentIndexStop(0),
m_ocrContentIndexStop(0) m_ocrContentIndexStop(0),
m_aiIndexStop(0)
{ {
qRegisterMetaType<IndexerState>("IndexerState"); qRegisterMetaType<IndexerState>("IndexerState");
qRegisterMetaType<BatchIndexer::WorkMode>("BatchIndexer::WorkMode"); qRegisterMetaType<BatchIndexer::WorkMode>("BatchIndexer::WorkMode");
@ -43,6 +45,7 @@ IndexScheduler::IndexScheduler(QObject *parent) :
connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable); connect(m_config, &FileIndexerConfig::fileIndexEnableStatusChanged, this, &IndexScheduler::fileIndexEnable);
connect(m_config, &FileIndexerConfig::contentIndexEnableStatusChanged, this, &IndexScheduler::contentIndexEnable); connect(m_config, &FileIndexerConfig::contentIndexEnableStatusChanged, this, &IndexScheduler::contentIndexEnable);
connect(m_config, &FileIndexerConfig::contentIndexEnableOcrStatusChanged, this, &IndexScheduler::ocrContentIndexEnable); connect(m_config, &FileIndexerConfig::contentIndexEnableOcrStatusChanged, this, &IndexScheduler::ocrContentIndexEnable);
connect(m_config, &FileIndexerConfig::aiIndexEnableStatusChanged, this, &IndexScheduler::aiIndexEnable);
connect(m_config, &FileIndexerConfig::appendIndexDir, this, &IndexScheduler::addNewPath); connect(m_config, &FileIndexerConfig::appendIndexDir, this, &IndexScheduler::addNewPath);
connect(m_config, &FileIndexerConfig::removeIndexDir, this, &IndexScheduler::removeIndex); connect(m_config, &FileIndexerConfig::removeIndexDir, this, &IndexScheduler::removeIndex);
@ -63,6 +66,11 @@ IndexScheduler::IndexScheduler(QObject *parent) :
} else { } else {
m_ocrContentIndexStop.fetchAndStoreRelaxed(1); m_ocrContentIndexStop.fetchAndStoreRelaxed(1);
} }
if(m_config->isAiIndexEnable()) {
targets |= BatchIndexer::Target::Ai;
} else {
m_aiIndexStop.fetchAndStoreRelaxed(1);
}
start(targets); start(targets);
} }
@ -119,7 +127,12 @@ void IndexScheduler::stop(BatchIndexer::Targets target)
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Off); m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Off);
qDebug() << "File ocr content index has been stopped."; qDebug() << "File ocr content index has been stopped.";
} }
if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD) { if(target & BatchIndexer::Target::Ai) {
m_aiIndexStop.fetchAndStoreRelaxed(1);
m_statusRecorder->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Off);
qDebug() << "Ai index has been stopped.";
}
if(m_indexStop.LOAD && m_contentIndexStop.LOAD && m_ocrContentIndexStop.LOAD && m_aiIndexStop.LOAD) {
m_fileWatcher.removeWatch(); m_fileWatcher.removeWatch();
m_watchInstalled = false; m_watchInstalled = false;
m_threadPool.clear(); m_threadPool.clear();
@ -139,7 +152,7 @@ IndexScheduler::IndexerState IndexScheduler::getIndexState()
void IndexScheduler::forceUpdate(BatchIndexer::Targets target) void IndexScheduler::forceUpdate(BatchIndexer::Targets target)
{ {
m_config->sync(); m_config->syncIndexFileTypes();
stop(target); stop(target);
if(target & BatchIndexer::Basic) { if(target & BatchIndexer::Basic) {
fileIndexEnable(true); fileIndexEnable(true);
@ -150,6 +163,9 @@ void IndexScheduler::forceUpdate(BatchIndexer::Targets target)
if(target & BatchIndexer::Ocr) { if(target & BatchIndexer::Ocr) {
ocrContentIndexEnable(true); ocrContentIndexEnable(true);
} }
if(target & BatchIndexer::Ai) {
aiIndexEnable(true);
}
} }
void IndexScheduler::start(BatchIndexer::Targets target) void IndexScheduler::start(BatchIndexer::Targets target)
@ -166,6 +182,9 @@ void IndexScheduler::start(BatchIndexer::Targets target)
if(target & BatchIndexer::Ocr) { if(target & BatchIndexer::Ocr) {
m_ocrContentIndexStop.fetchAndStoreRelaxed(0); m_ocrContentIndexStop.fetchAndStoreRelaxed(0);
} }
if(target & BatchIndexer::Ai) {
m_aiIndexStop.fetchAndStoreRelaxed(0);
}
if(target == BatchIndexer::None) { if(target == BatchIndexer::None) {
return; return;
} }
@ -189,6 +208,10 @@ void IndexScheduler::start(BatchIndexer::Targets target)
startTarget |= BatchIndexer::Target::Ocr; startTarget |= BatchIndexer::Target::Ocr;
m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating); m_statusRecorder->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
} }
if(target & BatchIndexer::Ai && !(rebuiltTarget & BatchIndexer::Target::Ai)) {
startTarget |= BatchIndexer::Target::Ai;
m_statusRecorder->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Updating);
}
startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, startTarget); startIndexJob(m_config->currentIndexableDir(), m_config->currentBlackListOfIndex(), mode, startTarget);
//启动监听 //启动监听
@ -239,19 +262,26 @@ void IndexScheduler::startIndexJob(const QStringList& folders,const QStringList&
if(target & BatchIndexer::Ocr) { if(target & BatchIndexer::Ocr) {
m_ocrContentIndexPendingWorkCount++; m_ocrContentIndexPendingWorkCount++;
} }
if(target & BatchIndexer::Ai) {
m_aiIndexPendingWorkCount++;
}
m_state = Running; m_state = Running;
Q_EMIT stateChange(m_state); Q_EMIT stateChange(m_state);
auto *indexer = new BatchIndexer(folders, blackList, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop, mode, target); auto *indexer = new BatchIndexer(folders, blackList,
m_indexStop, m_contentIndexStop, m_ocrContentIndexStop, m_aiIndexStop,
mode, target);
connect(indexer, &BatchIndexer::done, this, &IndexScheduler::batchIndexerFinished, Qt::QueuedConnection); connect(indexer, &BatchIndexer::done, this, &IndexScheduler::batchIndexerFinished, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::progress, this, &IndexScheduler::process, Qt::QueuedConnection); connect(indexer, &BatchIndexer::progress, this, &IndexScheduler::process, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::basicIndexDone, this, &IndexScheduler::onBasicIndexDone, Qt::QueuedConnection); connect(indexer, &BatchIndexer::basicIndexDone, this, &IndexScheduler::onBasicIndexDone, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::contentIndexDone, this, &IndexScheduler::onContentIndexDone, Qt::QueuedConnection); connect(indexer, &BatchIndexer::contentIndexDone, this, &IndexScheduler::onContentIndexDone, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::ocrContentIndexDone, this, &IndexScheduler::onOcrContentIndexDone, Qt::QueuedConnection); connect(indexer, &BatchIndexer::ocrContentIndexDone, this, &IndexScheduler::onOcrContentIndexDone, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::aiIndexDone, this, &IndexScheduler::onAiIndexDone, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::basicIndexStart, this, &IndexScheduler::basicIndexStart, Qt::QueuedConnection); connect(indexer, &BatchIndexer::basicIndexStart, this, &IndexScheduler::basicIndexStart, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::contentIndexStart, this, &IndexScheduler::contentIndexStart, Qt::QueuedConnection); connect(indexer, &BatchIndexer::contentIndexStart, this, &IndexScheduler::contentIndexStart, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::ocrContentIndexStart, this, &IndexScheduler::ocrContentIndexStart, Qt::QueuedConnection); connect(indexer, &BatchIndexer::ocrContentIndexStart, this, &IndexScheduler::ocrContentIndexStart, Qt::QueuedConnection);
connect(indexer, &BatchIndexer::aiIndexStart, this, &IndexScheduler::aiIndexStart, Qt::QueuedConnection);
m_threadPool.start(indexer); m_threadPool.start(indexer);
} }
} }
@ -295,6 +325,19 @@ void IndexScheduler::ocrContentIndexEnable(bool enable)
} }
} }
void IndexScheduler::aiIndexEnable(bool enable)
{
if(enable) {
if(m_aiIndexPendingWorkCount == 0) {
start(BatchIndexer::Ai);
} else {
m_aiIndexStartWaiting = true;
}
} else {
stop(BatchIndexer::Ai);
}
}
void IndexScheduler::updateIndex(const QVector<PendingFile> &files) void IndexScheduler::updateIndex(const QVector<PendingFile> &files)
{ {
qDebug() << "updateIndex====="; qDebug() << "updateIndex=====";
@ -303,7 +346,7 @@ void IndexScheduler::updateIndex(const QVector<PendingFile> &files)
m_state = Running; m_state = Running;
Q_EMIT stateChange(m_state); Q_EMIT stateChange(m_state);
auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop); auto *updateJob = new IndexUpdater(files, m_indexStop, m_contentIndexStop, m_ocrContentIndexStop, m_aiIndexStop);
connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection); connect(updateJob, &IndexUpdater::done, this, &IndexScheduler::updateFinished, Qt::QueuedConnection);
m_threadPoolForRTUpdate.start(updateJob); m_threadPoolForRTUpdate.start(updateJob);
} }
@ -333,6 +376,7 @@ bool IndexScheduler::isIdle()
return m_basicIndexPendingWorkCount == 0 return m_basicIndexPendingWorkCount == 0
&& m_contentIndexPendingWorkCount == 0 && m_contentIndexPendingWorkCount == 0
&& m_ocrContentIndexPendingWorkCount == 0 && m_ocrContentIndexPendingWorkCount == 0
&& m_aiIndexPendingWorkCount == 0
&& m_updatePendingWorkCount == 0 && m_updatePendingWorkCount == 0
&& m_addNewPathPendingWorkCount == 0; && m_addNewPathPendingWorkCount == 0;
} }
@ -387,6 +431,22 @@ void IndexScheduler::onOcrContentIndexDone(BatchIndexer::WorkMode mode)
Q_EMIT ocrContentIndexDone(success); Q_EMIT ocrContentIndexDone(success);
checkIfStartsWaiting(); checkIfStartsWaiting();
} }
void IndexScheduler::onAiIndexDone(BatchIndexer::WorkMode mode)
{
Q_UNUSED(mode)
m_aiIndexPendingWorkCount--;
bool success = false;
if(m_statusRecorder->getStatus(AI_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Initializing ||
m_statusRecorder->getStatus(AI_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::Updating) {
m_statusRecorder->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
success = true;
} else if (m_statusRecorder->getStatus(AI_INDEX_DATABASE_STATE_KEY).toInt() == IndexStatusRecorder::State::RealTimeUpdating ) {
success = true;
}
Q_EMIT aiIndexDone(success);
checkIfStartsWaiting();
}
void IndexScheduler::checkIfStartsWaiting() void IndexScheduler::checkIfStartsWaiting()
{ {
@ -403,6 +463,10 @@ void IndexScheduler::checkIfStartsWaiting()
targets |= BatchIndexer::Ocr; targets |= BatchIndexer::Ocr;
m_ocrContentIndexStartWaiting = false; m_ocrContentIndexStartWaiting = false;
} }
if(m_aiIndexStartWaiting && !m_aiIndexPendingWorkCount) {
targets |= BatchIndexer::Ai;
m_aiIndexStartWaiting = false;
}
if(targets != BatchIndexer::None) { if(targets != BatchIndexer::None) {
start(targets); start(targets);
} }

View File

@ -58,9 +58,11 @@ Q_SIGNALS:
void basicIndexStart(); void basicIndexStart();
void contentIndexStart(); void contentIndexStart();
void ocrContentIndexStart(); void ocrContentIndexStart();
void aiIndexStart();
void basicIndexDone(bool success); void basicIndexDone(bool success);
void contentIndexDone(bool success); void contentIndexDone(bool success);
void ocrContentIndexDone(bool success); void ocrContentIndexDone(bool success);
void aiIndexDone(bool success);
void done(); void done();
private Q_SLOTS: private Q_SLOTS:
@ -80,6 +82,7 @@ private Q_SLOTS:
void fileIndexEnable(bool enable); void fileIndexEnable(bool enable);
void contentIndexEnable(bool enable); void contentIndexEnable(bool enable);
void ocrContentIndexEnable(bool enable); void ocrContentIndexEnable(bool enable);
void aiIndexEnable(bool enable);
void updateIndex(const QVector<PendingFile>& files); void updateIndex(const QVector<PendingFile>& files);
void batchIndexerFinished(BatchIndexer::WorkMode mode, BatchIndexer::Targets targets); void batchIndexerFinished(BatchIndexer::WorkMode mode, BatchIndexer::Targets targets);
void updateFinished(); void updateFinished();
@ -87,6 +90,7 @@ private Q_SLOTS:
void onBasicIndexDone(BatchIndexer::WorkMode mode); void onBasicIndexDone(BatchIndexer::WorkMode mode);
void onContentIndexDone(BatchIndexer::WorkMode mode); void onContentIndexDone(BatchIndexer::WorkMode mode);
void onOcrContentIndexDone(BatchIndexer::WorkMode mode); void onOcrContentIndexDone(BatchIndexer::WorkMode mode);
void onAiIndexDone(BatchIndexer::WorkMode mode);
void checkIfStartsWaiting(); void checkIfStartsWaiting();
void installWatches(); void installWatches();
@ -105,6 +109,7 @@ private:
QAtomicInt m_indexStop; QAtomicInt m_indexStop;
QAtomicInt m_contentIndexStop; QAtomicInt m_contentIndexStop;
QAtomicInt m_ocrContentIndexStop; QAtomicInt m_ocrContentIndexStop;
QAtomicInt m_aiIndexStop;
QThreadPool m_threadPool; QThreadPool m_threadPool;
QThreadPool m_threadPoolForRTUpdate; //用于实时更新索引的线程池 QThreadPool m_threadPoolForRTUpdate; //用于实时更新索引的线程池
@ -112,6 +117,7 @@ private:
quint64 m_basicIndexPendingWorkCount = 0; quint64 m_basicIndexPendingWorkCount = 0;
quint64 m_contentIndexPendingWorkCount = 0; quint64 m_contentIndexPendingWorkCount = 0;
quint64 m_ocrContentIndexPendingWorkCount= 0; quint64 m_ocrContentIndexPendingWorkCount= 0;
quint64 m_aiIndexPendingWorkCount= 0;
quint64 m_updatePendingWorkCount = 0; quint64 m_updatePendingWorkCount = 0;
quint64 m_addNewPathPendingWorkCount = 0; quint64 m_addNewPathPendingWorkCount = 0;
@ -119,6 +125,7 @@ private:
bool m_basicIndexStartWaiting = false; bool m_basicIndexStartWaiting = false;
bool m_contentIndexStartWaiting = false; bool m_contentIndexStartWaiting = false;
bool m_ocrContentIndexStartWaiting = false; bool m_ocrContentIndexStartWaiting = false;
bool m_aiIndexStartWaiting = false;
//监听是否已安装 //监听是否已安装
bool m_watchInstalled = false; bool m_watchInstalled = false;

View File

@ -27,6 +27,7 @@
#define CONTENT_INDEX_DATABASE_STATE_KEY "content_index_database_state" #define CONTENT_INDEX_DATABASE_STATE_KEY "content_index_database_state"
#define OCR_CONTENT_INDEX_DATABASE_STATE_KEY "ocr_content_index_database_state" #define OCR_CONTENT_INDEX_DATABASE_STATE_KEY "ocr_content_index_database_state"
#define INDEX_DATABASE_STATE_KEY "index_database_state" #define INDEX_DATABASE_STATE_KEY "index_database_state"
#define AI_INDEX_DATABASE_STATE_KEY "ai_index_database_state"
#define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf" #define INDEX_STATUS QDir::homePath() + "/.config/org.ukui/ukui-search/ukui-search-index-status.conf"
#define INDEX_DATABASE_VERSION_KEY "index_database_version" #define INDEX_DATABASE_VERSION_KEY "index_database_version"
#define CONTENT_DATABASE_VERSION_KEY "content_database_version" #define CONTENT_DATABASE_VERSION_KEY "content_database_version"

View File

@ -19,6 +19,7 @@
*/ */
#include "index-updater.h" #include "index-updater.h"
#include <malloc.h> #include <malloc.h>
#include <QJsonArray>
#include "writable-database.h" #include "writable-database.h"
#include "basic-indexer.h" #include "basic-indexer.h"
#include "file-indexer-config.h" #include "file-indexer-config.h"
@ -27,12 +28,18 @@
#include "file-utils.h" #include "file-utils.h"
#include "compatible-define.h" #include "compatible-define.h"
#include "index-status-recorder.h" #include "index-status-recorder.h"
#include "ai-indexer.h"
using namespace UkuiSearch; using namespace UkuiSearch;
IndexUpdater::IndexUpdater(const QVector<PendingFile>& files, QAtomicInt& indexstop, QAtomicInt& contentIndexstop, QAtomicInt& contentIndexOcrStop) IndexUpdater::IndexUpdater(const QVector<PendingFile>& files,
QAtomicInt& indexstop,
QAtomicInt& contentIndexstop,
QAtomicInt& contentIndexOcrStop,
QAtomicInt& aiIndexStop)
: m_cache(files), : m_cache(files),
m_indexStop(&indexstop), m_indexStop(&indexstop),
m_contentIndexStop(&contentIndexstop), m_contentIndexStop(&contentIndexstop),
m_contentIndexOcrStop(&contentIndexOcrStop) m_contentIndexOcrStop(&contentIndexOcrStop),
m_aiIndexStop(&aiIndexStop)
{ {
} }
void IndexUpdater::updateIndex() void IndexUpdater::updateIndex()
@ -83,6 +90,10 @@ void IndexUpdater::run()
if(IndexStatusRecorder::getInstance()->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) { if(IndexStatusRecorder::getInstance()->getStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) {
IndexStatusRecorder::getInstance()->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready); IndexStatusRecorder::getInstance()->setStatus(OCR_CONTENT_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
} }
updateAiIndex();
if(IndexStatusRecorder::getInstance()->getStatus(AI_INDEX_DATABASE_STATE_KEY) == IndexStatusRecorder::State::RealTimeUpdating) {
IndexStatusRecorder::getInstance()->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::Ready);
}
m_cache.clear(); m_cache.clear();
m_cache.shrink_to_fit(); m_cache.shrink_to_fit();
malloc_trim(0); malloc_trim(0);
@ -192,3 +203,56 @@ void IndexUpdater::updateOcrContentIndex()
qDebug() << "===finish update ocr content index==="; qDebug() << "===finish update ocr content index===";
} }
} }
void IndexUpdater::updateAiIndex()
{
if(FileIndexerConfig::getInstance()->isAiIndexEnable() && !m_aiIndexStop->LOAD) {
IndexStatusRecorder::getInstance()->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating);
AiIndexer indexer;
if(!indexer.creatSession()) {
qWarning() << "Creat session failed, fail to update ai index";
return;
}
qDebug() << "===update ai index===";
for(PendingFile file : m_cache) {
if(m_aiIndexStop->LOAD) {
qDebug() << "Ai index update interrupted";
return;
}
auto update = [&](bool(AiIndexer::*creat)(const QJsonArray &)){
QString type;
if(!indexer.checkFileSupported(file.path(), type)) {
if(file.isModified() || file.isMoveTo()) {
qDebug() << "| remove:" <<file.path();
indexer.deleteFileIndex({file.path()});
}
return;
}
qDebug() << "| index:" <<file.path();
if(indexer.addTextFileIndex(QJsonArray{
QJsonObject{
{QStringLiteral("filepath"), file.path()},
{QStringLiteral("filetype"), type}
}}))
{
} else {
qWarning() << "Ai index failed for" << file.path();
}
};
QString suffix = QFileInfo(file.path()).suffix();
if(file.shouldRemoveIndex()) {
if(file.isDir() || (FileIndexerConfig::getInstance()->aiIndexFileTarget()[suffix] || FileIndexerConfig::getInstance()->aiIndexImageTarget()[suffix])) {
qDebug() << "| remove:" <<file.path();
indexer.deleteFileIndex({file.path()});
}
} else if(FileIndexerConfig::getInstance()->aiIndexFileTarget()[suffix] && !file.isDir()) {
update(&AiIndexer::addTextFileIndex);
} else if (FileIndexerConfig::getInstance()->aiIndexImageTarget()[suffix] && !file.isDir()) {
update(&AiIndexer::addImageFileIndex);
}
}
qDebug() << "===finish update ai content index===";
}
}

View File

@ -31,7 +31,11 @@ class IndexUpdater : public QObject, public QRunnable
{ {
Q_OBJECT Q_OBJECT
public: public:
explicit IndexUpdater(const QVector<PendingFile>& files, QAtomicInt& indexstop, QAtomicInt& contentIndexstop, QAtomicInt& contentIndexOcrStop); explicit IndexUpdater(const QVector<PendingFile>& files,
QAtomicInt& indexstop,
QAtomicInt& contentIndexstop,
QAtomicInt& contentIndexOcrStop,
QAtomicInt& aiIndexStop);
void run() override; void run() override;
Q_SIGNALS: Q_SIGNALS:
@ -41,11 +45,13 @@ private:
void updateIndex(); void updateIndex();
void updateContentIndex(); void updateContentIndex();
void updateOcrContentIndex(); void updateOcrContentIndex();
void updateAiIndex();
QVector<PendingFile> m_cache; QVector<PendingFile> m_cache;
QAtomicInt *m_contentIndexStop = nullptr; QAtomicInt *m_contentIndexStop = nullptr;
QAtomicInt *m_indexStop = nullptr; QAtomicInt *m_indexStop = nullptr;
QAtomicInt *m_contentIndexOcrStop = nullptr; QAtomicInt *m_contentIndexOcrStop = nullptr;
QAtomicInt *m_aiIndexStop = nullptr;
}; };
} }
#endif // INDEXUPDATER_H #endif // INDEXUPDATER_H

View File

@ -6,19 +6,25 @@ class Monitor
PROP(uint basicIndexSize READONLY); PROP(uint basicIndexSize READONLY);
PROP(uint contentIndexSize READONLY); PROP(uint contentIndexSize READONLY);
PROP(uint ocrContentIndexSize READONLY); PROP(uint ocrContentIndexSize READONLY);
PROP(uint aiIndexSize READONLY);
PROP(uint basicIndexProgress READONLY); PROP(uint basicIndexProgress READONLY);
PROP(uint contentIndexProgress READONLY); PROP(uint contentIndexProgress READONLY);
PROP(uint ocrContentIndexProgress READONLY); PROP(uint ocrContentIndexProgress READONLY);
PROP(uint aiIndexProgress READONLY);
PROP(uint basicIndexDocNum READONLY); PROP(uint basicIndexDocNum READONLY);
PROP(uint contentIndexDocNum READONLY); PROP(uint contentIndexDocNum READONLY);
PROP(uint ocrContentIndexDocNum READONLY); PROP(uint ocrContentIndexDocNum READONLY);
PROP(uint aiIndexDocNum READONLY);
PROP(bool basicIndexUpdating READONLY); PROP(bool basicIndexUpdating READONLY);
PROP(bool contentIndexUpdating READONLY); PROP(bool contentIndexUpdating READONLY);
PROP(bool ocrContentIndexUpdating READONLY); PROP(bool ocrContentIndexUpdating READONLY);
PROP(bool aiIndexUpdating READONLY);
SIGNAL(basicIndexStart()); SIGNAL(basicIndexStart());
SIGNAL(contentIndexStart()); SIGNAL(contentIndexStart());
SIGNAL(ocrContentIndexStart()); SIGNAL(ocrContentIndexStart());
SIGNAL(aiIndexStart());
SIGNAL(basicIndexDone(bool success)); SIGNAL(basicIndexDone(bool success));
SIGNAL(contentIndexDone(bool success)); SIGNAL(contentIndexDone(bool success));
SIGNAL(ocrContentIndexDone(bool success)); SIGNAL(ocrContentIndexDone(bool success));
SIGNAL(aiIndexDone(bool success));
}; };

View File

@ -1,23 +0,0 @@
/*
* Copyright (C) 2020, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#include "libsearch.h"
using namespace UkuiSearch;
// Encapsulate advanced interfaces here.

View File

@ -1,44 +0,0 @@
/*
* Copyright (C) 2020, KylinSoft Co., Ltd.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* Authors: zhangpengfei <zhangpengfei@kylinos.cn>
*
*/
#ifndef LIBSEARCH_H
#define LIBSEARCH_H
#include "libsearch_global.h"
#include "file-utils.h"
#include "global-settings.h"
#include "plugininterface/search-plugin-iface.h"
#include "plugininterface/data-queue.h"
#include "index/ukui-search-qdbus.h"
#include "index/search-manager.h"
namespace UkuiSearch {
//class LIBSEARCH_EXPORT GlobalSearch {
//public:
// static QStringList fileSearch(QString keyword, int begin = 0, int num = -1);
//private:
// GlobalSearch();
//};
}
#endif // LIBSEARCH_H

View File

@ -44,6 +44,11 @@ Monitor::Monitor(IndexScheduler *scheduler, QObject *parent) : MonitorSource(par
m_ocrContentIndexUpdating = false; m_ocrContentIndexUpdating = false;
Q_EMIT ocrContentIndexUpdatingChanged(m_ocrContentIndexUpdating); Q_EMIT ocrContentIndexUpdatingChanged(m_ocrContentIndexUpdating);
}); });
connect(scheduler, &IndexScheduler::aiIndexDone, this, [&](bool success){
Q_EMIT aiIndexDone(success);
m_aiIndexUpdating = false;
Q_EMIT aiIndexUpdatingChanged(m_aiIndexUpdating);
});
connect(scheduler, &IndexScheduler::basicIndexStart, this, [&](){ connect(scheduler, &IndexScheduler::basicIndexStart, this, [&](){
Q_EMIT basicIndexStart(); Q_EMIT basicIndexStart();
m_basicIndexUpdating = true; m_basicIndexUpdating = true;
@ -59,6 +64,11 @@ Monitor::Monitor(IndexScheduler *scheduler, QObject *parent) : MonitorSource(par
m_ocrContentIndexUpdating = true; m_ocrContentIndexUpdating = true;
Q_EMIT ocrContentIndexUpdatingChanged(m_ocrContentIndexUpdating); Q_EMIT ocrContentIndexUpdatingChanged(m_ocrContentIndexUpdating);
}); });
connect(scheduler, &IndexScheduler::aiIndexStart, this, [&](){
Q_EMIT aiIndexStart();
m_aiIndexUpdating = true;
Q_EMIT aiIndexUpdatingChanged(m_aiIndexUpdating);
});
connect(FileIndexerConfig::getInstance(), &FileIndexerConfig::appendIndexDir, this, [&](){ connect(FileIndexerConfig::getInstance(), &FileIndexerConfig::appendIndexDir, this, [&](){
m_currentIndexPaths = FileIndexerConfig::getInstance()->currentIndexableDir(); m_currentIndexPaths = FileIndexerConfig::getInstance()->currentIndexableDir();
Q_EMIT currentIndexPathsChanged(m_currentIndexPaths); Q_EMIT currentIndexPathsChanged(m_currentIndexPaths);
@ -92,7 +102,12 @@ uint Monitor::contentIndexSize() const
uint Monitor::ocrContentIndexSize() const uint Monitor::ocrContentIndexSize() const
{ {
return m_ocrIndexSize; return m_ocrContentIndexSize;
}
uint Monitor::aiIndexSize() const
{
return m_aiIndexSize;
} }
uint Monitor::basicIndexProgress() const uint Monitor::basicIndexProgress() const
@ -107,9 +122,15 @@ uint Monitor::contentIndexProgress() const
uint Monitor::ocrContentIndexProgress() const uint Monitor::ocrContentIndexProgress() const
{ {
return m_ocrIndexProgress; return m_ocrContentIndexProgress;
} }
uint Monitor::aiIndexProgress() const
{
return m_aiIndexProgress;
}
uint Monitor::basicIndexDocNum() const uint Monitor::basicIndexDocNum() const
{ {
return m_basicIndexDocNum; return m_basicIndexDocNum;
@ -125,6 +146,11 @@ uint Monitor::ocrContentIndexDocNum() const
return m_ocrContentIndexDocNum; return m_ocrContentIndexDocNum;
} }
uint Monitor::aiIndexDocNum() const
{
return m_aiIndexDocNum;
}
void Monitor::onIndexStateChanged(IndexScheduler::IndexerState state) void Monitor::onIndexStateChanged(IndexScheduler::IndexerState state)
{ {
if(state == IndexScheduler::IndexerState::Idle || state == IndexScheduler::IndexerState::Stop) { if(state == IndexScheduler::IndexerState::Idle || state == IndexScheduler::IndexerState::Stop) {
@ -175,13 +201,13 @@ void Monitor::processUpdate(IndexType type, uint all, uint finished)
break; break;
} }
case IndexType::OCR: { case IndexType::OCR: {
if(m_ocrIndexSize != all) { if(m_ocrContentIndexSize != all) {
m_ocrIndexSize = all; m_ocrContentIndexSize = all;
Q_EMIT ocrContentIndexSizeChanged(m_ocrIndexSize); Q_EMIT ocrContentIndexSizeChanged(m_ocrContentIndexSize);
} }
if(m_ocrIndexProgress != finished) { if(m_ocrContentIndexProgress != finished) {
m_ocrIndexProgress = finished; m_ocrContentIndexProgress = finished;
Q_EMIT ocrContentIndexProgressChanged(m_ocrIndexProgress); Q_EMIT ocrContentIndexProgressChanged(m_ocrContentIndexProgress);
} }
uint count = m_contentDatabase.getIndexDocCount(); uint count = m_contentDatabase.getIndexDocCount();
if(m_ocrContentIndexDocNum != count) { if(m_ocrContentIndexDocNum != count) {
@ -190,6 +216,18 @@ void Monitor::processUpdate(IndexType type, uint all, uint finished)
} }
break; break;
} }
case IndexType::Ai: {
if(m_aiIndexSize != all) {
m_aiIndexSize = all;
Q_EMIT aiIndexSizeChanged(m_aiIndexSize);
}
if(m_aiIndexProgress != finished) {
m_aiIndexProgress = finished;
Q_EMIT aiIndexProgressChanged(m_aiIndexProgress);
}
//AI索引暂不支持文档数量信号
break;
}
default: default:
break; break;
} }
@ -208,4 +246,9 @@ bool Monitor::contentIndexUpdating() const
bool Monitor::ocrContentIndexUpdating() const bool Monitor::ocrContentIndexUpdating() const
{ {
return m_ocrContentIndexUpdating; return m_ocrContentIndexUpdating;
} }
bool Monitor::aiIndexUpdating() const
{
return m_aiIndexUpdating;
}

View File

@ -60,6 +60,11 @@ public:
* @return OCR索引数量 * @return OCR索引数量
*/ */
uint ocrContentIndexSize() const override; uint ocrContentIndexSize() const override;
/**
* @brief aiIndexSize
* @return AI索引数量
*/
uint aiIndexSize() const override;
/** /**
* @brief basicIndexProgress * @brief basicIndexProgress
* @return * @return
@ -75,6 +80,11 @@ public:
* @return ocr索引进度 * @return ocr索引进度
*/ */
uint ocrContentIndexProgress() const override; uint ocrContentIndexProgress() const override;
/**
* @brief aiIndexProgress
* @return ai索引进度
*/
uint aiIndexProgress() const override;
/** /**
* @brief basicIndexDocNum * @brief basicIndexDocNum
* @return * @return
@ -90,21 +100,31 @@ public:
* @return ocr容索引完成的总文档数 * @return ocr容索引完成的总文档数
*/ */
uint ocrContentIndexDocNum() const override; uint ocrContentIndexDocNum() const override;
/**
* @brief aiIndexDocNum
* @return ai索引完成的总文档数
*/
uint aiIndexDocNum() const override;
/** /**
* basicIndexUpdating * basicIndexUpdating
* @return * @return
*/ */
virtual bool basicIndexUpdating() const override; bool basicIndexUpdating() const override;
/** /**
* basicIndexUpdating * basicIndexUpdating
* @return * @return
*/ */
virtual bool contentIndexUpdating() const override; bool contentIndexUpdating() const override;
/** /**
* basicIndexUpdating * basicIndexUpdating
* @return ocr索引是否正在执行增量更新 * @return ocr索引是否正在执行增量更新
*/ */
virtual bool ocrContentIndexUpdating() const override; bool ocrContentIndexUpdating() const override;
/**
* aiIndexUpdating
* @return ai索引是否正在执行增量更新
*/
bool aiIndexUpdating() const override;
private Q_SLOTS: private Q_SLOTS:
void onIndexStateChanged(IndexScheduler::IndexerState); void onIndexStateChanged(IndexScheduler::IndexerState);
@ -118,16 +138,21 @@ private:
Database m_ocrContentDatabase; Database m_ocrContentDatabase;
uint m_basicIndexSize = 0; uint m_basicIndexSize = 0;
uint m_contentIndexSize = 0; uint m_contentIndexSize = 0;
uint m_ocrIndexSize = 0; uint m_ocrContentIndexSize = 0;
uint m_aiIndexSize = 0;
uint m_basicIndexProgress = 0; uint m_basicIndexProgress = 0;
uint m_contentIndexProgress = 0; uint m_contentIndexProgress = 0;
uint m_ocrIndexProgress = 0; uint m_ocrContentIndexProgress = 0;
uint m_aiIndexProgress = 0;
uint m_basicIndexDocNum = 0; uint m_basicIndexDocNum = 0;
uint m_contentIndexDocNum = 0; uint m_contentIndexDocNum = 0;
uint m_ocrContentIndexDocNum = 0; uint m_ocrContentIndexDocNum = 0;
uint m_aiIndexDocNum = 0;
bool m_basicIndexUpdating = false; bool m_basicIndexUpdating = false;
bool m_contentIndexUpdating = false; bool m_contentIndexUpdating = false;
bool m_ocrContentIndexUpdating = false; bool m_ocrContentIndexUpdating = false;
bool m_aiIndexUpdating = false;
}; };
} }
#endif // MONITOR_H #endif // MONITOR_H

View File

@ -21,7 +21,7 @@ import QtQuick 2.0
import QtQuick.Layouts 1.12 import QtQuick.Layouts 1.12
Rectangle { Rectangle {
height: 180 height: 250
width: 320 width: 320
Column { Column {
visible: true visible: true
@ -43,6 +43,11 @@ Rectangle {
keyText: qsTr("OCR content index document number") keyText: qsTr("OCR content index document number")
valueText: monitor.ocrContentIndexDocNum valueText: monitor.ocrContentIndexDocNum
} }
StatusKeyValue {
id: aiIndexDocumentNum
keyText: qsTr("ai index document number")
valueText: monitor.aiIndexDocNum
}
StatusKeyValue { StatusKeyValue {
id: indexState id: indexState
@ -79,14 +84,25 @@ Rectangle {
value: monitor.ocrContentIndexProgress value: monitor.ocrContentIndexProgress
visible: true visible: true
} }
IndexProgressBar {
id: aiIndexProgress
name: "Ai progress"
width: parent.width;
from: 0
to: monitor.aiIndexSize
value: monitor.aiIndexProgress
visible: true
}
} }
Component.onCompleted: { Component.onCompleted: {
monitor.basicIndexDone.connect(onBasicIndexDone); monitor.basicIndexDone.connect(onBasicIndexDone);
monitor.contentIndexDone.connect(onContentIndexDone); monitor.contentIndexDone.connect(onContentIndexDone);
monitor.ocrContentIndexDone.connect(onOcrContentIndexDone); monitor.ocrContentIndexDone.connect(onOcrContentIndexDone);
monitor.aiIndexDone.connect(onAiIndexDone);
monitor.basicIndexStart.connect(onBasicIndexStart); monitor.basicIndexStart.connect(onBasicIndexStart);
monitor.contentIndexStart.connect(onContentIndexStart); monitor.contentIndexStart.connect(onContentIndexStart);
monitor.ocrContentIndexStart.connect(onOcrContentIndexStart); monitor.ocrContentIndexStart.connect(onOcrContentIndexStart);
monitor.aiIndexStart.connect(onAiIndexStart);
} }
function onBasicIndexStart() { function onBasicIndexStart() {
basicIndexProgress.state = "Running" basicIndexProgress.state = "Running"
@ -97,6 +113,9 @@ Rectangle {
function onOcrContentIndexStart() { function onOcrContentIndexStart() {
ocrContentIndexProgress.state = "Running" ocrContentIndexProgress.state = "Running"
} }
function onAiIndexStart() {
aiIndexProgress.state = "Running"
}
function onBasicIndexDone() { function onBasicIndexDone() {
basicIndexProgress.state = "Done" basicIndexProgress.state = "Done"
} }
@ -106,4 +125,7 @@ Rectangle {
function onOcrContentIndexDone() { function onOcrContentIndexDone() {
ocrContentIndexProgress.state = "Done" ocrContentIndexProgress.state = "Done"
} }
function onAiIndexDone() {
aiIndexProgress.state = "Done"
}
} }

View File

@ -104,6 +104,8 @@ void UkuiSearchService::parseCmd(const QString& msg, bool isPrimary)
targets = BatchIndexer::Content; targets = BatchIndexer::Content;
} else if (target == "ocr") { } else if (target == "ocr") {
targets = BatchIndexer::Ocr; targets = BatchIndexer::Ocr;
} else if (target == "ai") {
target = BatchIndexer::Ai;
} }
m_indexScheduler->forceUpdate(targets); m_indexScheduler->forceUpdate(targets);
} }

View File

@ -151,7 +151,7 @@ int SysdbusRegister::AddInotifyMaxUserInstance(int addNum)
// autoSettings->setValue("autologin-user", username); // autoSettings->setValue("autologin-user", username);
// autoSettings->endGroup(); // autoSettings->endGroup();
// autoSettings->sync(); // autoSettings->syncIndexFileTypes();
//} //}
//QString SysdbusRegister::getSuspendThenHibernate() { //QString SysdbusRegister::getSuspendThenHibernate() {
@ -160,7 +160,7 @@ int SysdbusRegister::AddInotifyMaxUserInstance(int addNum)
// QString time = mHibernateSet->value("HibernateDelaySec").toString(); // QString time = mHibernateSet->value("HibernateDelaySec").toString();
// mHibernateSet->endGroup(); // mHibernateSet->endGroup();
// mHibernateSet->sync(); // mHibernateSet->syncIndexFileTypes();
// return time; // return time;
//} //}