From 2758390e2e8956459af626dbf33cd520a3614433 Mon Sep 17 00:00:00 2001 From: iaom Date: Fri, 26 Jul 2024 11:44:43 +0800 Subject: [PATCH] =?UTF-8?q?perf(ai-index):=E4=BC=98=E5=8C=96=E9=92=88?= =?UTF-8?q?=E5=AF=B9ai=E7=B4=A2=E5=BC=95=E5=92=8C=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E7=9A=84=E5=BC=82=E5=B8=B8=E5=A4=84=E7=90=86=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libsearch/aisearch/ai-search-plugin.cpp | 16 ++++- libsearch/index/ai-indexer.cpp | 79 ++++++++++++++++++++++--- libsearch/index/ai-indexer.h | 3 +- libsearch/index/batch-indexer.cpp | 2 +- libsearch/index/index-updater.cpp | 2 +- 5 files changed, 90 insertions(+), 12 deletions(-) diff --git a/libsearch/aisearch/ai-search-plugin.cpp b/libsearch/aisearch/ai-search-plugin.cpp index a586b27..b6f0687 100644 --- a/libsearch/aisearch/ai-search-plugin.cpp +++ b/libsearch/aisearch/ai-search-plugin.cpp @@ -292,7 +292,19 @@ void AiSearch::keywordSearch() QJsonObject keyword; keyword.insert(QStringLiteral("text"), m_keyword); char* results = nullptr; - data_management_similarity_search(s_session, QJsonDocument(keyword).toJson().data(), &results); + DataManagementResult re = data_management_similarity_search(s_session, QJsonDocument(keyword).toJson().data(), &results); + if(re == DATA_MANAGEMENT_SESSION_ERROR) { + qWarning() << "AiSearch:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session"; + destroySession(); + if(!createSession()) { + qWarning() << "AiSearch:Failed to create session"; + data_management_free_search_result(s_session, results); + return; + } + qDebug() << "Creating session success."; + data_management_similarity_search(s_session, QJsonDocument(keyword).toJson().data(), &results); + } + for (const auto &aResult : QJsonDocument::fromJson(results).array()) { if (aResult.isObject()) { bool isBlocked(false); @@ -357,7 +369,9 @@ bool AiSearch::createSession() { } void AiSearch::destroySession() { + QMutexLocker locker(&s_sessionMutex); if (data_management_destroy_session(s_session) != DataManagementResult::DATA_MANAGEMENT_SUCCESS) { qWarning() << "===fail to destroy session==="; } + s_session = nullptr; } diff --git a/libsearch/index/ai-indexer.cpp b/libsearch/index/ai-indexer.cpp index a409c18..9465152 100644 --- a/libsearch/index/ai-indexer.cpp +++ b/libsearch/index/ai-indexer.cpp @@ -24,33 +24,34 @@ #include #include #include -#include "kylin-ai/ai-base/datamanagement.h" +#include namespace UkuiSearch { class AiIndexerPrivate { public: - QMutex m_mutex; + static QMutex m_mutex; static bool m_sessionValid; static DataManagementSession m_session; }; DataManagementSession AiIndexerPrivate::m_session = nullptr; bool AiIndexerPrivate::m_sessionValid = false; +QMutex AiIndexerPrivate::m_mutex; AiIndexer::AiIndexer(QObject *parent) : QObject(parent), d(new AiIndexerPrivate) { - d->m_mutex.lock(); + AiIndexerPrivate::m_mutex.lock(); } AiIndexer::~AiIndexer() { - d->m_mutex.unlock(); + AiIndexerPrivate::m_mutex.unlock(); if(d) { delete d; d = nullptr; } } -bool AiIndexer::creatSession() +bool AiIndexer::createSession() { if(!AiIndexerPrivate::m_sessionValid) { DataManagementResult result = data_management_create_session(&AiIndexerPrivate::m_session, DataManagementType::SYS_SEARCH, getuid()); @@ -65,6 +66,12 @@ bool AiIndexer::creatSession() } } +bool AiIndexer::reCreateSession() +{ + destroySession(); + return createSession(); +} + bool AiIndexer::destroySession() { AiIndexerPrivate::m_sessionValid = false; @@ -77,14 +84,23 @@ bool AiIndexer::checkFileSupported(const QString &filePath, QString &result) return false; } char *originalResult; - if(data_management_check_file_format(AiIndexerPrivate::m_session, filePath.toLocal8Bit().data(), &originalResult) == DataManagementResult::DATA_MANAGEMENT_SUCCESS) { + DataManagementResult re = data_management_check_file_format(AiIndexerPrivate::m_session, filePath.toLocal8Bit().data(), &originalResult); + if(re == DATA_MANAGEMENT_SESSION_ERROR) { + qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session"; + if(!reCreateSession()) { + qWarning() << "AiIndexer:Failed to create session"; + data_management_free_check_file_format_result(AiIndexerPrivate::m_session, originalResult); + return false; + } + qDebug() << "Creating session success."; + re = data_management_check_file_format(AiIndexerPrivate::m_session, filePath.toLocal8Bit().data(), &originalResult); + } + if(re == DataManagementResult::DATA_MANAGEMENT_SUCCESS) { QJsonObject jo = QJsonDocument::fromJson(QByteArray(originalResult)).object(); data_management_free_check_file_format_result(AiIndexerPrivate::m_session, originalResult); if(jo.value(QStringLiteral("supported")).toBool()) { result = jo.value(QStringLiteral("format")).toString(); return true; - } else { - return false; } }; return false; @@ -96,6 +112,15 @@ bool AiIndexer::addTextFileIndex(const QJsonArray &object) return false; } DataManagementResult result = data_management_add_text_files(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data()); + if(result == DATA_MANAGEMENT_SESSION_ERROR) { + qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session"; + if(!reCreateSession()) { + qWarning() << "AiIndexer:Failed to create session"; + return false; + } + qDebug() << "Creating session success."; + result = data_management_add_text_files(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data()); + } if(result != DataManagementResult::DATA_MANAGEMENT_SUCCESS) { qWarning() << "Fail to call data_management_add_text_files, input: " << QJsonDocument(object).toJson().data() << "result:" << result; @@ -110,6 +135,15 @@ bool AiIndexer::addImageFileIndex(const QJsonArray &object) return false; } DataManagementResult result = data_management_add_image_files(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data()); + if(result == DATA_MANAGEMENT_SESSION_ERROR) { + qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session"; + if(!reCreateSession()) { + qWarning() << "AiIndexer:Failed to create session"; + return false; + } + qDebug() << "Creating session success."; + result = data_management_add_image_files(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data()); + } if(result != DataManagementResult::DATA_MANAGEMENT_SUCCESS) { qWarning() << "Fail to call data_management_add_image_files, input: " << QJsonDocument(object).toJson().data() << "result:" << result; @@ -132,6 +166,15 @@ bool AiIndexer::deleteFileIndex(const QStringList &files) tmpArray.append(oneFile); } DataManagementResult result = data_management_delete_files(AiIndexerPrivate::m_session, QJsonDocument(tmpArray).toJson().data()); + if(result == DATA_MANAGEMENT_SESSION_ERROR) { + qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session"; + if(!reCreateSession()) { + qWarning() << "AiIndexer:Failed to create session"; + return false; + } + qDebug() << "Creating session success."; + result = data_management_delete_files(AiIndexerPrivate::m_session, QJsonDocument(tmpArray).toJson().data()); + } if(result != DataManagementResult::DATA_MANAGEMENT_SUCCESS) { qWarning() << "Fail to call data_management_delete_files, input: " << QJsonDocument(tmpArray).toJson().data() << "result:" << result; @@ -147,8 +190,19 @@ bool AiIndexer::getAllIndexedFiles(QJsonObject &object) } char *originalResult; DataManagementResult result = data_management_get_all_fileinfos(AiIndexerPrivate::m_session, &originalResult); + if(result == DATA_MANAGEMENT_SESSION_ERROR) { + qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session"; + if(!reCreateSession()) { + qWarning() << "AiIndexer:Failed to create session"; + data_management_free_get_all_fileinfos_result(AiIndexerPrivate::m_session, originalResult); + return false; + } + qDebug() << "Creating session success."; + result = data_management_get_all_fileinfos(AiIndexerPrivate::m_session, &originalResult); + } if(result == DataManagementResult::DATA_MANAGEMENT_SUCCESS) { QJsonArray array = QJsonDocument::fromJson(QByteArray(originalResult)).array(); + data_management_free_get_all_fileinfos_result(AiIndexerPrivate::m_session, originalResult); for(const QJsonValueRef value : array) { object.insert(value.toObject().value(QStringLiteral("filepath")).toString(), value.toObject().value(QStringLiteral("timestamp")).toInt()); } @@ -163,6 +217,15 @@ bool AiIndexer::update(const QJsonArray &object) return false; } DataManagementResult result = data_management_update_files_content(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data()); + if(result == DATA_MANAGEMENT_SESSION_ERROR) { + qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re creating session"; + if(!reCreateSession()) { + qWarning() << "AiIndexer:Failed to create session"; + return false; + } + qDebug() << "Creating session success."; + result = data_management_update_files_content(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data()); + } if(result != DataManagementResult::DATA_MANAGEMENT_SUCCESS) { qWarning() << "Fail to call data_management_update_files_content, input: " << QJsonDocument(object).toJson().data() << "result:" << result; diff --git a/libsearch/index/ai-indexer.h b/libsearch/index/ai-indexer.h index f665c26..20d1078 100644 --- a/libsearch/index/ai-indexer.h +++ b/libsearch/index/ai-indexer.h @@ -30,7 +30,8 @@ class AiIndexer : public QObject public: explicit AiIndexer(QObject *parent = nullptr); ~AiIndexer(); - bool creatSession(); + bool createSession(); + bool reCreateSession(); bool checkFileSupported(const QString &filePath, QString &result); bool addTextFileIndex(const QJsonArray &object); bool addImageFileIndex(const QJsonArray &object); diff --git a/libsearch/index/batch-indexer.cpp b/libsearch/index/batch-indexer.cpp index 195be23..ea3e6e2 100644 --- a/libsearch/index/batch-indexer.cpp +++ b/libsearch/index/batch-indexer.cpp @@ -399,7 +399,7 @@ void BatchIndexer::aiIndex() return; } AiIndexer indexer; - if(!indexer.creatSession()) { + if(!indexer.createSession()) { qWarning() << "Creat session failed, fail to run ai index!"; return; } diff --git a/libsearch/index/index-updater.cpp b/libsearch/index/index-updater.cpp index 6b63d68..9bb0564 100644 --- a/libsearch/index/index-updater.cpp +++ b/libsearch/index/index-updater.cpp @@ -216,7 +216,7 @@ void IndexUpdater::updateAiIndex() if(FileIndexerConfig::getInstance()->isAiIndexEnable() && !m_aiIndexStop->LOAD) { IndexStatusRecorder::getInstance()->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating); AiIndexer indexer; - if(!indexer.creatSession()) { + if(!indexer.createSession()) { qWarning() << "Creat session failed, fail to update ai index"; return; }