openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

perf(ai-index):优化针对ai索引和搜索的异常处理机制

This commit is contained in:
iaom 2024-07-26 11:44:43 +08:00
parent 7153aee119
commit eb4eca39f9
5 changed files with 90 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
libsearch/aisearch/ai-search-plugin.cpp
View File

@ -292,7 +292,19 @@ void AiSearch::keywordSearch()
QJsonObject keyword;
keyword.insert(QStringLiteral("text"), m_keyword);
char* results = nullptr;
data_management_similarity_search(s_session, QJsonDocument(keyword).toJson().data(), &results);
DataManagementResult re = data_management_similarity_search(s_session, QJsonDocument(keyword).toJson().data(), &results);
if(re == DATA_MANAGEMENT_SESSION_ERROR) {
qWarning() << "AiSearch:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session";
destroySession();
if(!createSession()) {
qWarning() << "AiSearch:Failed to create session";
data_management_free_search_result(s_session, results);
return;
}
qDebug() << "Creating session success.";
data_management_similarity_search(s_session, QJsonDocument(keyword).toJson().data(), &results);
}
for (const auto &aResult : QJsonDocument::fromJson(results).array()) {
if (aResult.isObject()) {
bool isBlocked(false);
@ -357,7 +369,9 @@ bool AiSearch::createSession() {
}
void AiSearch::destroySession() {
QMutexLocker locker(&s_sessionMutex);
if (data_management_destroy_session(s_session) != DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
qWarning() << "===fail to destroy session===";
}
s_session = nullptr;
}

79
libsearch/index/ai-indexer.cpp
View File

@ -24,33 +24,34 @@
#include <QJsonValue>
#include <QJsonArray>
#include <QMutex>
#include "kylin-ai/ai-base/datamanagement.h"
#include <kylin-ai/ai-base/datamanagement.h>
namespace UkuiSearch {
class AiIndexerPrivate
{
public:
QMutex m_mutex;
static QMutex m_mutex;
static bool m_sessionValid;
static DataManagementSession m_session;
};
DataManagementSession AiIndexerPrivate::m_session = nullptr;
bool AiIndexerPrivate::m_sessionValid = false;
QMutex AiIndexerPrivate::m_mutex;
AiIndexer::AiIndexer(QObject *parent) : QObject(parent), d(new AiIndexerPrivate)
{
d->m_mutex.lock();
AiIndexerPrivate::m_mutex.lock();
}
AiIndexer::~AiIndexer()
{
d->m_mutex.unlock();
AiIndexerPrivate::m_mutex.unlock();
if(d) {
delete d;
d = nullptr;
}
}
bool AiIndexer::creatSession()
bool AiIndexer::createSession()
{
if(!AiIndexerPrivate::m_sessionValid) {
DataManagementResult result = data_management_create_session(&AiIndexerPrivate::m_session, DataManagementType::SYS_SEARCH, getuid());
@ -65,6 +66,12 @@ bool AiIndexer::creatSession()
}
}
bool AiIndexer::reCreateSession()
{
destroySession();
return createSession();
}
bool AiIndexer::destroySession()
{
AiIndexerPrivate::m_sessionValid = false;
@ -77,14 +84,23 @@ bool AiIndexer::checkFileSupported(const QString &filePath, QString &result)
return false;
}
char *originalResult;
if(data_management_check_file_format(AiIndexerPrivate::m_session, filePath.toLocal8Bit().data(), &originalResult) == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
DataManagementResult re = data_management_check_file_format(AiIndexerPrivate::m_session, filePath.toLocal8Bit().data(), &originalResult);
if(re == DATA_MANAGEMENT_SESSION_ERROR) {
qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session";
if(!reCreateSession()) {
qWarning() << "AiIndexer:Failed to create session";
data_management_free_check_file_format_result(AiIndexerPrivate::m_session, originalResult);
return false;
}
qDebug() << "Creating session success.";
re = data_management_check_file_format(AiIndexerPrivate::m_session, filePath.toLocal8Bit().data(), &originalResult);
}
if(re == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
QJsonObject jo = QJsonDocument::fromJson(QByteArray(originalResult)).object();
data_management_free_check_file_format_result(AiIndexerPrivate::m_session, originalResult);
if(jo.value(QStringLiteral("supported")).toBool()) {
result = jo.value(QStringLiteral("format")).toString();
return true;
} else {
return false;
}
};
return false;
@ -96,6 +112,15 @@ bool AiIndexer::addTextFileIndex(const QJsonArray &object)
return false;
}
DataManagementResult result = data_management_add_text_files(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data());
if(result == DATA_MANAGEMENT_SESSION_ERROR) {
qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session";
if(!reCreateSession()) {
qWarning() << "AiIndexer:Failed to create session";
return false;
}
qDebug() << "Creating session success.";
result = data_management_add_text_files(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data());
}
if(result != DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
qWarning() << "Fail to call data_management_add_text_files, input: " << QJsonDocument(object).toJson().data()
<< "result:" << result;
@ -110,6 +135,15 @@ bool AiIndexer::addImageFileIndex(const QJsonArray &object)
return false;
}
DataManagementResult result = data_management_add_image_files(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data());
if(result == DATA_MANAGEMENT_SESSION_ERROR) {
qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session";
if(!reCreateSession()) {
qWarning() << "AiIndexer:Failed to create session";
return false;
}
qDebug() << "Creating session success.";
result = data_management_add_image_files(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data());
}
if(result != DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
qWarning() << "Fail to call data_management_add_image_files, input: " << QJsonDocument(object).toJson().data()
<< "result:" << result;
@ -132,6 +166,15 @@ bool AiIndexer::deleteFileIndex(const QStringList &files)
tmpArray.append(oneFile);
}
DataManagementResult result = data_management_delete_files(AiIndexerPrivate::m_session, QJsonDocument(tmpArray).toJson().data());
if(result == DATA_MANAGEMENT_SESSION_ERROR) {
qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session";
if(!reCreateSession()) {
qWarning() << "AiIndexer:Failed to create session";
return false;
}
qDebug() << "Creating session success.";
result = data_management_delete_files(AiIndexerPrivate::m_session, QJsonDocument(tmpArray).toJson().data());
}
if(result != DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
qWarning() << "Fail to call data_management_delete_files, input: " << QJsonDocument(tmpArray).toJson().data()
<< "result:" << result;
@ -147,8 +190,19 @@ bool AiIndexer::getAllIndexedFiles(QJsonObject &object)
}
char *originalResult;
DataManagementResult result = data_management_get_all_fileinfos(AiIndexerPrivate::m_session, &originalResult);
if(result == DATA_MANAGEMENT_SESSION_ERROR) {
qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re Creating session";
if(!reCreateSession()) {
qWarning() << "AiIndexer:Failed to create session";
data_management_free_get_all_fileinfos_result(AiIndexerPrivate::m_session, originalResult);
return false;
}
qDebug() << "Creating session success.";
result = data_management_get_all_fileinfos(AiIndexerPrivate::m_session, &originalResult);
}
if(result == DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
QJsonArray array = QJsonDocument::fromJson(QByteArray(originalResult)).array();
data_management_free_get_all_fileinfos_result(AiIndexerPrivate::m_session, originalResult);
for(const QJsonValueRef value : array) {
object.insert(value.toObject().value(QStringLiteral("filepath")).toString(), value.toObject().value(QStringLiteral("timestamp")).toInt());
}
@ -163,6 +217,15 @@ bool AiIndexer::update(const QJsonArray &object)
return false;
}
DataManagementResult result = data_management_update_files_content(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data());
if(result == DATA_MANAGEMENT_SESSION_ERROR) {
qWarning() << "AiIndexer:DATA_MANAGEMENT_SESSION_ERROR occured, re creating session";
if(!reCreateSession()) {
qWarning() << "AiIndexer:Failed to create session";
return false;
}
qDebug() << "Creating session success.";
result = data_management_update_files_content(AiIndexerPrivate::m_session, QJsonDocument(object).toJson().data());
}
if(result != DataManagementResult::DATA_MANAGEMENT_SUCCESS) {
qWarning() << "Fail to call data_management_update_files_content, input: " << QJsonDocument(object).toJson().data()
<< "result:" << result;

3
libsearch/index/ai-indexer.h
View File

@ -30,7 +30,8 @@ class AiIndexer : public QObject
public:
explicit AiIndexer(QObject *parent = nullptr);
~AiIndexer();
bool creatSession();
bool createSession();
bool reCreateSession();
bool checkFileSupported(const QString &filePath, QString &result);
bool addTextFileIndex(const QJsonArray &object);
bool addImageFileIndex(const QJsonArray &object);

2
libsearch/index/batch-indexer.cpp
View File

@ -399,7 +399,7 @@ void BatchIndexer::aiIndex()
return;
}
AiIndexer indexer;
if(!indexer.creatSession()) {
if(!indexer.createSession()) {
qWarning() << "Creat session failed, fail to run ai index!";
return;
}

2
libsearch/index/index-updater.cpp
View File

@ -216,7 +216,7 @@ void IndexUpdater::updateAiIndex()
if(FileIndexerConfig::getInstance()->isAiIndexEnable() && !m_aiIndexStop->LOAD) {
IndexStatusRecorder::getInstance()->setStatus(AI_INDEX_DATABASE_STATE_KEY, IndexStatusRecorder::State::RealTimeUpdating);
AiIndexer indexer;
if(!indexer.creatSession()) {
if(!indexer.createSession()) {
qWarning() << "Creat session failed, fail to update ai index";
return;
}