openkylin
/
ukui-search
mirror of https://gitee.com/openkylin/ukui-search.git
9
0
Fork 1
Code Issues Projects Releases Wiki Activity

Merge pull request #7 from MouseZhangZh/main 

add pinyin index
This commit is contained in:
iaom 2020-12-24 20:50:24 +08:00 committed by GitHub
parent 7113f68435 36c675e701
commit daab516a27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 83159 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
index/chinesecharacterstopinyin.cpp Normal file
View File

@ -0,0 +1,8 @@
#include "chinesecharacterstopinyin.h"
chineseCharactersToPinyin::chineseCharactersToPinyin(QObject *parent) : QObject(parent)
{
}

54
index/chinesecharacterstopinyin.h Normal file
View File

@ -0,0 +1,54 @@
#ifndef CHINESECHARACTERSTOPINYIN_H
#define CHINESECHARACTERSTOPINYIN_H
#include <QObject>
#include <QMap>
#include <QFile>
class chineseCharactersToPinyin : public QObject
{
Q_OBJECT
public:
explicit chineseCharactersToPinyin(QObject *parent = nullptr);
static QString find(const QString &hanzi)
{
static QMap<QString, QStringList> map = loadHanziTable("://index/pinyinWithoutTone.txt");
QString output;
QStringList stringList = hanzi.split("");
/* 遍历查找汉字-拼音对照表的内容并将汉字替换为拼音 */
for (const QString &str : stringList) {
if (map.contains(str))
output += map[str].first();
else
output += str;
}
return output;
}
Q_SIGNALS:
private:
/* 加载汉字对照表 */
static QMap<QString, QStringList> loadHanziTable(const QString &fileName)
{
QMap<QString, QStringList> map;
QFile file(fileName);
if (!file.open(QFile::ReadOnly | QFile::Text)) {
qDebug("File: '%s' open failed!", file.fileName().toStdString().c_str());
return map;
}
/* 读取汉字对照表文件并转换为QMap存储 */
while(!file.atEnd()) {
QString content = QString::fromUtf8(file.readLine());
map[content.split(" ").last().trimmed()] = content.split(" ").first().split(",");
}
file.close();
return map;
}
};
#endif // CHINESECHARACTERSTOPINYIN_H

31
index/index-generator.cpp
View File

@ -1,11 +1,13 @@
#include "index-generator.h"
#include <QFile>
#include <QStandardPaths>
#include <QFileInfo>
#include <QDebug>
#include "index-generator.h"
#include "chinesecharacterstopinyin.h"
using namespace std;
#define INDEX_PATH "/home/zpf/.config/org.ukui/index_data"
#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString()
static IndexGenerator *global_instance = nullptr;
IndexGenerator *IndexGenerator::getInstance()
@ -31,14 +33,15 @@ bool IndexGenerator::creatAllIndex(QStringList *pathlist)
// m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING);
m_indexer->set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
QMap<QString, QString>::const_iterator i;
QString *indexStr;
QMap<QString, QStringList>::const_iterator i;
QStringList *indexStrList;
QString *docStr;
for(i=m_index_map->constBegin();i!=m_index_map->constEnd();++i)
// for(auto i : *m_index_map)
{
docStr = new QString(i.key());
indexStr = new QString(i.value());
insertIntoDatabase(indexStr,docStr);
indexStrList = new QStringList(i.value());
insertIntoDatabase(indexStrList,docStr);
}
m_datebase->commit();
}
@ -63,11 +66,11 @@ IndexGenerator::~IndexGenerator()
{
}
void IndexGenerator::insertIntoDatabase(QString *indexText,QString *doc)
void IndexGenerator::insertIntoDatabase(QStringList *indexTextList,QString *doc)
{
qDebug()<< "--index start--";
m_docstr = doc->toStdString();
m_index_text_str = indexText->toStdString();
//m_index_text_str = indexTextList->toStdString();
std::string uniqueterm = m_cryp->hash(doc->toUtf8(),QCryptographicHash::Md5).toStdString();
@ -75,7 +78,11 @@ void IndexGenerator::insertIntoDatabase(QString *indexText,QString *doc)
document.set_data(m_docstr);
document.add_term(uniqueterm);
m_indexer->set_document(document);
m_indexer->index_text(m_index_text_str);
for(auto i : *indexTextList){
m_indexer->index_text(i.toStdString());
}
// m_indexer->index_text(m_index_text_str);
Xapian::docid innerId= m_datebase->replace_document(uniqueterm,document);
@ -89,7 +96,7 @@ void IndexGenerator::insertIntoDatabase(QString *indexText,QString *doc)
void IndexGenerator::HandlePathList(QStringList *pathlist)
{
qDebug()<<"Begin HandlePathList!";
m_index_map = new QMap<QString,QString>;
m_index_map = new QMap<QString,QStringList>;
QStringList *list = pathlist;
for(int i = 0;i<list->size();i++)
{
@ -97,10 +104,12 @@ void IndexGenerator::HandlePathList(QStringList *pathlist)
//提取文件名并用空格分割,同时去除'.'
QString filename = info->fileName();
QString index_test = filename.replace(".","").replace(""," ");
QString pinyin_test = chineseCharactersToPinyin::find(filename.replace(".", "")).replace("", " ");
// index_text.simplified();
// qDebug()<<"index_test"<<index_test;
m_index_map->insert(info->absoluteFilePath(),index_test);
m_index_map->insert(info->absoluteFilePath(),QStringList() << index_test << pinyin_test);
// qDebug()<<m_index_map->value(index_test);
}

4
index/index-generator.h
View File

@ -27,10 +27,10 @@ private:
explicit IndexGenerator(QObject *parent = nullptr);
void HandlePathList(QStringList *pathlist);
//add one data in database
void insertIntoDatabase(QString *indexText,QString *doc);
void insertIntoDatabase(QStringList *indexText,QString *doc);
~IndexGenerator();
QMap<QString,QString> *m_index_map;
QMap<QString,QStringList> *m_index_map;
QCryptographicHash *m_cryp;
QString *m_index_data_path;
Xapian::WritableDatabase *m_datebase;

2
index/index.pri
View File

@ -1,11 +1,13 @@
INCLUDEPATH += $$PWD
HEADERS += \
$$PWD/chinesecharacterstopinyin.h \
$$PWD/index-generator.h \
$$PWD/inotify-manager.h \ \
$$PWD/messagelist-manager.h
SOURCES += \
$$PWD/chinesecharacterstopinyin.cpp \
$$PWD/index-generator.cpp \
$$PWD/inotify-manager.cpp \
$$PWD/messagelist-manager.cpp \

29
index/inotify-manager.cpp
View File

@ -2,11 +2,9 @@
#include "index-generator.h"
#include "messagelist-manager.h"
bool InotifyManager::Traverse_BFS(const QString& path, const bool& CREATORDELETE){
bool InotifyManager::Traverse_BFS(const QString& path, int autoSendMessageLength){
qDebug() << "BFS start-----------------------------";
int total = 0;
MessageListManager mlm;
mlm.SetAutoSendMessageLength(80000);
this->mlm->SetAutoSendMessageLength(autoSendMessageLength);
QQueue<QString> bfs;
bfs.enqueue(path);
QFileInfoList list;
@ -23,19 +21,19 @@ bool InotifyManager::Traverse_BFS(const QString& path, const bool& CREATORDELETE
bfs.enqueue(i.absoluteFilePath());
}
else{
mlm.AddMessage(i.absoluteFilePath());
total++;
this->mlm->AddMessage(i.absoluteFilePath());
//continue;
}
}
}
mlm.SendMessage();
qDebug()<<total;
this->mlm->SendMessage();
qDebug() << "BFS end-----------------------------";
return true;
}
bool InotifyManager::Traverse(const QString& path, const bool& CREATORDELETE){
//the DFS method is aborted
bool InotifyManager::Traverse_DFS(const QString& path, const bool& CREATORDELETE){
QDir dir(path);
if (!dir.exists()) {
@ -60,7 +58,7 @@ bool InotifyManager::Traverse(const QString& path, const bool& CREATORDELETE){
if (CREATORDELETE){
AddWatch(fileInfo.filePath());
}
Traverse(fileInfo.filePath(), CREATORDELETE);
Traverse_DFS(fileInfo.filePath(), CREATORDELETE);
if (!CREATORDELETE){
RemoveWatch(fileInfo.filePath());
}
@ -69,6 +67,7 @@ bool InotifyManager::Traverse(const QString& path, const bool& CREATORDELETE){
// .arg(fileInfo.path())
// .arg(fileInfo.fileName());
//IndexGenerator::getInstance()->creatAllIndex(new QStringList(fileInfo.filePath()));
}
}
i++;
@ -82,6 +81,7 @@ bool InotifyManager::AddWatch(const QString &path){
//int ret = inotify_add_watch(m_fd, path.toStdString().c_str(), IN_ALL_EVENTS);
int ret = inotify_add_watch(m_fd, path.toStdString().c_str(), (IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_DELETE));
if (ret == -1) {
qDebug() << "AddWatch error:" << path;
return false;
}
currentPath[ret] = path;
@ -169,10 +169,12 @@ void InotifyManager::run(){
//添加监视要先序遍历，先添加top节点
if (event->mask & IN_ISDIR){
AddWatch(currentPath[event->wd] + '/' + event->name);
Traverse_BFS(currentPath[event->wd] + '/' + event->name, true);
Traverse_BFS(currentPath[event->wd] + '/' + event->name);
}
else {
//IndexGenerator::getInstance()->creatAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
this->mlm->AddMessage(currentPath[event->wd] + '/' + event->name);
this->mlm->SendMessage();
}
}
else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){
@ -181,7 +183,9 @@ void InotifyManager::run(){
}
else {
//这里调用删除索引
IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
this->mlm->AddMessage(currentPath[event->wd] + '/' + event->name);
this->mlm->SendDeleteMessage();
// IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
}
}
/*--------------------------------*/
@ -214,6 +218,7 @@ InotifyManager::InotifyManager()
num2string.insert(IN_UNMOUNT, "IN_UNMOUNT");
num2string.insert(IN_Q_OVERFLOW, "IN_Q_OVERFLOW");
num2string.insert(IN_IGNORED, "IN_IGNORED");
this->mlm = new MessageListManager();
return;
}

8
index/inotify-manager.h
View File

@ -8,6 +8,7 @@
#include <QDebug>
#include <QDir>
#include <QQueue>
#include "messagelist-manager.h"
//#define EVENT_NUM 12
#define BUF_LEN 1024
@ -17,8 +18,9 @@ class InotifyManager : public QThread
Q_OBJECT
public:
explicit InotifyManager();
bool Traverse(const QString&, const bool&);//true->create, false->delete
bool Traverse_BFS(const QString&, const bool&);
//the DFS method is aborted
bool Traverse_DFS(const QString&, const bool&);//true->create, false->delete
bool Traverse_BFS(const QString&, int autoSendMessageLength = 80000);
//typedef bool (*AddWatch)(const QString&);
//AddWatch cmp;
@ -33,6 +35,8 @@ private:
int m_fd;
QMap<int, QString> currentPath;
QMap<int, QString> num2string;
MessageListManager* mlm;
};
void testTraverse(void);

29
index/messagelist-manager.cpp
View File

@ -1,16 +1,30 @@
#include "messagelist-manager.h"
#include <QDebug>
#include <QThread>
#include <QStringList>
#include <QTimer>
//#include <unistd.h>
MessageListManager::MessageListManager(){
this->messageList = new QStringList();
this->ig = IndexGenerator::getInstance();
// indexGeneratorThread = new QThread();
// this->ig->moveToThread(indexGeneratorThread);
// connect(this,&MessageListManager::Send, this->ig, &IndexGenerator::creatAllIndex/*, Qt::QueuedConnection*/);
// connect(this,&MessageListManager::Send1, this->ig, [=](QStringList *l){
// qDebug()<<"send"<<*l;
// });
}
MessageListManager::~MessageListManager(){
delete this->messageList;
delete this->indexGeneratorThread;
//delete this->ig;
this->messageList = nullptr;
this->ig = nullptr;
this->indexGeneratorThread = nullptr;
}
void MessageListManager::AddMessage(const QString& path){
@ -21,11 +35,26 @@ void MessageListManager::AddMessage(const QString& path){
}
bool MessageListManager::SendMessage(){
//Q_EMIT Send(this->messageList);
if (this->messageList->empty()){
return true;
}
// Q_EMIT Send(this->messageList);
// qDebug() << "emit";
this->ig->creatAllIndex(this->messageList);
//sleep(1);
this->messageList->clear();
return true;
}
bool MessageListManager::SendDeleteMessage(){
if (this->messageList->empty()){
return true;
}
this->ig->deleteAllIndex(this->messageList);
this->messageList->clear();
return true;
}

5
index/messagelist-manager.h
View File

@ -13,14 +13,17 @@ public:
void AddMessage(const QString&);
bool SendMessage();
bool SendDeleteMessage();
void SetAutoSendMessageLength(const size_t&);
private:
QStringList* messageList;
size_t length = 0;
IndexGenerator* ig;
Q_SIGNALS:
QThread* indexGeneratorThread;
Q_SIGNALS:
bool Send(QStringList*);
};
#endif // MESSAGELISTMANAGER_H

41451
index/pinyinWithTone.txt Normal file
View File

File diff suppressed because it is too large Load Diff

41451
index/pinyinWithoutTone.txt Normal file
View File

File diff suppressed because it is too large Load Diff

117
index/test-Inotify-Manager.cpp
View File

@ -1,17 +1,120 @@
#include "src/mainwindow.h"
#include "inotify-manager.h"
#include <QTime>
#include <QDebug>
#include "src/mainwindow.h"
#include "inotify-manager.h"
#include "chinesecharacterstopinyin.h"
void testTraverse(void){
/*-------------Inotify Test Start---------------*/
// QTime t1 = QTime::currentTime();
// InotifyManager* im = new InotifyManager();
// im->AddWatch("/home");
// im->Traverse_BFS("/home", true);
// QTime t2 = QTime::currentTime();
// qDebug() << t1;
// qDebug() << t2;
// im->start();
/*-------------Inotify Test End-----------------*/
/*-------------PinyinSearch Test Start---------------*/
QTime t1 = QTime::currentTime();
InotifyManager* im = new InotifyManager();
im->AddWatch("/home/zpf");
im->Traverse_BFS("/home/zpf", true);
QString test("gongzuo");
qDebug() << IndexGenerator::IndexSearch(test);
QTime t2 = QTime::currentTime();
qDebug() << t1;
qDebug() << t2;
im->start();
/*-------------PinyinSearch Test End-----------------*/
//exit(0);
exit(0);
}
void removeTone(){
/*-------------Remove Tone Start---------------*/
qDebug() << chineseCharactersToPinyin::find("z测试策士xl123123");
QFile file("://index/pinyinWithTone.txt");
QFile fileOut("/home/zhangzihao/ukui/ns/ukui-search/index/pinyinWithoutTone.txt");
fileOut.open(QIODevice::WriteOnly/* | QIODevice::Text*/);
if (!file.open(QFile::ReadOnly | QFile::Text)) {
qDebug("File: '%s' open failed!", file.fileName().toStdString().c_str());
exit(-1);
}
while(!file.atEnd()) {
QString content = QString::fromUtf8(file.readLine());
content.replace("ā", "a")
.replace("á", "a")
.replace("ǎ", "a")
.replace("à", "a")
.replace("ō", "o")
.replace("ó", "o")
.replace("ǒ", "o")
.replace("ò", "o")
.replace("ê", "e")
.replace("ē", "e")
.replace("é", "e")
.replace("ě", "e")
.replace("è", "e")
.replace("ī", "i")
.replace("í", "i")
.replace("ǐ", "i")
.replace("ì", "i")
.replace("ū", "u")
.replace("ú", "u")
.replace("ǔ", "u")
.replace("ù", "u")
//l和n后面的ü写作v
.replace("lǖ", "lv")
.replace("lǘ", "lv")
.replace("lǚ", "lv")
.replace("lǜ", "lv")
.replace("lü", "lv")
.replace("nǖ", "nv")
.replace("nǘ", "nv")
.replace("nǚ", "nv")
.replace("nǜ", "nv")
.replace("nü", "nv")
//l和n后面的ü替换之后，其他的ü替换为u
.replace("ǖ", "u")
.replace("ǘ", "u")
.replace("ǚ", "u")
.replace("ǜ", "u")
.replace("ü", "u")
.replace("ê", "e")
.replace("ɑ", "a")
.replace("", "m")
.replace("ń", "n")
.replace("", "n")
.replace("ɡ", "g");
//去除同音不同调
//QString content = QString::fromUtf8(file.readLine());
QStringList temp = content.split(" ").first().split(",").toSet().toList();
QString outContent;
for (auto i : temp){
outContent += i;
outContent += ",";
}
outContent = outContent.left(outContent.size() - 1);
outContent += " ";
outContent += content.split(" ").last().trimmed();
outContent += "\n";
fileOut.write(outContent.toUtf8());
// temp.toSet().toList();
//content.split(" ").first().split(",")
//map[content.split(" ").last().trimmed()] = content.split(" ").first().split(",");
// ā á ǎ à ō ó ǒ ò ê ē é ě è ī í ǐ ì ū ú ǔ ù ǖ ǘ ǚ ǜ ü ê ɑ  ń ň  ɡ
// fileOut.write(content.toUtf8());
qDebug() << content;
}
file.close();
fileOut.close();
/*-------------Remove Tone End-----------------*/
}

2
resource.qrc
View File

@ -6,5 +6,7 @@
<file>res/translations/tr.ts</file>
<file>res/translations/zh_CN.ts</file>
<file>res/icons/desktop.png</file>
<file>index/pinyinWithTone.txt</file>
<file>index/pinyinWithoutTone.txt</file>
</qresource>
</RCC>

3
src/mainwindow.cpp
View File

@ -45,6 +45,9 @@ extern void qt_blurImage(QImage &blurImage, qreal radius, bool quality, int tran
MainWindow::MainWindow(QWidget *parent) :
QMainWindow(parent)
{
//testBackServe
testTraverse();
this->setWindowFlags(Qt::CustomizeWindowHint | Qt::FramelessWindowHint | Qt::X11BypassWindowManagerHint);
this->setAttribute(Qt::WA_TranslucentBackground, true);
this->setAutoFillBackground(false);