add pinyin index

index/chinesecharacterstopinyin.cpp

@@ -0,0 +1,8 @@
+#include "chinesecharacterstopinyin.h"
+
+chineseCharactersToPinyin::chineseCharactersToPinyin(QObject *parent) : QObject(parent)
+{
+
+}
+
+

index/chinesecharacterstopinyin.h

@@ -0,0 +1,54 @@
+#ifndef CHINESECHARACTERSTOPINYIN_H
+#define CHINESECHARACTERSTOPINYIN_H
+
+#include <QObject>
+#include <QMap>
+#include <QFile>
+
+class chineseCharactersToPinyin : public QObject
+{
+    Q_OBJECT
+public:
+    explicit chineseCharactersToPinyin(QObject *parent = nullptr);
+    static QString find(const QString &hanzi)
+    {
+        static QMap<QString, QStringList> map = loadHanziTable("://index/pinyinWithoutTone.txt");
+        QString output;
+        QStringList stringList = hanzi.split("");
+
+        /* 遍历查找汉字-拼音对照表的内容并将汉字替换为拼音 */
+        for (const QString &str : stringList) {
+            if (map.contains(str))
+                output += map[str].first();
+            else
+                output += str;
+        }
+
+        return output;
+    }
+
+Q_SIGNALS:
+private:
+    /* 加载汉字对照表 */
+    static QMap<QString, QStringList> loadHanziTable(const QString &fileName)
+    {
+        QMap<QString, QStringList> map;
+        QFile file(fileName);
+        if (!file.open(QFile::ReadOnly | QFile::Text)) {
+            qDebug("File: '%s' open failed!", file.fileName().toStdString().c_str());
+            return map;
+        }
+
+        /* 读取汉字对照表文件并转换为QMap存储 */
+        while(!file.atEnd()) {
+            QString content = QString::fromUtf8(file.readLine());
+            map[content.split(" ").last().trimmed()] = content.split(" ").first().split(",");
+        }
+
+        file.close();
+
+        return map;
+    }
+};
+
+#endif // CHINESECHARACTERSTOPINYIN_H

index/index-generator.cpp

@@ -1,11 +1,13 @@
-#include "index-generator.h"
 #include <QFile>
+#include <QStandardPaths>
 #include <QFileInfo>
 #include <QDebug>
+#include "index-generator.h"
+#include "chinesecharacterstopinyin.h"
 
 using namespace std;
 
-#define INDEX_PATH "/home/zpf/.config/org.ukui/index_data"
+#define INDEX_PATH (QStandardPaths::writableLocation(QStandardPaths::HomeLocation)+"/.config/org.ukui/index_data").toStdString()
 static IndexGenerator *global_instance = nullptr;
 
 IndexGenerator *IndexGenerator::getInstance()
@@ -31,14 +33,15 @@ bool IndexGenerator::creatAllIndex(QStringList *pathlist)
 //        m_indexer->set_flags(Xapian::TermGenerator::FLAG_SPELLING);
         m_indexer->set_stemming_strategy(Xapian::TermGenerator::STEM_SOME);
 
-        QMap<QString, QString>::const_iterator i;
+        QMap<QString, QStringList>::const_iterator i;
-        QString *indexStr;
+        QStringList *indexStrList;
         QString *docStr;
         for(i=m_index_map->constBegin();i!=m_index_map->constEnd();++i)
+//        for(auto i : *m_index_map)
         {
             docStr = new QString(i.key());
-            indexStr = new QString(i.value());
+            indexStrList = new QStringList(i.value());
-            insertIntoDatabase(indexStr,docStr);
+            insertIntoDatabase(indexStrList,docStr);
         }
         m_datebase->commit();
     }
@@ -63,11 +66,11 @@ IndexGenerator::~IndexGenerator()
 {
 }
 
-void IndexGenerator::insertIntoDatabase(QString *indexText,QString *doc)
+void IndexGenerator::insertIntoDatabase(QStringList *indexTextList,QString *doc)
 {
     qDebug()<< "--index start--";
     m_docstr = doc->toStdString();
-    m_index_text_str = indexText->toStdString();
+    //m_index_text_str = indexTextList->toStdString();
 
     std::string uniqueterm = m_cryp->hash(doc->toUtf8(),QCryptographicHash::Md5).toStdString();
 
@@ -75,7 +78,11 @@ void IndexGenerator::insertIntoDatabase(QString *indexText,QString *doc)
     document.set_data(m_docstr);
     document.add_term(uniqueterm);
     m_indexer->set_document(document);
-    m_indexer->index_text(m_index_text_str);
+
+    for(auto i : *indexTextList){
+        m_indexer->index_text(i.toStdString());
+    }
+//    m_indexer->index_text(m_index_text_str);
 
     Xapian::docid innerId= m_datebase->replace_document(uniqueterm,document);
 
@@ -89,7 +96,7 @@ void IndexGenerator::insertIntoDatabase(QString *indexText,QString *doc)
 void IndexGenerator::HandlePathList(QStringList *pathlist)
 {
     qDebug()<<"Begin HandlePathList!";
-    m_index_map = new QMap<QString,QString>;
+    m_index_map = new QMap<QString,QStringList>;
     QStringList *list = pathlist;
     for(int i = 0;i<list->size();i++)
     {
@@ -97,10 +104,12 @@ void IndexGenerator::HandlePathList(QStringList *pathlist)
         //提取文件名并用空格分割,同时去除'.'
         QString filename = info->fileName();
         QString index_test = filename.replace(".","").replace(""," ");
+        QString pinyin_test = chineseCharactersToPinyin::find(filename.replace(".", "")).replace("", " ");
+
 
 //        index_text.simplified();
         //            qDebug()<<"index_test"<<index_test;
-        m_index_map->insert(info->absoluteFilePath(),index_test);
+        m_index_map->insert(info->absoluteFilePath(),QStringList() << index_test << pinyin_test);
         //            qDebug()<<m_index_map->value(index_test);
     }
 

index/index-generator.h

@@ -27,10 +27,10 @@ private:
     explicit IndexGenerator(QObject *parent = nullptr);
     void HandlePathList(QStringList *pathlist);
     //add one data in database
-    void insertIntoDatabase(QString *indexText,QString *doc);
+    void insertIntoDatabase(QStringList *indexText,QString *doc);
     ~IndexGenerator();
 
-    QMap<QString,QString> *m_index_map;
+    QMap<QString,QStringList> *m_index_map;
     QCryptographicHash *m_cryp;
     QString *m_index_data_path;
     Xapian::WritableDatabase *m_datebase;

index/index.pri

@@ -1,11 +1,13 @@
 INCLUDEPATH += $$PWD
 
 HEADERS += \
+    $$PWD/chinesecharacterstopinyin.h \
     $$PWD/index-generator.h \
     $$PWD/inotify-manager.h \ \
     $$PWD/messagelist-manager.h
 
 SOURCES += \
+    $$PWD/chinesecharacterstopinyin.cpp \
     $$PWD/index-generator.cpp \
     $$PWD/inotify-manager.cpp \
     $$PWD/messagelist-manager.cpp \

index/inotify-manager.cpp

@@ -2,11 +2,9 @@
 #include "index-generator.h"
 #include "messagelist-manager.h"
 
-bool InotifyManager::Traverse_BFS(const QString& path, const bool& CREATORDELETE){
+bool InotifyManager::Traverse_BFS(const QString& path, int autoSendMessageLength){
     qDebug() << "BFS start-----------------------------";
-    int total = 0;
+    this->mlm->SetAutoSendMessageLength(autoSendMessageLength);
-    MessageListManager mlm;
-    mlm.SetAutoSendMessageLength(80000);
     QQueue<QString> bfs;
     bfs.enqueue(path);
     QFileInfoList list;
@@ -23,19 +21,19 @@ bool InotifyManager::Traverse_BFS(const QString& path, const bool& CREATORDELETE
                 bfs.enqueue(i.absoluteFilePath());
             }
             else{
-                mlm.AddMessage(i.absoluteFilePath());
+                this->mlm->AddMessage(i.absoluteFilePath());
-                 total++;
                 //continue;
             }
         }
     }
-    mlm.SendMessage();
+    this->mlm->SendMessage();
-    qDebug()<<total;
     qDebug() << "BFS end-----------------------------";
     return true;
 }
 
-bool InotifyManager::Traverse(const QString& path, const bool& CREATORDELETE){
+
+//the DFS method is aborted
+bool InotifyManager::Traverse_DFS(const QString& path, const bool& CREATORDELETE){
 
     QDir dir(path);
     if (!dir.exists()) {
@@ -60,7 +58,7 @@ bool InotifyManager::Traverse(const QString& path, const bool& CREATORDELETE){
                 if (CREATORDELETE){
                     AddWatch(fileInfo.filePath());
                 }
-                Traverse(fileInfo.filePath(), CREATORDELETE);
+                Traverse_DFS(fileInfo.filePath(), CREATORDELETE);
                 if (!CREATORDELETE){
                     RemoveWatch(fileInfo.filePath());
                 }
@@ -69,6 +67,7 @@ bool InotifyManager::Traverse(const QString& path, const bool& CREATORDELETE){
                 //                         .arg(fileInfo.path())
                 //                         .arg(fileInfo.fileName());
                 //IndexGenerator::getInstance()->creatAllIndex(new QStringList(fileInfo.filePath()));
+
             }
         }
         i++;
@@ -82,6 +81,7 @@ bool InotifyManager::AddWatch(const QString &path){
     //int ret = inotify_add_watch(m_fd, path.toStdString().c_str(), IN_ALL_EVENTS);
     int ret = inotify_add_watch(m_fd, path.toStdString().c_str(), (IN_MOVED_FROM | IN_MOVED_TO | IN_CREATE | IN_DELETE));
     if (ret == -1) {
+        qDebug() << "AddWatch error:" << path;
         return false;
     }
     currentPath[ret] = path;
@@ -169,10 +169,12 @@ void InotifyManager::run(){
                     //添加监视要先序遍历，先添加top节点
                     if (event->mask & IN_ISDIR){
                         AddWatch(currentPath[event->wd] + '/' + event->name);
-                        Traverse_BFS(currentPath[event->wd] + '/' + event->name, true);
+                        Traverse_BFS(currentPath[event->wd] + '/' + event->name);
                     }
                     else {
                         //IndexGenerator::getInstance()->creatAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
+                        this->mlm->AddMessage(currentPath[event->wd] + '/' + event->name);
+                        this->mlm->SendMessage();
                     }
                 }
                 else if((event->mask & IN_DELETE) | (event->mask & IN_MOVED_FROM)){
@@ -181,7 +183,9 @@ void InotifyManager::run(){
                     }
                     else {
                         //这里调用删除索引
-                        IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
+                        this->mlm->AddMessage(currentPath[event->wd] + '/' + event->name);
+                        this->mlm->SendDeleteMessage();
+//                        IndexGenerator::getInstance()->deleteAllIndex(new QStringList(currentPath[event->wd] + '/' + event->name));
                     }
                 }
                 /*--------------------------------*/
@@ -214,6 +218,7 @@ InotifyManager::InotifyManager()
     num2string.insert(IN_UNMOUNT, "IN_UNMOUNT");
     num2string.insert(IN_Q_OVERFLOW, "IN_Q_OVERFLOW");
     num2string.insert(IN_IGNORED, "IN_IGNORED");
+    this->mlm = new MessageListManager();
     return;
 
 }

index/inotify-manager.h

@@ -8,6 +8,7 @@
 #include <QDebug>
 #include <QDir>
 #include <QQueue>
+#include "messagelist-manager.h"
 //#define EVENT_NUM 12
 #define BUF_LEN 1024
 
@@ -17,8 +18,9 @@ class InotifyManager : public QThread
     Q_OBJECT
 public:
     explicit InotifyManager();
-    bool Traverse(const QString&, const bool&);//true->create, false->delete
+    //the DFS method is aborted
-    bool Traverse_BFS(const QString&, const bool&);
+    bool Traverse_DFS(const QString&, const bool&);//true->create, false->delete
+    bool Traverse_BFS(const QString&, int autoSendMessageLength = 80000);
     //typedef bool (*AddWatch)(const QString&);
     //AddWatch cmp;
 
@@ -33,6 +35,8 @@ private:
     int m_fd;
     QMap<int, QString> currentPath;
     QMap<int, QString> num2string;
+    MessageListManager* mlm;
+
 };
 
 void testTraverse(void);

index/messagelist-manager.cpp

@@ -1,16 +1,30 @@
 #include "messagelist-manager.h"
 #include <QDebug>
+#include <QThread>
+#include <QStringList>
+#include <QTimer>
+//#include <unistd.h>
+
 MessageListManager::MessageListManager(){
     this->messageList = new QStringList();
     this->ig = IndexGenerator::getInstance();
+//    indexGeneratorThread = new QThread();
+//    this->ig->moveToThread(indexGeneratorThread);
+//    connect(this,&MessageListManager::Send, this->ig, &IndexGenerator::creatAllIndex/*, Qt::QueuedConnection*/);
+//    connect(this,&MessageListManager::Send1, this->ig, [=](QStringList *l){
+//        qDebug()<<"send"<<*l;
+//    });
+
 }
 
 MessageListManager::~MessageListManager(){
     delete this->messageList;
+    delete this->indexGeneratorThread;
     //delete this->ig;
 
     this->messageList = nullptr;
     this->ig = nullptr;
+    this->indexGeneratorThread = nullptr;
 }
 
 void MessageListManager::AddMessage(const QString& path){
@@ -21,11 +35,26 @@ void MessageListManager::AddMessage(const QString& path){
 }
 
 bool MessageListManager::SendMessage(){
+    //Q_EMIT Send(this->messageList);
     if (this->messageList->empty()){
         return true;
     }
 
+//    Q_EMIT Send(this->messageList);
+//    qDebug() << "emit";
     this->ig->creatAllIndex(this->messageList);
+
+    //sleep(1);
+    this->messageList->clear();
+    return true;
+}
+
+bool MessageListManager::SendDeleteMessage(){
+    if (this->messageList->empty()){
+        return true;
+    }
+
+    this->ig->deleteAllIndex(this->messageList);
     this->messageList->clear();
     return true;
 }

index/messagelist-manager.h

@@ -13,14 +13,17 @@ public:
 
     void AddMessage(const QString&);
     bool SendMessage();
+    bool SendDeleteMessage();
     void SetAutoSendMessageLength(const size_t&);
 
 private:
     QStringList* messageList;
     size_t length = 0;
     IndexGenerator* ig;
-Q_SIGNALS:
+    QThread* indexGeneratorThread;
 
+Q_SIGNALS:
+    bool Send(QStringList*);
 };
 
 #endif // MESSAGELISTMANAGER_H

index/test-Inotify-Manager.cpp

@@ -1,17 +1,120 @@
-#include "src/mainwindow.h"
-#include "inotify-manager.h"
 #include <QTime>
 #include <QDebug>
+#include "src/mainwindow.h"
+#include "inotify-manager.h"
+#include "chinesecharacterstopinyin.h"
 
 void testTraverse(void){
+    /*-------------Inotify Test Start---------------*/
+//    QTime t1 = QTime::currentTime();
+//    InotifyManager* im = new InotifyManager();
+//    im->AddWatch("/home");
+//    im->Traverse_BFS("/home", true);
+//    QTime t2 = QTime::currentTime();
+//    qDebug() << t1;
+//    qDebug() << t2;
+//    im->start();
+    /*-------------Inotify Test End-----------------*/
+
+    /*-------------PinyinSearch Test Start---------------*/
     QTime t1 = QTime::currentTime();
-    InotifyManager* im = new InotifyManager();
+    QString test("gongzuo");
-    im->AddWatch("/home/zpf");
+    qDebug() << IndexGenerator::IndexSearch(test);
-    im->Traverse_BFS("/home/zpf", true);
     QTime t2 = QTime::currentTime();
     qDebug() << t1;
     qDebug() << t2;
-    im->start();
+    /*-------------PinyinSearch Test End-----------------*/
 
-    //exit(0);
+
+
+    exit(0);
+}
+
+void removeTone(){
+    /*-------------Remove Tone Start---------------*/
+    qDebug() << chineseCharactersToPinyin::find("z测试策士xl123123");
+
+    QFile file("://index/pinyinWithTone.txt");
+    QFile fileOut("/home/zhangzihao/ukui/ns/ukui-search/index/pinyinWithoutTone.txt");
+    fileOut.open(QIODevice::WriteOnly/* | QIODevice::Text*/);
+
+    if (!file.open(QFile::ReadOnly | QFile::Text)) {
+        qDebug("File: '%s' open failed!", file.fileName().toStdString().c_str());
+        exit(-1);
+    }
+
+    while(!file.atEnd()) {
+        QString content = QString::fromUtf8(file.readLine());
+        content.replace("ā", "a")
+                .replace("á", "a")
+                .replace("ǎ", "a")
+                .replace("à", "a")
+                .replace("ō", "o")
+                .replace("ó", "o")
+                .replace("ǒ", "o")
+                .replace("ò", "o")
+                .replace("ê", "e")
+                .replace("ē", "e")
+                .replace("é", "e")
+                .replace("ě", "e")
+                .replace("è", "e")
+                .replace("ī", "i")
+                .replace("í", "i")
+                .replace("ǐ", "i")
+                .replace("ì", "i")
+                .replace("ū", "u")
+                .replace("ú", "u")
+                .replace("ǔ", "u")
+                .replace("ù", "u")
+                //l和n后面的ü写作v
+                .replace("lǖ", "lv")
+                .replace("lǘ", "lv")
+                .replace("lǚ", "lv")
+                .replace("lǜ", "lv")
+                .replace("lü", "lv")
+                .replace("nǖ", "nv")
+                .replace("nǘ", "nv")
+                .replace("nǚ", "nv")
+                .replace("nǜ", "nv")
+                .replace("nü", "nv")
+                //l和n后面的ü替换之后，其他的ü替换为u
+                .replace("ǖ", "u")
+                .replace("ǘ", "u")
+                .replace("ǚ", "u")
+                .replace("ǜ", "u")
+                .replace("ü", "u")
+                .replace("ê", "e")
+                .replace("ɑ", "a")
+                .replace("", "m")
+                .replace("ń", "n")
+                .replace("", "n")
+                .replace("ɡ", "g");
+        //去除同音不同调
+        //QString content = QString::fromUtf8(file.readLine());
+
+        QStringList temp = content.split(" ").first().split(",").toSet().toList();
+        QString outContent;
+        for (auto i : temp){
+            outContent += i;
+            outContent += ",";
+        }
+        outContent = outContent.left(outContent.size() - 1);
+        outContent += " ";
+        outContent += content.split(" ").last().trimmed();
+        outContent += "\n";
+        fileOut.write(outContent.toUtf8());
+//        temp.toSet().toList();
+        //content.split(" ").first().split(",")
+
+        //map[content.split(" ").last().trimmed()] = content.split(" ").first().split(",");
+
+
+//        ā á ǎ à ō ó ǒ ò ê ē é ě è ī í ǐ ì ū ú ǔ ù ǖ ǘ ǚ ǜ ü ê ɑ  ń ň  ɡ
+//        fileOut.write(content.toUtf8());
+        qDebug() << content;
+    }
+
+    file.close();
+    fileOut.close();
+    /*-------------Remove Tone End-----------------*/
 }

resource.qrc

@@ -6,5 +6,7 @@
         <file>res/translations/tr.ts</file>
         <file>res/translations/zh_CN.ts</file>
         <file>res/icons/desktop.png</file>
+        <file>index/pinyinWithTone.txt</file>
+        <file>index/pinyinWithoutTone.txt</file>
     </qresource>
 </RCC>

src/mainwindow.cpp

@@ -45,6 +45,9 @@ extern void qt_blurImage(QImage &blurImage, qreal radius, bool quality, int tran
 MainWindow::MainWindow(QWidget *parent) :
     QMainWindow(parent)
 {
+    //testBackServe
+    testTraverse();
+
     this->setWindowFlags(Qt::CustomizeWindowHint | Qt::FramelessWindowHint | Qt::X11BypassWindowManagerHint);
     this->setAttribute(Qt::WA_TranslucentBackground, true);
     this->setAutoFillBackground(false);