forked from openkylin/ukui-search
361 lines
14 KiB
C++
361 lines
14 KiB
C++
/*
|
||
* Copyright (C) 2022, KylinSoft Co., Ltd.
|
||
*
|
||
* This program is free software: you can redistribute it and/or modify
|
||
* it under the terms of the GNU General Public License as published by
|
||
* the Free Software Foundation, either version 3 of the License, or
|
||
* (at your option) any later version.
|
||
*
|
||
* This program is distributed in the hope that it will be useful,
|
||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
* GNU General Public License for more details.
|
||
*
|
||
* You should have received a copy of the GNU General Public License
|
||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||
*
|
||
* Authors: jixiaoxu <jixiaoxu@kylinos.cn>
|
||
*
|
||
*/
|
||
|
||
#include <mutex>
|
||
#include <cctype>
|
||
#include "hanzi-to-pinyin.h"
|
||
#include "hanzi-to-pinyin-private.h"
|
||
#include "chinese-segmentation.h"
|
||
#include "cppjieba/Unicode.hpp"
|
||
|
||
HanZiToPinYin * HanZiToPinYin::g_pinYinManager = nullptr;
|
||
std::once_flag g_singleFlag;
|
||
|
||
bool HanZiToPinYinPrivate::contains(string &word)
|
||
{
|
||
return m_pinYinTrie.Contains(word);
|
||
}
|
||
|
||
int HanZiToPinYinPrivate::getResults(string &word, QStringList &results)
|
||
{
|
||
results.clear();
|
||
|
||
string directResult = m_pinYinTrie.Find(word);
|
||
|
||
if (directResult == string()) {
|
||
if (m_segType == SegType::NoSegmentation) {//无分词、无结果直接返回-1
|
||
return -1;
|
||
} else {//无结果、启用分词
|
||
vector<string> segResults = ChineseSegmentation::getInstance()->callMixSegmentCutStr(word);
|
||
string data;
|
||
for (string &info : segResults) {
|
||
if (info == string()) {
|
||
continue;
|
||
}
|
||
data = m_pinYinTrie.Find(info);
|
||
if (data == string()) {//分词后无结果
|
||
if (cppjieba::IsSingleWord(info)) {//单个字符
|
||
if (m_exDataProcessType == ExDataProcessType::Default) {//原数据返回
|
||
results.append(QString().fromStdString(info));
|
||
} else if (m_exDataProcessType == ExDataProcessType::Delete) {//忽略
|
||
continue;
|
||
}
|
||
} else {//多个字符
|
||
string oneWord;
|
||
cppjieba::RuneStrArray runeArray;
|
||
cppjieba::DecodeRunesInString(info, runeArray);
|
||
for (auto i = runeArray.begin(); i != runeArray.end(); ++i) {
|
||
oneWord = cppjieba::GetStringFromRunes(info, i, i);
|
||
data = m_pinYinTrie.Find(oneWord);
|
||
if (data == string()) {//单字无结果则按设置返回
|
||
if (m_exDataProcessType == ExDataProcessType::Default) {//原数据返回
|
||
results.append(QString().fromStdString(oneWord));
|
||
} else if (m_exDataProcessType == ExDataProcessType::Delete) {//忽略
|
||
continue;
|
||
}
|
||
}
|
||
if (m_polyphoneType == PolyphoneType::Enable) {//启用多音字
|
||
results.append(QString().fromStdString(data));
|
||
} else if (m_polyphoneType == PolyphoneType::Disable) {//不启用多音字
|
||
if (limonp::IsInStr(data, ',')) {
|
||
results.append(QString().fromStdString(data.substr(0, data.find_first_of(",", 0))));
|
||
} else {
|
||
results.append(QString().fromStdString(data));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {//分词后有结果
|
||
if (cppjieba::IsSingleWord(info)) {//单个字符
|
||
if (m_polyphoneType == PolyphoneType::Enable) {//启用多音字
|
||
results.append(QString().fromStdString(data));
|
||
} else if (m_polyphoneType == PolyphoneType::Disable) {//不启用多音字
|
||
if (limonp::IsInStr(data, ',')) {
|
||
results.append(QString().fromStdString(data.substr(0, data.find_first_of(",", 0))));
|
||
} else {
|
||
results.append(QString().fromStdString(data));
|
||
}
|
||
}
|
||
} else {//多个字符
|
||
vector<string> dataVec = limonp::Split(data, "/");
|
||
if (dataVec.size() == 1) {//无多音词
|
||
vector<string> dataVec = limonp::Split(data, ",");
|
||
for (auto &oneResult : dataVec) {
|
||
results.append(QString().fromStdString(oneResult));
|
||
}
|
||
} else {
|
||
if (m_polyphoneType == PolyphoneType::Enable) {//启用多音字
|
||
int wordSize = limonp::Split(dataVec[0], ",").size();
|
||
for (int i = 0; i < wordSize; ++i) {
|
||
QStringList oneResult;
|
||
for (size_t j = 0; j < dataVec.size(); ++j) {
|
||
oneResult.append(QString().fromStdString(limonp::Split(dataVec[j], ",")[i]));
|
||
}
|
||
results.append(oneResult.join('/'));
|
||
}
|
||
} else if (m_polyphoneType == PolyphoneType::Disable) {//不启用多音字
|
||
vector<string> tmp = limonp::Split(dataVec[0], ",");
|
||
for (auto &oneResult : tmp) {
|
||
results.append(QString().fromStdString(oneResult));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {//可以直接查到结果
|
||
if (cppjieba::IsSingleWord(word)) {//单个字符
|
||
if (m_polyphoneType == PolyphoneType::Enable) {//启用多音字
|
||
results.append(QString().fromStdString(directResult));
|
||
} else if (m_polyphoneType == PolyphoneType::Disable) {//不启用多音字
|
||
if (limonp::IsInStr(directResult, ',')) {
|
||
results.append(QString().fromStdString(directResult.substr(0, directResult.find_first_of(",", 0))));
|
||
} else {
|
||
results.append(QString().fromStdString(directResult));
|
||
}
|
||
}
|
||
} else {//多个字符
|
||
vector<string> dataVec = limonp::Split(directResult, "/");
|
||
if (dataVec.size() == 1) {//无多音词
|
||
vector<string> dataVec = limonp::Split(directResult, ",");
|
||
for (auto &oneResult : dataVec) {
|
||
results.append(QString().fromStdString(oneResult));
|
||
}
|
||
} else {
|
||
if (m_polyphoneType == PolyphoneType::Enable) {//启用多音字
|
||
int wordSize = limonp::Split(dataVec[0], ",").size();
|
||
for (int i = 0; i < wordSize; ++i) {
|
||
QStringList oneResult;
|
||
for (size_t j = 0; j < dataVec.size(); ++j) {
|
||
oneResult.append(QString().fromStdString(limonp::Split(dataVec[j], ",")[i]));
|
||
}
|
||
results.append(oneResult.join('/'));
|
||
}
|
||
} else if (m_polyphoneType == PolyphoneType::Disable) {//不启用多音字
|
||
vector<string> tmp = limonp::Split(dataVec[0], ",");
|
||
for (auto &oneResult : tmp) {
|
||
results.append(QString().fromStdString(oneResult));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
convertDataStyle(results);
|
||
return 0;//todo
|
||
}
|
||
|
||
void HanZiToPinYinPrivate::setConfig(PinyinDataStyle dataStyle, SegType segType, PolyphoneType polyphoneType, ExDataProcessType processType)
|
||
{
|
||
m_pinyinDataStyle = dataStyle;
|
||
m_segType = segType;
|
||
m_polyphoneType = polyphoneType;
|
||
m_exDataProcessType = processType;
|
||
}
|
||
|
||
void HanZiToPinYinPrivate::convertDataStyle(QStringList &results)
|
||
{
|
||
QString value;
|
||
if (m_pinyinDataStyle == PinyinDataStyle::Default) {
|
||
for (QString &info : results) {
|
||
if(info == ",") {
|
||
continue;
|
||
}
|
||
//if info's length was been changed, there's someting wrong while traverse the chars of info
|
||
for (const QChar &c : info) {
|
||
if (!isalpha(c.toLatin1())) {
|
||
value = PhoneticSymbol.value(c);
|
||
if (!value.isEmpty()) {
|
||
info.replace(c, value.at(0));
|
||
}
|
||
}
|
||
}
|
||
|
||
QStringList tmpList = info.split(',', QString::SkipEmptyParts); //去重(保持原顺序)
|
||
QStringList tmpValue;
|
||
for (auto &str : tmpList) {
|
||
if (!tmpValue.contains(str)) {
|
||
tmpValue.push_back(str);
|
||
}
|
||
}
|
||
info = tmpValue.join(",");
|
||
}
|
||
} else if (m_pinyinDataStyle == PinyinDataStyle::Tone) {
|
||
//无需处理
|
||
} else if (m_pinyinDataStyle == PinyinDataStyle::Tone2) {
|
||
for (QString &info : results) {
|
||
for (int i = 0; i < info.size();) {
|
||
auto c = info.at(i);
|
||
if (!isalpha(c.toLatin1())) {
|
||
value = PhoneticSymbol.value(c);
|
||
if (!value.isEmpty()) {
|
||
info.replace(c, PhoneticSymbol.value(c));
|
||
i += PhoneticSymbol.value(c).size();
|
||
continue;
|
||
}
|
||
}
|
||
i++;
|
||
}
|
||
}
|
||
} else if (m_pinyinDataStyle == PinyinDataStyle::Tone3) {
|
||
for (QString &info : results) {
|
||
if(info == "/") {
|
||
continue;
|
||
}
|
||
bool isPolyphoneWords(false);
|
||
if (info.contains("/")) {
|
||
isPolyphoneWords = true;
|
||
info.replace("/", ",");
|
||
}
|
||
|
||
for (int i = 0; i < info.size();) {
|
||
auto c = info.at(i);
|
||
if (!isalpha(c.toLatin1())) {
|
||
value = PhoneticSymbol.value(c);
|
||
if (!value.isEmpty()) {
|
||
info.replace(i, 1, value.at(0));
|
||
//多音词模式
|
||
if (info.contains(",")) {
|
||
int pos = info.indexOf(',', i);
|
||
if (isPolyphoneWords) {
|
||
info.replace(",", "/");
|
||
}
|
||
//最后一个读音时
|
||
if (pos == -1) {
|
||
info.append(value.at(1));
|
||
break;
|
||
}
|
||
info.insert(pos, value.at(1));
|
||
i = pos + 1; //insert导致','的位置加一，将i行进到','的位置
|
||
i++;
|
||
continue;
|
||
} else {
|
||
info.append(value.at(1));
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
i++;
|
||
}
|
||
|
||
}
|
||
} else if (m_pinyinDataStyle == PinyinDataStyle::FirstLetter) {
|
||
for (QString &info : results) {
|
||
if(info == "," or info == "/") {
|
||
continue;
|
||
}
|
||
|
||
bool isPolyphoneWords(false);
|
||
if (info.contains("/")) {
|
||
isPolyphoneWords = true;
|
||
info.replace("/", ",");
|
||
}
|
||
|
||
for (int i = 0; i < info.size();i++) {
|
||
auto c = info.at(i);
|
||
if (!isalpha(c.toLatin1())) {
|
||
value = PhoneticSymbol.value(c);
|
||
if (!value.isEmpty()) {
|
||
info.replace(c, value.at(0));
|
||
}
|
||
}
|
||
}
|
||
|
||
QStringList tmpList = info.split(',', QString::SkipEmptyParts); //去重(保持原顺序)
|
||
QStringList tmpValue;
|
||
for (auto &str : tmpList) {
|
||
if (!tmpValue.contains(str)) {
|
||
tmpValue.push_back(str.at(0));
|
||
}
|
||
}
|
||
if (isPolyphoneWords) {
|
||
info = tmpValue.join("/");
|
||
} else {
|
||
info = tmpValue.join(",");
|
||
}
|
||
}
|
||
} else if (m_pinyinDataStyle == PinyinDataStyle::English) {
|
||
//暂不支持
|
||
}
|
||
}
|
||
|
||
HanZiToPinYinPrivate::HanZiToPinYinPrivate(HanZiToPinYin *parent) : q(parent)
|
||
{
|
||
//const char * const SINGLE_WORD_PINYIN_PATH = "/usr/share/ukui-search/res/dict/singleWordPinyin.txt";
|
||
//const char * const WORDS_PINYIN_PATH = "/usr/share/ukui-search/res/dict/wordsPinyin.txt";
|
||
//m_pinYinTrie = new Pinyin4cppDictTrie(SINGLE_WORD_PINYIN_PATH, WORDS_PINYIN_PATH);
|
||
//m_pinYinTrie = new Pinyin4cppTrie;
|
||
}
|
||
|
||
HanZiToPinYinPrivate::~HanZiToPinYinPrivate()
|
||
{
|
||
// if (m_pinYinTrie){
|
||
// delete m_pinYinTrie;
|
||
// m_pinYinTrie = nullptr;
|
||
// }
|
||
}
|
||
|
||
HanZiToPinYin * HanZiToPinYin::getInstance()
|
||
{
|
||
call_once(g_singleFlag, []() {
|
||
g_pinYinManager = new HanZiToPinYin;
|
||
});
|
||
return g_pinYinManager;
|
||
}
|
||
|
||
bool HanZiToPinYin::contains(string &word)
|
||
{
|
||
return d->contains(word);
|
||
}
|
||
|
||
bool HanZiToPinYin::isMultiTone(string &word)
|
||
{
|
||
return d->isMultiTone(word);
|
||
}
|
||
|
||
bool HanZiToPinYin::isMultiTone(string &&word)
|
||
{
|
||
return d->isMultiTone(word);
|
||
}
|
||
|
||
bool HanZiToPinYin::isMultiTone(const string &word)
|
||
{
|
||
return d->isMultiTone(word);
|
||
}
|
||
|
||
bool HanZiToPinYin::isMultiTone(const string &&word)
|
||
{
|
||
return d->isMultiTone(word);
|
||
}
|
||
|
||
int HanZiToPinYin::getResults(string word, QStringList &results)
|
||
{
|
||
return d->getResults(word, results);
|
||
}
|
||
|
||
void HanZiToPinYin::setConfig(PinyinDataStyle dataStyle, SegType segType, PolyphoneType polyphoneType, ExDataProcessType processType)
|
||
{
|
||
d->setConfig(dataStyle, segType, polyphoneType, processType);
|
||
}
|
||
|
||
HanZiToPinYin::HanZiToPinYin() : d(new HanZiToPinYinPrivate)
|
||
{
|
||
}
|