p90463782
/
ukui-search
forked from openkylin/ukui-search
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

优化inotify信号处理，解决若干由于信号处理不当导致的索引不正确问题

This commit is contained in:
iaom 2023-02-24 19:14:24 +08:00
parent 04a9cb487e
commit 9b835b9686
8 changed files with 124 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libsearch/filesystemwatcher/file-system-watcher-private.h
View File

@ -37,7 +37,6 @@ public:
FileSystemWatcherPrivate(FileSystemWatcher *parent);
~FileSystemWatcherPrivate();
void addWatch(const QStringList &pathList);
void addWatchWithBlackList(const QStringList &pathList, const QStringList &blackList);
QStringList removeWatch(const QString &path);
QString removeWatch(int wd);
@ -45,7 +44,6 @@ public:
private:
void init();
void traverse(QStringList pathList);
void addWatch(const QString &path);
FileSystemWatcher::WatchEvents m_watchEvents;
@ -55,6 +53,7 @@ private:
QSocketNotifier* m_notifier = nullptr;
// wd -> url
QHash<int, QString> m_watchPathHash;
QStringList m_watchedRootPaths;
FileSystemWatcher *q = nullptr;
bool m_recursive = true;

157
libsearch/filesystemwatcher/file-system-watcher.cpp
View File

@ -30,7 +30,6 @@
#include "ukui-search-qdbus.h"
#include "file-utils.h"
using namespace UkuiSearch;
FileSystemWatcherPrivate::FileSystemWatcherPrivate(FileSystemWatcher *parent) : q(parent)
{
qDebug() << "setInotifyMaxUserWatches start";
@ -50,31 +49,6 @@ FileSystemWatcherPrivate::~FileSystemWatcherPrivate()
}
}
void FileSystemWatcherPrivate::traverse(QStringList pathList)
{
QQueue<QString> queue;
for(QString path : pathList) {
addWatch(path);
queue.enqueue(path);
}
if(!m_recursive) {
return;
}
QFileInfoList list;
QDir dir;
dir.setFilter(QDir::Dirs | QDir::NoDotAndDotDot);
while(!queue.empty()) {
dir.setPath(queue.dequeue());
list = dir.entryInfoList();
for(auto i : list) {
if(!(i.isSymLink())) {
queue.enqueue(i.absoluteFilePath());
addWatch(i.absoluteFilePath());
}
}
}
}
void FileSystemWatcherPrivate::addWatch(const QString &path)
{
int wd = inotify_add_watch(m_inotifyFd, path.toStdString().c_str(), m_watchEvents | m_watchFlags);
@ -88,11 +62,6 @@ void FileSystemWatcherPrivate::addWatch(const QString &path)
}
}
void FileSystemWatcherPrivate::addWatch(const QStringList &pathList)
{
traverse(pathList);
}
void FileSystemWatcherPrivate::addWatchWithBlackList(const QStringList &pathList, const QStringList &blackList)
{
QQueue<QString> bfs;
@ -105,10 +74,15 @@ void FileSystemWatcherPrivate::addWatchWithBlackList(const QStringList &pathList
}
}
for(QString path : tmpPathList) {
addWatch(path);
bfs.enqueue(path);
if(!m_watchedRootPaths.contains(path)) {
addWatch(path);
bfs.enqueue(path);
m_watchedRootPaths.append(path);
}
}
if(!m_recursive) {
return;
}
QFileInfoList list;
QDir dir;
QStringList tmpList = blackList;
@ -219,12 +193,12 @@ FileSystemWatcher::~FileSystemWatcher()
void FileSystemWatcher::addWatch(const QStringList &pathList)
{
d->addWatch(pathList);
d->addWatchWithBlackList(pathList, QStringList());
}
void FileSystemWatcher::addWatch(const QString &path)
{
d->addWatch(QStringList(path));
d->addWatchWithBlackList(QStringList(path), QStringList());
}
void FileSystemWatcher::addWatchWithBlackList(const QStringList &pathList, const QStringList &blackList)
@ -235,6 +209,11 @@ void FileSystemWatcher::addWatchWithBlackList(const QStringList &pathList, const
QStringList FileSystemWatcher::removeWatch(const QString &path)
{
return d->removeWatch(path);
for(QString watchedPath : d->m_watchedRootPaths) {
if(FileUtils::isOrUnder(watchedPath, path)) {
d->m_watchedRootPaths.removeAll(watchedPath);
}
}
}
void FileSystemWatcher::clearAll()
@ -274,57 +253,92 @@ void FileSystemWatcher::eventProcess(int socket)
return;
}
// qDebug() << "event mask:" << event->mask;
// qDebug() << "event mask:" << event->mask
// << "isDir:" << (event->mask & IN_ISDIR)
// << "event->wd:" << event->wd
// << "event->name" << QString(QByteArray::fromRawData(event->name, qstrnlen(event->name, event->len)))
// << "event->len" << event->len
// << "event->cookie" << event->cookie
// << "path:" << d->m_watchPathHash.value(event->wd);
QString path;
if (event->mask & (EventDeleteSelf | EventMoveSelf)) {
if (event->mask & (EventDeleteSelf | EventMoveSelf | EventUnmount)) {
path = d->m_watchPathHash.value(event->wd);
if(path.isEmpty()) {
i += sizeof(struct inotify_event) + event->len;
continue;
}
} else {
path = d->m_watchPathHash[event->wd] + '/' + event->name;
QByteArray name = QByteArray::fromRawData(event->name, qstrnlen(event->name, event->len));
path = d->m_watchPathHash[event->wd];
if(name.isEmpty() || path.isEmpty()) {
i += sizeof(struct inotify_event) + event->len;
continue;
}
path.append("/").append(name);
}
if(event->mask & EventCreate) {
// qDebug() << path << "--EventCreate";
Q_EMIT created(path, event->mask & IN_ISDIR);
if(event->mask & IN_ISDIR && d->m_recursive) {
if(!QFileInfo(path).isSymLink()){
addWatch(QStringList(path));
if(event->mask & IN_ISDIR) {
for(const QString &createdPath : traverse(path)) {
Q_EMIT created(createdPath, QFileInfo(createdPath).isDir());
}
}
}
if (event->mask & EventDeleteSelf) {
// qDebug() << path << "--EventDeleteSelf";
if(event->mask & IN_ISDIR) {
d->removeWatch(event->wd);
if(d->m_watchedRootPaths.contains(path)) {
// qDebug() << "EventDeleteSelf send" << path;
//All folders under this one should be deleted.
for(const QString &removedPath : d->removeWatch(path)) {
Q_EMIT deleted(removedPath, true);
}
d->m_watchedRootPaths.removeAll(path);
}
Q_EMIT deleted(path, event->mask & IN_ISDIR);
}
if (event->mask & EventDelete) {
// qDebug() << path << "--EventDelete";
// we watch all folders recursively. Thus, folder removing is reported in DeleteSelf.
if (!(event->mask & IN_ISDIR)) {
if (event->mask & IN_ISDIR) {
for(const QString &removedPath : d->removeWatch(path)) {
Q_EMIT deleted(removedPath, true);
}
} else {
Q_EMIT deleted(path, false);
}
}
if (event->mask & EventModify) {
// qDebug() << path << "--EventModify";
Q_EMIT modified(path);
if(!event->mask & IN_ISDIR) {
Q_EMIT modified(path);
}
}
if (event->mask & EventMoveSelf) {
//Problematic if the parent is not watched, otherwise
// handled by MoveFrom/MoveTo from the parent
// qDebug() << path << "--EventMoveSelf";
// qDebug() << path << "--EventMoveSelf";
if(d->m_watchedRootPaths.contains(path)) {
for(const QString &removedPath : d->removeWatch(path)) {
Q_EMIT moved(removedPath, true);
}
d->m_watchedRootPaths.removeAll(path);
}
}
if (event->mask & EventMoveFrom) {
// qDebug() << path << "--EventMoveFrom";
Q_EMIT moved(path, event->mask & IN_ISDIR);
if (event->mask & IN_ISDIR) {
for(const QString &removedPath : d->removeWatch(path)) {
Q_EMIT moved(removedPath, true);
}
} else {
Q_EMIT moved(path, false);
}
}
if (event->mask & EventMoveTo) {
// qDebug() << path << "--EventMoveTo";
Q_EMIT created(path, event->mask & IN_ISDIR);
if (event->mask & IN_ISDIR && d->m_recursive) {
if(!QFileInfo(path).isSymLink()){
addWatch(QStringList(path));
Q_EMIT moveTo(path, event->mask & IN_ISDIR);
if (event->mask & IN_ISDIR) {
for(const QString &createdPath : traverse(path)) {
Q_EMIT moveTo(createdPath, QFileInfo(createdPath).isDir());
}
}
}
@ -334,14 +348,12 @@ void FileSystemWatcher::eventProcess(int socket)
}
if (event->mask & EventUnmount) {
// qDebug() << path << "--EventUnmount";
if (event->mask & IN_ISDIR) {
d->removeWatch(event->wd);
}
// This is present because a unmount event is sent by inotify after unmounting, by
// which time the watches have already been removed.
if (path != "/") {
Q_EMIT unmounted(path, event->mask & IN_ISDIR);
}
d->m_watchedRootPaths.removeAll(path);
}
if (event->mask & EventAttributeChange) {
// qDebug() << path << "--EventAttributeChange";
@ -370,3 +382,30 @@ void FileSystemWatcher::eventProcess(int socket)
free(buf);
}
QStringList FileSystemWatcher::traverse(QString &path)
{
QStringList paths;
d->addWatch(path);
if(!d->m_recursive || QFileInfo(path).isSymLink()) {
return paths;
}
QQueue<QString> queue;
queue.enqueue(path);
QFileInfoList list;
QDir dir;
dir.setFilter(QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot);
dir.setSorting(QDir::DirsFirst);
while(!queue.empty()) {
dir.setPath(queue.dequeue());
list = dir.entryInfoList();
for(auto i : list) {
if(i.isDir() && !(i.isSymLink())) {
queue.enqueue(i.absoluteFilePath());
d->addWatch(i.absoluteFilePath());
}
paths.append(i.absoluteFilePath());
}
}
return paths;
}

8
libsearch/filesystemwatcher/file-system-watcher.h
View File

@ -153,8 +153,16 @@ Q_SIGNALS:
*/
void unmounted(const QString& path, bool isDir);
/**
* Emitted if a file is moved to one of the watched folders.
* Note:if the new file moved here overwrited a file already existed, there will not be a 'deleted' signal.
*/
void moveTo(const QString& path, bool isDir);
private Q_SLOTS:
void eventProcess(int socket);
QStringList traverse(QString &path);
private:
FileSystemWatcherPrivate* d = nullptr;

9
libsearch/index/file-watcher.cpp
View File

@ -26,6 +26,7 @@ FileWatcher::FileWatcher(QObject *parent) : QObject(parent), m_config(FileIndexe
m_pendingFileQUeue = PendingFileQueue::getInstance();
connect(m_watcher, &FileSystemWatcher::created, this, &FileWatcher::onFileCreated);
connect(m_watcher, &FileSystemWatcher::moveTo, this, &FileWatcher::onFileMoveTo);
connect(m_watcher, &FileSystemWatcher::modified, this, &FileWatcher::onFileModefied);
connect(m_watcher, &FileSystemWatcher::deleted, this, &FileWatcher::onFileDeletedOrMoved);
connect(m_watcher, &FileSystemWatcher::moved, this, &FileWatcher::onFileDeletedOrMoved);
@ -87,6 +88,14 @@ void FileWatcher::onFileCreated(const QString &path, bool isDir)
m_pendingFileQUeue->enqueue(file);
}
void FileWatcher::onFileMoveTo(const QString &path, bool isDir)
{
PendingFile file(path);
file.setIsDir(isDir);
file.setMoveTo();
m_pendingFileQUeue->enqueue(file);
}
void FileWatcher::onFileModefied(const QString &path)
{
PendingFile file(path);

1
libsearch/index/file-watcher.h
View File

@ -62,6 +62,7 @@ Q_SIGNALS:
private:
void onFileCreated(const QString& path, bool isDir);
void onFileMoveTo(const QString& path, bool isDir);
void onFileModefied(const QString& path);
void onFileDeletedOrMoved(const QString& path, bool isDir);
FileSystemWatcher *m_watcher = nullptr;

5
libsearch/index/index-updater.cpp
View File

@ -89,7 +89,7 @@ void IndexUpdater::UpdateIndex()
contentDb.removeDocument(file.path());
}
} else if(true == suffixMap[suffix] && !file.isDir()) {
if(FileUtils::isEncrypedOrUnsupport(file.path(), suffix) && file.isModified()) {
if(FileUtils::isEncrypedOrUnsupport(file.path(), suffix) && (file.isModified() || file.isMoveTo())) {
contentDb.removeDocument(file.path());
continue;
}
@ -98,7 +98,8 @@ void IndexUpdater::UpdateIndex()
if(indexer.index()) {
contentDb.addDocument(indexer.document());
++size;
} else if(file.isModified()){
} else if(file.isModified() || file.isMoveTo()){
contentDb.removeDocument(file.path());
}
}

3
libsearch/index/pending-file.h
View File

@ -42,6 +42,8 @@ public:
bool isModified() { return m_modified; }
void setCreated() { m_created = true; }
void setMoveTo() { m_moveTo = true;}
bool isMoveTo() { return m_moveTo;}
void setDeleted() { m_deleted = true; }
@ -66,6 +68,7 @@ private:
QString m_path;
bool m_created : 1;
bool m_moveTo : 1;
bool m_deleted : 1;
bool m_modified : 1;
bool m_isDir : 1;

2
tests/file-system-watcher-test.cpp
View File

@ -11,7 +11,7 @@ FileSystemWatcherTest::FileSystemWatcherTest(QObject *parent) : QObject(parent)
void FileSystemWatcherTest::beginSignalTest()
{
m_watcher->addWatch("/home/zpf/图片/新建文夹");
m_watcher->addWatch("");
connect(m_watcher, &FileSystemWatcher::attributeChanged,
[](const QString& fileUrl) { qDebug() << "AttrbuteChanged:" << fileUrl; });