diff --git a/backup-daemon/backup-daemon.pro b/backup-daemon/backup-daemon.pro index 6e77156..c290d8b 100644 --- a/backup-daemon/backup-daemon.pro +++ b/backup-daemon/backup-daemon.pro @@ -29,6 +29,7 @@ HEADERS += \ backupmanager_adaptor.h \ mybackupmanager.h \ mymountproxy.h \ + myprocess/calcbackupsize.h \ myprocess/mountbackupprocess.h \ parsebackuplist.h \ systembackupproxy.h \ @@ -44,6 +45,7 @@ SOURCES += \ main.cpp \ mybackupmanager.cpp \ mymountproxy.cpp \ + myprocess/calcbackupsize.cpp \ myprocess/mountbackupprocess.cpp \ parsebackuplist.cpp \ systembackupproxy.cpp \ diff --git a/backup-daemon/myprocess/calcbackupsize.cpp b/backup-daemon/myprocess/calcbackupsize.cpp new file mode 100644 index 0000000..4083365 --- /dev/null +++ b/backup-daemon/myprocess/calcbackupsize.cpp @@ -0,0 +1,101 @@ +#include "calcbackupsize.h" +#include + +CalcBackupSize::CalcBackupSize(QObject* parent) : + QObject(parent), + m_process(new QProcess(this)) +{ + connect(m_process, &QProcess::readyReadStandardError, this, [&]() { + QByteArray err = m_process->readAllStandardError(); + qCritical("backup process error: %s", err.data()); + }); + + connect(m_process, &QProcess::errorOccurred, this, [&](QProcess::ProcessError error) { + m_process->kill(); + }); + + connect(m_process, SIGNAL(finished(int, QProcess::ExitStatus)), this, SLOT(getCalcResult(int, QProcess::ExitStatus))); +} + +CalcBackupSize::~CalcBackupSize() +{ + +} + +/** + * @brief 计算备份大小,单位字节 + * @param args rsync参数列表 + * @param block 是否阻塞计算模式。默认true——阻塞计算模式 + * @return block为true时返回值为待备份数据大小,单位字节;block为false时,默认返回0; + */ +qint64 CalcBackupSize::calcBackupSize(QStringList args, bool block) +{ + QString cmd("rsync "); + for (const QString& item : args) { + cmd += " "; + cmd += item; + } + qDebug() << cmd; + + m_size = 0; + + m_process->start("rsync", args); + if (!m_process->waitForStarted()) { + qCritical("rsync start failed"); + return m_size; + } + + if (block) { + m_process->waitForFinished(); + } + + return m_size; +} + +/** + * @brief 解析结果 + * @param exitCode + */ +void CalcBackupSize::getCalcResult(int exitCode, QProcess::ExitStatus) +{ + // 解析结果,结果内容样例如下: + /* + Number of files: 8 (reg: 4, dir: 4) + Number of created files: 7 (reg: 4, dir: 3) + Number of deleted files: 0 + Number of regular files transferred: 4 + Total file size: 20 bytes + Total transferred file size: 20 bytes + Literal data: 0 bytes + Matched data: 0 bytes + File list size: 0 + File list generation time: 0.001 seconds + File list transfer time: 0.000 seconds + Total bytes sent: 248 + Total bytes received: 43 + + sent 248 bytes received 43 bytes 582.00 bytes/sec + total size is 20 speedup is 0.07 (DRY RUN) + */ + QString out(m_process->readAll()); + QStringList lines = out.split("\n"); + for (QString& line : lines) { + line.trimmed(); + // 获取文件夹数目 + if (line.startsWith("Number of files:")) { + int indexOfLastColon = line.lastIndexOf(":"); + int indexOfLastLable = line.lastIndexOf(")"); + int numOfDirs = line.mid(indexOfLastColon+1, indexOfLastLable-indexOfLastColon).trimmed().toInt(); + // 每个目录下还都有.和..,故总数还要*3 + numOfDirs *= 3; + // ext4格式的目录本身要占用4K空间(4096) + m_size += numOfDirs * 4096; + } + if (line.startsWith("Total file size:")) { + m_size += line.replace("Total file size:", "").replace("bytes", "").trimmed().toLongLong(); + break ; + } + } + + emit calcFinished(m_size); +} diff --git a/backup-daemon/myprocess/calcbackupsize.h b/backup-daemon/myprocess/calcbackupsize.h new file mode 100644 index 0000000..1850679 --- /dev/null +++ b/backup-daemon/myprocess/calcbackupsize.h @@ -0,0 +1,38 @@ +#ifndef CALCBACKUPSIZE_H +#define CALCBACKUPSIZE_H + +#include +#include + +#define GB (1024 * 1024 * 1024) +#define MB (1024 * 1024) +#define KB (1024) + +class CalcBackupSize : public QObject +{ + Q_OBJECT +public: + CalcBackupSize(QObject* parent = nullptr); + ~CalcBackupSize(); + + /** + * @brief 计算备份大小,单位字节 + * @param args rsync参数列表 + * @param block 是否阻塞计算模式。默认true——阻塞计算模式 + * @return block为true时返回值为待备份数据大小,单位字节;block为false时,默认返回0 + */ + qint64 calcBackupSize(QStringList args, bool block = true); + +signals: + void calcFinished(qint64 size); + +private slots: + void getCalcResult(int exitCode, QProcess::ExitStatus); + +private: + QProcess * m_process; + + qint64 m_size = 0; // 备份大小,单位字节 +}; + +#endif // CALCBACKUPSIZE_H diff --git a/backup-daemon/systembackupproxy.cpp b/backup-daemon/systembackupproxy.cpp index f0876ed..93033bb 100644 --- a/backup-daemon/systembackupproxy.cpp +++ b/backup-daemon/systembackupproxy.cpp @@ -5,6 +5,7 @@ #include "../common/mydusizetool.h" #include "mymountproxy.h" #include "parsebackuplist.h" +#include "myprocess/calcbackupsize.h" IMPLEMENT_DYNCREATE(SystemBackupProxy) @@ -29,11 +30,10 @@ void SystemBackupProxy::checkEnvEx() } // 2、检测备份是否增量备份 - bool bInc = checkIsIncBackup(); - m_backupWrapper.m_bIncrement = bInc; + checkIsIncBackup(); // 3、检测空间是否满足备份 - checkFreeCapacity(bInc); + checkFreeCapacity(); qDebug() << "SystemBackupProxy::checkEnv invoke end"; } @@ -51,11 +51,12 @@ void SystemBackupProxy::cancelEx() bool SystemBackupProxy::checkIsIncBackup() { QString backupPath; + ParseBackupList::BackupPoint point; if (m_backupWrapper.m_uuid.isEmpty()) { QString xmlPath(Utils::getSysRootPath() + BACKUP_XML_PATH); xmlPath.replace("//", "/"); ParseBackupList parser(xmlPath); - ParseBackupList::BackupPoint point = parser.getLastSysBackupPoint(); + point = parser.getLastSysBackupPoint(); if (point.m_uuid.isEmpty()) return false; backupPath = Utils::getSysRootPath() + BACKUP_SNAPSHOTS_PATH + "/" + point.m_uuid + "/data"; @@ -64,30 +65,70 @@ bool SystemBackupProxy::checkIsIncBackup() } backupPath.replace("//", "/"); - return Utils::isDirExist(backupPath); + if (Utils::isDirExist(backupPath)) { + m_backupWrapper.m_baseUuid = point.m_uuid; + m_backupWrapper.m_bIncrement = true; + return true; + } + return false; } /** * @brief 校验剩余空间是否满足备份 - * @param bInc,是否增量备份 - * @note 增量备份和全量备份计算空间方法不同 */ -void SystemBackupProxy::checkFreeCapacity(bool bInc) +void SystemBackupProxy::checkFreeCapacity() { // 1、计算待备份数据的大小 - MyDuSizeTool du; - qint64 itotalSize = du.Do(m_backupWrapper.m_backupPaths, m_backupWrapper.m_backupExcludePaths, true); - if (-1 == itotalSize) { - qCritical() << "du system backup failed"; - emit checkResult(int(BackupResult::DU_ERR)); - return ; - } + qint64 itotalSize = calcSizeForBackup(); + // 备份过程中会有一些临时文件产生,会占用一部分空间,故我们预留500M的空间 + itotalSize += 500 * MB; // 2、计算备份分区剩余空间大小 QString backupPath(Utils::getSysRootPath() + BACKUP_PATH); backupPath.replace("//", "/"); QStorageInfo backupDisk(backupPath); qint64 freeSize = backupDisk.bytesFree(); + + // 3、校验空间是否足够 + if (itotalSize > freeSize) { + emit checkResult(int(BackupResult::BACKUP_CAPACITY_IS_NOT_ENOUGH)); + return ; + } +} + +/** + * @brief 计算备份所需空间大小 + * @return 计算备份所需空间大小,单位字节 + */ +qint64 SystemBackupProxy::calcSizeForBackup() +{ + QStringList args; + if (m_backupWrapper.m_bIncrement) { + if (m_backupWrapper.m_uuid.isEmpty()) { + // 新增增量备份点场景 + args = getRsyncArgs(SystemBackupScene::TRY_INC_SYSTEM_BACKUP); + } else { + // 在原来的备份点上增量备份场景 + args = getRsyncArgs(SystemBackupScene::TRY_INC_SYSTEM_BACKUP_AT_BASE); + } + } else { + // 全量备份场景 + args = getRsyncArgs(SystemBackupScene::TRY_SYSTEM_BACKUP); + } + + // 拼接备份源路径和目标路径 + QString srcPath = Utils::getSysRootPath(); + srcPath += "/"; + srcPath.replace("//", "/"); + args << srcPath; + QString destPath = Utils::getSysRootPath(); + destPath += CHECK_PATH; + destPath.replace("//", "/"); + args << destPath; + Utils::mkpath(destPath); + + CalcBackupSize calcator; + return calcator.calcBackupSize(args); } /** @@ -103,22 +144,44 @@ QStringList SystemBackupProxy::getRsyncArgs(SystemBackupScene scene) case SystemBackupScene::SYSTEM_BACKUP : args << "-avAXW"; args << "--progress"; + args << "--ignore-missing-args"; break ; case SystemBackupScene::INC_SYSTEM_BACKUP : args << "-avAXr"; args << "--progress"; - args << "--link-dest"; + args << "--ignore-missing-args"; + args << QString("--link-dest=../%1/data").arg(m_baseUuid); + break ; + case SystemBackupScene::INC_SYSTEM_BACKUP_AT_BASE : + args << "-avAXr"; + args << "--progress"; + args << "--ignore-missing-args"; + args << "--delete"; break ; case SystemBackupScene::TRY_SYSTEM_BACKUP : - args << "-avAXWn"; + args << "-aAXWn"; args << "--stats"; + args << "--ignore-missing-args"; break ; case SystemBackupScene::TRY_INC_SYSTEM_BACKUP : - args << "-avAXrn"; + args << "-aAXrn"; args << "--stats"; - args << "--link-dest"; + args << "--ignore-missing-args"; + args << QString("--link-dest=../%1/data").arg(m_baseUuid); + break ; + case SystemBackupScene::TRY_INC_SYSTEM_BACKUP_AT_BASE : + args << "-aAXrn"; + args << "--stats"; + args << "--ignore-missing-args"; + args << "--delete"; break ; default: + return args; + } + + // --exclude=排除路径设置 + for (const QString & item : m_backupWrapper.m_backupExcludePaths) { + args << QString("--exclude=%1").arg(item); } return args; diff --git a/backup-daemon/systembackupproxy.h b/backup-daemon/systembackupproxy.h index 23d67d0..540c385 100644 --- a/backup-daemon/systembackupproxy.h +++ b/backup-daemon/systembackupproxy.h @@ -12,8 +12,10 @@ public: enum SystemBackupScene { SYSTEM_BACKUP, // 系统备份 INC_SYSTEM_BACKUP, // 增量系统备份 + INC_SYSTEM_BACKUP_AT_BASE, // 在原备份点里增量备份 TRY_SYSTEM_BACKUP, // 测试系统备份,可用于计算备份传输数据大小 TRY_INC_SYSTEM_BACKUP, // 测试增量系统备份,可用于计算备份传输数据大小 + TRY_INC_SYSTEM_BACKUP_AT_BASE, // 测试增量系统备份,在原备份点里增量备份,可用于计算备份传输数据大小 }; explicit SystemBackupProxy(); @@ -34,17 +36,22 @@ private: bool checkIsIncBackup(); // 校验剩余空间是否满足备份 - void checkFreeCapacity(bool bInc); + void checkFreeCapacity(); + + // 计算备份所需空间大小 + qint64 calcSizeForBackup(); /** * @brief 根据场景获取rsync命令参数 * @param scene,场景 * @return 组装好的rsync的参数信息 */ - QString getRsyncArgs(SystemBackupScene scene); + QStringList getRsyncArgs(SystemBackupScene scene); // 是否增量备份 bool m_bIncrement = false; + // 基准备份点,新增增量备份点时用 + QString m_baseUuid; }; #endif // SYSTEMBACKUPPROXY_H diff --git a/common/mydefine.h b/common/mydefine.h index 32cef9c..62eac61 100644 --- a/common/mydefine.h +++ b/common/mydefine.h @@ -12,6 +12,7 @@ #define BACKUP_SNAPSHOTS_PATH "/backup/snapshots" #define BACKUP_XML_PATH "/backup/snapshots/backuplist.xml" #define EXCLUDE_FILE_PATH "/backup/snapshots/.exclude" +#define CHECK_PATH "/backup/snapshots/check/data/" #define BOOTINFO_PATH "/etc/.bootinfo" #define FSTAB_PATH "/etc/fstab" @@ -82,6 +83,8 @@ struct BackupWrapper { int m_gid = -1; // 是否增量备份 bool m_bIncrement = false; + // 新增备份点时增量备份的基准uuid + QString m_baseUuid; static void registerMetaType(); }; diff --git a/common/mydusizetool.cpp b/common/mydusizetool.cpp index 629cdc1..693b1ca 100644 --- a/common/mydusizetool.cpp +++ b/common/mydusizetool.cpp @@ -51,12 +51,12 @@ qint64 MyDuSizeTool::_Do(QStringList paths, QStringList excludePaths, bool needE } QString cmd = "du"; - for (QString& x : paths) { + for (const QString& x : paths) { cmd.append(QString(" \"%1\"").arg(x)); } if (needExclude) { - for (QString& item : excludePaths) { - QString arg = QString(" --exclude=\"%1\"").arg(x); + for (const QString& item : excludePaths) { + QString arg = QString(" --exclude=\"%1\"").arg(item); cmd.append(arg); } } diff --git a/common/utils.cpp b/common/utils.cpp index 7e4e0e2..d03ca76 100644 --- a/common/utils.cpp +++ b/common/utils.cpp @@ -287,6 +287,7 @@ bool Utils::generateExcludePathsFile() in << "/swap_file" << endl; in << "/sys" << endl; //添加*(/sys/*),表示如果/sys目录不存在,则会拷贝/sys,但不会拷贝/sys下的内容 in << "/tmp" << endl; + in << "/var/lib/docker/overlay2" << endl; // 安卓兼容的这个里面很多文件都是设置了特殊扩展文件属性,lsetxattr无法设置成功,听取安卓兼容模块同事的意见不用管这个文件夹,其实是从home下挂载的 in << "/var/lib/kmre" << endl; in << "/var/lib/udisks2" << endl; @@ -343,6 +344,7 @@ QStringList Utils::getFromExcludePathsFile() list << "/swap_file"; list << "/sys"; list << "/tmp"; + list << "/var/lib/docker/overlay2"; list << "/var/lib/kmre"; list << "/var/lib/udisks2"; list << "/var/log";