openkylin
/
yhkylin-backup-tools
mirror of https://gitee.com/openkylin/yhkylin-backup-tools.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

再提一把

This commit is contained in:
zhaominyong 2021-08-19 19:24:49 +08:00
parent 272456a31d
commit ac0d321be6
8 changed files with 240 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backup-daemon/backup-daemon.pro
View File

@ -29,6 +29,7 @@ HEADERS += \
backupmanager_adaptor.h \
mybackupmanager.h \
mymountproxy.h \
myprocess/calcbackupsize.h \
myprocess/mountbackupprocess.h \
parsebackuplist.h \
systembackupproxy.h \
@ -44,6 +45,7 @@ SOURCES += \
main.cpp \
mybackupmanager.cpp \
mymountproxy.cpp \
myprocess/calcbackupsize.cpp \
myprocess/mountbackupprocess.cpp \
parsebackuplist.cpp \
systembackupproxy.cpp \

101
backup-daemon/myprocess/calcbackupsize.cpp Normal file
View File

@ -0,0 +1,101 @@
#include "calcbackupsize.h"
#include <QDebug>
CalcBackupSize::CalcBackupSize(QObject* parent) :
QObject(parent),
m_process(new QProcess(this))
{
connect(m_process, &QProcess::readyReadStandardError, this, [&]() {
QByteArray err = m_process->readAllStandardError();
qCritical("backup process error: %s", err.data());
});
connect(m_process, &QProcess::errorOccurred, this, [&](QProcess::ProcessError error) {
m_process->kill();
});
connect(m_process, SIGNAL(finished(int, QProcess::ExitStatus)), this, SLOT(getCalcResult(int, QProcess::ExitStatus)));
}
CalcBackupSize::~CalcBackupSize()
{
}
/**
* @brief 计算备份大小，单位字节
* @param args rsync参数列表
* @param block 是否阻塞计算模式。默认true——阻塞计算模式
* @return block为true时返回值为待备份数据大小，单位字节；block为false时，默认返回0;
*/
qint64 CalcBackupSize::calcBackupSize(QStringList args, bool block)
{
QString cmd("rsync ");
for (const QString& item : args) {
cmd += " ";
cmd += item;
}
qDebug() << cmd;
m_size = 0;
m_process->start("rsync", args);
if (!m_process->waitForStarted()) {
qCritical("rsync start failed");
return m_size;
}
if (block) {
m_process->waitForFinished();
}
return m_size;
}
/**
* @brief 解析结果
* @param exitCode
*/
void CalcBackupSize::getCalcResult(int exitCode, QProcess::ExitStatus)
{
// 解析结果，结果内容样例如下：
/*
Number of files: 8 (reg: 4, dir: 4)
Number of created files: 7 (reg: 4, dir: 3)
Number of deleted files: 0
Number of regular files transferred: 4
Total file size: 20 bytes
Total transferred file size: 20 bytes
Literal data: 0 bytes
Matched data: 0 bytes
File list size: 0
File list generation time: 0.001 seconds
File list transfer time: 0.000 seconds
Total bytes sent: 248
Total bytes received: 43
sent 248 bytes received 43 bytes 582.00 bytes/sec
total size is 20 speedup is 0.07 (DRY RUN)
*/
QString out(m_process->readAll());
QStringList lines = out.split("\n");
for (QString& line : lines) {
line.trimmed();
// 获取文件夹数目
if (line.startsWith("Number of files:")) {
int indexOfLastColon = line.lastIndexOf(":");
int indexOfLastLable = line.lastIndexOf(")");
int numOfDirs = line.mid(indexOfLastColon+1, indexOfLastLable-indexOfLastColon).trimmed().toInt();
// 每个目录下还都有.和..，故总数还要*3
numOfDirs *= 3;
// ext4格式的目录本身要占用4K空间（4096）
m_size += numOfDirs * 4096;
}
if (line.startsWith("Total file size:")) {
m_size += line.replace("Total file size:", "").replace("bytes", "").trimmed().toLongLong();
break ;
}
}
emit calcFinished(m_size);
}

38
backup-daemon/myprocess/calcbackupsize.h Normal file
View File

@ -0,0 +1,38 @@
#ifndef CALCBACKUPSIZE_H
#define CALCBACKUPSIZE_H
#include <QStringList>
#include <QProcess>
#define GB (1024 * 1024 * 1024)
#define MB (1024 * 1024)
#define KB (1024)
class CalcBackupSize : public QObject
{
Q_OBJECT
public:
CalcBackupSize(QObject* parent = nullptr);
~CalcBackupSize();
/**
* @brief 计算备份大小，单位字节
* @param args rsync参数列表
* @param block 是否阻塞计算模式。默认true——阻塞计算模式
* @return block为true时返回值为待备份数据大小，单位字节；block为false时，默认返回0
*/
qint64 calcBackupSize(QStringList args, bool block = true);
signals:
void calcFinished(qint64 size);
private slots:
void getCalcResult(int exitCode, QProcess::ExitStatus);
private:
QProcess * m_process;
qint64 m_size = 0; // 备份大小，单位字节
};
#endif // CALCBACKUPSIZE_H

101
backup-daemon/systembackupproxy.cpp
View File

@ -5,6 +5,7 @@
#include "../common/mydusizetool.h"
#include "mymountproxy.h"
#include "parsebackuplist.h"
#include "myprocess/calcbackupsize.h"
IMPLEMENT_DYNCREATE(SystemBackupProxy)
@ -29,11 +30,10 @@ void SystemBackupProxy::checkEnvEx()
}
// 2、检测备份是否增量备份
bool bInc = checkIsIncBackup();
m_backupWrapper.m_bIncrement = bInc;
checkIsIncBackup();
// 3、检测空间是否满足备份
checkFreeCapacity(bInc);
checkFreeCapacity();
qDebug() << "SystemBackupProxy::checkEnv invoke end";
}
@ -51,11 +51,12 @@ void SystemBackupProxy::cancelEx()
bool SystemBackupProxy::checkIsIncBackup()
{
QString backupPath;
ParseBackupList::BackupPoint point;
if (m_backupWrapper.m_uuid.isEmpty()) {
QString xmlPath(Utils::getSysRootPath() + BACKUP_XML_PATH);
xmlPath.replace("//", "/");
ParseBackupList parser(xmlPath);
ParseBackupList::BackupPoint point = parser.getLastSysBackupPoint();
point = parser.getLastSysBackupPoint();
if (point.m_uuid.isEmpty())
return false;
backupPath = Utils::getSysRootPath() + BACKUP_SNAPSHOTS_PATH + "/" + point.m_uuid + "/data";
@ -64,30 +65,70 @@ bool SystemBackupProxy::checkIsIncBackup()
}
backupPath.replace("//", "/");
return Utils::isDirExist(backupPath);
if (Utils::isDirExist(backupPath)) {
m_backupWrapper.m_baseUuid = point.m_uuid;
m_backupWrapper.m_bIncrement = true;
return true;
}
return false;
}
/**
* @brief 校验剩余空间是否满足备份
* @param bInc，是否增量备份
* @note 增量备份和全量备份计算空间方法不同
*/
void SystemBackupProxy::checkFreeCapacity(bool bInc)
void SystemBackupProxy::checkFreeCapacity()
{
// 1、计算待备份数据的大小
MyDuSizeTool du;
qint64 itotalSize = du.Do(m_backupWrapper.m_backupPaths, m_backupWrapper.m_backupExcludePaths, true);
if (-1 == itotalSize) {
qCritical() << "du system backup failed";
emit checkResult(int(BackupResult::DU_ERR));
return ;
}
qint64 itotalSize = calcSizeForBackup();
// 备份过程中会有一些临时文件产生，会占用一部分空间，故我们预留500M的空间
itotalSize += 500 * MB;
// 2、计算备份分区剩余空间大小
QString backupPath(Utils::getSysRootPath() + BACKUP_PATH);
backupPath.replace("//", "/");
QStorageInfo backupDisk(backupPath);
qint64 freeSize = backupDisk.bytesFree();
// 3、校验空间是否足够
if (itotalSize > freeSize) {
emit checkResult(int(BackupResult::BACKUP_CAPACITY_IS_NOT_ENOUGH));
return ;
}
}
/**
* @brief 计算备份所需空间大小
* @return 计算备份所需空间大小,单位字节
*/
qint64 SystemBackupProxy::calcSizeForBackup()
{
QStringList args;
if (m_backupWrapper.m_bIncrement) {
if (m_backupWrapper.m_uuid.isEmpty()) {
// 新增增量备份点场景
args = getRsyncArgs(SystemBackupScene::TRY_INC_SYSTEM_BACKUP);
} else {
// 在原来的备份点上增量备份场景
args = getRsyncArgs(SystemBackupScene::TRY_INC_SYSTEM_BACKUP_AT_BASE);
}
} else {
// 全量备份场景
args = getRsyncArgs(SystemBackupScene::TRY_SYSTEM_BACKUP);
}
// 拼接备份源路径和目标路径
QString srcPath = Utils::getSysRootPath();
srcPath += "/";
srcPath.replace("//", "/");
args << srcPath;
QString destPath = Utils::getSysRootPath();
destPath += CHECK_PATH;
destPath.replace("//", "/");
args << destPath;
Utils::mkpath(destPath);
CalcBackupSize calcator;
return calcator.calcBackupSize(args);
}
/**
@ -103,22 +144,44 @@ QStringList SystemBackupProxy::getRsyncArgs(SystemBackupScene scene)
case SystemBackupScene::SYSTEM_BACKUP :
args << "-avAXW";
args << "--progress";
args << "--ignore-missing-args";
break ;
case SystemBackupScene::INC_SYSTEM_BACKUP :
args << "-avAXr";
args << "--progress";
args << "--link-dest";
args << "--ignore-missing-args";
args << QString("--link-dest=../%1/data").arg(m_baseUuid);
break ;
case SystemBackupScene::INC_SYSTEM_BACKUP_AT_BASE :
args << "-avAXr";
args << "--progress";
args << "--ignore-missing-args";
args << "--delete";
break ;
case SystemBackupScene::TRY_SYSTEM_BACKUP :
args << "-avAXWn";
args << "-aAXWn";
args << "--stats";
args << "--ignore-missing-args";
break ;
case SystemBackupScene::TRY_INC_SYSTEM_BACKUP :
args << "-avAXrn";
args << "-aAXrn";
args << "--stats";
args << "--link-dest";
args << "--ignore-missing-args";
args << QString("--link-dest=../%1/data").arg(m_baseUuid);
break ;
case SystemBackupScene::TRY_INC_SYSTEM_BACKUP_AT_BASE :
args << "-aAXrn";
args << "--stats";
args << "--ignore-missing-args";
args << "--delete";
break ;
default:
return args;
}
// --exclude=排除路径设置
for (const QString & item : m_backupWrapper.m_backupExcludePaths) {
args << QString("--exclude=%1").arg(item);
}
return args;

11
backup-daemon/systembackupproxy.h
View File

@ -12,8 +12,10 @@ public:
enum SystemBackupScene {
SYSTEM_BACKUP, // 系统备份
INC_SYSTEM_BACKUP, // 增量系统备份
INC_SYSTEM_BACKUP_AT_BASE, // 在原备份点里增量备份
TRY_SYSTEM_BACKUP, // 测试系统备份，可用于计算备份传输数据大小
TRY_INC_SYSTEM_BACKUP, // 测试增量系统备份，可用于计算备份传输数据大小
TRY_INC_SYSTEM_BACKUP_AT_BASE, // 测试增量系统备份，在原备份点里增量备份，可用于计算备份传输数据大小
};
explicit SystemBackupProxy();
@ -34,17 +36,22 @@ private:
bool checkIsIncBackup();
// 校验剩余空间是否满足备份
void checkFreeCapacity(bool bInc);
void checkFreeCapacity();
// 计算备份所需空间大小
qint64 calcSizeForBackup();
/**
* @brief 根据场景获取rsync命令参数
* @param scene，场景
* @return 组装好的rsync的参数信息
*/
QString getRsyncArgs(SystemBackupScene scene);
QStringList getRsyncArgs(SystemBackupScene scene);
// 是否增量备份
bool m_bIncrement = false;
// 基准备份点，新增增量备份点时用
QString m_baseUuid;
};
#endif // SYSTEMBACKUPPROXY_H

3
common/mydefine.h
View File

@ -12,6 +12,7 @@
#define BACKUP_SNAPSHOTS_PATH "/backup/snapshots"
#define BACKUP_XML_PATH "/backup/snapshots/backuplist.xml"
#define EXCLUDE_FILE_PATH "/backup/snapshots/.exclude"
#define CHECK_PATH "/backup/snapshots/check/data/"
#define BOOTINFO_PATH "/etc/.bootinfo"
#define FSTAB_PATH "/etc/fstab"
@ -82,6 +83,8 @@ struct BackupWrapper {
int m_gid = -1;
// 是否增量备份
bool m_bIncrement = false;
// 新增备份点时增量备份的基准uuid
QString m_baseUuid;
static void registerMetaType();
};

6
common/mydusizetool.cpp
View File

@ -51,12 +51,12 @@ qint64 MyDuSizeTool::_Do(QStringList paths, QStringList excludePaths, bool needE
}
QString cmd = "du";
for (QString& x : paths) {
for (const QString& x : paths) {
cmd.append(QString(" \"%1\"").arg(x));
}
if (needExclude) {
for (QString& item : excludePaths) {
QString arg = QString(" --exclude=\"%1\"").arg(x);
for (const QString& item : excludePaths) {
QString arg = QString(" --exclude=\"%1\"").arg(item);
cmd.append(arg);
}
}

2
common/utils.cpp
View File

@ -287,6 +287,7 @@ bool Utils::generateExcludePathsFile()
in << "/swap_file" << endl;
in << "/sys" << endl; //添加*(/sys/*)，表示如果/sys目录不存在，则会拷贝/sys，但不会拷贝/sys下的内容
in << "/tmp" << endl;
in << "/var/lib/docker/overlay2" << endl;
// 安卓兼容的这个里面很多文件都是设置了特殊扩展文件属性，lsetxattr无法设置成功，听取安卓兼容模块同事的意见不用管这个文件夹，其实是从home下挂载的
in << "/var/lib/kmre" << endl;
in << "/var/lib/udisks2" << endl;
@ -343,6 +344,7 @@ QStringList Utils::getFromExcludePathsFile()
list << "/swap_file";
list << "/sys";
list << "/tmp";
list << "/var/lib/docker/overlay2";
list << "/var/lib/kmre";
list << "/var/lib/udisks2";
list << "/var/log";