[新增][定时任务]ApiAccessLog、ApiErrorLog、JobLog 3个日志的定时清理

This commit is contained in:
j-sentinel 2023-10-01 16:53:17 +08:00
parent 3875c80471
commit 11921fab5a
29 changed files with 391 additions and 0 deletions

View File

@ -1,9 +1,12 @@
package cn.iocoder.yudao.framework.quartz.config;
import cn.iocoder.yudao.framework.quartz.core.job.JobLogJobHandler;
import cn.iocoder.yudao.framework.quartz.core.job.LogJobProperties;
import cn.iocoder.yudao.framework.quartz.core.scheduler.SchedulerManager;
import lombok.extern.slf4j.Slf4j;
import org.quartz.Scheduler;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
@ -15,6 +18,7 @@ import java.util.Optional;
@AutoConfiguration
@EnableScheduling // 开启 Spring 自带的定时任务
@Slf4j
@EnableConfigurationProperties(LogJobProperties.class)
public class YudaoQuartzAutoConfiguration {
@Bean
@ -26,4 +30,9 @@ public class YudaoQuartzAutoConfiguration {
return new SchedulerManager(scheduler.get());
}
@Bean
public JobLogJobHandler jobLogJobHandler(LogJobProperties logJobProperties){
return new JobLogJobHandler(logJobProperties);
}
}

View File

@ -0,0 +1,33 @@
package cn.iocoder.yudao.framework.quartz.core.job;
import cn.iocoder.yudao.framework.quartz.core.handler.JobHandler;
import cn.iocoder.yudao.framework.quartz.core.service.JobLogFrameworkService;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.Resource;
/**
* @Author: j-sentinel
* @Date: 2023/9/30 20:40
*/
@Slf4j
@AllArgsConstructor
public class JobLogJobHandler implements JobHandler {
private LogJobProperties logJobProperties;
public JobLogJobHandler(LogJobProperties logJobProperties) {
this.logJobProperties = logJobProperties;
}
@Resource
private JobLogFrameworkService jobLogFrameworkService;
@Override
public String execute(String param) throws Exception {
Integer integer = jobLogFrameworkService.timingJobCleanLog(logJobProperties.getJobCleanRetainDay());
log.info("定时执行清理定时任务日志数量({})个",integer);
return String.format("定时执行清理定时任务日志数量 %s 个", integer);
}
}

View File

@ -0,0 +1,20 @@
package cn.iocoder.yudao.framework.quartz.core.job;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* @Author: j-sentinel
* @Date: 2023/9/30 16:17
*/
@Data
@ConfigurationProperties(prefix = "yudao.clean-job")
public class LogJobProperties {
private int accessRetainDay = 7;
private int errorRetainDay = 8;
private int jobCleanRetainDay = 7;
}

View File

@ -41,4 +41,10 @@ public interface JobLogFrameworkService {
@NotNull(message = "运行时长不能为空") Integer duration,
boolean success, String result);
/**
* 清理 @param jobCleanRetainDay 天的访问日志
*
* @param jobCleanRetainDay 超过多少天就进行清理
*/
Integer timingJobCleanLog(Integer jobCleanRetainDay);
}

View File

@ -61,6 +61,12 @@
<artifactId>jsoup</artifactId>
</dependency>
<!-- Job 定时任务相关 -->
<dependency>
<groupId>cn.iocoder.boot</groupId>
<artifactId>yudao-spring-boot-starter-job</artifactId>
</dependency>
</dependencies>
</project>

View File

@ -1,11 +1,14 @@
package cn.iocoder.yudao.framework.apilog.config;
import cn.iocoder.yudao.framework.apilog.core.filter.ApiAccessLogFilter;
import cn.iocoder.yudao.framework.apilog.core.job.ApiAccessLogJobHandler;
import cn.iocoder.yudao.framework.apilog.core.job.ApiErrorLogJobHandler;
import cn.iocoder.yudao.framework.apilog.core.service.ApiAccessLogFrameworkService;
import cn.iocoder.yudao.framework.apilog.core.service.ApiAccessLogFrameworkServiceImpl;
import cn.iocoder.yudao.framework.apilog.core.service.ApiErrorLogFrameworkService;
import cn.iocoder.yudao.framework.apilog.core.service.ApiErrorLogFrameworkServiceImpl;
import cn.iocoder.yudao.framework.common.enums.WebFilterOrderEnum;
import cn.iocoder.yudao.framework.quartz.core.job.LogJobProperties;
import cn.iocoder.yudao.framework.web.config.WebProperties;
import cn.iocoder.yudao.framework.web.config.YudaoWebAutoConfiguration;
import cn.iocoder.yudao.module.infra.api.logger.ApiAccessLogApi;
@ -31,6 +34,16 @@ public class YudaoApiLogAutoConfiguration {
return new ApiErrorLogFrameworkServiceImpl(apiErrorLogApi);
}
@Bean
public ApiAccessLogJobHandler apiAccessLogJobHandler(LogJobProperties logJobProperties) {
return new ApiAccessLogJobHandler(logJobProperties);
}
@Bean
public ApiErrorLogJobHandler apiErrorLogJobHandler(LogJobProperties logJobProperties) {
return new ApiErrorLogJobHandler(logJobProperties);
}
/**
* 创建 ApiAccessLogFilter Bean记录 API 请求日志
*/

View File

@ -0,0 +1,34 @@
package cn.iocoder.yudao.framework.apilog.core.job;
import cn.iocoder.yudao.framework.apilog.core.service.ApiAccessLogFrameworkService;
import cn.iocoder.yudao.framework.quartz.core.handler.JobHandler;
import cn.iocoder.yudao.framework.quartz.core.job.LogJobProperties;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.Resource;
/**
* @Author: j-sentinel
* @Date: 2023/9/30 16:13
*/
@Slf4j
@AllArgsConstructor
public class ApiAccessLogJobHandler implements JobHandler {
private LogJobProperties logJobProperties;
public ApiAccessLogJobHandler(LogJobProperties logJobProperties) {
this.logJobProperties = logJobProperties;
}
@Resource
private ApiAccessLogFrameworkService apiAccessLogFrameworkService;
@Override
public String execute(String param) throws Exception {
apiAccessLogFrameworkService.jobCleanAccessLog(logJobProperties.getAccessRetainDay());
return "";
}
}

View File

@ -0,0 +1,34 @@
package cn.iocoder.yudao.framework.apilog.core.job;
import cn.iocoder.yudao.framework.apilog.core.service.ApiErrorLogFrameworkService;
import cn.iocoder.yudao.framework.quartz.core.handler.JobHandler;
import cn.iocoder.yudao.framework.quartz.core.job.LogJobProperties;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.Resource;
/**
* @Author: j-sentinel
* @Date: 2023/9/30 16:13
*/
@Slf4j
@AllArgsConstructor
public class ApiErrorLogJobHandler implements JobHandler {
private LogJobProperties logJobProperties;
public ApiErrorLogJobHandler(LogJobProperties logJobProperties) {
this.logJobProperties = logJobProperties;
}
@Resource
private ApiErrorLogFrameworkService apiErrorLogFrameworkService;
@Override
public String execute(String param) throws Exception {
apiErrorLogFrameworkService.jobCleanErrorLog(logJobProperties.getErrorRetainDay());
return "";
}
}

View File

@ -14,4 +14,10 @@ public interface ApiAccessLogFrameworkService {
*/
void createApiAccessLog(ApiAccessLog apiAccessLog);
/**
* 清理 @param accessLogJobDay 天的访问日志
*
* @param accessLogJobDay 超过多少天就进行清理
*/
void jobCleanAccessLog(Integer accessLogJobDay);
}

View File

@ -25,4 +25,9 @@ public class ApiAccessLogFrameworkServiceImpl implements ApiAccessLogFrameworkSe
apiAccessLogApi.createApiAccessLog(reqDTO);
}
@Override
public void jobCleanAccessLog(Integer accessLogJobDay) {
apiAccessLogApi.jobCleanAccessLog(accessLogJobDay);
}
}

View File

@ -14,4 +14,10 @@ public interface ApiErrorLogFrameworkService {
*/
void createApiErrorLog(ApiErrorLog apiErrorLog);
/**
* 清理 @param errorLogJobDay 天的访问日志
*
* @param errorLogJobDay 超过多少天就进行清理
*/
void jobCleanErrorLog(Integer errorLogJobDay);
}

View File

@ -25,4 +25,9 @@ public class ApiErrorLogFrameworkServiceImpl implements ApiErrorLogFrameworkServ
apiErrorLogApi.createApiErrorLog(reqDTO);
}
@Override
public void jobCleanErrorLog(Integer errorLogJobDay) {
apiErrorLogApi.jobCleanErrorLog(errorLogJobDay);
}
}

View File

@ -18,4 +18,10 @@ public interface ApiAccessLogApi {
*/
void createApiAccessLog(@Valid ApiAccessLogCreateReqDTO createDTO);
/**
* 清理 @param accessLogJobDay 天的访问日志
*
* @param accessLogJobDay 超过多少天就进行清理
*/
void jobCleanAccessLog(Integer accessLogJobDay);
}

View File

@ -18,4 +18,10 @@ public interface ApiErrorLogApi {
*/
void createApiErrorLog(@Valid ApiErrorLogCreateReqDTO createDTO);
/**
* 清理 @param errorLogJobDay 天的访问日志
*
* @param errorLogJobDay 超过多少天就进行清理
*/
void jobCleanErrorLog(Integer errorLogJobDay);
}

View File

@ -35,6 +35,10 @@
<groupId>cn.iocoder.boot</groupId>
<artifactId>yudao-spring-boot-starter-biz-operatelog</artifactId>
</dependency>
<dependency>
<groupId>cn.iocoder.boot</groupId>
<artifactId>yudao-spring-boot-starter-biz-tenant</artifactId>
</dependency>
<!-- Web 相关 -->
<dependency>

View File

@ -24,4 +24,9 @@ public class ApiAccessLogApiImpl implements ApiAccessLogApi {
apiAccessLogService.createApiAccessLog(createDTO);
}
@Override
public void jobCleanAccessLog(Integer accessLogJobDay) {
apiAccessLogService.jobCleanAccessLog(accessLogJobDay);
}
}

View File

@ -24,4 +24,9 @@ public class ApiErrorLogApiImpl implements ApiErrorLogApi {
apiErrorLogService.createApiErrorLog(createDTO);
}
@Override
public void jobCleanErrorLog(Integer errorLogJobDay) {
apiErrorLogService.jobCleanErrorLog(errorLogJobDay);
}
}

View File

@ -7,6 +7,7 @@ import cn.iocoder.yudao.module.infra.controller.admin.job.vo.log.JobLogExportReq
import cn.iocoder.yudao.module.infra.controller.admin.job.vo.log.JobLogPageReqVO;
import cn.iocoder.yudao.module.infra.dal.dataobject.job.JobLogDO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@ -40,4 +41,7 @@ public interface JobLogMapper extends BaseMapperX<JobLogDO> {
);
}
Integer timingJobCleanLog(@Param("jobCleanRetainDay") Integer jobCleanRetainDay);
void optimizeTable();
}

View File

@ -7,6 +7,7 @@ import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apiaccesslog.Api
import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apiaccesslog.ApiAccessLogPageReqVO;
import cn.iocoder.yudao.module.infra.dal.dataobject.logger.ApiAccessLogDO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@ -44,4 +45,7 @@ public interface ApiAccessLogMapper extends BaseMapperX<ApiAccessLogDO> {
);
}
Integer jobCleanAccessLog(@Param("accessLogJobDay") Integer accessLogJobDay);
void optimizeTable();
}

View File

@ -7,6 +7,7 @@ import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apierrorlog.ApiE
import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apierrorlog.ApiErrorLogPageReqVO;
import cn.iocoder.yudao.module.infra.dal.dataobject.logger.ApiErrorLogDO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
@ -42,4 +43,7 @@ public interface ApiErrorLogMapper extends BaseMapperX<ApiErrorLogDO> {
);
}
Integer jobCleanErrorLog(@Param("errorLogJobDay") Integer errorLogJobDay);
void optimizeTable();
}

View File

@ -26,6 +26,8 @@ import java.util.List;
@Slf4j
public class JobLogServiceImpl implements JobLogService {
private static final Integer DELETE_LIMIT = 1;
@Resource
private JobLogMapper jobLogMapper;
@ -50,6 +52,22 @@ public class JobLogServiceImpl implements JobLogService {
}
}
@Override
public Integer timingJobCleanLog(Integer jobCleanRetainDay) {
Integer result = null;
int count = 0;
while (result == null || DELETE_LIMIT.equals(result)){
result = jobLogMapper.timingJobCleanLog(jobCleanRetainDay);
count += result;
}
if(count > 0){
// ALTER TABLE...FORCE 会导致表重建发生,这会根据主键索引对表空间中的物理页进行排序
// 它将行压缩到页面上并消除可用空间同时确保数据处于主键查找的最佳顺序
jobLogMapper.optimizeTable();
}
return count;
}
@Override
public JobLogDO getJobLog(Long id) {
return jobLogMapper.selectById(id);

View File

@ -38,4 +38,10 @@ public interface ApiAccessLogService {
*/
List<ApiAccessLogDO> getApiAccessLogList(ApiAccessLogExportReqVO exportReqVO);
/**
* 清理 @param accessLogJobDay 天的访问日志
*
* @param accessLogJobDay 超过多少天就进行清理
*/
void jobCleanAccessLog(Integer accessLogJobDay);
}

View File

@ -1,12 +1,14 @@
package cn.iocoder.yudao.module.infra.service.logger;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.tenant.core.util.TenantUtils;
import cn.iocoder.yudao.module.infra.api.logger.dto.ApiAccessLogCreateReqDTO;
import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apiaccesslog.ApiAccessLogExportReqVO;
import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apiaccesslog.ApiAccessLogPageReqVO;
import cn.iocoder.yudao.module.infra.convert.logger.ApiAccessLogConvert;
import cn.iocoder.yudao.module.infra.dal.dataobject.logger.ApiAccessLogDO;
import cn.iocoder.yudao.module.infra.dal.mysql.logger.ApiAccessLogMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
@ -18,10 +20,13 @@ import java.util.List;
*
* @author 芋道源码
*/
@Slf4j
@Service
@Validated
public class ApiAccessLogServiceImpl implements ApiAccessLogService {
private static final Integer DELETE_LIMIT = 100;
@Resource
private ApiAccessLogMapper apiAccessLogMapper;
@ -41,4 +46,22 @@ public class ApiAccessLogServiceImpl implements ApiAccessLogService {
return apiAccessLogMapper.selectList(exportReqVO);
}
@Override
public void jobCleanAccessLog(Integer accessLogJobDay) {
TenantUtils.executeIgnore(() -> {
Integer result = null;
int count = 0;
while (result == null || DELETE_LIMIT.equals(result)) {
result = apiAccessLogMapper.jobCleanAccessLog(accessLogJobDay);
count += result;
}
if (count > 0) {
// ALTER TABLE...FORCE 会导致表重建发生,这会根据主键索引对表空间中的物理页进行排序
// 它将行压缩到页面上并消除可用空间同时确保数据处于主键查找的最佳顺序
apiAccessLogMapper.optimizeTable();
}
log.info("定时执行清理访问日志数量({})个", count);
});
}
}

View File

@ -47,4 +47,10 @@ public interface ApiErrorLogService {
*/
void updateApiErrorLogProcess(Long id, Integer processStatus, Long processUserId);
/**
* 清理 @param errorLogJobDay 天的访问日志
*
* @param errorLogJobDay 超过多少天就进行清理
*/
void jobCleanErrorLog(Integer errorLogJobDay);
}

View File

@ -1,6 +1,7 @@
package cn.iocoder.yudao.module.infra.service.logger;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.tenant.core.util.TenantUtils;
import cn.iocoder.yudao.module.infra.api.logger.dto.ApiErrorLogCreateReqDTO;
import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apierrorlog.ApiErrorLogExportReqVO;
import cn.iocoder.yudao.module.infra.controller.admin.logger.vo.apierrorlog.ApiErrorLogPageReqVO;
@ -8,6 +9,7 @@ import cn.iocoder.yudao.module.infra.convert.logger.ApiErrorLogConvert;
import cn.iocoder.yudao.module.infra.dal.dataobject.logger.ApiErrorLogDO;
import cn.iocoder.yudao.module.infra.dal.mysql.logger.ApiErrorLogMapper;
import cn.iocoder.yudao.module.infra.enums.logger.ApiErrorLogProcessStatusEnum;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
@ -24,10 +26,13 @@ import static cn.iocoder.yudao.module.infra.enums.ErrorCodeConstants.API_ERROR_L
*
* @author 芋道源码
*/
@Slf4j
@Service
@Validated
public class ApiErrorLogServiceImpl implements ApiErrorLogService {
private static final Integer DELETE_LIMIT = 100;
@Resource
private ApiErrorLogMapper apiErrorLogMapper;
@ -62,4 +67,22 @@ public class ApiErrorLogServiceImpl implements ApiErrorLogService {
.processUserId(processUserId).processTime(LocalDateTime.now()).build());
}
@Override
public void jobCleanErrorLog(Integer errorLogJobDay) {
TenantUtils.executeIgnore(() -> {
Integer result = null;
int count = 0;
while (result == null || DELETE_LIMIT.equals(result)) {
result = apiErrorLogMapper.jobCleanErrorLog(errorLogJobDay);
count += result;
}
if (count > 0) {
// ALTER TABLE...FORCE 会导致表重建发生,这会根据主键索引对表空间中的物理页进行排序
// 它将行压缩到页面上并消除可用空间同时确保数据处于主键查找的最佳顺序
apiErrorLogMapper.optimizeTable();
}
log.info("定时执行清理错误日志数量({})个",count);
});
}
}

View File

@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cn.iocoder.yudao.module.infra.dal.mysql.logger.ApiAccessLogMapper">
<!--
一般情况下,尽可能使用 Mapper 进行 CRUD 增删改查即可。
无法满足的场景,例如说多表关联查询,才使用 XML 编写 SQL。
代码生成器暂时只生成 Mapper XML 文件本身,更多推荐 MybatisX 快速开发插件来生成查询。
文档可见https://www.iocoder.cn/MyBatis/x-plugins/
-->
<delete id="jobCleanAccessLog" parameterType="integer">
DELETE FROM infra_api_access_log
WHERE id IN (
SELECT * FROM (
SELECT id FROM infra_api_access_log
WHERE create_time &lt; DATE_SUB(CURDATE(), INTERVAL #{accessLogJobDay} DAY)
ORDER BY id ASC LIMIT 100
) AS a
)
</delete>
<select id="optimizeTable">
ALTER TABLE infra_job_log FORCE
/*
ALTER TABLE infra_job_log ENGINE = INNODB,
ALGORITHM = INPLACE,
LOCK = NONE
*/
</select>
</mapper>

View File

@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cn.iocoder.yudao.module.infra.dal.mysql.logger.ApiErrorLogMapper">
<!--
一般情况下,尽可能使用 Mapper 进行 CRUD 增删改查即可。
无法满足的场景,例如说多表关联查询,才使用 XML 编写 SQL。
代码生成器暂时只生成 Mapper XML 文件本身,更多推荐 MybatisX 快速开发插件来生成查询。
文档可见https://www.iocoder.cn/MyBatis/x-plugins/
-->
<delete id="jobCleanErrorLog" parameterType="integer">
DELETE FROM infra_api_error_log
WHERE id IN (
SELECT * FROM (
SELECT id FROM infra_api_error_log
WHERE create_time &lt; DATE_SUB(CURDATE(), INTERVAL #{errorLogJobDay} DAY)
ORDER BY id ASC LIMIT 100
) AS a
)
</delete>
<select id="optimizeTable">
ALTER TABLE infra_api_error_log FORCE
/*
ALTER TABLE infra_api_error_log ENGINE = INNODB,
ALGORITHM = INPLACE,
LOCK = NONE
*/
</select>
</mapper>

View File

@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cn.iocoder.yudao.module.infra.dal.mysql.job.JobLogMapper">
<!--
一般情况下,尽可能使用 Mapper 进行 CRUD 增删改查即可。
无法满足的场景,例如说多表关联查询,才使用 XML 编写 SQL。
代码生成器暂时只生成 Mapper XML 文件本身,更多推荐 MybatisX 快速开发插件来生成查询。
文档可见https://www.iocoder.cn/MyBatis/x-plugins/
-->
<delete id="timingJobCleanLog" parameterType="integer">
DELETE FROM infra_job_log
WHERE id IN (
SELECT * FROM (
SELECT id FROM infra_job_log
WHERE create_time &lt; DATE_SUB(CURDATE(), INTERVAL #{jobCleanRetainDay} DAY)
ORDER BY id ASC LIMIT 100
) AS a
)
</delete>
<select id="optimizeTable">
ALTER TABLE infra_job_log FORCE
/*
ALTER TABLE infra_job_log ENGINE = INNODB,
ALGORITHM = INPLACE,
LOCK = NONE
*/
</select>
</mapper>

View File

@ -213,6 +213,10 @@ yudao:
error-code: # 错误码相关配置项
enable: false
demo: false # 关闭演示模式
clean-job:
access-retain-day: 7
error-retain-day: 8
job-clean-retain-day: 8
justauth:
enabled: true