p78659104
/
ossean
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

gather_program solved some bugs

This commit is contained in:
zhanyun 2016-10-17 20:58:01 +08:00
parent 79141911f1
commit 91c0f0ce66
4 changed files with 40 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
gather_program/bin/resources/log4j.xml
View File

@ -2,60 +2,39 @@
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="threshold" value="ERROR" />
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<!-- 邮件（只有ERROR时才会发送！） -->
<appender name="MAIL" class="org.apache.log4j.net.SMTPAppender">
<param name="threshold" value="debug" />
<!-- 日志的错误级别 <param name="threshold" value="fatal"/> -->
<!-- 缓存文件大小，日志达到512K时发送Email -->
<param name="BufferSize" value="1" />
<!-- 单位K -->
<param name="From" value="ossean_debug@163.com" />
<param name="SMTPHost" value="smtp.163.com" />
<param name="Subject" value="ossean-crawler-debug-log4jMessage" />
<param name="To" value="cloud_zhan@163.com" />
<param name="SMTPUsername" value="ossean_debug" />
<param name="SMTPPassword" value="goodwell123" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%-d{yyyy-MM-dd HH:mm:ss.SSS} [%p]-[%c] %m%n" />
</layout>
</appender>
<appender name="MAIL" class="org.apache.log4j.net.SMTPAppender">
<param name="threshold" value="info" />
<param name="BufferSize" value="512" />
<param name="From" value="gcm365111@126.com" />
<param name="SMTPHost" value="SMTP.126.com" />
<param name="Subject" value="this is test" />
<param name="SMTPUsername" value="gcm365111@126.com" />
<param name="SMTPPassword" value="03023651gcm" />
<param name="to" value="gcm3651@126.com" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<!-- 数据库状态-->
<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
<param name="URL" value="jdbc:mysql://127.0.0.1:3306/webmagic?characterEncoding=UTF-8"/>
<param name="driver" value="com.mysql.jdbc.Driver"/>
<param name="user" value="root"/>
<param name="password" value="1234"/>
<param name="sql" value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES ('%d{yyyy-MM-dd HH:mm:ss}','%t','%p','%c','%m','%l')"/>
<!-- <layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES ('%d{yyyy-MM-dd HH:mm:ss}','%t','%.50p','%.50c','%.1000m','%.50l')" />
</layout>-->
<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都 FATAL，所以输出DEBUG级别到FATAL级别的LOG数据-->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="LevelMin" value="DEBUG"/>
<param name="LevelMax" value="FATAL"/>
</filter>
</appender>
<appender name="file" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="./log/info.log" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<logger name="org.apache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="info" />
<appender-ref ref="stdout" />
<appender-ref ref="file" />
<appender-ref ref="file_log" />
<!-- <appender-ref ref="MAIL" />-->
<!-- <appender-ref ref="DATABASE" /> -->
</root>
<root>
<level value="info" />
<appender-ref ref="file" />
<appender-ref ref="stdout" />
</root>
</log4j:configuration>

2
gather_program/src/main/java/org/ossean/gather/process/GatherThread.java
View File

@ -78,7 +78,7 @@ public class GatherThread implements Runnable {
idsIncrement = conf.getIdsIncrement();
idsBegin = readPointer(conf.getPointerTableName(), sourceTableName,
conf.getTargetTableName());
idsEnd = maxId = pointerDao.getMaxId(sourceTableName);
idsEnd = maxId = gatherDao.getMaxId(sourceTableName);
while (idsBegin < idsEnd) {
beginId = idsBegin;
endId = beginId + idsIncrement - 1; // 取数据时两边都取等号

14
gather_program/src/main/java/org/ossean/gather/targetDao/PointerDao.java
View File

@ -7,13 +7,13 @@ import org.apache.ibatis.annotations.Update;
public interface PointerDao {
// 读取表中最小id值
@Select("select min(id) from ${table}")
public int getMinId(@Param("table") String table);
// 读取表中最大id值
@Select("select max(id) from ${table}")
public int getMaxId(@Param("table") String table);
// // 读取表中最小id值
// @Select("select min(id) from ${table}")
// public int getMinId(@Param("table") String table);
//
// // 读取表中最大id值
// @Select("select max(id) from ${table}")
// public int getMaxId(@Param("table") String table);
@Select("select Pointer from ${table} where SourceTableName=#{SourceTableName} and TargetTableName=#{TargetTableName}")
public int getPointer(@Param("table") String table,

4
gather_program/src/main/resource/applicationContext-myBatis.xml
View File

@ -21,7 +21,7 @@
<property name="url"
value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull&amp;autoReconnect=true" />
<property name="username" value="root" />
<property name="password" value="1234" />
<property name="password" value="123456" />
<property name="validationQuery" value="SELECT 1" />
<property name="testOnBorrow" value="true"/>
</bean>
@ -41,7 +41,7 @@
<property name="url"
value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull&amp;autoReconnect=true" />
<property name="username" value="root" />
<property name="password" value="1234" />
<property name="password" value="123456" />
<property name="validationQuery" value="SELECT 1" />
<property name="testOnBorrow" value="true"/>
</bean>