p78659104
/
ossean
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

gather_program solved some bugs

This commit is contained in:
zhanyun 2016-10-17 20:58:01 +08:00
parent 79141911f1
commit 91c0f0ce66
4 changed files with 40 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
gather_program/bin/resources/log4j.xml
View File

@ -2,60 +2,39 @@
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> <!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"> <log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender"> <appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout"> <layout class="org.apache.log4j.PatternLayout">
<param name="threshold" value="ERROR" /> <param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" /> </layout>
</layout> </appender>
</appender>
<!-- 邮件（只有ERROR时才会发送！） --> <appender name="MAIL" class="org.apache.log4j.net.SMTPAppender">
<appender name="MAIL" class="org.apache.log4j.net.SMTPAppender"> <param name="threshold" value="info" />
<param name="threshold" value="debug" /> <param name="BufferSize" value="512" />
<!-- 日志的错误级别 <param name="threshold" value="fatal"/> --> <param name="From" value="gcm365111@126.com" />
<!-- 缓存文件大小，日志达到512K时发送Email --> <param name="SMTPHost" value="SMTP.126.com" />
<param name="BufferSize" value="1" /> <param name="Subject" value="this is test" />
<!-- 单位K --> <param name="SMTPUsername" value="gcm365111@126.com" />
<param name="From" value="ossean_debug@163.com" /> <param name="SMTPPassword" value="03023651gcm" />
<param name="SMTPHost" value="smtp.163.com" /> <param name="to" value="gcm3651@126.com" />
<param name="Subject" value="ossean-crawler-debug-log4jMessage" /> <layout class="org.apache.log4j.PatternLayout">
<param name="To" value="cloud_zhan@163.com" /> <param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
<param name="SMTPUsername" value="ossean_debug" /> </layout>
<param name="SMTPPassword" value="goodwell123" /> </appender>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%-d{yyyy-MM-dd HH:mm:ss.SSS} [%p]-[%c] %m%n" />
</layout>
</appender>
<!-- 数据库状态--> <appender name="file" class="org.apache.log4j.DailyRollingFileAppender">
<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender"> <param name="File" value="./log/info.log" />
<param name="URL" value="jdbc:mysql://127.0.0.1:3306/webmagic?characterEncoding=UTF-8"/> <layout class="org.apache.log4j.PatternLayout">
<param name="driver" value="com.mysql.jdbc.Driver"/> <param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
<param name="user" value="root"/> </layout>
<param name="password" value="1234"/> </appender>
<param name="sql" value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES ('%d{yyyy-MM-dd HH:mm:ss}','%t','%p','%c','%m','%l')"/>
<!-- <layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES ('%d{yyyy-MM-dd HH:mm:ss}','%t','%.50p','%.50c','%.1000m','%.50l')" />
</layout>-->
<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都 FATAL，所以输出DEBUG级别到FATAL级别的LOG数据-->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="LevelMin" value="DEBUG"/>
<param name="LevelMax" value="FATAL"/>
</filter>
</appender>
<logger name="org.apache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="info" /> <root>
<appender-ref ref="stdout" /> <level value="info" />
<appender-ref ref="file" /> <appender-ref ref="file" />
<appender-ref ref="file_log" /> <appender-ref ref="stdout" />
<!-- <appender-ref ref="MAIL" />--> </root>
<!-- <appender-ref ref="DATABASE" /> -->
</root>
</log4j:configuration> </log4j:configuration>

2
gather_program/src/main/java/org/ossean/gather/process/GatherThread.java
View File

@ -78,7 +78,7 @@ public class GatherThread implements Runnable {
idsIncrement = conf.getIdsIncrement(); idsIncrement = conf.getIdsIncrement();
idsBegin = readPointer(conf.getPointerTableName(), sourceTableName, idsBegin = readPointer(conf.getPointerTableName(), sourceTableName,
conf.getTargetTableName()); conf.getTargetTableName());
idsEnd = maxId = pointerDao.getMaxId(sourceTableName); idsEnd = maxId = gatherDao.getMaxId(sourceTableName);
while (idsBegin < idsEnd) { while (idsBegin < idsEnd) {
beginId = idsBegin; beginId = idsBegin;
endId = beginId + idsIncrement - 1; // 取数据时两边都取等号 endId = beginId + idsIncrement - 1; // 取数据时两边都取等号

14
gather_program/src/main/java/org/ossean/gather/targetDao/PointerDao.java
View File

@ -7,13 +7,13 @@ import org.apache.ibatis.annotations.Update;
public interface PointerDao { public interface PointerDao {
// 读取表中最小id值 // // 读取表中最小id值
@Select("select min(id) from ${table}") // @Select("select min(id) from ${table}")
public int getMinId(@Param("table") String table); // public int getMinId(@Param("table") String table);
//
// 读取表中最大id值 // // 读取表中最大id值
@Select("select max(id) from ${table}") // @Select("select max(id) from ${table}")
public int getMaxId(@Param("table") String table); // public int getMaxId(@Param("table") String table);
@Select("select Pointer from ${table} where SourceTableName=#{SourceTableName} and TargetTableName=#{TargetTableName}") @Select("select Pointer from ${table} where SourceTableName=#{SourceTableName} and TargetTableName=#{TargetTableName}")
public int getPointer(@Param("table") String table, public int getPointer(@Param("table") String table,

4
gather_program/src/main/resource/applicationContext-myBatis.xml
View File

@ -21,7 +21,7 @@
<property name="url" <property name="url"
value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull&amp;autoReconnect=true" /> value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull&amp;autoReconnect=true" />
<property name="username" value="root" /> <property name="username" value="root" />
<property name="password" value="1234" /> <property name="password" value="123456" />
<property name="validationQuery" value="SELECT 1" /> <property name="validationQuery" value="SELECT 1" />
<property name="testOnBorrow" value="true"/> <property name="testOnBorrow" value="true"/>
</bean> </bean>
@ -41,7 +41,7 @@
<property name="url" <property name="url"
value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull&amp;autoReconnect=true" /> value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull&amp;autoReconnect=true" />
<property name="username" value="root" /> <property name="username" value="root" />
<property name="password" value="1234" /> <property name="password" value="123456" />
<property name="validationQuery" value="SELECT 1" /> <property name="validationQuery" value="SELECT 1" />
<property name="testOnBorrow" value="true"/> <property name="testOnBorrow" value="true"/>
</bean> </bean>