initializeHEADmain

3 files changed, 263 insertions, 0 deletions

diff --git a/src/main/java/com/mesasoft/cn/sketch/config/ApplicationConfig.java b/src/main/java/com/mesasoft/cn/sketch/config/ApplicationConfig.java
new file mode 100644
index 0000000..099205e
--- /dev/null
+++ b/src/main/java/com/mesasoft/cn/sketch/config/ApplicationConfig.java
@@ -0,0 +1,54 @@
+package com.mesasoft.cn.sketch.config;
+
+
+import com.mesasoft.cn.util.ConfigUtils;
+
+public class ApplicationConfig {
+    public static final String QUERY_OUTPUT_DIR = ConfigUtils.getStringProperty("query.output.dir");
+
+    public static final String QUERY_TYPES_DOMAIN = ConfigUtils.getStringProperty("query.types.domain");
+    public static final String QUERY_TYPES_IP = ConfigUtils.getStringProperty("query.types.ip");
+
+    public static final Integer UPDATE_EXPIRED_DAY = ConfigUtils.getIntProperty("update.expired.day"); // 更新任务中过期时间长度（天数）
+
+    public static final Integer QUERY_READIN_BATCH = ConfigUtils.getIntProperty("query.readin.batch");
+    public static final Integer QUERY_LOG_FILE_LINE_INTERVAL = ConfigUtils.getIntProperty("query.log.file.line.interval"); // 文件查询时，打印log的读取行数间隔
+
+    // api参数
+    // brightcloud
+    public static final String API_BC_OEMID = ConfigUtils.getStringProperty("bc.oemid");
+    public static final String API_BC_DEVICEID = ConfigUtils.getStringProperty("bc.deviceid");
+    public static final String API_BC_UID = ConfigUtils.getStringProperty("bc.uid");
+    public static final String API_BC_URL = ConfigUtils.getStringProperty("bc.url");
+    public static final String API_BC_METHOD = ConfigUtils.getStringProperty("bc.method");
+
+    public static final String API_BC_ISA1CAT = ConfigUtils.getStringProperty("bc.isa1cat");
+    public static final String API_BC_ISREPU = ConfigUtils.getStringProperty("bc.isReputation");
+    public static final String API_BC_ISXML = ConfigUtils.getStringProperty("bc.isxml");
+
+
+    public static final Integer API_BC_MAXIMUM_QUERYNUM = ConfigUtils.getIntProperty("bc.maximum.query.num"); // brightcloud单次查询条数上线
+    public static final String API_BC_QUERYTYPE = ConfigUtils.getStringProperty("bc.queryType");
+
+    public static final String API_BC_USE_REPORT_FILE = ConfigUtils.getStringProperty("bc.usereport.filepath"); // brightcloud使用报告导出文件目录
+    public static final String API_BC_CATEINFO_FILE = ConfigUtils.getStringProperty("bc.cateinfo.filepath"); // brightcloud使用报告导出文件目录
+
+
+    // chinaz
+    public static final String API_CHINAZ_URL_SINGLE = ConfigUtils.getStringProperty("chinaz.url.single");
+    public static final String API_CHINAZ_URL_BATCH = ConfigUtils.getStringProperty("chinaz.url.batch");
+    public static final String API_CHINAZ_KEY = ConfigUtils.getStringProperty("chinaz.key");
+    public static final Integer API_CHINAZ_MAXIMUM_QUERYNUM = ConfigUtils.getIntProperty("chinaz.maximum.query.num");
+    public static final String API_CHINAZ_USE_REPORT_FILE = ConfigUtils.getStringProperty("chinaz.usereport.filepath");
+
+    // Mariadb
+    public static final String DATABASE = ConfigUtils.getStringProperty("database");
+    public static final String DOMAIN_CATE_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.category");
+    public static final String DOMAIN_WHOIS_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.whois");
+    public static final String DNS_SERVER_TABLENAME = ConfigUtils.getStringProperty("tablename.dns.server");
+    public static final Integer DB_QUERY_BATCH_SIZE = ConfigUtils.getIntProperty("db.query.batch.size");
+
+    // 其他
+    public static final String TLD_FILE = ConfigUtils.getStringProperty("tld.file"); // 顶级域名公开列表文件
+
+}
diff --git a/src/main/java/com/mesasoft/cn/sketch/config/MariaDbBase.java b/src/main/java/com/mesasoft/cn/sketch/config/MariaDbBase.java
new file mode 100644
index 0000000..ba617ce
--- /dev/null
+++ b/src/main/java/com/mesasoft/cn/sketch/config/MariaDbBase.java
@@ -0,0 +1,84 @@
+package com.mesasoft.cn.sketch.config;
+
+import com.mesasoft.cn.util.TimeUtils;
+import org.apache.log4j.Logger;
+
+import java.sql.*;
+import java.util.Date;
+import java.util.Properties;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: joy
+ * Date: 2021/12/28
+ * Time: 2:56 PM
+ * Description: No Description
+ */
+public class MariaDbBase {
+
+    private static final Logger LOG = Logger.getLogger(MariaDbBase.class);
+    private static final Properties props = new Properties();
+
+    private final Statement statement;
+
+    public MariaDbBase(Connection conn, Statement stat) {
+        statement = stat;
+    }
+
+    /**
+     * 执行写入sql
+     */
+    public void writeSqlExecute(String sql){
+        try {
+            statement.executeUpdate(sql);
+        } catch (SQLIntegrityConstraintViolationException e){
+            LOG.error("Duplicated entry for key 'PRIMARY'");
+        } catch (SQLException exception) {
+            LOG.error("Sql : " + sql);
+            exception.printStackTrace();
+        }
+    }
+
+    /**
+     * 执行查询sql
+     */
+    public ResultSet querySqlExecute(String sql){
+        ResultSet set = null;
+        try {
+            set = statement.executeQuery(sql);
+        } catch (SQLException exception) {
+            exception.printStackTrace();
+        }
+        return set;
+    }
+
+
+    /**
+     * 获得指定表格、按指定时间字段的过期记录
+     * @param tableName 库表名称
+     * @param timeColumnName 时间列名
+     * @return 查询结果
+     */
+    public ResultSet getExpiredRecord(String tableName, String timeColumnName){
+        Date lastUpdateTime = new Timestamp(getExpiredTime(ApplicationConfig.UPDATE_EXPIRED_DAY).getTime());
+
+        String resSql = "SELECT *"
+                + " FROM " + ApplicationConfig.DATABASE + "." + tableName
+                + " WHERE " + timeColumnName + " < '" + lastUpdateTime + '\'';
+
+        LOG.debug("Update task: expired query sql" + resSql);
+
+        return querySqlExecute(resSql);
+    }
+
+    /**
+     * TODO: getUnlabeledRecord() 考虑多个来源的情况
+     */
+
+    /**
+     * 获得过期时间, 当前时间的expiredRangeDays天之前的日期为过期日期
+     */
+    public static Date getExpiredTime(int expiredRangeDays){
+        return new Timestamp(TimeUtils.getStartOfDay(-expiredRangeDays).getTime());
+    }
+}
diff --git a/src/main/java/com/mesasoft/cn/sketch/config/SketchDatabaseConfig.java b/src/main/java/com/mesasoft/cn/sketch/config/SketchDatabaseConfig.java
new file mode 100644
index 0000000..dc69bbd
--- /dev/null
+++ b/src/main/java/com/mesasoft/cn/sketch/config/SketchDatabaseConfig.java
@@ -0,0 +1,125 @@
+//package com.zhazhapan.efo.sketch.config;
+//
+//import com.alibaba.druid.pool.DruidDataSource;
+//import lombok.Data;
+//import org.apache.ibatis.session.SqlSessionFactory;
+//import org.mybatis.spring.SqlSessionFactoryBean;
+//import org.mybatis.spring.annotation.MapperScan;
+//import org.springframework.beans.factory.annotation.Qualifier;
+//import org.springframework.beans.factory.annotation.Value;
+//import org.springframework.boot.context.properties.ConfigurationProperties;
+//import org.springframework.context.annotation.Bean;
+//import org.springframework.context.annotation.Configuration;
+//import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+//import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+//
+//import javax.sql.DataSource;
+//import java.sql.SQLException;
+//
+///**
+// * @ProjectName
+// * @Description: 后台数据源配置类
+// */
+//@Data
+//@Configuration
+//@ConfigurationProperties(prefix = "sketch.datasource.druid")
+//@MapperScan(basePackages = SketchDatabaseConfig.PACKAGE, sqlSessionFactoryRef = "sketchSqlSessionFactory")
+//public class SketchDatabaseConfig {
+//    /**
+//     * dao层的包路径
+//     */
+//    static final String PACKAGE = "com.mao.mysqlhive.demomh.mapper.sketch";
+//
+//    /**
+//     * mapper文件的相对路径
+//     */
+//    private static final String MAPPER_LOCATION = "classpath:mappers/sketch/*Mapper.xml";
+//
+//    @Value("${sketch.datasource.druid.filters}")
+//    private String filters;
+//    @Value("${sketch.datasource.druid.driverClassName}")
+//    private String url;
+//    @Value("${sketch.datasource.druid.url}")
+//    private String username;
+//    @Value("${sketch.datasource.druid.username}")
+//    private String password;
+//    @Value("${sketch.datasource.druid.password}")
+//    private String driverClassName;
+//    @Value("${sketch.datasource.druid.initialSize}")
+//    private int initialSize;
+//    @Value("${sketch.datasource.druid.minIdle}")
+//    private int minIdle;
+//    @Value("${sketch.datasource.druid.maxActive}")
+//    private int maxActive;
+//    @Value("${sketch.datasource.druid.maxWait}")
+//    private long maxWait;
+//    @Value("${sketch.datasource.druid.timeBetweenEvictionRunsMillis}")
+//    private long timeBetweenEvictionRunsMillis;
+//    @Value("${sketch.datasource.druid.minEvictableIdleTimeMillis}")
+//    private long minEvictableIdleTimeMillis;
+//    @Value("${sketch.datasource.druid.validationQuery}")
+//    private String validationQuery;
+//    @Value("${sketch.datasource.druid.testWhileIdle}")
+//    private boolean testWhileIdle;
+//    @Value("${sketch.datasource.druid.testOnBorrow}")
+//    private boolean testOnBorrow;
+//    @Value("${sketch.datasource.druid.testOnReturn}")
+//    private boolean testOnReturn;
+//    @Value("${sketch.datasource.druid.poolPreparedStatements}")
+//    private boolean poolPreparedStatements;
+//    @Value("${sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize}")
+//    private int maxPoolPreparedStatementPerConnectionSize;
+//
+//
+//    @Bean(name = "sketchDataSource")
+//    public DataSource sketchDataSource() throws SQLException {
+//        DruidDataSource druid = new DruidDataSource();
+//        // 监控统计拦截的filters
+//        druid.setFilters(filters);
+//
+//        // 配置基本属性
+//        druid.setDriverClassName(driverClassName);
+//        druid.setUsername(username);
+//        druid.setPassword(password);
+//        druid.setUrl(url);
+//
+//        //初始化时建立物理连接的个数
+//        druid.setInitialSize(initialSize);
+//        //最大连接池数量
+//        druid.setMaxActive(maxActive);
+//        //最小连接池数量
+//        druid.setMinIdle(minIdle);
+//        //获取连接时最大等待时间，单位毫秒。
+//        druid.setMaxWait(maxWait);
+//        //间隔多久进行一次检测，检测需要关闭的空闲连接
+//        druid.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
+//        //一个连接在池中最小生存的时间
+//        druid.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
+//        //用来检测连接是否有效的sql
+//        druid.setValidationQuery(validationQuery);
+//        //建议配置为true，不影响性能，并且保证安全性。
+//        druid.setTestWhileIdle(testWhileIdle);
+//        //申请连接时执行validationQuery检测连接是否有效
+//        druid.setTestOnBorrow(testOnBorrow);
+//        druid.setTestOnReturn(testOnReturn);
+//        //是否缓存preparedStatement，也就是PSCache，oracle设为true，mysql设为false。分库分表较多推荐设置为false
+//        druid.setPoolPreparedStatements(poolPreparedStatements);
+//        // 打开PSCache时，指定每个连接上PSCache的大小
+//        druid.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize);
+//        return druid;
+//    }
+//
+//    @Bean(name = "sketchTransactionManager")
+//    public DataSourceTransactionManager sketchTransactionManager() throws SQLException {
+//        return new DataSourceTransactionManager(sketchDataSource());
+//    }
+//
+//    @Bean(name = "sketchSqlSessionFactory")
+//    public SqlSessionFactory sketchSqlSessionFactory(@Qualifier("sketchDataSource") DataSource sketchDataSource) throws Exception {
+//        final SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
+//        sessionFactory.setDataSource(sketchDataSource);
+//        sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(SketchDatabaseConfig.MAPPER_LOCATION));
+//
+//        return sessionFactory.getObject();
+//    }
+//}