[优化][知识库] CN知识库同步任务IPv4,IPv6文件合并

11 files changed, 172 insertions, 309 deletions

diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/Constant.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/Constant.java
index d316e9b..cc559c0 100644
--- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/Constant.java
+++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/Constant.java
@@ -29,7 +29,7 @@ public class Constant {
     public static final String WORK_PATH = "/home/tsg/galaxy/galaxy-job/data/";
     public static final String ZK_KNOWLEDGE_PATH = "/knowledgeBase/metadata/ids/";
 //    public static String KNOWLEDGE_WORK_DIR = "C:\\home\\tsg\\galaxy\\galaxy-job\\plugins\\mmdb\\";
-    public static String KNOWLEDGE_WORK_DIR = "/home/tsg/galaxy/galaxy-job/galaxy-job-executor/plugins/mmdb/";
+    public static String KNOWLEDGE_WORK_DIR = "/opt/schedule-service/mmdb/";
     public static final Integer DB_BATCH = 200000;
 
 
diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataflowJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataflowJob.java
index 86d827a..33e2fa5 100644
--- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataflowJob.java
+++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataflowJob.java
@@ -164,7 +164,7 @@ public class DataflowJob {
                 defVsysId = TypeUtils.castToInt(requestBody.get("vsysId"));
             }
 
-            String insertTemplate = "INSERT INTO {0} (common_log_id,common_recv_time,assessment_date,lot_number,file_name,assessment_file,assessment_type,features,`size`,file_checksum_sha,vsys_id) " +
+            String insertTemplate = "INSERT INTO {0} (log_id,recv_time,assessment_date,lot_number,file_name,assessment_file,assessment_type,features,`size`,file_checksum_sha,vsys_id) " +
                     "VALUES({1,number,#},{2,number,#},{3,number,#},''{4}'',''{5}'',''{6}'',''{7}'',''{8}'',{9,number,#},''{10}'',{11,number,#});";
             long logId = SnowflakeId.generateId(1, 1);
             long now = System.currentTimeMillis() / 1000;
diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/IpLocateLibraryJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/IpLocateLibraryJob.java
index 996c584..541db75 100644
--- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/IpLocateLibraryJob.java
+++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/IpLocateLibraryJob.java
@@ -147,7 +147,10 @@ public class IpLocateLibraryJob {
             JobUtil.errorLog("perl query failed result: status {}, message {}.", mmdbStatus.getStatus(), mmdbStatus.getMessage());
             return false;
         }
-
+        if (mmdbStatus.getTotal() == 0 ) {
+            JobUtil.errorLog("perl query failed result: total {}.", mmdbStatus.getTotal());
+            return false;
+        }
         String builtinPath = "";
         String userDefinedPath = "";
         String filePath = newFile.getAbsolutePath();
diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/MMDBStatus.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/MMDBStatus.java
index 701030c..a22c81d 100644
--- a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/MMDBStatus.java
+++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/MMDBStatus.java
@@ -10,6 +10,7 @@ public class MMDBStatus {
 
     private Integer status;
     private String message;
+    private Integer total;
     private Long updateTime;
     private List<String> ipSample = new ArrayList<>();
 
diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/knowledgeBase/KnowledgeBaseService.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/knowledgeBase/KnowledgeBaseService.java
index 6bfd737..99df735 100644
--- a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/knowledgeBase/KnowledgeBaseService.java
+++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/knowledgeBase/KnowledgeBaseService.java
@@ -161,7 +161,7 @@ public class KnowledgeBaseService {
                 ipLocation = false;
             }
             int executeResult = buildMMDB(knowledgeBase, script);
-            String filename = StringUtil.join(script, ".", knowledgeBase.getFormat());
+            String filename = StringUtil.join(script, "_builtin.", knowledgeBase.getFormat());
 
             if (executeResult == 0) {
                 if (ipLocation && !checkSample(KNOWLEDGE_WORK_DIR, filename, script)) {
@@ -184,7 +184,7 @@ public class KnowledgeBaseService {
     }
 
     private int buildMMDB(KnowledgeBase knowledgeBase, String script) throws Exception {
-        final String workDirCmd = "docker inspect --format '{{range.Mounts}}  {{.Source}} {{end}}' galaxy-job-executor |xargs -n 1|grep plugins";
+        final String workDirCmd = "docker inspect --format '{{range.Mounts}}  {{.Source}} {{end}}' galaxy-job-executor |xargs -n 1|grep schedule-service";
         final String imagesNameCmd = "docker images --format '{{.Repository}}:{{.Tag}}' | grep '^perl:' | sort -r | head -n 1";
 
         String workDir = JobUtil.exeCmd(workDirCmd);
@@ -244,16 +244,20 @@ public class KnowledgeBaseService {
      */
     public static boolean checkSample(String path, String filename, String script) throws Exception {
 
-        final String builtinStatusFile = "builtin-status.json";
+        final String builtinStatusFile = script+"_builtin_status.json";
 
-        MMDBStatus builtMMDBStatus = JobUtil.readJsonFile(path + builtinStatusFile, MMDBStatus.class);
-        JobUtil.infoLog("builtin ip locate library result: status {}, message {}", builtMMDBStatus.getStatus(), builtMMDBStatus.getMessage());
-        if (200 != builtMMDBStatus.getStatus()) {
-            JobUtil.errorLog("builtin status fail {}", builtMMDBStatus.getStatus());
+        MMDBStatus mmdbStatus = JobUtil.readJsonFile(path + builtinStatusFile, MMDBStatus.class);
+        JobUtil.infoLog("builtin ip locate library result: status {}, message {}", mmdbStatus.getStatus(), mmdbStatus.getMessage());
+        if (200 != mmdbStatus.getStatus()) {
+            JobUtil.errorLog("builtin status fail {}", mmdbStatus.getStatus());
+            return false;
+        }
+        if (mmdbStatus.getTotal() == 0 ) {
+            JobUtil.errorLog("perl query failed result: total {}.", mmdbStatus.getTotal());
             return false;
         }
         //测试抽样数据能查到结果说明打包成功
-        List<String> ipSample = builtMMDBStatus.getIpSample();
+        List<String> ipSample = mmdbStatus.getIpSample();
         String samplePath = path + File.separator + filename;
         if ("ip".equals(script)) {
            return JobUtil.ipKnowledgeNotEmpty(samplePath,"",ipSample);
diff --git a/galaxy-job-executor/src/main/resources/plugins/mmdb/cn/asn.pl b/galaxy-job-executor/src/main/resources/plugins/mmdb/cn/asn.pl
index 10ef82a..30b79c9 100644
--- a/galaxy-job-executor/src/main/resources/plugins/mmdb/cn/asn.pl
+++ b/galaxy-job-executor/src/main/resources/plugins/mmdb/cn/asn.pl
@@ -2,181 +2,97 @@
 # This source code is subject to the terms of the MIT License. If a
 # copy of the MIT License was not distributed with this file, you can
 # obtain one at http://opensource.org/licenses/MIT.
+# MMDB format docs: https://maxmind.github.io/MaxMind-DB/
+# MaxMind::DB:Writer::Tree API docs: https://metacpan.org/pod/MaxMind::DB::Writer::Tree
 
 use strict;
 use warnings;
 
 use Data::Dumper;
 use Getopt::Long qw(GetOptions);
+use List::Util;
 use MaxMind::DB::Writer::Tree;
 use Net::Works::Network;
 use Getopt::Long;
-use JSON;
 use DBI;
 use utf8;
 use POSIX qw(strftime);
 binmode(STDIN,":encoding(gbk)");
 
+###################数据库连接信息###################
+my $host = "192.168.44.53";     # 主机地址
+my $database = "cn_api";        # 数据库
+my $username = "root";          # 数据库用户名
+my $password = "111111";        # 数据库密码
 
-#接收参数，第一个参数为addr_type,缺省为4
 my @args = @ARGV;
-my $addrType = 4;
-$addrType= @args && $args[0] ?$args[0]:4;
-my $whereKnowledgeId= @args && $args[1] ? " AND knowledge_id=".$args[1]:"";
-my $where="WHERE addr_type=$addrType".$whereKnowledgeId;
-###################数据库连接信息###################
-my $host = "192.168.44.53";         # 主机地址
-my $driver = "mysql";           # 接口类型 默认为 localhost
-my $database = "cn-dev-2306";        # 数据库
-my $sql = "SELECT addr_type,ip1,ip2,asn,organization FROM `cn_ip_asn_built_in` $where";
+my $where= @args ? " WHERE knowledge_id=".$args[0]:"";
+my $sql = "SELECT ip1,ip2,asn,organization FROM `cn_ip_asn_built_in` $where";
 # 驱动程序对象的句柄
-my $dsn = "DBI:$driver:database=$database:$host";
-my $userid = "root";            # 数据库用户名
-my $password = "111111";        # 数据库密码
+my $dsn = "DBI:mysql:database=$database:$host";
+my $outfile = 'asn_builtin.mmdb';
 ###################数据库连接信息###################
 
-###################日志文件信息###################
-printf "sql : $sql \n\n";
-
-my $help = 0;
-my $outfile = 'asn'.$addrType.'.mmdb';;
+my $dbh = DBI->connect($dsn, $username, $password ) or die $DBI::errstr;
+print "HOST: $host, DB: $database connect: ". ($dbh->ping() ? "success" : "fail")."
+";
+$dbh->do("SET NAMES utf8");
+$dbh->{'mysql_enable_utf8mb4'} = 1;
+my $sth = $dbh->prepare("$sql");
+$sth->execute();
 
-my $mmdbLog = 'built-mmdb.log';
-my $mmdbStatus = 'built-status.json';
-print "The built-in perl script is running!\n";
+my $record_size = 24;
+my $db_type = "ASNLOCATOR";
+my $ip_version = 6;
 
-GetOptions("db=s" => \$outfile, 'help|?' => \$help, ) or &help();
-&help() if $help;
+GetOptions(
+	'db-type=s' => \$db_type,
+	'ipv=i' => \$ip_version,
+	'record-size=i' => \$record_size,
+) or die "Usage: $0 --db-type TYPE --ipv VERSION --record-size SIZE\n";
 
-my $db_name = "IPLOCATOR";
+my $db_name = "ASNLOCATOR";
 my $db_description = "IP Geolocation Library";
 my %types = (
-	ASN				=> 'utf8_string',
-	COUNTRY_CODE 			=> 'utf8_string',
-	ORGANIZATION      		=> 'utf8_string',
+	ASN				      => 'utf8_string',
+	COUNTRY_CODE 		  => 'utf8_string',
+	ORGANIZATION      	  => 'utf8_string',
 );
+
 my $tree = MaxMind::DB::Writer::Tree->new(
 	remove_reserved_networks => 0,
-	ip_version            => $addrType,
-	record_size           => 24,
+	ip_version            => $ip_version,
+	record_size           => $record_size,
 	database_type         => $db_name,
 	languages             => ['en', 'zh'],
-	description           => { en => $db_description, zh => 'IP定位库'},
+	description           => { en => $db_description, zh => 'ASN定位库'},
 	map_key_type_callback => sub { $types{ $_[0] } },
 );
 
-my $dbh = DBI->connect($dsn, $userid, $password ) or die $DBI::errstr;
-print "HOST: $host, DB: $database connect: ". ($dbh->ping() ? "success" : "fail")."
-";
-$dbh->do("SET NAMES utf8");
-$dbh->{'mysql_enable_utf8mb4'} = 1;
-my $sth = $dbh->prepare("$sql LIMIT ?,?");   # 预处理 SQL  语句
-
-my $pageNo = 0;
-my $pageSize = 200000;
-my $totalv4 = 0;
-my $totalv6 = 0;
-my $total= 0;
-my $requestMsg= 'success';
-my $requestCode= 0;
-my @ipv4Sample= ();
-my @ipv6Sample= ();
-
-while(1==1){
-
-my $offset = $pageNo * $pageSize;
-$sth->execute($offset,$pageSize);
-print "$sql LIMIT $offset,$pageSize; \n";
-
-$pageNo+=1;
-#记录日志
-my $gmtDatestring = strftime "%Y-%m-%d %H:%M:%S", gmtime;
-open(MMDBLOG, "+>>$mmdbLog");
-
-my $resCount = 0;
 while (my $ref = $sth->fetchrow_hashref()) {
-      $resCount++;
-      if ($addrType == 4) {
-              if($resCount == 1){
-                 push(@ipv4Sample, $ref->{'ip1'});
-              }
-              $totalv4++;
-      } elsif ($addrType == 6) {
-            $totalv6++;
-            if($resCount == 1){
-                 push(@ipv6Sample, $ref->{'ip1'});
-            }
-      };
-      my $start_ip = $ref->{'ip1'};
-      my $mask = $ref->{'ip2'};
-      my $netblock = $start_ip.'/'.$mask;
-      if ($start_ip) {
-          my $network = Net::Works::Network->new_from_string( string => $netblock );
-          my %data = ('ASN'=>isNotBlank($ref->{'asn'}), 'ORGANIZATION'=>isNotBlank($ref->{'organization'}));
-        $tree->insert_network(
-            $network,
-              \%data
-      );
-     }
-};
-
-$sth->finish();
-printf MMDBLOG "$gmtDatestring MMDB write success , pageNo: $pageNo, resultCount: $resCount\n";
 
-if($resCount < $pageSize){
-  $requestCode= 200;
-  print MMDBLOG "IPv4 total: $totalv4. IPv6 total: $totalv6\n";
-  close( MMDBLOG );
-        last;
-  }
-  $resCount = 0;
-}
-$dbh->disconnect();
+    my $start_ip = $ref->{'ip1'};
+    my $mask = $ref->{'ip2'};
+    my $netblock = $start_ip.'/'.$mask;
 
-open(MMDBLOG, "+>>$mmdbLog");
-my $gmtDateLog = strftime "%Y-%m-%d %H:%M:%S", gmtime;
+    if ($start_ip) {
+        my $network = Net::Works::Network->new_from_string( string => $netblock );
+        my %data = ('ASN'=>isNotBlank($ref->{'asn'}),'ORGANIZATION'=>isNotBlank($ref->{'organization'}));
+        $tree->insert_network($network, \%data);
+    }
+};
 
 open my $fhfile, '>:raw', $outfile;
 $tree->write_tree($fhfile);
 close($outfile);
-printf MMDBLOG "$gmtDateLog The built-in perl script is finished !\n\n";
-close( MMDBLOG );
-
-#状态
-my $totalv46 = $totalv4+$totalv6;
-$requestMsg = "$requestMsg . Getting result total $totalv46, ipv4 total  $totalv4, ipv6 total $totalv6.";
-
-print "The built-in perl script is finished!\n";
-
-my %statusHash = ('status'=>$requestCode, 'updateTime'=>time(), 'ipv4Total'=>$totalv4, 'ipv6Total'=>$totalv6, 'message'=>$requestMsg,'ipv4Sample'=> \@ipv4Sample, 'ipv6Sample'=> \@ipv6Sample);
-my $statusJson = encode_json \%statusHash;
-open my $statusFile, '>', $mmdbStatus;
-print $statusFile $statusJson;
-
-sub changeSplit {
-  my $n = scalar(@_);
-  my $i = $_[0];
-  $i =~ s/\\b/ /g;
-
-  return "$i"
-}
+print "The perl script is finished!\n";
 
 sub isNotBlank {
    my $tmp = $_[0];
-  return $tmp ? "$tmp" : " ";
+   if(defined $tmp){
+     return $tmp;
+   } else{
+     return "";
+   }
 }
 
-
-sub help {
-    print "Usage: $0 -f file.json -d my-ip-data.mmdb\n";
-    print 'Example json:
-{
-  "records": [
-    {
-      "address": "6.6.6.6/28",
-      "service_area": "LAX"
-    }
-  ]
-}' . "\n";
-    exit;
-};
diff --git a/galaxy-job-executor/src/main/resources/plugins/mmdb/cn/ip.pl b/galaxy-job-executor/src/main/resources/plugins/mmdb/cn/ip.pl
index e8d0120..e6fbc2f 100644
--- a/galaxy-job-executor/src/main/resources/plugins/mmdb/cn/ip.pl
+++ b/galaxy-job-executor/src/main/resources/plugins/mmdb/cn/ip.pl
@@ -1,7 +1,4 @@
 #!/usr/bin/perl
-# This source code is subject to the terms of the MIT License. If a
-# copy of the MIT License was not distributed with this file, you can
-# obtain one at http://opensource.org/licenses/MIT.
 use strict;      #use this all times
 use warnings;    #this too - helps a lot!
 use JSON;
@@ -12,43 +9,28 @@ use utf8;
 use POSIX qw(strftime);
 binmode(STDIN,":encoding(gbk)");
 
+
 ###################数据库连接信息###################
+my $host = "192.168.44.53";    # 主机地址
+my $database = "cn_api";       # 数据库
+my $userid = "root";
+my $password = "111111";
 
-#接收参数，第一个参数为addr_type,缺省为4
 my @args = @ARGV;
-my $addrType = 4;
-$addrType= @args && $args[0] ?$args[0]:4;
-my $whereKnowledgeId= @args && $args[1] ? " AND knowledge_id=".$args[1]:"";
-
-my $where="WHERE addr_type=$addrType".$whereKnowledgeId;
-
-my $host = "192.168.44.53";         # 主机地址
-my $driver = "mysql";           # 接口类型 默认为 localhost
-my $database = "cn-dev-2306";        # 数据库
-my $sql = "SELECT addr_type,start_ip,end_ip,latwgs,lngwgs,isp,continent,country,province,city FROM cn_ip_location_built_in $where";
-# 驱动程序对象的句柄
-my $dsn = "DBI:$driver:database=$database:$host";
-my $userid = "root";            # 数据库用户名
-my $password = "111111";        # 数据库密码
+my $where= @args && $args[0] ? " WHERE knowledge_id=".$args[0]:"";
+my $sql = "SELECT start_ip,end_ip,latwgs,lngwgs,isp,continent,country,province,city FROM cn_ip_location_built_in $where";
+my $dsn = "DBI:mysql:database=$database:$host";
 ###################数据库连接信息###################
 
 ###################日志文件信息###################
 
 printf "sql : $sql \n\n";
 
-my $help = 0;
-my $outfileV6 = 'ip6.mmdb';;
-my $outfileV4 = 'ip4.mmdb';
-
-my $mmdbLog = 'built-mmdb.log';
-my $mmdbStatus = 'built-status.json';
-print "The built-in perl script is running!\n";
-
-GetOptions("db=s" => \$outfileV6, 'help|?' => \$help, ) or &help();
-
-GetOptions("db=s" => \$outfileV4, 'help|?' => \$help, ) or &help();
-
-&help() if $help;
+my $fileName='ip_builtin';
+my $outfile = "$fileName.mmdb";
+my $mmdbLog = "$fileName.log";
+my $mmdbStatus = $fileName.'_status.json';
+print "The perl script is running, output to $outfile \n";
 
 my %types = (
         AREA_CODE => 'utf8_string',
@@ -75,133 +57,90 @@ my $treeV6 = MaxMind::DB::Writer::Tree->new(
        description           => { en => 'My database of IP data' },
        map_key_type_callback => sub { $types{ $_[0] } },
 );
-# V4databse info
-my $treeV4 = MaxMind::DB::Writer::Tree->new(
-       ip_version            => 4,
-       record_size           => 32,
-       database_type         => 'IPLOCATOR',
-       languages             => ['en'],
-       description           => { en => 'My database of IP data' },
-       map_key_type_callback => sub { $types{ $_[0] } },
-       remove_reserved_networks => 0,
-);
 
 my $dbh = DBI->connect($dsn, $userid, $password ) or die $DBI::errstr;
 print "HOST: $host, DB: $database connect: ". ($dbh->ping() ? "success" : "fail")."
 ";
 $dbh->do("SET NAMES utf8");
 $dbh->{'mysql_enable_utf8mb4'} = 1;
-my $sth = $dbh->prepare("$sql LIMIT ?,?");   # 预处理 SQL  语句
+my $sth = $dbh->prepare("$sql LIMIT ?,?");
 
 my $pageNo = 0;
-my $pageSize = 200000;
-my $totalv4 = 0;
-my $totalv6 = 0;
+my $pageSize = 500000;
 my $total= 0;
 my $requestMsg= 'success';
 my $requestCode= 0;
-my @ipv4Sample= ();
-my @ipv6Sample= ();
+my @ipSample= ();
 
 while(1==1){
 
-my $offset = $pageNo * $pageSize;
-$sth->execute($offset,$pageSize);
-
-print "$sql LIMIT $offset,$pageSize; \n";
-
-$pageNo+=1;
-#记录日志
-my $gmtDatestring = strftime "%Y-%m-%d %H:%M:%S", gmtime;
-open(MMDBLOG, "+>>$mmdbLog");
-printf MMDBLOG "$gmtDatestring MMDB request success , pageNo: $pageNo \n";
-
-    my $resCount = 0;
-    while (my $ref = $sth->fetchrow_hashref()) {
-      $resCount++;
-      if(($ref->{'addr_type'}) == 4) {
-              if($resCount == 1){
-                 push(@ipv4Sample, $ref->{'start_ip'});
-              }
-              $totalv4++;
-              $treeV4->insert_range(
-                   $ref->{'start_ip'},
-                   $ref->{'end_ip'},
-              {
-                  AREA_CODE => isNotBlank($ref->{'areacode'}),
-                  ASN       => isNotBlank($ref->{'asnumber'}),
-                  ISP       => isNotBlank($ref->{'isp'}),
-                  LATITUDE  => isNotBlank($ref->{'latwgs'}),
-                  LONGITUDE => isNotBlank($ref->{'lngwgs'}),
-                  CONTINENT => isNotBlank($ref->{'continent'}),
-                  COUNTRY   => changeSplit(isNotBlank($ref->{'country'})),
-                  SUPER_ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'province'})),
-                  ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'city'})),
-               },
-            );
-      }elsif (($ref->{'addr_type'}) == 6) {
-            if($resCount == 1){
-                 push(@ipv6Sample, $ref->{'start_ip'});
-            }
-            $totalv6++;
-            $treeV6->insert_range(
-                   $ref->{'start_ip'},
-                   $ref->{'end_ip'},
-                {
-                  AREA_CODE => isNotBlank($ref->{'areacode'}),
-                  ASN       => isNotBlank($ref->{'asnumber'}),
-                  ISP       => isNotBlank($ref->{'isp'}),
-                  LATITUDE  => isNotBlank($ref->{'latwgs'}),
-                  LONGITUDE => isNotBlank($ref->{'lngwgs'}),
-                  CONTINENT => isNotBlank($ref->{'continent'}),
-                  COUNTRY   => changeSplit(isNotBlank($ref->{'country'})),
-                  SUPER_ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'province'})),
-                  ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'city'})),
-                },
-            );
-        };
-    };
-
-$sth->finish();
-printf MMDBLOG "$gmtDatestring MMDB write success , pageNo: $pageNo, resultCount: $resCount\n";
-
-if($resCount < $pageSize){
-  $requestCode= 200;
-  print MMDBLOG "IPv4 total: $totalv4. IPv6 total: $totalv6\n";
-  close( MMDBLOG );
-        last;
+  my $offset = $pageNo * $pageSize;
+  $sth->execute($offset,$pageSize);
+  print "$sql LIMIT $offset,$pageSize; \n";
+
+  $pageNo+=1;
+  #记录日志
+  my $gmtDatestring = strftime "%Y-%m-%d %H:%M:%S", gmtime;
+  open(MMDBLOG, "+>>$mmdbLog");
+  printf MMDBLOG "$gmtDatestring MMDB request success , pageNo: $pageNo \n";
+
+  my $resCount = 0;
+  while (my $ref = $sth->fetchrow_hashref()) {
+        $resCount++;
+        if($resCount == 1 && $ref->{'lngwgs'} ){
+           push(@ipSample, $ref->{'start_ip'});
+        }
+        $total++;
+        $treeV6->insert_range(
+             $ref->{'start_ip'},
+             $ref->{'end_ip'},
+        {
+            AREA_CODE => isNotBlank($ref->{'areacode'}),
+            ASN       => isNotBlank($ref->{'asnumber'}),
+            ISP       => isNotBlank($ref->{'isp'}),
+            LATITUDE  => isNotBlank($ref->{'latwgs'}),
+            LONGITUDE => isNotBlank($ref->{'lngwgs'}),
+            CONTINENT => isNotBlank($ref->{'continent'}),
+            COUNTRY   => changeSplit(isNotBlank($ref->{'country'})),
+            SUPER_ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'province'})),
+            ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'city'})),
+         },
+      );
+  };
+  $sth->finish();
+  printf MMDBLOG "$gmtDatestring MMDB write success , pageNo: $pageNo, resultCount: $resCount\n";
+
+  if($resCount < $pageSize){
+      $requestCode= 200;
+      print MMDBLOG "IP total: $total.\n";
+      close( MMDBLOG );
+      last;
   }
   $resCount = 0;
 }
-
 $dbh->disconnect();
 
 open(MMDBLOG, "+>>$mmdbLog");
 my $gmtDateLog = strftime "%Y-%m-%d %H:%M:%S", gmtime;
 
-if($addrType == 6) {
-    open my $fhV6, '>:raw', $outfileV6;
-    $treeV6->write_tree($fhV6);
-    close($outfileV6);
-    printf MMDBLOG "$gmtDateLog MMDB IPv6 Write Over !\n";
-
-} elsif($addrType == 4) {
-    open my $fhV4, '>:raw', $outfileV4;
-    $treeV4->write_tree($fhV4);
-    close($outfileV4);
-    printf MMDBLOG "$gmtDateLog MMDB IPv4 Write Over !\n";
+if($total==0){
+ @ipSample=();
 }
 
+open my $fhV6, '>:raw', $outfile;
+$treeV6->write_tree($fhV6);
+close($outfile);
+printf MMDBLOG "$gmtDateLog MMDB IP Write Over !\n";
+
 printf MMDBLOG "$gmtDateLog The built-in perl script is finished !\n\n";
 close( MMDBLOG );
 
 #状态
-my $totalv46 = $totalv4+$totalv6;
-$requestMsg = "$requestMsg . Getting result total $totalv46, ipv4 total  $totalv4, ipv6 total $totalv6.";
+$requestMsg = "$requestMsg . Getting result total $total";
 
 print "The built-in perl script is finished!\n";
 
-my %statusHash = ('status'=>$requestCode, 'updateTime'=>time(), 'ipv4Total'=>$totalv4, 'ipv6Total'=>$totalv6, 'message'=>$requestMsg,'ipv4Sample'=> \@ipv4Sample, 'ipv6Sample'=> \@ipv6Sample);
+my %statusHash = ('status'=>$requestCode, 'updateTime'=>time(), 'total'=>$total,'message'=>$requestMsg,'ipSample'=> \@ipSample);
 my $statusJson = encode_json \%statusHash;
 open my $statusFile, '>', $mmdbStatus;
 print $statusFile $statusJson;
@@ -216,20 +155,10 @@ sub changeSplit {
 
 sub isNotBlank {
    my $tmp = $_[0];
-  return $tmp ? "$tmp" : " ";
+   if(defined $tmp){
+     return $tmp;
+   } else{
+     return "";
+   }
 }
 
-
-sub help {
-    print "Usage: $0 -f file.json -d my-ip-data.mmdb\n";
-    print 'Example json:
-{
-  "records": [
-    {
-      "address": "6.6.6.6/28",
-      "service_area": "LAX"
-    }
-  ]
-}' . "\n";
-    exit;
-};
\ No newline at end of file
diff --git a/galaxy-job-executor/src/main/resources/plugins/mmdb/tsg/mmdb-perl.pl b/galaxy-job-executor/src/main/resources/plugins/mmdb/tsg/mmdb-perl.pl
index b3ee792..01fcb4d 100644
--- a/galaxy-job-executor/src/main/resources/plugins/mmdb/tsg/mmdb-perl.pl
+++ b/galaxy-job-executor/src/main/resources/plugins/mmdb/tsg/mmdb-perl.pl
@@ -17,12 +17,12 @@ my $token="{{ cm_server_token }}";
 #自定义：0. 内置：1
 my @args = @ARGV;
 my $isBuiltin= @args?$args[0]:1;
-my $url="http://$serverHost:8080/v1/global/geolocation/dict/detail?is_builtin=$isBuiltin&vsys_id=0";
+my $url="http://$serverHost:8080/v1/global/geolocation/dict/detail?is_builtin=$isBuiltin&vsys_vim id=0";
 my $fileName=$isBuiltin==0?'ip_user_defined':'ip_builtin';
-my $outfileV6="$fileName.mmdb";
+my $outfile="$fileName.mmdb";
 my $mmdbLog="$fileName.log";
 my $mmdbStatus=$fileName.'_status.json';
-print "The perl script is running, output to $outfileV6 \n";
+print "The perl script is running, output to $outfile \n";
 
 my %types = (
         AREA_CODE => 'utf8_string',
@@ -111,9 +111,9 @@ if(@$arraydata < $pageSize){
 }
 
 if($total>0){
-    open my $fhV6, '>:raw', $outfileV6;
+    open my $fhV6, '>:raw', $outfile;
     $treeV6->write_tree($fhV6);
-    close($outfileV6);
+    close($outfile);
 }
 
 my %statusHash = ('status'=>$requestCode, 'updateTime'=>time(), 'total'=>$total, 'message'=>$requestMsg,'ipSample'=> \@ipSample);
@@ -121,7 +121,7 @@ my $statusJson = encode_json \%statusHash;
 open my $statusFile, '>', $mmdbStatus;
 print $statusFile $statusJson;
 
-my $gmtDateLog = strftime "%Y-%m-%d %H:%M:%S", gmtime;
+my $gmtDateLog = strftime "%Y-%m-%d %H:%M:%S", gmtimvim m   e;
 print MMDBLOG "$gmtDateLog MMDB IP Write Over. \n\n";
 close(MMDBLOG);
 print "The perl script is finished!\n";
@@ -135,11 +135,11 @@ sub changeSplit {
 }
 
 sub isNotBlank {
-  my $tmp = $_[0];
-
-  if($tmp) {
-     return "$tmp"
-  }else{
-     return " "
-  }
+   my $tmp = $_[0];
+   if($tmp){
+     return $tmp;
+   } else{
+     return " ";
+   }
 }
+
diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/EncryptConfigUtil.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/EncryptConfigUtil.java
index b93a951..3153c43 100644
--- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/EncryptConfigUtil.java
+++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/EncryptConfigUtil.java
@@ -21,6 +21,6 @@ public class EncryptConfigUtil {
 //        String pin = textEncryptor.decrypt("galaxy2019");
 //        System.err.println("user: "+user);
 //        System.err.println("pin: "+pin);
-        System.err.println("pin: "+textEncryptor.decrypt("nPNttVO8Tp/G16BFdg/+18dpU04BRnzY"));
+        System.err.println("pin: "+textEncryptor.decrypt("vrvDJu5crIzkrALX+O8v3lSuBhwzCoVNCHkh94VuubzhXW/7z1crClpUT9AZY1jL"));
     }
 }
\ No newline at end of file
diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/KnowledgeJobTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/KnowledgeJobTest.java
index 861cdde..bd5328f 100644
--- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/KnowledgeJobTest.java
+++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/KnowledgeJobTest.java
@@ -10,7 +10,7 @@ import com.geedgenetworks.utils.IpLookupV2;
 import com.google.common.collect.ImmutableMap;
 import com.mesalab.executor.core.utils.HttpClientUtils;
 import com.mesalab.executor.core.utils.JobUtil;
-import com.mesalab.executor.service.knowledgeBase.KnowledgeBaseService;
+import com.mesalab.executor.pojo.MMDBStatus;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -34,16 +34,18 @@ public class KnowledgeJobTest {
 //                .loadDataFilePrivateV4("C:\\code\\galaxy-qgw-service\\dat\\ip_v6_built_in.mmdb")
 //                .loadDataFile("C:\\test\\1.mmdb")
 //                .loadDataFileV4("C:\\test\\ip4.mmdb")
-                .loadAsnDataFile("C:\\test\\knowledge\\asn_builtin.mmdb")
+//                .loadAsnDataFile("C:\\test\\knowledge\\asn_builtin.mmdb")
+                .loadDataFile("C:\\test\\knowledge\\old4.mmdb")
 //                .loadDataFilePrivateV6("")
                 .build();
 
 //        String latLngLookup = ipLookup.latLngLookup("0.0.0.0");
 //        System.err.println(latLngLookup);
 //        System.err.println(ipLookup.latLngLookup("192.168.1.1"));
-        System.err.println(ipLookup.asnLookupInfo("2c0f:fc89:ec::"));
-        System.err.println(ipLookup.asnLookupInfo("49.142.208.0"));
-//        System.err.println(ipLookup.countryLookup("0.0.0.0"));
+//        System.err.println(ipLookup.asnLookupInfo("2c0f:fc89:ec::"));
+//        System.err.println(ipLookup.asnLookupInfo("49.142.208.0"));
+        System.err.println(ipLookup.infoLookup("0.1.0.0"));
+        System.err.println(ipLookup.infoLookup("0.0.0.0"));
 //        System.err.println(ipLookup.cityLookup("192.168.1.1"));
 //        System.err.println(ipLookup.asnLookupDetail("1.0.0.0"));
 //        System.err.println(ipLookup.asnLookupDetail("8.6.146.255"));
@@ -135,8 +137,16 @@ public class KnowledgeJobTest {
     @Test
     public void asnTest(){
         try {
-            boolean asn = KnowledgeBaseService.checkSample("C:\\test\\knowledge\\", "asn6.mmdb", "asn");
-            Assert.assertTrue(asn);
+            String script = "ip";
+            String path = "C:\\test\\knowledge\\";
+            final String builtinStatusFile = script+"_builtin_status.json";
+
+            MMDBStatus builtMMDBStatus = JobUtil.readJsonFile(path + builtinStatusFile, MMDBStatus.class);
+            JobUtil.infoLog("builtin ip locate library result: status {}, message {}", builtMMDBStatus.getStatus(), builtMMDBStatus.getMessage());
+            //测试抽样数据能查到结果说明打包成功
+            List<String> ipSample = builtMMDBStatus.getIpSample();
+            String samplePath = path + File.separator + "ip_builtin2.mmdb";
+            JobUtil.ipKnowledgeNotEmpty(samplePath,"",ipSample);
         } catch (Exception e) {
             JobUtil.errorLog(e);
         }
diff --git a/pom.xml b/pom.xml
index 36a0cc3..c783aea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,7 +56,7 @@
 		<docker.registry.port>9080</docker.registry.port>
 		<docker.image.prefix>galaxy</docker.image.prefix>
 		<xxl-job-core.version>1.0.0</xxl-job-core.version>
-		<galaxy.version>1.1.0</galaxy.version>
+		<galaxy.version>1.2</galaxy.version>
 		<httpclient.version>4.5.6</httpclient.version>
 		<lombok.version>1.16.20</lombok.version>
 		<nacos.config.version>0.2.7</nacos.config.version>