diff options
| author | wcc <[email protected]> | 2021-01-05 11:22:29 +0800 |
|---|---|---|
| committer | wcc <[email protected]> | 2021-01-05 11:22:29 +0800 |
| commit | cc9713565dcdd879e189d80013ea90833446a13a (patch) | |
| tree | c22f9d22f53b1d7600314a077ea667863d33998f | |
| parent | 4e31003602322d435195de275072633bca482f88 (diff) | |
2021.1.5
47 files changed, 1008 insertions, 136 deletions
diff --git a/.idea/workspace.xml b/.idea/workspace.xml index c2cc025..6d9331a 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,78 +2,21 @@ <project version="4"> <component name="ChangeListManager"> <list default="true" id="78427964-fb77-4044-a8c8-48f499a0f9df" name="Default Changelist" comment=""> - <change beforePath="$PROJECT_DIR$/.idea/encodings.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/encodings.xml" afterDir="false" /> + <change afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKDNSBolt.java" afterDir="false" /> + <change afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKHTTPBolt.java" afterDir="false" /> + <change afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKIPBolt.java" afterDir="false" /> + <change afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKSSLBolt.java" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bean/SessionRecordLog.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bean/SessionRecordLog.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingDNSBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingDNSBolt.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingFTPBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingFTPBolt.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingHTTPBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingHTTPBolt.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingIPBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingIPBolt.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingJSTXBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingJSTXBolt.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingMAILBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingMAILBolt.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingSSLBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingSSLBolt.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingVPNBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/FloodingVPNBolt.java" afterDir="false" /> + <change beforePath="$PROJECT_DIR$/.idea/xj-log-etl.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/xj-log-etl.iml" afterDir="false" /> + <change beforePath="$PROJECT_DIR$/pom.xml" beforeDir="false" afterPath="$PROJECT_DIR$/pom.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.java" afterDir="false" /> <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/index/CountBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/index/CountBolt.java" afterDir="false" /> <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/spout/FloodingKafkaSpout.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/spout/FloodingKafkaSpout.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/spout/SafeKafkaSpout.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/spout/SafeKafkaSpout.java" afterDir="false" /> <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.java" afterDir="false" /> <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfigurations.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfigurations.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/general/App_Coding.java" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/general/TransFormUtils.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/general/TransFormUtils.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/general/URL_HOS_Interactive.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/general/URL_HOS_Interactive.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/http/HttpClientService.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/cn/ac/iie/storm/utils/http/HttpClientService.java" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/floodingschema/DNS_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/floodingschema/FTP_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/floodingschema/HTTP_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/floodingschema/IP_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/floodingschema/JSTX_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/floodingschema/MAIL_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/floodingschema/SSL_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/floodingschema/VPN_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/src/main/resources/service_flow_config.properties" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/resources/service_flow_config.properties" afterDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bean/SessionRecordLog.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bean/Traffic_Forward.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bean/UserRegion.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingDNSBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingFTPBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingHTTPBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingIPBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingJSTXBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingMAILBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingSSLBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingVPNBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/index/CountBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/spout/FloodingKafkaSpout.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/spout/SafeKafkaSpout.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/topology/StormRunner.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/topology/StreamAggregateTopology.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/combine/flooding/FloodingProcess_RZ.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/config/StreamAggregateConfig.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/config/StreamAggregateConfigurations.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/general/App_Coding.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/general/TransFormUtils.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/general/URL_HOS_Interactive.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/http/HttpClientService.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/cn/ac/iie/storm/utils/http/HttpConfig.class" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/common_app_id_coding.properties" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/common_app_label_coding.properties" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/floodingschema/DNS_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/floodingschema/FTP_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/floodingschema/HTTP_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/floodingschema/IP_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/floodingschema/JSTX_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/floodingschema/MAIL_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/floodingschema/SSL_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/floodingschema/VPN_RZZB.avsc" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/classes/service_flow_config.properties" beforeDir="false" /> - <change beforePath="$PROJECT_DIR$/target/test-classes/UrlTest.class" beforeDir="false" /> + <change beforePath="$PROJECT_DIR$/src/main/resources/common_app_id_coding.properties" beforeDir="false" afterPath="$PROJECT_DIR$/src/resources/common_app_id_coding.properties" afterDir="false" /> + <change beforePath="$PROJECT_DIR$/src/main/resources/common_app_label_coding.properties" beforeDir="false" afterPath="$PROJECT_DIR$/src/resources/common_app_label_coding.properties" afterDir="false" /> + <change beforePath="$PROJECT_DIR$/src/main/resources/service_flow_config.properties" beforeDir="false" afterPath="$PROJECT_DIR$/src/resources/service_flow_config.properties" afterDir="false" /> </list> <option name="SHOW_DIALOG" value="false" /> <option name="HIGHLIGHT_CONFLICTS" value="true" /> @@ -136,14 +79,15 @@ <recent name="C:\Users\ZDjizhi\IdeaProjects\xj-log-etl\.idea" /> </key> <key name="CopyClassDialog.RECENTS_KEY"> + <recent name="cn.ac.iie.storm.bolt.safe.gk" /> <recent name="cn.ac.iie.storm.bolt.safe" /> <recent name="cn.ac.iie.storm.utils.config" /> <recent name="cn.ac.iie.storm.topology" /> </key> <key name="MoveFile.RECENT_KEYS"> + <recent name="C:\Users\ZDjizhi\IdeaProjects\xj-log-etl\src" /> <recent name="C:\Users\ZDjizhi\IdeaProjects\xj-log-etl\src\main" /> <recent name="C:\Users\ZDjizhi\IdeaProjects\xj-log-etl" /> - <recent name="C:\Users\ZDjizhi\IdeaProjects\xj-log-etl\src" /> <recent name="C:\Users\ZDjizhi\IdeaProjects\xj-log-etl\src\resources" /> <recent name="C:\Users\ZDjizhi\IdeaProjects\xj-log-etl\src\main\resources" /> </key> @@ -282,7 +226,14 @@ <option name="project" value="LOCAL" /> <updated>1609325509992</updated> </task> - <option name="localTasksCounter" value="5" /> + <task id="LOCAL-00005" summary="2021.1.3"> + <created>1609665281547</created> + <option name="number" value="00005" /> + <option name="presentableId" value="LOCAL-00005" /> + <option name="project" value="LOCAL" /> + <updated>1609665281547</updated> + </task> + <option name="localTasksCounter" value="6" /> <servers /> </component> <component name="VcsManagerConfiguration"> @@ -290,7 +241,8 @@ <MESSAGE value="cluster Successful version" /> <MESSAGE value="2020.12.30" /> <MESSAGE value="2020.12.30(2)" /> - <option name="LAST_COMMIT_MESSAGE" value="2020.12.30(2)" /> + <MESSAGE value="2021.1.3" /> + <option name="LAST_COMMIT_MESSAGE" value="2021.1.3" /> </component> <component name="WindowStateProjectService"> <state x="1145" y="421" key="#Project_Structure" timestamp="1609385603196"> @@ -317,34 +269,34 @@ <screen x="0" y="0" width="2560" height="1400" /> </state> <state x="633" y="713" key="#com.intellij.ide.util.projectWizard.JdkChooserPanel.MyDialog/[email protected]" timestamp="1608791306166" /> - <state x="822" y="293" key="CommitChangelistDialog2" timestamp="1609325502475"> + <state x="822" y="293" key="CommitChangelistDialog2" timestamp="1609665279019"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state x="822" y="293" key="CommitChangelistDialog2/[email protected]" timestamp="1609325502475" /> - <state x="248" y="121" width="2048" height="1149" key="DiffContextDialog" timestamp="1609565866047"> + <state x="822" y="293" key="CommitChangelistDialog2/[email protected]" timestamp="1609665279019" /> + <state x="248" y="121" width="2048" height="1149" key="DiffContextDialog" timestamp="1609665302513"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state x="248" y="121" width="2048" height="1149" key="DiffContextDialog/[email protected]" timestamp="1609565866047" /> + <state x="248" y="121" width="2048" height="1149" key="DiffContextDialog/[email protected]" timestamp="1609665302513" /> <state x="567" y="579" key="FileChooserDialogImpl" timestamp="1609408096748"> <screen x="0" y="0" width="2560" height="1400" /> </state> <state x="567" y="579" key="FileChooserDialogImpl/[email protected]" timestamp="1609408096748" /> - <state width="2224" height="342" key="GridCell.Tab.0.bottom" timestamp="1609665199079"> + <state width="2224" height="342" key="GridCell.Tab.0.bottom" timestamp="1609748293121"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state width="2224" height="342" key="GridCell.Tab.0.bottom/[email protected]" timestamp="1609665199079" /> - <state width="2224" height="342" key="GridCell.Tab.0.center" timestamp="1609665199079"> + <state width="2224" height="342" key="GridCell.Tab.0.bottom/[email protected]" timestamp="1609748293121" /> + <state width="2224" height="342" key="GridCell.Tab.0.center" timestamp="1609748293121"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state width="2224" height="342" key="GridCell.Tab.0.center/[email protected]" timestamp="1609665199079" /> - <state width="2224" height="342" key="GridCell.Tab.0.left" timestamp="1609665199079"> + <state width="2224" height="342" key="GridCell.Tab.0.center/[email protected]" timestamp="1609748293121" /> + <state width="2224" height="342" key="GridCell.Tab.0.left" timestamp="1609748293121"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state width="2224" height="342" key="GridCell.Tab.0.left/[email protected]" timestamp="1609665199079" /> - <state width="2224" height="342" key="GridCell.Tab.0.right" timestamp="1609665199079"> + <state width="2224" height="342" key="GridCell.Tab.0.left/[email protected]" timestamp="1609748293121" /> + <state width="2224" height="342" key="GridCell.Tab.0.right" timestamp="1609748293121"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state width="2224" height="342" key="GridCell.Tab.0.right/[email protected]" timestamp="1609665199079" /> + <state width="2224" height="342" key="GridCell.Tab.0.right/[email protected]" timestamp="1609748293121" /> <state width="2224" height="342" key="GridCell.Tab.1.bottom" timestamp="1609663965154"> <screen x="0" y="0" width="2560" height="1400" /> </state> @@ -373,14 +325,14 @@ <screen x="0" y="0" width="2560" height="1400" /> </state> <state x="1126" y="615" key="VCS.ChangelistChooser/[email protected]" timestamp="1609579293994" /> - <state x="872" y="432" key="Vcs.Push.Dialog.v2" timestamp="1609320726723"> + <state x="872" y="432" key="Vcs.Push.Dialog.v2" timestamp="1609665314503"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state x="872" y="432" key="Vcs.Push.Dialog.v2/[email protected]" timestamp="1609320726723" /> - <state x="468" y="648" key="com.intellij.ide.util.TipDialog" timestamp="1609638132852"> + <state x="872" y="432" key="Vcs.Push.Dialog.v2/[email protected]" timestamp="1609665314503" /> + <state x="468" y="648" key="com.intellij.ide.util.TipDialog" timestamp="1609816867764"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state x="468" y="648" key="com.intellij.ide.util.TipDialog/[email protected]" timestamp="1609638132852" /> + <state x="468" y="648" key="com.intellij.ide.util.TipDialog/[email protected]" timestamp="1609816867764" /> <state x="1069" y="564" key="git4idea.rebase.GitRebaseDialog" timestamp="1608798913713"> <screen x="0" y="0" width="2560" height="1400" /> </state> @@ -389,9 +341,9 @@ <screen x="0" y="0" width="2560" height="1400" /> </state> <state x="936" y="330" key="run.anything.popup/[email protected]" timestamp="1608798671106" /> - <state x="1119" y="463" width="672" height="678" key="search.everywhere.popup" timestamp="1609661566324"> + <state x="1119" y="463" width="672" height="678" key="search.everywhere.popup" timestamp="1609750763584"> <screen x="0" y="0" width="2560" height="1400" /> </state> - <state x="1119" y="463" width="672" height="678" key="search.everywhere.popup/[email protected]" timestamp="1609661566324" /> + <state x="1119" y="463" width="672" height="678" key="search.everywhere.popup/[email protected]" timestamp="1609750763584" /> </component> </project>
\ No newline at end of file diff --git a/.idea/xj-log-etl.iml b/.idea/xj-log-etl.iml index 63039c5..8a21208 100644 --- a/.idea/xj-log-etl.iml +++ b/.idea/xj-log-etl.iml @@ -27,7 +27,7 @@ <content url="file://$MODULE_DIR$"> <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" /> <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" /> - <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" /> + <sourceFolder url="file://$MODULE_DIR$/src/resources" type="java-resource" /> <excludeFolder url="file://$MODULE_DIR$/target" /> </content> <orderEntry type="inheritedJdk" /> @@ -42,19 +42,19 @@ <orderEntry type="library" name="Maven: com.yammer.metrics:metrics-core:2.2.0" level="project" /> <orderEntry type="library" name="Maven: org.scala-lang:scala-library:2.11.11" level="project" /> <orderEntry type="library" name="Maven: com.101tec:zkclient:0.10" level="project" /> - <orderEntry type="library" name="Maven: org.apache.storm:storm-core:1.0.2" level="project" /> - <orderEntry type="library" name="Maven: com.esotericsoftware:kryo:3.0.3" level="project" /> - <orderEntry type="library" name="Maven: com.esotericsoftware:reflectasm:1.10.1" level="project" /> - <orderEntry type="library" name="Maven: org.ow2.asm:asm:5.0.3" level="project" /> - <orderEntry type="library" name="Maven: com.esotericsoftware:minlog:1.3.0" level="project" /> - <orderEntry type="library" name="Maven: org.objenesis:objenesis:2.1" level="project" /> - <orderEntry type="library" name="Maven: org.clojure:clojure:1.7.0" level="project" /> - <orderEntry type="library" name="Maven: com.lmax:disruptor:3.3.2" level="project" /> - <orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.1" level="project" /> - <orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-core:2.1" level="project" /> - <orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-slf4j-impl:2.1" level="project" /> - <orderEntry type="library" name="Maven: org.slf4j:log4j-over-slf4j:1.6.6" level="project" /> - <orderEntry type="library" name="Maven: javax.servlet:servlet-api:2.5" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.storm:storm-core:1.0.2" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: com.esotericsoftware:kryo:3.0.3" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: com.esotericsoftware:reflectasm:1.10.1" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: org.ow2.asm:asm:5.0.3" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: com.esotericsoftware:minlog:1.3.0" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: org.objenesis:objenesis:2.1" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: org.clojure:clojure:1.7.0" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: com.lmax:disruptor:3.3.2" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.logging.log4j:log4j-api:2.1" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.logging.log4j:log4j-core:2.1" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: org.apache.logging.log4j:log4j-slf4j-impl:2.1" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: org.slf4j:log4j-over-slf4j:1.6.6" level="project" /> + <orderEntry type="library" scope="PROVIDED" name="Maven: javax.servlet:servlet-api:2.5" level="project" /> <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.7" level="project" /> <orderEntry type="library" name="Maven: org.apache.storm:storm-kafka:1.0.2" level="project" /> <orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.10.0" level="project" /> @@ -165,7 +165,7 @@ <groupId>org.apache.storm</groupId> <artifactId>storm-core</artifactId> <version>${storm.version}</version> -<!-- <scope>provided</scope>--> + <scope>provided</scope> </dependency> <dependency> diff --git a/src/main/java/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.java b/src/main/java/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.java index 82ad3fd..53d1bf3 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.java @@ -45,7 +45,6 @@ public class ParseFloodingBolt extends BaseBasicBolt { String common_schema_type = sessionRecordLog.getCommon_schema_type(); ybInputSize=line.getBytes().length; //按common_schema_type的值对数据进行分类发送 - System.out.println(line); switch(common_schema_type){ case "BASE": if(sessionRecordLog.getCommon_schema_type()==null){ diff --git a/src/main/java/cn/ac/iie/storm/bolt/index/CountBolt.java b/src/main/java/cn/ac/iie/storm/bolt/index/CountBolt.java index a5df6b3..3569c3e 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/index/CountBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/index/CountBolt.java @@ -83,7 +83,6 @@ public class CountBolt extends BaseBasicBolt { YBInputLog.labels("flooding-etl",localip,String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(floodingcount); YBOutputLog.labels("flooding-etl",localip,String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).set(floodingsucesscount); Timestamp.labels("flooding-etl",localip,String.valueOf(StreamAggregateConfig.COUNT_TOPOLOGY_TICK_TUPLE_FREQ_SECS/60)).setToCurrentTime(); - System.out.println(inputrate); try { push(); } catch (IOException e) { diff --git a/src/main/java/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.java b/src/main/java/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.java index c9adc38..74da49d 100644 --- a/src/main/java/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.java +++ b/src/main/java/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.java @@ -25,8 +25,8 @@ public class ParseSafeBolt extends BaseBasicBolt { private SessionRecordLog hmdrzzbSource; private SessionRecordLog dnsgkzbSource; private SessionRecordLog httpgkzbSource; - private SessionRecordLog tdzsgkzbSource; - private SessionRecordLog wygjzgkrzSource; + private SessionRecordLog sslgkzbSource; + private SessionRecordLog wygjcgkrzSource; private SessionRecordLog mailgkzbSource; private SessionRecordLog ftpgjzgkzbSouce; private SessionRecordLog vpngkzbSource; @@ -38,8 +38,8 @@ public class ParseSafeBolt extends BaseBasicBolt { private SessionRecordLog ipjczbSource; private SessionRecordLog dnsjczbSource; private SessionRecordLog httpjczbSource; - private SessionRecordLog tdzsjczbSource; - private SessionRecordLog wygjzjcrzSource; + private SessionRecordLog ssljczbSource; + private SessionRecordLog wygjcjcrzSource; private SessionRecordLog mailgjzjczbSource; private SessionRecordLog ftpgjzjczbSource; private SessionRecordLog vpnjczbSource; @@ -106,12 +106,12 @@ public class ParseSafeBolt extends BaseBasicBolt { gkInputLog = line.getBytes().length; break; case 405: //特定证书封堵 - tdzsgkzbSource = sessionRecordLog; + sslgkzbSource = sessionRecordLog; gkInputLog = line.getBytes().length; break; case 406: //网站关键字封堵 case 409: //搜索词封堵 - wygjzgkrzSource = sessionRecordLog; + wygjcgkrzSource = sessionRecordLog; gkInputLog = line.getBytes().length; break; case 407: //邮件关键字封堵 @@ -147,12 +147,12 @@ public class ParseSafeBolt extends BaseBasicBolt { jcInputLog = line.getBytes().length; break; case 505: //特定证书监测 - tdzsjczbSource = sessionRecordLog; + ssljczbSource = sessionRecordLog; jcInputLog = line.getBytes().length; break; case 506: //网站关键词监测 case 509: //搜索词监测 - wygjzjcrzSource = sessionRecordLog; + wygjcjcrzSource = sessionRecordLog; jcInputLog = line.getBytes().length; break; case 507: //邮件监测 @@ -188,8 +188,8 @@ public class ParseSafeBolt extends BaseBasicBolt { basicOutputCollector.emit(StreamAggregateConfig.GK_IP_STREAMID,new Values(hmdrzzbSource)); basicOutputCollector.emit(StreamAggregateConfig.GK_DNS_STREAMID,new Values(dnsgkzbSource)); basicOutputCollector.emit(StreamAggregateConfig.GK_HTTP_STREAMID,new Values(httpgkzbSource)); - basicOutputCollector.emit(StreamAggregateConfig.GK_TDZS_STREAMID,new Values(tdzsgkzbSource)); - basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJZ_STREAMID,new Values(wygjzgkrzSource)); + basicOutputCollector.emit(StreamAggregateConfig.GK_SSL_STREAMID,new Values(sslgkzbSource)); + basicOutputCollector.emit(StreamAggregateConfig.GK_WYGJC_STREAMID,new Values(wygjcgkrzSource)); basicOutputCollector.emit(StreamAggregateConfig.GK_MAIL_STREAMID,new Values(mailgkzbSource)); basicOutputCollector.emit(StreamAggregateConfig.GK_FTP_STREAMID,new Values(ftpgjzgkzbSouce)); basicOutputCollector.emit(StreamAggregateConfig.GK_VPN_STREAMID,new Values(vpngkzbSource)); @@ -199,8 +199,8 @@ public class ParseSafeBolt extends BaseBasicBolt { basicOutputCollector.emit(StreamAggregateConfig.JC_IP_STREAMID,new Values(ipjczbSource)); basicOutputCollector.emit(StreamAggregateConfig.JC_DNS_STREAMID,new Values(dnsjczbSource)); basicOutputCollector.emit(StreamAggregateConfig.JC_HTTP_STREAMID,new Values(httpjczbSource)); - basicOutputCollector.emit(StreamAggregateConfig.JC_TDZS_STREAMID,new Values(tdzsjczbSource)); - basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJZ_STREAMID,new Values(wygjzjcrzSource)); + basicOutputCollector.emit(StreamAggregateConfig.JC_SSL_STREAMID,new Values(ssljczbSource)); + basicOutputCollector.emit(StreamAggregateConfig.JC_WYGJC_STREAMID,new Values(wygjcjcrzSource)); basicOutputCollector.emit(StreamAggregateConfig.JC_MAIL_STREAMID,new Values(mailgjzjczbSource)); basicOutputCollector.emit(StreamAggregateConfig.JC_FTP_STREAMID,new Values(ftpgjzjczbSource)); basicOutputCollector.emit(StreamAggregateConfig.JC_VPN_STREAMID,new Values(vpnjczbSource)); @@ -221,8 +221,8 @@ public class ParseSafeBolt extends BaseBasicBolt { outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_IP_STREAMID,new Fields(StreamAggregateConfig.GK_IP_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_DNS_STREAMID,new Fields(StreamAggregateConfig.GK_DNS_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_HTTP_STREAMID,new Fields(StreamAggregateConfig.GK_HTTP_STREAMID)); - outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_TDZS_STREAMID,new Fields(StreamAggregateConfig.GK_TDZS_STREAMID)); - outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_WYGJZ_STREAMID,new Fields(StreamAggregateConfig.GK_WYGJZ_STREAMID)); + outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_SSL_STREAMID,new Fields(StreamAggregateConfig.GK_SSL_STREAMID)); + outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_WYGJC_STREAMID,new Fields(StreamAggregateConfig.GK_WYGJC_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_MAIL_STREAMID,new Fields(StreamAggregateConfig.GK_MAIL_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_FTP_STREAMID,new Fields(StreamAggregateConfig.GK_FTP_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.GK_VPN_STREAMID,new Fields(StreamAggregateConfig.GK_VPN_STREAMID)); @@ -232,10 +232,10 @@ public class ParseSafeBolt extends BaseBasicBolt { outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_IP_STREAMID,new Fields(StreamAggregateConfig.JC_IP_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_DNS_STREAMID,new Fields(StreamAggregateConfig.JC_DNS_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_HTTP_STREAMID,new Fields(StreamAggregateConfig.JC_HTTP_STREAMID)); - outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_TDZS_STREAMID,new Fields(StreamAggregateConfig.JC_TDZS_STREAMID)); - outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_WYGJZ_STREAMID,new Fields(StreamAggregateConfig.JC_WYGJZ_STREAMID)); + outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_SSL_STREAMID,new Fields(StreamAggregateConfig.JC_SSL_STREAMID)); + outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_WYGJC_STREAMID,new Fields(StreamAggregateConfig.JC_WYGJC_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_MAIL_STREAMID,new Fields(StreamAggregateConfig.JC_MAIL_STREAMID)); - outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_FTP_STREAMID,new Fields(StreamAggregateConfig.JC_VPN_STREAMID)); + outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_FTP_STREAMID,new Fields(StreamAggregateConfig.JC_FTP_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_VPN_STREAMID,new Fields(StreamAggregateConfig.JC_VPN_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_JSTX_STREAMID,new Fields(StreamAggregateConfig.JC_JSTX_STREAMID)); outputFieldsDeclarer.declareStream(StreamAggregateConfig.JC_PIC_STREAMID,new Fields(StreamAggregateConfig.JC_PIC_STREAMID)); diff --git a/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKDNSBolt.java b/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKDNSBolt.java new file mode 100644 index 0000000..89bbd3e --- /dev/null +++ b/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKDNSBolt.java @@ -0,0 +1,140 @@ +package cn.ac.iie.storm.bolt.safe.gk; + +import cn.ac.iie.storm.bean.SessionRecordLog; +import cn.ac.iie.storm.utils.config.StreamAggregateConfig; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; +import org.apache.storm.utils.TupleUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static cn.ac.iie.storm.utils.general.TransFormUtils.*; + +public class GKDNSBolt extends BaseBasicBolt { + private final static Logger logger = Logger.getLogger(GKDNSBolt.class); + private static final long serialVersionUID = -6795251425357896415L; + private List<GenericRecord> dnsList = new ArrayList<GenericRecord>(); + private Schema schema; + private SessionRecordLog sessionRecordLog; + private Long dnscount = 0L; + private Long dnsloadfail = 0L; + private long dnsdatasize = 0L; + private byte[] generate=null; + + + @Override + public void prepare(Map stormConf, TopologyContext context) { + schema = new Schema.Parser().parse(StreamAggregateConfig.DNS_GKZB_SCHEMA); + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + + if (TupleUtils.isTick(tuple)&&!dnsList.isEmpty()){ //定时 + dnsdatasize = dnsdatasize + getValidLength(generate); + + dnsList.clear(); + dnsdatasize = 0L; + dnscount = 0L; + dnsloadfail = 0L; + }else if(dnsList.size()>=StreamAggregateConfig.SAFE_BATCH_INTER_NUM){ //分批 + dnsdatasize = dnsdatasize + getValidLength(generate); + dnsList.clear(); + } + else { + if(tuple.getSourceStreamId().equals(StreamAggregateConfig.GK_DNS_STREAMID)){ + sessionRecordLog =(SessionRecordLog)tuple.getValue(0); + } + if (sessionRecordLog != null) { + try { + GenericRecord dns_gkzb = new GenericData.Record(schema); + setDNS_RZZB(dns_gkzb, sessionRecordLog); + dnsList.add(dns_gkzb); + generate = generate(schema, dnsList); + dnscount++; + } catch (Exception e) { + logger.error("dns_gkzb数据封装处理异常"+e); + dnsloadfail++; + + } + } + } + +} + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + + } + private void setDNS_RZZB(GenericRecord dns_gkzb, SessionRecordLog sessionRecordLog) throws IOException { + dns_gkzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); + dns_gkzb.put("YDY","ff"); + dns_gkzb.put("MDDY",sessionRecordLog.getCommon_server_location()); + // dns_gkzb.put("QDJIP",sessionRecordLog.getCommon_sled_ip()); + dns_gkzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_server_ip())); //--其实是使用网络序列 + dns_gkzb.put("SIP",111); + dns_gkzb.put("DIP",sessionRecordLog.getCommon_server_ip()); + dns_gkzb.put("SPORT",sessionRecordLog.getCommon_client_port()); + dns_gkzb.put("DPORT",sessionRecordLog.getCommon_server_port()); + //字段类型不一致，还未处理 + dns_gkzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ///--转化表 还没弄 + dns_gkzb.put("DETAIL_ID","1"); //同上 + dns_gkzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); //-common_l4_protocol其实是这个字段 + dns_gkzb.put("YYS",1); //-**common_isp这个字段 + dns_gkzb.put("yl1",sessionRecordLog.getCommon_address_type()); + //dns特有 + dns_gkzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); + dns_gkzb.put("CS_BYTES",sessionRecordLog.getCommon_c2s_byte_num()); + dns_gkzb.put("SC_PKTS",sessionRecordLog.getCommon_s2c_byte_num()); + dns_gkzb.put("SC_BYTES",sessionRecordLog.getCommon_s2c_byte_num()); + dns_gkzb.put("DNS_NAME",sessionRecordLog.getDns_qname()); + dns_gkzb.put("DNS_VALUE",sessionRecordLog.getDns_rr()); + } + @Override + public Map<String, Object> getComponentConfiguration() { + Map<String, Object> conf = new HashMap<String, Object>(); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, StreamAggregateConfig.SAFE_TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + + private byte[] generate(Schema schema, List<GenericRecord> genericRecordList){ + DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(); + DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter); + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + dataFileWriter.create(schema, out); + if (genericRecordList != null && genericRecordList.size() > 0) { + for (int i = 0; i < genericRecordList.size(); i++) { + dataFileWriter.append(genericRecordList.get(i)); + } + } + dataFileWriter.flush(); + return out.toByteArray(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + dataFileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } +} diff --git a/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKHTTPBolt.java b/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKHTTPBolt.java new file mode 100644 index 0000000..e4fe8da --- /dev/null +++ b/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKHTTPBolt.java @@ -0,0 +1,138 @@ +package cn.ac.iie.storm.bolt.safe.gk; + +import cn.ac.iie.storm.bean.SessionRecordLog; +import cn.ac.iie.storm.utils.config.StreamAggregateConfig; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.utils.TupleUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static cn.ac.iie.storm.utils.general.TransFormUtils.*; + +public class GKHTTPBolt extends BaseBasicBolt { + private final static Logger logger = Logger.getLogger(GKHTTPBolt.class); + private static final long serialVersionUID = -6795251425357896415L; + private List<GenericRecord> httpList = new ArrayList<GenericRecord>(); + private Schema schema; + private SessionRecordLog sessionRecordLog; + private Long httpcount = 0L; + private Long httploadfail = 0L; + private long httpdatasize = 0L; + private byte[] generate=null; + + + @Override + public void prepare(Map stormConf, TopologyContext context) { + schema = new Schema.Parser().parse(StreamAggregateConfig.HTTP_GKZB_SCHEMA); + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + + if (TupleUtils.isTick(tuple)&&!httpList.isEmpty()){ //定时 + httpdatasize = httpdatasize + getValidLength(generate); + + httpList.clear(); + httpdatasize = 0L; + httpcount = 0L; + httploadfail = 0L; + }else if(httpList.size()>=StreamAggregateConfig.SAFE_BATCH_INTER_NUM){ //分批 + httpdatasize = httpdatasize + getValidLength(generate); + httpList.clear(); + } + else { + if(tuple.getSourceStreamId().equals(StreamAggregateConfig.GK_HTTP_STREAMID)){ + sessionRecordLog =(SessionRecordLog)tuple.getValue(0); + } + if (sessionRecordLog != null) { + try { + GenericRecord http_gkzb = new GenericData.Record(schema); + setDNS_RZZB(http_gkzb, sessionRecordLog); + httpList.add(http_gkzb); + generate = generate(schema, httpList); + httpcount++; + } catch (Exception e) { + logger.error("http_gkzb数据封装处理异常"+e); + httploadfail++; + + } + } + } + +} + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + + } + private void setDNS_RZZB(GenericRecord http_gkzb, SessionRecordLog sessionRecordLog) throws IOException { + http_gkzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); + http_gkzb.put("YDY","ff"); + http_gkzb.put("MDDY",sessionRecordLog.getCommon_server_location()); + // http_gkzb.put("QDJIP",sessionRecordLog.getCommon_sled_ip()); + http_gkzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_server_ip())); //--其实是使用网络序列 + http_gkzb.put("SIP",111); + http_gkzb.put("DIP",sessionRecordLog.getCommon_server_ip()); + http_gkzb.put("SPORT",sessionRecordLog.getCommon_client_port()); + http_gkzb.put("DPORT",sessionRecordLog.getCommon_server_port()); + //字段类型不一致，还未处理 + http_gkzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ///--转化表 还没弄 + http_gkzb.put("DETAIL_ID","1"); //同上 + http_gkzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); //-common_l4_protocol其实是这个字段 + http_gkzb.put("YYS",1); //-**common_isp这个字段 + http_gkzb.put("yl1",sessionRecordLog.getCommon_address_type()); + //dns特有 + http_gkzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); + http_gkzb.put("CS_BYTES",sessionRecordLog.getCommon_c2s_byte_num()); + http_gkzb.put("SC_PKTS",sessionRecordLog.getCommon_s2c_byte_num()); + http_gkzb.put("SC_BYTES",sessionRecordLog.getCommon_s2c_byte_num()); + http_gkzb.put("DNS_NAME",sessionRecordLog.getDns_qname()); + http_gkzb.put("DNS_VALUE",sessionRecordLog.getDns_rr()); + } + @Override + public Map<String, Object> getComponentConfiguration() { + Map<String, Object> conf = new HashMap<String, Object>(); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, StreamAggregateConfig.SAFE_TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + + private byte[] generate(Schema schema, List<GenericRecord> genericRecordList){ + DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(); + DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter); + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + dataFileWriter.create(schema, out); + if (genericRecordList != null && genericRecordList.size() > 0) { + for (int i = 0; i < genericRecordList.size(); i++) { + dataFileWriter.append(genericRecordList.get(i)); + } + } + dataFileWriter.flush(); + return out.toByteArray(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + dataFileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } +} diff --git a/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKIPBolt.java b/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKIPBolt.java new file mode 100644 index 0000000..a1c9248 --- /dev/null +++ b/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKIPBolt.java @@ -0,0 +1,142 @@ +package cn.ac.iie.storm.bolt.safe.gk; + +import cn.ac.iie.storm.bean.SessionRecordLog; +import cn.ac.iie.storm.utils.config.StreamAggregateConfig; +import cn.ac.iie.storm.utils.http.HttpClientService; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; +import org.apache.storm.utils.TupleUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static cn.ac.iie.storm.utils.general.TransFormUtils.*; + +public class GKIPBolt extends BaseBasicBolt { + private final static Logger logger = Logger.getLogger(GKIPBolt.class); + private static final long serialVersionUID = -6795251425357896415L; + private List<GenericRecord> ipList = new ArrayList<GenericRecord>(); + private Schema schema; + private SessionRecordLog sessionRecordLog; + private Long ipcount = 0L; + private Long iploadfail = 0L; + private long ipdatasize = 0L; + private byte[] generate=null; + + + @Override + public void prepare(Map stormConf, TopologyContext context) { + schema = new Schema.Parser().parse(StreamAggregateConfig.IP_GKRZ_SCHEMA); + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + + if (TupleUtils.isTick(tuple)&&!ipList.isEmpty()){ //定时 + ipdatasize = ipdatasize + getValidLength(generate); + basicOutputCollector.emit(new Values(ipcount,iploadfail,ipdatasize)); + ipList.clear(); + ipdatasize = 0L; + ipcount = 0L; + iploadfail = 0L; + }else if(ipList.size()>=StreamAggregateConfig.SAFE_BATCH_INTER_NUM){ //分批 + + ipdatasize = ipdatasize + getValidLength(generate); + ipList.clear(); + } + else { + if(tuple.getSourceStreamId().equals(StreamAggregateConfig.GK_IP_STREAMID)){ + sessionRecordLog =(SessionRecordLog)tuple.getValue(0); + } + if (sessionRecordLog != null) { + try { + GenericRecord ip_gkrz = new GenericData.Record(schema); + setDNS_RZZB(ip_gkrz, sessionRecordLog); + ipList.add(ip_gkrz); + generate = generate(schema, ipList); + ipcount++; + } catch (Exception e) { + logger.error("ip_gkrz数据封装处理异常"+e); + iploadfail++; + + } + } + } + +} + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + + } + private void setDNS_RZZB(GenericRecord ip_gkrz, SessionRecordLog sessionRecordLog) throws IOException { + ip_gkrz.put("BHSJ",sessionRecordLog.getCommon_recv_time()); + ip_gkrz.put("YDY","ff"); + ip_gkrz.put("MDDY",sessionRecordLog.getCommon_server_location()); + // IP_rzzb.put("QDJIP",sessionRecordLog.getCommon_sled_ip()); + ip_gkrz.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_server_ip())); //--其实是使用网络序列 + ip_gkrz.put("SIP",111); + ip_gkrz.put("DIP",sessionRecordLog.getCommon_server_ip()); + ip_gkrz.put("SPORT",sessionRecordLog.getCommon_client_port()); + ip_gkrz.put("DPORT",sessionRecordLog.getCommon_server_port()); + //字段类型不一致，还未处理 + ip_gkrz.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ///--转化表 还没弄 + ip_gkrz.put("DETAIL_ID","1"); //同上 + ip_gkrz.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); //-common_l4_protocol其实是这个字段 + ip_gkrz.put("YYS",1); //-**common_isp这个字段 + ip_gkrz.put("yl1",sessionRecordLog.getCommon_address_type()); + //dns特有 + ip_gkrz.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); + ip_gkrz.put("CS_BYTES",sessionRecordLog.getCommon_c2s_byte_num()); + ip_gkrz.put("SC_PKTS",sessionRecordLog.getCommon_s2c_byte_num()); + ip_gkrz.put("SC_BYTES",sessionRecordLog.getCommon_s2c_byte_num()); + ip_gkrz.put("DNS_NAME",sessionRecordLog.getDns_qname()); + ip_gkrz.put("DNS_VALUE",sessionRecordLog.getDns_rr()); + } + @Override + public Map<String, Object> getComponentConfiguration() { + Map<String, Object> conf = new HashMap<String, Object>(); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, StreamAggregateConfig.SAFE_TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + + private byte[] generate(Schema schema, List<GenericRecord> genericRecordList){ + DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(); + DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter); + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + dataFileWriter.create(schema, out); + if (genericRecordList != null && genericRecordList.size() > 0) { + for (int i = 0; i < genericRecordList.size(); i++) { + dataFileWriter.append(genericRecordList.get(i)); + } + } + dataFileWriter.flush(); + return out.toByteArray(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + dataFileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } +} diff --git a/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKSSLBolt.java b/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKSSLBolt.java new file mode 100644 index 0000000..c9ba3a2 --- /dev/null +++ b/src/main/java/cn/ac/iie/storm/bolt/safe/gk/GKSSLBolt.java @@ -0,0 +1,138 @@ +package cn.ac.iie.storm.bolt.safe.gk; + +import cn.ac.iie.storm.bean.SessionRecordLog; +import cn.ac.iie.storm.utils.config.StreamAggregateConfig; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.log4j.Logger; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.BasicOutputCollector; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseBasicBolt; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.utils.TupleUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static cn.ac.iie.storm.utils.general.TransFormUtils.*; + +public class GKSSLBolt extends BaseBasicBolt { + private final static Logger logger = Logger.getLogger(GKSSLBolt.class); + private static final long serialVersionUID = -6795251425357896415L; + private List<GenericRecord> sslList = new ArrayList<GenericRecord>(); + private Schema schema; + private SessionRecordLog sessionRecordLog; + private Long sslcount = 0L; + private Long sslloadfail = 0L; + private long ssldatasize = 0L; + private byte[] generate=null; + + + @Override + public void prepare(Map stormConf, TopologyContext context) { + schema = new Schema.Parser().parse(StreamAggregateConfig.SSL_GKZB_SCHEMA); + } + + @Override + public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { + + if (TupleUtils.isTick(tuple)&&!sslList.isEmpty()){ //定时 + ssldatasize = ssldatasize + getValidLength(generate); + + sslList.clear(); + ssldatasize = 0L; + sslcount = 0L; + sslloadfail = 0L; + }else if(sslList.size()>=StreamAggregateConfig.SAFE_BATCH_INTER_NUM){ //分批 + ssldatasize = ssldatasize + getValidLength(generate); + sslList.clear(); + } + else { + if(tuple.getSourceStreamId().equals(StreamAggregateConfig.GK_SSL_STREAMID)){ + sessionRecordLog =(SessionRecordLog)tuple.getValue(0); + } + if (sessionRecordLog != null) { + try { + GenericRecord ssl_gkzb = new GenericData.Record(schema); + setDNS_RZZB(ssl_gkzb, sessionRecordLog); + sslList.add(ssl_gkzb); + generate = generate(schema, sslList); + sslcount++; + } catch (Exception e) { + logger.error("dns_gkzb数据封装处理异常"+e); + sslloadfail++; + + } + } + } + +} + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + + } + private void setDNS_RZZB(GenericRecord dns_gkzb, SessionRecordLog sessionRecordLog) throws IOException { + dns_gkzb.put("BHSJ",sessionRecordLog.getCommon_recv_time()); + dns_gkzb.put("YDY","ff"); + dns_gkzb.put("MDDY",sessionRecordLog.getCommon_server_location()); + // dns_gkzb.put("QDJIP",sessionRecordLog.getCommon_sled_ip()); + dns_gkzb.put("QDJIP",getQDJIP(sessionRecordLog.getCommon_server_ip())); //--其实是使用网络序列 + dns_gkzb.put("SIP",111); + dns_gkzb.put("DIP",sessionRecordLog.getCommon_server_ip()); + dns_gkzb.put("SPORT",sessionRecordLog.getCommon_client_port()); + dns_gkzb.put("DPORT",sessionRecordLog.getCommon_server_port()); + //字段类型不一致，还未处理 + dns_gkzb.put("PROTO_ID",getProto_id(sessionRecordLog.getCommon_schema_type())); ///--转化表 还没弄 + dns_gkzb.put("DETAIL_ID","1"); //同上 + dns_gkzb.put("PROTOCOL",getProtocol(sessionRecordLog.getCommon_l4_protocol())); //-common_l4_protocol其实是这个字段 + dns_gkzb.put("YYS",1); //-**common_isp这个字段 + dns_gkzb.put("yl1",sessionRecordLog.getCommon_address_type()); + //dns特有 + dns_gkzb.put("CS_PKTS",sessionRecordLog.getCommon_c2s_pkt_num()); + dns_gkzb.put("CS_BYTES",sessionRecordLog.getCommon_c2s_byte_num()); + dns_gkzb.put("SC_PKTS",sessionRecordLog.getCommon_s2c_byte_num()); + dns_gkzb.put("SC_BYTES",sessionRecordLog.getCommon_s2c_byte_num()); + dns_gkzb.put("DNS_NAME",sessionRecordLog.getDns_qname()); + dns_gkzb.put("DNS_VALUE",sessionRecordLog.getDns_rr()); + } + @Override + public Map<String, Object> getComponentConfiguration() { + Map<String, Object> conf = new HashMap<String, Object>(); + conf.put(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, StreamAggregateConfig.SAFE_TOPOLOGY_TICK_TUPLE_FREQ_SECS); + return conf; + } + + private byte[] generate(Schema schema, List<GenericRecord> genericRecordList){ + DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(); + DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter); + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + dataFileWriter.create(schema, out); + if (genericRecordList != null && genericRecordList.size() > 0) { + for (int i = 0; i < genericRecordList.size(); i++) { + dataFileWriter.append(genericRecordList.get(i)); + } + } + dataFileWriter.flush(); + return out.toByteArray(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + dataFileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } +} diff --git a/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java b/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java index 8b84fa2..7f1589e 100644 --- a/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java +++ b/src/main/java/cn/ac/iie/storm/topology/StreamAggregateTopology.java @@ -67,11 +67,11 @@ public class StreamAggregateTopology { -// //设置拓扑关系(Spout) -// topologyBuilder.setSpout(StreamAggregateConfig.SAFE_KAFKA_SPOUT,new FloodingKafkaSpout()); -// -// //设置拓扑关系(Bolt) --切分行 -// topologyBuilder.setBolt("ParseSafeBolt",new ParseSafeBolt()).shuffleGrouping(StreamAggregateConfig.SAFE_KAFKA_SPOUT); + //设置拓扑关系(Spout) + topologyBuilder.setSpout(StreamAggregateConfig.SAFE_KAFKA_SPOUT,new FloodingKafkaSpout()); + + //设置拓扑关系(Bolt) --切分行 + topologyBuilder.setBolt("ParseSafeBolt",new ParseSafeBolt()).shuffleGrouping(StreamAggregateConfig.SAFE_KAFKA_SPOUT); diff --git a/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java b/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java index be0ff55..355ac12 100644 --- a/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java +++ b/src/main/java/cn/ac/iie/storm/utils/config/StreamAggregateConfig.java @@ -25,14 +25,14 @@ public class StreamAggregateConfig { public static final String KAFKA_FLOODING_TOPIC = StreamAggregateConfigurations.getStringProperty(0, "kafka.flooding.topic"); public static final String KAFKA_SAFE_TOPIC = StreamAggregateConfigurations.getStringProperty(0, "kafka.safe.topic"); public static final String AUTO_OFFSET_RESET = StreamAggregateConfigurations.getStringProperty(0, "auto.offset.reset"); - - - - + /** + * flooding + */ //Flooding-bolt设置 public static final int FLOODING_TOPOLOGY_TICK_TUPLE_FREQ_SECS = StreamAggregateConfigurations.getIntProperty(0,"flooding.topology.tick.tuple.freq.secs"); //定时 //分批 - //Flooding-spout-field + public static final int FLOODING_BATCH_INTER_NUM = StreamAggregateConfigurations.getIntProperty(0,"flooding.batch.inter.num"); + public static final String FLOODING_SPOUT_FIELD = StreamAggregateConfigurations.getStringProperty(0, "flooding.spout.field"); //Flooding-ComponentId-spout public static final String FLOODING_KAFKA_SPOUT = StreamAggregateConfigurations.getStringProperty(0, "flooding.kafka.spout"); @@ -120,6 +120,15 @@ public class StreamAggregateConfig { * safa */ + + + //Flooding-bolt设置 + //定时 + public static final int SAFE_TOPOLOGY_TICK_TUPLE_FREQ_SECS = StreamAggregateConfigurations.getIntProperty(0,"safe.topology.tick.tuple.freq.secs"); + //分批 + public static final int SAFE_BATCH_INTER_NUM = StreamAggregateConfigurations.getIntProperty(0,"safe.batch.inter.num"); + + //Flooding-spout-field public static final String SAFE_SPOUT_FIELD = StreamAggregateConfigurations.getStringProperty(0, "safe.spout.field"); //Flooding-ComponentId-spout @@ -129,19 +138,25 @@ public class StreamAggregateConfig { public static final String GK_IP_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.ip.streamid"); public static final String GK_DNS_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.dns.streamid"); public static final String GK_HTTP_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.http.streamid"); - public static final String GK_TDZS_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.tdzs.streamid"); - public static final String GK_WYGJZ_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.wygjz.streamid"); + public static final String GK_SSL_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.ssl.streamid"); + public static final String GK_WYGJC_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.wygjc.streamid"); public static final String GK_MAIL_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.mail.streamid"); public static final String GK_FTP_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.ftp.streamid"); public static final String GK_VPN_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.vpn.streamid"); public static final String GK_JSTX_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "gk.jstx.streamid"); + //gk-schema + public static final String IP_GKRZ_SCHEMA = StreamAggregateConfigurations.getStringProperty(0, "ip.gkrz.schema"); + public static final String DNS_GKZB_SCHEMA = StreamAggregateConfigurations.getStringProperty(0, "dns.gkzb.schema"); + public static final String HTTP_GKZB_SCHEMA = StreamAggregateConfigurations.getStringProperty(0, "http.gkzb.schema"); + public static final String SSL_GKZB_SCHEMA = StreamAggregateConfigurations.getStringProperty(0, "ssl.gkzb.schema"); + //jc-source-streamid public static final String JC_IP_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.ip.streamid"); public static final String JC_DNS_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.dns.streamid"); public static final String JC_HTTP_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.http.streamid"); - public static final String JC_TDZS_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.tdzs.streamid"); - public static final String JC_WYGJZ_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.wygjz.streamid"); + public static final String JC_SSL_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.ssl.streamid"); + public static final String JC_WYGJC_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.wygjc.streamid"); public static final String JC_MAIL_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.mail.streamid"); public static final String JC_FTP_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.ftp.streamid"); public static final String JC_VPN_STREAMID = StreamAggregateConfigurations.getStringProperty(0, "jc.vpn.streamid"); diff --git a/src/main/resources/common_app_id_coding.properties b/src/resources/common_app_id_coding.properties index f234055..f234055 100644 --- a/src/main/resources/common_app_id_coding.properties +++ b/src/resources/common_app_id_coding.properties diff --git a/src/main/resources/common_app_label_coding.properties b/src/resources/common_app_label_coding.properties index 043b8eb..043b8eb 100644 --- a/src/main/resources/common_app_label_coding.properties +++ b/src/resources/common_app_label_coding.properties diff --git a/src/resources/service_flow_config.properties b/src/resources/service_flow_config.properties new file mode 100644 index 0000000..1d023dd --- /dev/null +++ b/src/resources/service_flow_config.properties @@ -0,0 +1,182 @@ +#\u7BA1\u7406kafka\u5730\u5740 +bootstrap.servers=192.168.44.11:9092,192.168.44.14:9092,192.168.44.15:9092 +#latest/earliest +auto.offset.reset=latest + +#\u538B\u7F29\u6A21\u5F0F none or snappy +kafka.compression.type=none + +#kafka broker\u4E0B\u7684\u6CDB\u6536topic\u540D\u79F0 +kafka.flooding.topic=CONNECTION-RECORD-COMPLETED-LOG + +#kafka broker\u4E0B\u7684\u5B89\u5168topic\u540D\u79F0 +kafka.safe.topic=SECURITY-EVENT-LOG + +#\u8BFB\u53D6topic,\u5B58\u50A8\u8BE5spout id\u7684\u6D88\u8D39offset\u4FE1\u606F\uFF0C\u53EF\u901A\u8FC7\u8BE5\u62D3\u6251\u547D\u540D;\u5177\u4F53\u5B58\u50A8offset\u7684\u4F4D\u7F6E\uFF0C\u786E\u5B9A\u4E0B\u6B21\u8BFB\u53D6\u4E0D\u91CD\u590D\u7684\u6570\u636E\uFF1B +group.id=test-consumer-group121 +group.safe.id=test-consumer-group11 + +#storm topology workers +topology.workers=1 + +#spout\u5E76\u884C\u5EA6 \u5EFA\u8BAE\u4E0Ekafka\u5206\u533A\u6570\u76F8\u540C +spout.parallelism=2 + + +#ack\u8BBE\u7F6E 1\u542F\u52A8ack 0\u4E0D\u542F\u52A8ack +topology.num.acks=0 + +#spout\u63A5\u6536\u7761\u7720\u65F6\u95F4 +topology.spout.sleep.time=1 + + +#flooding-bolt\u6BCF\u6279\u6B21\u5904\u7406\u7684\u65F6\u95F4\u95F4\u9694 +flooding.topology.tick.tuple.freq.secs=10 +#flooding-bolt\u6BCF\u6279\u5904\u7406\u6761\u6570 +flooding.batch.inter.num=200 + + +#safe-bolt\u6BCF\u6279\u6B21\u5904\u7406\u7684\u65F6\u95F4\u95F4\u9694 +safe.topology.tick.tuple.freq.secs=10 +#flooding-bolt\u6BCF\u6279\u5904\u7406\u6761\u6570 +safe.batch.inter.num=200 + + + +#Flooding-spout-field +flooding.spout.field=Flooding-SOURCE + +#Flooding-bolt-StreamID +flooding.dns.streamid=FLOODING-DNS-SOURCE +flooding.ftp.streamid=FLOODING-FTP-SOURCE +flooding.http.streamid=FLOODING-HTTP-SOURCE +flooding.ip.streamid=FLOODING-IP-SOURCE +flooding.jstx.streamid=FLOODING-JSTX-SOURCE +flooding.mail.streamid=FLOODING-MAIL-SOURCE +flooding.ssl.streamid=FLOODING-SSL-SOURCE +flooding.vpn.streamid=FLOODING-VPN-SOURCE + + +#Flooding-ComponentId-spout +flooding.kafka.spout=FLOODING-KAFKA-SPOUT + +#Flooding-ComponentId-bolt +flooding.parse.componentid=FLODING-PARSE-COMPONENTID +flooding.dns.componentid=FLOODING-DNS-COMPONENTID +flooding.ftp.componentid=FLOODING-FTP-COMPONENTID +flooding.http.componentid=FLOODING-HTTP-COMPONENTID +flooding.ip.componentid=FLOODING-IP-COMPONENTID +flooding.jstx.componentid=FLOODING-JSTX-COMPONENTID +flooding.mail.componentid=FLOODING-MAIL-COMPONENTID +flooding.ssl.componentid=FLOODING-SSL-COMPONENTID +flooding.vpn.componentid=FLOODING-VPN-COMPONENTID + +floding.ybinputsize.componentid=FLODING-YBInputSize-COMPONENTID + + + +#Flooding-Fields-ID +flooding.dnscount.field=FLOODING-DNSCOUNT-FIELD +flooding.dnsloadfail.field=FLOODING-DNSLOADFAIL-FIELD +flooding.dnsdatasize.field=FLOODING-DNSDATASIZE-FIELD + +flooding.ftpcount.field=FLOODING-FTPCOUNT-FIELD +flooding.ftploadfail.field=FLOODING-FTPLOADFAIL-FIELD +flooding.ftpdatasize.field=FLOODING-FTPDATASIZE-FIELD + +flooding.httpcount.field=FLOODING-HTTPCOUNT-FIELD +flooding.httploadfail.field=FLOODING-HTTPLOADFAIL-FIELD +flooding.httpdatasize.field=FLOODING-HTTPDATASIZE-FIELD + + +flooding.ipcount.field=FLOODING-IPCOUNT-FIELD +flooding.iploadfail.field=FLOODING-IPLOADFAIL-FIELD +flooding.ipdatasize.field=FLOODING-IPDATASIZE-FIELD + +flooding.jstxcount.field=FLOODING-JSTXCOUNT-FIELD +flooding.jstxloadfail.field=FLOODING-JSTXLOADFAIL-FIELD +flooding.jstxdatasize.field=FLOODING-JSTXDATASIZE-FIELD + +flooding.mailcount.field=FLOODING-MAILCOUNT-FIELD +flooding.mailloadfail.field=FLOODING-MAILLOADFAIL-FIELD +flooding.maildatasize.field=FLOODING-MAILDATASIZE-FIELD + +flooding.sslcount.field=FLOODING-SSLCOUNT-FIELD +flooding.sslloadfail.field=FLOODING-SSLLOADFAIL-FIELD +flooding.ssldatasize.field=FLOODING-SSLDATASIZE-FIELD + +flooding.vpncount.field=FLOODING-VPNCOUNT-FIELD +flooding.vpnloadfail.field=FLOODING-VPNLOADFAIL-FIELD +flooding.vpndatasize.field=FLOODING-VPNDATASIZE-FIELD + + +#flooding-schema +ftp.rzzb.schema={"type":"record","name":"FTP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +dns.rzzb.schema={"type":"record","name":"DNS_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +http.rzzb.schema={"type":"record","name":"HTTP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":["int","null"]},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +ip.rzzb.schema={"type":"record","name":"IP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +jstx.rzzb.schema={"type":"record","name":"JSTX_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"URL_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +mail.rzzb.schema={"type":"record","name":"MAIL_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":"string"},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["int","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +ssl.rzzb.schema={"type":"record","name":"SSL_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +vpn.rzzb.schema={"type":"record","name":"VPN_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["string","null"]},{"name":"URL","type":["int","null"]},{"name":"USERNAME","type":["int","null"]},{"name":"PWD","type":["int","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} + +#flooding-bolt-parallelism_hint +flooding.parse.parallelism=1 +flooding.dns.parallelism=1 +flooding.ftp.parallelism=1 +flooding.http.parallelism=1 +flooding.ip.parallelism=1 +flooding.jstx.parallelism=1 +flooding.mail.parallelism=1 +flooding.ssl.parallelism=1 +flooding.vpn.parallelism=1 + +#Flooding-http-post-url +flooding.http.post.url=http://192.168.40.119:6699 + +#Flooding-http-post-socketTimeOut +flooding.http.post.socketTimeOut=60 + + + +#Flooding-ComponentId-spout +safe.kafka.spout=SAFE-KAFKA-SPOUT + +#GK-SOURCE-STREAMID + +gk.ip.streamid=GK-IP-STREAMID +gk.dns.streamid=GK-DNS-STREAMID +gk.http.streamid=GK-HTTP-STREAMID +gk.ssl.streamid=GK-SSL-STREAMID +gk.wygjc.streamid=GK-WYGJC-STREAMID +gk.mail.streamid=GK-MAIL-STREAMID +gk.ftp.streamid=GK-FTP-STREAMID +gk.vpn.streamid=GK-VPN-STREAMID +gk.jstx.streamid=GK-JSTX-STREAMID + +#gk-schema +ip.gkrz.schema={"type":"record","name":"IP_GKRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +dns.gkzb.schema={"type":"record","name":"DNS_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":["int","null"]},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +http.gkzb.schema={"type":"record","name":"HTTP_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} +ssl.gkzb.schema={"type":"record","name":"SSL_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"yl1","type":"int"},{"name":"yl2","type":["int","null"]},{"name":"yl3","type":["string","null"]},{"name":"yl4","type":["string","null"]},{"name":"yl5","type":["string","null"]}]} + + +#JC-SOURCE-STREAMID +jc.ip.streamid=JC-IP-STREAMID +jc.dns.streamid=JC-DNS-STREAMID +jc.http.streamid=JC-HTTP-STREAMID +jc.ssl.streamid=JC-SSL-STREAMID +jc.wygjc.streamid=JC-WYGJC-STREAMID +jc.mail.streamid=JC-MAIL-STREAMID +jc.ftp.streamid=JC-FTP-STREAMID +jc.vpn.streamid=JC-VPN-STREAMID +jc.jstx.streamid=JC-JSTX-STREAMID +jc.pic.streamid=JC-PIC-STREAMID +jc.media.streamid=JC-MEDIA-STREAMID + +#count\u51FA\u6307\u6807\u7684\u65F6\u95F4\u95F4\u9694(\u5355\u4F4D\u4E3A\u79D2\uFF0C\u6307\u6807\u662F\u4EE5\u5206\u949F\u4E3A\u5355\u4F4D\u6700\u597D\u4E3A60\u7684\u500D\u6570) +count.topology.tick.tuple.freq.secs=60 + +#prometheus-httpserver +prometheus.pushgateway.address=192.168.40.151:9091 + diff --git a/target/classes/cn/ac/iie/storm/bean/SessionRecordLog.class b/target/classes/cn/ac/iie/storm/bean/SessionRecordLog.class Binary files differnew file mode 100644 index 0000000..60a727b --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bean/SessionRecordLog.class diff --git a/target/classes/cn/ac/iie/storm/bean/Traffic_Forward.class b/target/classes/cn/ac/iie/storm/bean/Traffic_Forward.class Binary files differnew file mode 100644 index 0000000..245a39a --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bean/Traffic_Forward.class diff --git a/target/classes/cn/ac/iie/storm/bean/UserRegion.class b/target/classes/cn/ac/iie/storm/bean/UserRegion.class Binary files differnew file mode 100644 index 0000000..b1eeb87 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bean/UserRegion.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingDNSBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingDNSBolt.class Binary files differnew file mode 100644 index 0000000..436feca --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingDNSBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingFTPBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingFTPBolt.class Binary files differnew file mode 100644 index 0000000..fc4f2b0 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingFTPBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingHTTPBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingHTTPBolt.class Binary files differnew file mode 100644 index 0000000..783d568 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingHTTPBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingIPBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingIPBolt.class Binary files differnew file mode 100644 index 0000000..63558ff --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingIPBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingJSTXBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingJSTXBolt.class Binary files differnew file mode 100644 index 0000000..45ebd72 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingJSTXBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingMAILBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingMAILBolt.class Binary files differnew file mode 100644 index 0000000..27bdf08 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingMAILBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingSSLBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingSSLBolt.class Binary files differnew file mode 100644 index 0000000..49df490 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingSSLBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingVPNBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingVPNBolt.class Binary files differnew file mode 100644 index 0000000..2214f96 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/FloodingVPNBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.class b/target/classes/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.class Binary files differnew file mode 100644 index 0000000..6dd53c5 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/flooding/ParseFloodingBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/index/CountBolt.class b/target/classes/cn/ac/iie/storm/bolt/index/CountBolt.class Binary files differnew file mode 100644 index 0000000..f458e2b --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/index/CountBolt.class diff --git a/target/classes/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.class b/target/classes/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.class Binary files differnew file mode 100644 index 0000000..13865ba --- /dev/null +++ b/target/classes/cn/ac/iie/storm/bolt/safe/ParseSafeBolt.class diff --git a/target/classes/cn/ac/iie/storm/spout/FloodingKafkaSpout.class b/target/classes/cn/ac/iie/storm/spout/FloodingKafkaSpout.class Binary files differnew file mode 100644 index 0000000..9ae4f0d --- /dev/null +++ b/target/classes/cn/ac/iie/storm/spout/FloodingKafkaSpout.class diff --git a/target/classes/cn/ac/iie/storm/spout/SafeKafkaSpout.class b/target/classes/cn/ac/iie/storm/spout/SafeKafkaSpout.class Binary files differnew file mode 100644 index 0000000..e818b38 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/spout/SafeKafkaSpout.class diff --git a/target/classes/cn/ac/iie/storm/topology/StormRunner.class b/target/classes/cn/ac/iie/storm/topology/StormRunner.class Binary files differnew file mode 100644 index 0000000..0397459 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/topology/StormRunner.class diff --git a/target/classes/cn/ac/iie/storm/topology/StreamAggregateTopology.class b/target/classes/cn/ac/iie/storm/topology/StreamAggregateTopology.class Binary files differnew file mode 100644 index 0000000..93dfc72 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/topology/StreamAggregateTopology.class diff --git a/target/classes/cn/ac/iie/storm/utils/combine/flooding/FloodingProcess_RZ.class b/target/classes/cn/ac/iie/storm/utils/combine/flooding/FloodingProcess_RZ.class Binary files differnew file mode 100644 index 0000000..1e00449 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/utils/combine/flooding/FloodingProcess_RZ.class diff --git a/target/classes/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.class b/target/classes/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.class Binary files differnew file mode 100644 index 0000000..07ab887 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/utils/combine/httpurl/Http_URL_process.class diff --git a/target/classes/cn/ac/iie/storm/utils/config/StreamAggregateConfig.class b/target/classes/cn/ac/iie/storm/utils/config/StreamAggregateConfig.class Binary files differnew file mode 100644 index 0000000..66be65d --- /dev/null +++ b/target/classes/cn/ac/iie/storm/utils/config/StreamAggregateConfig.class diff --git a/target/classes/cn/ac/iie/storm/utils/config/StreamAggregateConfigurations.class b/target/classes/cn/ac/iie/storm/utils/config/StreamAggregateConfigurations.class Binary files differnew file mode 100644 index 0000000..2ddff20 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/utils/config/StreamAggregateConfigurations.class diff --git a/target/classes/cn/ac/iie/storm/utils/general/TransFormUtils.class b/target/classes/cn/ac/iie/storm/utils/general/TransFormUtils.class Binary files differnew file mode 100644 index 0000000..fafdb27 --- /dev/null +++ b/target/classes/cn/ac/iie/storm/utils/general/TransFormUtils.class diff --git a/target/classes/cn/ac/iie/storm/utils/http/HttpClientService.class b/target/classes/cn/ac/iie/storm/utils/http/HttpClientService.class Binary files differnew file mode 100644 index 0000000..778c27b --- /dev/null +++ b/target/classes/cn/ac/iie/storm/utils/http/HttpClientService.class diff --git a/target/classes/cn/ac/iie/storm/utils/http/HttpConfig.class b/target/classes/cn/ac/iie/storm/utils/http/HttpConfig.class Binary files differnew file mode 100644 index 0000000..c0e40ec --- /dev/null +++ b/target/classes/cn/ac/iie/storm/utils/http/HttpConfig.class diff --git a/target/classes/common_app_id_coding.properties b/target/classes/common_app_id_coding.properties new file mode 100644 index 0000000..f234055 --- /dev/null +++ b/target/classes/common_app_id_coding.properties @@ -0,0 +1,161 @@ +//common_app_id_conding +01999="HTTP" +02999="DNS" +03001="WEBMAIL" +03002="SMTP" +03003="POP3" +03004="IMAP" +04001="PPTP" +04002="L2TP" +04003="OpenVPN" +04004="IPSec" +05001="QQ" +05002="BlackLight" +05003="\u9489\u9489" +05004="\u7FFC\u804A" +05005="Hangouts" +05006="\u4E2A\u4FE1" +05007="\u5FAE\u4FE1\u7535\u8BDD\u672C" +05008="\u98DE\u804A" +05009="\u73AF\u804A" +05010="\u6709\u4FE1\u7535\u8BDD" +05011="\u53CB\u7EA6" +05012="\u9047\u89C1" +05013="Kik" +05014="\u5FAE\u7231" +05015="Tango" +05016="\u6562\u804A" +05017="\u5FAE\u4FE1" +05018="\u6613\u4FE1" +05019="\u65FA\u4FE1" +05020="Tumblr" +05021="Blued" +05022="\u7F51\u804A" +05023="\u5FEB\u4F20" +05024="\u5531\u5427" +05025="\u6BD4\u90BB" +05026="\u5343\u725B" +05027="\u6D3E\u6D3E" +05028="\u627E\u5BF9\u8C61" +05029="IM+" +05030="\u5FAE\u805A" +05031="BBM" +05032="LOFTER" +05033="Weico" +05034="\u7C73\u804A" +05035="YY\u8BED\u97F3" +05036="Skype" +05037="Path" +05038="\u7F51\u6613\u6CE1\u6CE1" +05039="\u6709\u4F60" +05040="\u5FAE\u4E50" +05041="GO\u77ED\u4FE1" +05042="\u548C\u901A\u8BAF\u5F55" +05043="\u7EA6\u7231" +05044="\u65E0\u79D8" +05045="Wicker" +05046="\u7EA6\u4F1A\u5427" +05047="hike messager" +05048="\u6469\u64E6" +05049="\u63A2\u63A2" +05050="\u964C\u964C" +05051="\u6765\u5F80/\u70B9\u70B9\u866B" +05052="\u767E\u5EA6Hi" +05053="ZANK" +05054="\u5C0F\u6069\u7231" +05055="\u98DE\u4FE1" +05056="ChatON" +05057="QT\u8BED\u97F3" +05058="\u7231\u804A" +05059="nice\u597D\u8D5E" +05060="\u670B\u53CB\u5370\u8C61" +05061="Googletalk" +05062="\u53CB\u52A0" +05063="Keechat" +05064="\u556A\u556A/\u556A\u556A\u97F3\u4E50\u5708" +05065="\u5FAE\u4F1A" +05066="Radicall" +05067="\u69FD\u5382" +05068="\u60F3\u604B\u7231" +05069="\u989C\u503C/\u989C\u503C\u70B9\u8BC4" +05070="LinkedIn" +05071="Telegram" +05072="DiDi" +05073="ICQ" +05074="Zello" +05075="NextDoor" +05076="Pinterest" +05077="\u53EE\u549A" +05078="\u5E8A\u4E0A" +05079="Nimbuzz" +05080="\u5FAE\u4EBA\u8109" +05081="\u966A\u966A" +05082="\u5FAE\u5BC6" +05083="Vine" +05084="Line" +05085="between" +05086="facebook" +05087="Viber" +05088="MeetUP" +05089="MeetMe" +05090="OkCupidDating" +05091="\u78B0\u78B0" +05092="Diaspora" +05093="\u549A\u5471" +05094="\u966A\u804A" +05095="Twitter" +05096="Kakao" +05097="MSN" +05098="Voxer" +05099="Airetalk" +05100="ooVoo" +05101="Mico" +05102="Banjo" +05103="TextNow" +05104="\u62B1\u62B1" +05105="\u591A\u591A\u7684\u9C7C" +05106="\u4E2A\u6027" +05107="\u95FA\u871C\u5708" +05108="Instagram" +05109="Whatsapp" +05110="Talkbox" +05111="ChatOn" +05112="Coco" +05113="Badoo" +05114="Match.com" +05115="TextFree" +05116="Tapatalk" +05117="KC\u7F51\u7EDC\u7535\u8BDD" +05118="\u7231\u5531" +05119="\u9177\u6211" +05120="\u8354\u679DFM" +05121="\u94C1\u8840" +05122="\u9177\u53CB\u793E\u533A" +05123="\u82B1\u7530\u4EA4\u53CB" +05124="\u7EA2\u5A18\u7F51" +05125="\u77E5\u4E4E" +05126="\u767E\u5EA6\u8D34\u5427" +05127="\u767E\u5408\u5A5A\u604B" +05128="\u73CD\u7231\u7F51" +05129="\u732B\u6251" +05130="\u65B0\u6D6A\u5FAE\u535A" +05131="\u5F00\u5FC3\u7F51" +05132="\u5929\u9645\u597D\u597D\u7EA6" +05133="\u5929\u6DAF\u793E\u533A" +05134="\u4EBA\u4EBA\u7F51" +05135="\u4E16\u7EAA\u4F73\u7F18" +05136="We heart it" +05137="Avocado" +05138="Tinder" +05139="Swarm" +05140="Snapchat" +05141="Dating for everyone" +05142="Skout" +05143="QQ\u7A7A\u95F4" +05144="Couple" +05145="Happn" +06999="FTP" +07999="SSL" +08001="IP\u9ED1" +08002="IP\u76D1\u6D4B" +08003="IP\u65E5\u5FD7\u603B\u8868"
\ No newline at end of file diff --git a/target/classes/common_app_label_coding.properties b/target/classes/common_app_label_coding.properties new file mode 100644 index 0000000..043b8eb --- /dev/null +++ b/target/classes/common_app_label_coding.properties @@ -0,0 +1 @@ +\u5168\u90E8=00
HTTP=01
DNS=02
MAIL=03
VPN=04
jstx=05
FTP=06
SSL=07
IP\u7C7B=08
\ No newline at end of file diff --git a/src/main/resources/service_flow_config.properties b/target/classes/service_flow_config.properties index 3fc5c11..8cc4bd6 100644 --- a/src/main/resources/service_flow_config.properties +++ b/target/classes/service_flow_config.properties @@ -163,5 +163,5 @@ jc.media.streamid=JC-MEDIA-STREAMID count.topology.tick.tuple.freq.secs=60 #prometheus-httpserver -prometheus.pushgateway.address=192.168.40.151:5678 +prometheus.pushgateway.address=192.168.40.151:9091 diff --git a/target/maven-archiver/pom.properties b/target/maven-archiver/pom.properties new file mode 100644 index 0000000..72d41f2 --- /dev/null +++ b/target/maven-archiver/pom.properties @@ -0,0 +1,5 @@ +#Generated by Maven +#Mon Jan 04 14:49:37 CST 2021 +version=1.0-SNAPSHOT +groupId=org.example +artifactId=xj-log-etl diff --git a/target/original-xj-log-etl-1.0-SNAPSHOT.jar b/target/original-xj-log-etl-1.0-SNAPSHOT.jar Binary files differnew file mode 100644 index 0000000..9e58331 --- /dev/null +++ b/target/original-xj-log-etl-1.0-SNAPSHOT.jar diff --git a/target/test-classes/UrlTest.class b/target/test-classes/UrlTest.class Binary files differnew file mode 100644 index 0000000..2244d4b --- /dev/null +++ b/target/test-classes/UrlTest.class diff --git a/target/xj-log-etl-1.0-SNAPSHOT.jar b/target/xj-log-etl-1.0-SNAPSHOT.jar Binary files differnew file mode 100644 index 0000000..948db7c --- /dev/null +++ b/target/xj-log-etl-1.0-SNAPSHOT.jar |