update template youtube.sft

7 files changed, 40 insertions, 245 deletions

diff --git a/bin/conf/frag_reassembly.conf b/bin/conf/frag_reassembly.conf
index f0f87db..7ccd557 100644
--- a/bin/conf/frag_reassembly.conf
+++ b/bin/conf/frag_reassembly.conf
@@ -3,10 +3,7 @@ RedisClusterSwitch=1
 RedisTimeout=5
 
 #cluster mode 
-RedisBrokers=10.0.6.192:9012
-RedisNet=192
-#HIRCLUSTER_FLAG_UNIQIP_FRAG_ONE=0x100=256,HIRCLUSTER_FLAG_UNIQIP_FRAG_TWO=0x200=512,HIRCLUSTER_FLAG_UNIQIP_FRAG_THREE=0x400=1024,HIRCLUSTER_FLAG_UNIQIP_FRAG_FOUR=0x800=2048
-RedisUniqIPFlag=2048
+RedisBrokers=192.168.10.205:9012
 
 #no cluster mode
 RedisIP=
@@ -21,11 +18,11 @@ ConvergeHashExpireTime=300
 
 [LOG]
 #frag reassembly log ����ƴ�ӵ�������־
-FragReassemblyLogLevel=10
+FragReassemblyLogLevel=30
 FragReassemblyLogName=./frag_reassembly.log
 
 #media log
-MediaLogLevel=10
+MediaLogLevel=30
 MediaLogName=./frag_reassembly_media.log
 
 #stat log
diff --git a/bin/conf/main.conf b/bin/conf/main.conf
index 6906ca9..b6a5258 100644
--- a/bin/conf/main.conf
+++ b/bin/conf/main.conf
@@ -1,7 +1,7 @@
 #--------------------------------------------------------------------NETWORK
 [NETWORK]
-LocalIP=em1
-FwdIP=em1
+LocalIP=ens33
+FwdIP=ens33
 
 #recv bizman param, recv data from sapp
 BizmanAckSmoothTime = 10
@@ -22,9 +22,9 @@ UnixSocketSendDestAddr=/home/mcger/un_recv;/home/audiorecognition/aufilter/un_re
 UnixSocketRecvAddr=/home/mesasoft/frag_rssb/un_recv
 
 #udp socket:send frag
-UdpSendIP=127.0.0.1;
-UdpSendIPNum=1
-UdpSendPort=33082
+UdpSendIP=127.0.0.1;127.0.0.1;
+UdpSendIPNum=2
+UdpSendPort=33082;33083
 
 #�����ڴ�����
 USM_ON_FLAG=0
@@ -53,19 +53,19 @@ WhiteListPort=22085
 CPZTpye=0
 
 ThreadNum=1
-BizmanQueueMode=0
+BizmanQueueMode=1
 BizmanQueueMaxnum=2000000
 
-AsmisSwitch=1
+AsmisSwitch=0
 
 #��ǰ�˷��ͽ�Ŀȷ��
 AckSwitch=0
 
 #IVIȥ�ؿ���
-IVISwitch=0
+IVISwitch=1
 
 #���ͽӿ��Ƿ�����AV_sendback
-AVSendbackSwitch=1
+AVSendbackSwitch=0
 
 #index�ڶ��η���ʱ��
 IndexQueryTime=10
@@ -78,6 +78,9 @@ RenewTimeMin=600
 RenewTimeStep=6
 
 #��ĿHASH��
+#MediaHashSize=8
+#MediaHashElemNum=16
+#MediaHashExpireTime=120
 MediaHashSize=65536
 MediaHashElemNum=1048576
 MediaHashExpireTime=300
@@ -97,16 +100,16 @@ MonitorFileRootDir=/home/yspdata/qd_monitor/
 MonitorFileDay=1
 
 #special media type, send windows system
-ForwardSpecialMediaSwitch=1
-SpecialMediaType=0x68
+ForwardSpecialMediaSwitch=0
+SpecialMediaType=0x63
 
 #AV record switch
-AVRecordFileSwitch=1
+AVRecordFileSwitch=0
 AVRecordFileRootDir=./AVrecord/
 AVRecordFileMaxNum=100000
 
 #ģ��hashժҪ���㿪��
-FuzzyDigestSwitch=1
+FuzzyDigestSwitch=0
 
 #Ԫ��Ϣѡ��capIP�Ƿ��޸� 1����Ϊ����capIP, 0:ѡ��Ϊǰ�˻�CapIP
 ModifyCapIPSwitch=1
@@ -124,7 +127,7 @@ DedupThriftTime=5
 DedupHtableArg=0
 
 #���ض�Դҵ�񿪹�
-AVDedupSwitch=1
+AVDedupSwitch=0
 AVDedupInvalid=0
 #td=(URL+ServerIP...)+data(<=65536)
 DedupTdDataSize=65536
@@ -146,7 +149,7 @@ MediaJSONSwitch=1
 
 #==============================================================VOIP
 [VOIP]
-VOIPFilterSwitch=1
+VOIPFilterSwitch=0
 
 #VOIP survey log �����־
 VOIPSurveyLogIP=
@@ -159,7 +162,7 @@ VOIPFullLogIPNum=
 VOIPFullLogPort=
 
 #voip log
-VOIPLogLevel= 30
+VOIPLogLevel= 20
 VOIPLogPath= ./log/voip.log
 
 #==============================================================APP
@@ -202,13 +205,13 @@ MediaExpireLogPath= ./log/media_expire.log
 #--------------------------------------------------------------------DEBUG
 [DEBUG]
 /*��Ŀ���б��ش洢*/
-FileStoreSwitch=0
+FileStoreSwitch=1
 FileStorePath=./log/file/
 
 /*���н�Ŀ�����б��ظ��ٻ�д*/
-AllHitMonitorSwitch=2
-AllHitMonitorCompleteRate=90
-AllHitMonitorFilenameType=1
+AllHitMonitorSwitch=0
+AllHitMonitorCompleteRate=0
+AllHitMonitorFilenameType=0
 
 #1: (reoffset-1)*content-length  0: accoffset
 HlsAboffsetInMode=1
@@ -221,7 +224,7 @@ JSONLocalSwitch=1
 
 
 [WLB]
-wlb_on=1
+wlb_on=0
 wlb_topic=AV
 wlb_group_name=xx
 user_tag=xx
@@ -232,5 +235,4 @@ capacity=32
 cost=32
 wlb_report_interval=10
 
-bfd_recv_port= 
-bfd_send_port=
\ No newline at end of file
+bfd_recv_port=33082
diff --git a/bin/conf/sifter/frag_reassembly.conf b/bin/conf/sifter/frag_reassembly.conf
deleted file mode 100644
index 33df3b6..0000000
--- a/bin/conf/sifter/frag_reassembly.conf
+++ /dev/null
@@ -1,57 +0,0 @@
-[NETWORK]
-RedisBrokers=10.0.6.80:9001,10.0.6.80:9002,10.0.6.80:9003,10.0.6.80:9004,10.0.6.80:9005
-RedisTimeout=5
-
-[SYSTEM]
-ConvergeHashSize=65536
-ConvergeHashElemNum=1048576
-ConvergeHashExpireTime=120
-
-#same with protocol in frag_monitor
-ProtocolNetdisk=17
-ProtocolWebmail=18
-
-#webmail param================================start
-WebmailHashSize=65536
-WebmailHashElemNum=1048576
-WebmailHashExpireTime=120
-
-StoreidAttachHashSize=65536
-StoreidAttachHashElemNum=1048576
-StoreidAttachHashExpireTime=120
-
-StoreidMidHashSize=65536
-StoreidMidHashElemNum=1048576
-StoreidMidHashExpireTime=120
-
-#0, not open iterate thread to output
-WebmailIterateTimeout=60
-WebmailAttachIterateTimeout=60
-
-WebmailService=123,163,sina,sohu,qq
-WebmailAction=send,recv
-#webmail param================================end
-
-[LOG]
-#frag reassembly log ����ƴ�ӵ�������־
-FragReassemblyLogLevel=30
-FragReassemblyLogName=./frag_reassembly.log
-
-#media log
-MediaLogLevel=30
-MediaLogName=./frag_reassembly_media.log
-
-#stat log
-StatInterval=5
-StatFile=./frag_reassembly_stat.log
-SysinfoInterval=5
-SysinfoFile=./frag_reassembly_sysinfo.log
-
-#webmail log
-WebmailLogLevel=10
-WebmailLogName=./webmail.log
-
-#Maat
-MaatStatSwitch=0
-MaatPerfSwitch=0
-#MaatStatFile=./maat_stat.log
diff --git a/bin/conf/sifter/main.conf b/bin/conf/sifter/main.conf
deleted file mode 100644
index 8d3b761..0000000
--- a/bin/conf/sifter/main.conf
+++ /dev/null
@@ -1,147 +0,0 @@
-[NETWORK]
-LocalIP=em1
-
-#recv bizman param, recv data from sapp
-BizmanAckSmoothTime = 10
-BizmanAckAccumulateTime=10
-BizmanAckAccumulateNum=5
-BizmanPort=22082
-
-#sapp recv resp msg port
-BizmanAckPort=22084
-
-#msg port
-MsgPort=22080
-
-#unix-domain socket file path ��send frag
-UnixSocketSendSrcAddr=/home/lishu/frag/frag_rssb/maskey/bin/un_send
-UnixSocketSendDestAddr=/home/lishu/frag/frag_rssb/maskey/bin/data_recv
-
-#udp socket recv port ��recv response msg
-UnixSocketRecvAddr=/home/lishu/frag/frag_rssb/maskey/bin/un_recv
-
-#usm
-USM_ON_FLAG=1
-USM_SHM_KEY=1635151988
-#USM_SHM_KEY=12346
-USM_SHM_SIZE=100000
-USM_READER_CNT=3
-USM_SMOOTH_TIME=1000000
-USM_Q_SIZE=1048576
-USM_READER_PATH=/home/mcger/un_recv;/home/audiorecognition/aufilter/un_recv;/home/mesasoft/un_test/un_recv
-USM_LOG_LEVEL=40
-USM_LOG_PATH=./log/usm_writer0;./log/usm_writer1;./log/usm_writer2
-
-#special media type, send windows system , sep: ";" : IP1+PORT1/PORT2 or IP2+PORT1/PORT2 
-SpecialMediaWindowsIP=10.0.6.5;10.0.6.1;
-SpecialMediaWindowsIPNum=2
-SpecialMediaWindowsPort=44096;44098
-
-#white list IP and PORT : IP1+PORT and IP2+PORT
-WhiteListIP = 10.52.178.15;
-WhiteListIPNum=1
-WhiteListPort=22085
-
-#av_dedup service ����ҵ�������������Ϣ
-ThriftIP=127.0.0.1;
-ThriftIPNum=1
-ThriftPort=9090
-DedupBalanceNum=1000
-DedupQueueNum=100000
-DedupStatInterval=60
-DedupThriftNum=5
-DedupHtableArg=0
-
-#JSON to kafka ��ý����Ϣ�ش�ҵ�����������
-MediaJSONKafkaBrokers=
-
-
-[SYSTEM]
-ThreadNum=4
-BizmanQueueMode=0
-
-#media renew time
-RenewTimeMax=7200
-RenewTimeMin=600
-RenewTimeStep=6
-
-#��ĿHASH��
-MediaHashSize=65536
-MediaHashElemNum=1048576
-MediaHashExpireTime=2
-
-#���ٻ�дhash�ļ�����DumpfileHashElemNum�������IJ����ļ���Ŀ
-DumpfileHashSize=1024
-DumpfileHashElemNum=512
-DumpfileHahsExpireTime=6
-
-#monitor service
-MonitorHashSize=65536
-MonitorHashElemNum=1048576
-MonitorHahsExpireTime=120
-
-MonitorFileSwitch=1
-MonitorFileRootDir=/home/yspdata/qd_monitor/
-MonitorFileDay=1
-
-#special media type, send windows system
-ForwardSpecialMediaSwitch=1
-SpecialMediaType=0x68
-
-#AV record switch
-AVRecordFileSwitch=0
-AVRecordFileRootDir=./AVrecord/
-AVRecordFileMaxNum=100000
-
-#����ҵ�񿪹�
-AVDedupSwitch=0
-#td=(URL+ServerIP...)+data(<=65536)
-DedupTdDataSize=0
-
-#media info send to kfaka ��ý��JSON��Ϣ�ش�KAFKA����
-MediaJSONSwitch=0
-
-#�ֻ�app����
-AppSwitch=0
-
-[LOG]
-#field_stat
-StatCycle=2
-StatFile=./log/rssb_stat.log
-
-SysinfoCycle=2
-SysinfoFile=./log/rssb_sysinfo.log
-
-#log in general
-LogLevel= 30
-LogPath= ./log/runtime.log
-
-#frag msg log
-FragLogLevel= 20
-FragLogPath= ./log/frag.log
-
-#av_dedup log
-DedupLogLevel= 10
-DedupLogPath= ./log/dedup.log
-
-#resp msg log
-RespLogPath= ./log/survey.log
-
-#media create log
-MediaCreateLogPath= ./log/media_create.log
-
-#media expire log
-MediaExpireLogPath= ./log/media_expire.log
-
-[DEBUG]
-/*����mid_trace.conf�Ľ�Ŀ���б��ش洢*/
-FileStoreSwitch=0
-FileStorePath=./log/file/
-
-/*���н�Ŀ�����б��ظ��ٻ�д*/
-AllHitMonitorSwitch=0
-#1: (reoffset-1)*content-length  0: accoffset
-HlsAboffsetInMode=1
-
-/*trace log path*/
-TracePath=./log/
diff --git a/bin/conf/sifter/mid_trace.conf b/bin/conf/sifter/mid_trace.conf
deleted file mode 100644
index 1235696..0000000
--- a/bin/conf/sifter/mid_trace.conf
+++ /dev/null
@@ -1 +0,0 @@
-#mid or pid
diff --git a/bin/conf/sifter/table_info.conf b/bin/conf/sifter/table_info.conf
deleted file mode 100644
index ae0c330..0000000
--- a/bin/conf/sifter/table_info.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-#each collumn seperate with '\t'
-#id	(0~65535(
-#name string
-#type one of ip,expr,compile or plugin
-#src_charset one of GBK,BIG5,UNICODE,UTF8
-#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
-#do_merege yes or no
-#id	name	type	src_charset	dst_charset	do_merge
-1	SIFTER_CONFIG_FULL	compile	GBK	GBK	no
-2	SIFTER_GROUP	group	GBK	GBK	no
-3	SIFTER_START	expr	GBK	GBK	yes
diff --git a/bin/conf/sifter/youtube_data.sft b/bin/conf/sifter/youtube_data.sft
new file mode 100644
index 0000000..518173b
--- /dev/null
+++ b/bin/conf/sifter/youtube_data.sft
@@ -0,0 +1,12 @@
+[INFOR]
+sft_id = 4122
+
+[EXTRACT]
+#[expect_type][expect_name] = extract_location$extract_method$start$mid$end$
+#expect_type and expect_name must be one
+#sifter_method = Tag | RawText | Boundary | Reposition
+#special character : \r\n = 0x0d0x0a, \x30 = no end , \x0 = null
+#expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT}
+#extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT}
+{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$range=$
+{4}{FRAG_UNIT_REOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$