添加inc和srcHEADmaster

9 files changed, 326 insertions, 0 deletions

diff --git a/src/rssb_statistics/all_len_st.py b/src/rssb_statistics/all_len_st.py
new file mode 100644
index 0000000..6e32518
--- /dev/null
+++ b/src/rssb_statistics/all_len_st.py
@@ -0,0 +1,17 @@
+import os
+import re
+import csv
+import bisect
+# cmd1 = "cat survey.log.2018* | grep \"recv survey\" > survey.recv_survey.log"
+# cmd2 = "cat media_expire.log.2018* > media_expire.log"
+# os.system(cmd1)
+# os.system(cmd2)
+breakpoints = [1048576,1310720,1572864,1835008,2097152,3145728,4194304]
+st_num = [0,0,0,0,0,0,0,0]
+
+with open("media_expire.log",'r') as infile:
+	for line in infile:
+		line_result = re.split(r",|MID:|media_len:",line)
+		j = bisect.bisect(breakpoints,int(line_result[6]))
+		st_num[j]+=1
+print st_num
\ No newline at end of file
diff --git a/src/rssb_statistics/delete_len_st.py b/src/rssb_statistics/delete_len_st.py
new file mode 100644
index 0000000..010dc84
--- /dev/null
+++ b/src/rssb_statistics/delete_len_st.py
@@ -0,0 +1,48 @@
+import os
+import re
+import csv
+import bisect
+cmd1 = "cat survey.log.2018* | grep \"recv survey\" > survey.recv_survey.log"
+cmd2 = "cat ../12.log/media_expire.log.2018* > ../12.log/media_expire.log"
+cmd3 = "cat media_expire.log.2018* > media_expire.log"
+os.system(cmd1)
+os.system(cmd2)
+os.system(cmd3)
+breakpoints = [1048576,1310720,1572864,1835008,2097152,3145728,4194304]
+st_num = [0,0,0,0,0,0,0,0]
+mid_13_set=set()
+mid_12_set=set()
+mid_12_file = "../12.log/media_expire.log"
+mid_13_file = "media_expire.log"
+
+i=0
+with open(mid_13_file,'r') as infile:
+	for line in infile:
+		i+=1
+		if(i%100000==0):
+			print i
+		line_result = re.split(r",|MID:|media_len:",line)
+		mid_13_set.add(int(line_result[3]))
+
+i=0
+with open(mid_12_file,'r') as infile:
+	for line in infile:
+		i+=1
+		if(i%100000==0):
+			print i
+		line_result = re.split(r",|MID:|media_len:",line)
+		mid_12_set.add(int(line_result[3]))
+
+different = mid_12_set.difference(mid_13_set)
+
+i=0	
+with open(mid_12_file,'r') as infile:
+	for line in infile:
+		i+=1
+		if(i%100000==0):
+			print i
+		line_result = re.split(r",|MID:|media_len:",line)
+		if((int(line_result[3]) in different)==True):
+			j = bisect.bisect(breakpoints,int(line_result[6]))
+			st_num[j]+=1
+print st_num
\ No newline at end of file
diff --git a/src/rssb_statistics/find_lost_td.conf b/src/rssb_statistics/find_lost_td.conf
new file mode 100644
index 0000000..6ab40d9
--- /dev/null
+++ b/src/rssb_statistics/find_lost_td.conf
@@ -0,0 +1,14 @@
+[file]
+raw_survey_file_13 = ../data/data_20180423/14.log/survey.recv.log
+raw_deup_file = ../data/data_20180423/13.log/dedup.log
+raw_survey_file_12 = ../data/data_20180423/12.log/survey.recv.log
+run_time_file = ../data/data_20180423/runtime_log
+lost_td_line = ../data/data_20180423/ripe_file/lost_td_line
+mid_12_file = ../data/data_20180423/ripe_file/mid_12_file
+mid_13_file = ../data/data_20180423/ripe_file/mid_14_file
+list_12_file = ../data/data_20180423/ripe_file/list_12_file
+list_13_file = ../data/data_20180423/ripe_file/list_14_file
+different_mid_file_13 = ../data/data_20180423/ripe_file/different_mid_file_14
+different_mid_file_12 = ../data/data_20180423/ripe_file/different_mid_file_12
+different_list_file_13 = ../data/data_20180423/ripe_file/different_list_file_14
+different_list_file_12 = ../data/data_20180423/ripe_file/different_list_file_12
\ No newline at end of file
diff --git a/src/rssb_statistics/find_lost_td.py b/src/rssb_statistics/find_lost_td.py
new file mode 100644
index 0000000..50f3fab
--- /dev/null
+++ b/src/rssb_statistics/find_lost_td.py
@@ -0,0 +1,147 @@
+import ConfigParser
+import re
+
+config = ConfigParser.RawConfigParser()
+config.read("find_lost_td.conf")
+raw_survey_file_13 = config.get("file","raw_survey_file_13")
+raw_deup_file = config.get("file","raw_deup_file")
+# run_time_file = config.get("file","run_time_file")
+raw_survey_file_12 = config.get("file","raw_survey_file_12")
+# lost_td_line = config.get("file","lost_td_line") 
+mid_12_file = config.get("file","mid_12_file")
+mid_13_file = config.get("file","mid_13_file")
+list_12_file = config.get("file","list_12_file")
+list_13_file = config.get("file","list_13_file")
+different_mid_file_13 = config.get("file","different_mid_file_13")
+different_mid_file_12 = config.get("file","different_mid_file_12")
+different_list_file_13 = config.get("file","different_list_file_13")
+different_list_file_12 = config.get("file","different_list_file_12")
+
+term = {'td_len':(lambda x : len(x)==32),
+		'data_num':(lambda x : len(x)>7),
+		'url':(lambda x : x.find['NUll']),
+		'sfh_len':(lambda x : len(x)>20),
+		'not_null':(lambda x : len(x)!=0)} 
+
+mid_13=dict()
+with open(raw_survey_file_13,'r') as infile:
+	for line in infile:
+		data_line_val = re.split(r',',line)
+		if(len(data_line_val)==8):
+			mid_string = (re.split(r"MID:",data_line_val[2]))[1]
+			mid_13[mid_string]=list()
+
+
+with open(mid_13_file,'w') as outfile:
+	for key in mid_13:
+		outfile.write(key+'\n')
+
+mid_12=dict()
+with open(raw_survey_file_12,'r') as infile:
+	for line in infile:
+		data_line_val = re.split(r',',line)
+		if(len(data_line_val)==8):
+			mid_string = (re.split(r"MID:",data_line_val[2]))[1]
+			mid_12[mid_string]=list()
+
+with open(mid_12_file,'w') as outfile:
+	for key in mid_12:
+		outfile.write(key+'\n')
+
+different_mid_13 = list()
+with open(different_mid_file_13,'w') as outfile:
+	for key in mid_13:
+		if(mid_12.has_key(key)==False):
+			different_mid_13.append(key)
+			outfile.write(key+'\n')
+
+different_mid_12 = list()
+with open(different_mid_file_12,'w') as outfile:
+	for key in mid_12:
+		if(mid_13.has_key(key)==False):
+			different_mid_12.append(key)
+			outfile.write(key+'\n')
+
+i=0
+with open(raw_deup_file,'r') as infile:
+	for line in infile:
+		i+=1
+		if(i%100000==0):
+			print i
+		data_line_val = re.split(r",|MID:|TD:",line)
+		if(term['data_num'](data_line_val) and \
+		mid_13.has_key(str(data_line_val[4])) == True):
+			mid_13[data_line_val[4]].append(data_line_val[6])
+		if(term['data_num'](data_line_val) and \
+		mid_12.has_key(str(data_line_val[4])) == True):
+			mid_12[data_line_val[4]].append(data_line_val[6])
+
+td_list_13 =list()
+with open(list_13_file,'w') as outfile:
+	for key in mid_13.keys():
+		for td in mid_13[key]:
+			if(term['not_null'](td) and td_list_13.count(td)==0):
+				td_list_13.append(td)
+				outfile.write(td+'\n')
+
+td_list_12 =list()
+with open(list_12_file,'w') as outfile:
+	for key in mid_12.keys():
+		for td in mid_12[key]:
+			if(term['not_null'](td) and td_list_12.count(td)==0):
+				td_list_12.append(td)
+				outfile.write(td+'\n')
+
+different_list_12 = list()
+with open(different_list_file_12,'w') as outfile:
+	for x in td_list_12:
+		if(td_list_13.count(x)==0):
+			different_list_12.append(x)
+			outfile.write(x+'\n')
+
+different_list_13 = list()
+with open(different_list_file_13,'w') as outfile:
+	for x in td_list_13:
+		if(td_list_12.count(x)==0):
+			different_list_13.append(x)
+			outfile.write(x+'\n')
+
+td_dict=dict()
+for i in different_list_12:
+	td_dict[i]=list()
+
+# i=0
+# with open(run_time_file,'r') as infile:
+# 	for line in infile:
+# 		i+=1
+# 		if(i%100000==0):
+# 			print i
+# 		if(line.find("NCHK_QUREY__KNOW")!=-1):
+# 			data_line_val = re.split(r',|TD:',line)
+# 			if(td_dict.has_key(data_line_val[6]) == True):
+# 				td_dict[data_line_val[6]].insert(0,"NCHK_QUREY__KNOW"+'\n')
+# 				td_dict[data_line_val[6]].append(line)
+# 		elif(line.find("NCHK_QUREY__UNKNOW")!=-1):
+# 			data_line_val = re.split(r',|TD:',line)
+# 			if(td_dict.has_key(data_line_val[6]) == True):
+# 				td_dict[data_line_val[6]].append(line)
+# 		elif(line.find("NCHK_REPORT__SUCC")!=-1):
+# 			data_line_val = re.split(r',|TD:',line)
+# 			if(td_dict.has_key(data_line_val[6]) == True):			
+# 				td_dict[data_line_val[6]].append(line)
+
+# 		else:
+# 			continue
+
+
+print len(different_list_12),len(different_list_13),\
+len(td_list_12),len(td_list_13),\
+len(mid_12),len(mid_13),len(different_mid_13),len(different_mid_12)
+
+# with open(lost_td_line,'w') as outfile:
+# 	for key in td_dict.keys():
+# 		if(len(td_dict[key])>2 and td_dict[key][0]=="NCHK_QUREY__KNOW"):
+# 			outfile.write(key+':\n')
+# 			for i in td_dict[key]:
+# 				outfile.write(i)
+
diff --git a/src/rssb_statistics/harm_len_st.py b/src/rssb_statistics/harm_len_st.py
new file mode 100644
index 0000000..8a372b3
--- /dev/null
+++ b/src/rssb_statistics/harm_len_st.py
@@ -0,0 +1,29 @@
+import os
+import re
+import csv
+import bisect
+cmd1 = "cat survey.log.2018* | grep \"recv survey\" > survey.recv_survey.log"
+cmd2 = "cat media_expire.log.2018* > media_expire.log"
+os.system(cmd1)
+os.system(cmd2)
+breakpoints = [1048576,1310720,1572864,1835008,2097152,3145728,4194304]
+st_num = [0,0,0,0,0,0,0,0]
+harm_mid_dic=dict()
+with open("survey.recv_survey.log",'r') as infile:
+	for line in infile:
+		data_line_val = re.split(r',',line)
+		if(len(data_line_val)==8):
+			mid_string = (re.split(r"MID:",data_line_val[2]))[1]
+			harm_mid_dic[mid_string]=list()
+
+with open("media_expire.log",'r') as infile:
+	for line in infile:
+		line_result = re.split(r",|MID:|media_len:",line)
+		if(harm_mid_dic.has_key(str(line_result[3]))==True):
+			# print int(line_result[6])
+			j = bisect.bisect(breakpoints,int(line_result[6]))
+			st_num[j]+=1
+print st_num
+# with open("un_recv_list.csv",'w') as csvfile:
+# 	writer = csv.writer(csvfile)
+# 	writer.writerow(un_recv_list)
\ No newline at end of file
diff --git a/src/rssb_statistics/recv_survey_mid_st.py b/src/rssb_statistics/recv_survey_mid_st.py
new file mode 100644
index 0000000..f7faaeb
--- /dev/null
+++ b/src/rssb_statistics/recv_survey_mid_st.py
@@ -0,0 +1,3 @@
+import os
+cmd = "cat survey.log.2018* | grep \"recv survey\" | wc -l"
+os.system(cmd)
\ No newline at end of file
diff --git a/src/rssb_statistics/service_st.py b/src/rssb_statistics/service_st.py
new file mode 100644
index 0000000..7fecd33
--- /dev/null
+++ b/src/rssb_statistics/service_st.py
@@ -0,0 +1,29 @@
+import re
+
+different_mid_12=set()
+different_mid_14=set()
+service_dic=dict()
+
+with open("different_mid_file_12",'r') as infile:
+	for line in infile:
+		different_mid_12.add(long(line))
+with open("different_mid_file_14",'r') as infile:
+	for line in infile:
+		different_mid_14.add(long(line))
+with open("../12.log/survey.recv_survey.log",'r') as infile:
+	for line in infile:
+		line_result = re.split(r",|MID:|service:",line)
+		if((long(line_result[3]) in different_mid_12)==True):
+			if(service_dic.has_key(line_result[5])==True):
+				service_dic[line_result[5]]+=1
+			else:
+				service_dic[line_result[5]]=0
+with open("../14.log/survey.recv.log",'r') as infile:
+	for line in infile:
+		line_result = re.split(r",|MID:|service:",line)
+		if((long(line_result[3]) in different_mid_14)==True):
+			if(service_dic.has_key(line_result[5])==True):
+				service_dic[line_result[5]]+=1
+			else:
+				service_dic[line_result[5]]=0
+print service_dic
\ No newline at end of file
diff --git a/src/rssb_statistics/un_recv_st.py b/src/rssb_statistics/un_recv_st.py
new file mode 100644
index 0000000..9d3d234
--- /dev/null
+++ b/src/rssb_statistics/un_recv_st.py
@@ -0,0 +1,36 @@
+import os
+import re
+import csv
+cmd = "cat rssb_stat.log.2018-04-16 | grep \"/home/audiorecognition/aufilter/un_recv\" > rssb_stat.log.un_recv"
+os.system(cmd)
+i=0
+last_len = 0
+add_len = 0
+num = 0
+un_recv_list = list()
+with open("rssb_stat.log.un_recv",'r') as infile:
+	for line in infile:
+		line_result = re.split(r'\t',line)
+		if(i==0):
+			last_len = long(line_result[2])
+			break
+
+with open("rssb_stat.log.un_recv",'r') as infile:
+	for line in infile:
+		line_result = re.split(r'\t',line)
+		if(last_len <= long(line_result[2])):
+			add_len = add_len+long(line_result[2])-last_len
+			last_len = long(line_result[2])
+		else:
+			add_len = add_len+long(line_result[2])
+			last_len = long(line_result[2])
+		i+=1
+		if(i>=120):
+			i=0
+			un_recv_list.append(str(add_len)) 
+			add_len=0
+
+with open("un_recv_list.csv",'w') as csvfile:
+	writer = csv.writer(csvfile)
+	writer.writerow(un_recv_list)
+
diff --git a/src/rssb_statistics/un_recv_survey_mid_st.py b/src/rssb_statistics/un_recv_survey_mid_st.py
new file mode 100644
index 0000000..e1e970f
--- /dev/null
+++ b/src/rssb_statistics/un_recv_survey_mid_st.py
@@ -0,0 +1,3 @@
+import os
+cmd = "cat survey.log.2018* | grep \"sync_audio\" | wc -l"
+os.system(cmd)
\ No newline at end of file