添加inc和srcHEADmaster

14 files changed, 913 insertions, 0 deletions

diff --git a/src/dataset_build/CMakeLists.txt b/src/dataset_build/CMakeLists.txt
new file mode 100644
index 0000000..8840a74
--- /dev/null
+++ b/src/dataset_build/CMakeLists.txt
@@ -0,0 +1,11 @@
+PROJECT (CALCULATE)
+SET (SRC_LIST get_lost.c)
+SET(CMAKE_BUILD_TYPE "Debug")
+SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -Wall -g -ggdb")
+SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall")
+MESSAGE(STATUS "This is BINARY dir" ${CALCULATE_BINARY_DIR})
+MESSAGE(STATUS "This is SOURCE dir" ${CALCULATE_SOURCE_DIR})
+#INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/../../include/)
+#LINK_DIRECTORIES(${PROJECT_SOURCE_DIR}/../../lib/)
+ADD_EXECUTABLE(get_lost ${SRC_LIST})
+TARGET_LINK_LIBRARIES(get_lost maatframe libMESA_htable.so pthread m)
diff --git a/src/dataset_build/based_sfh.conf b/src/dataset_build/based_sfh.conf
new file mode 100644
index 0000000..cdcf4cf
--- /dev/null
+++ b/src/dataset_build/based_sfh.conf
@@ -0,0 +1,3 @@
+[file]
+raw_file_address = ../../data/td_data_20171207/td_data/all_av_digest_nots
+ripe_file_address = ../../data/td_data_set/td_data_20171207/base_sfh_set
\ No newline at end of file
diff --git a/src/dataset_build/based_sfh.py b/src/dataset_build/based_sfh.py
new file mode 100644
index 0000000..b3281ce
--- /dev/null
+++ b/src/dataset_build/based_sfh.py
@@ -0,0 +1,44 @@
+import re
+import ConfigParser
+import bisect
+import random
+
+term = {'not_null':(lambda x : len(x)!=0)}
+	
+config = ConfigParser.RawConfigParser()
+config.read("based_sfh.conf")
+raw_file_address = config.get("file","raw_file_address")
+ripe_file_address = config.get("file","ripe_file_address")
+
+class sfh_fingerprint(object):
+	
+	def __init__(self,sfh):
+		self.sfh = sfh
+
+	@staticmethod
+	def get_hashed_len(sfh):
+		p = r"\[+\d+?:+\d+?\]"
+		pattern = re.compile(p)
+		hashed_len_set = pattern.findall(sfh)
+		if (term['not_null'](hashed_len_set)):
+			hashed_len = 0
+			for x in xrange(0,len(hashed_len_set)):
+				hashed_len_num = re.split(r"\[|\]|:",hashed_len_set[x])
+				hashed_len = hashed_len + int(hashed_len_num[2]) - int(hashed_len_num[1])
+			return hashed_len/len(hashed_len_set)
+		else :
+			return -1
+
+i=0
+sfh_set = list()
+with open(raw_file_address,'r') as infile:
+	with open(ripe_file_address,'w') as outfile: 
+		for line in infile:
+			i+=1
+			if(i%100000==0):
+				print i
+			result = re.split(r';',line)
+			if(term['not_null'](result[3]) and term['not_null'](result[19])):
+				hashed_len = sfh_fingerprint.get_hashed_len(result[19])
+				if(hashed_len/int(result[3])>0.8):
+					outfile.write(result[19]+'\n')
\ No newline at end of file
diff --git a/src/dataset_build/cal_information.conf b/src/dataset_build/cal_information.conf
new file mode 100644
index 0000000..1571b8b
--- /dev/null
+++ b/src/dataset_build/cal_information.conf
@@ -0,0 +1,5 @@
+[file]
+raw_file_address = ../../data/ripe_data/td_data_20171207/video_id.txt
+ripe_file_address = ../../data/td_data_set/td_data_20171207/td_data_set_statistic
+[feature]
+feature_name = URL,ServerIP,MediaType,MediaLen,Etag,LastModify
\ No newline at end of file
diff --git a/src/dataset_build/cal_information.py b/src/dataset_build/cal_information.py
new file mode 100644
index 0000000..19cd95c
--- /dev/null
+++ b/src/dataset_build/cal_information.py
@@ -0,0 +1,133 @@
+import re
+import numpy
+import ConfigParser
+import binascii
+term = {'td_len':(lambda x : len(x)==32),
+		'data_num':(lambda x : len(x)==4),
+		'url':(lambda x : x.find['NUll']),
+		'sfh_len':(lambda x : len(x)>20),
+		'not_null':(lambda x : len(x)!=0)} 
+
+class calculation(object):
+	"""docstring for calculation"""
+	def __init__(self, arg):
+		super(calculation, self).__init__()
+		self.arg = arg
+
+	@staticmethod
+	def cal_ent(x):
+		x_value_list = set([x[i] for i in range(x.shape[0])])
+		ent = 0.0
+		num_0 = x[x == 0].shape[0]
+		for x_value in x_value_list:
+			if(x_value==0):
+				continue
+			p = float(x[x == x_value].shape[0])/(x.shape[0]- num_0)
+			logp = numpy.log2(p)
+			ent -=p*logp
+		return ent
+
+class data_value(object):
+	"""docstring for data_value"""
+	def __init__(self, arg):
+		super(data_value, self).__init__()
+		self.arg = arg
+	
+	@staticmethod
+	def get_data_values(data):
+		data_set = re.split(r"URL:|ServerIP:|MediaType:|MediaLen:|Etag:|LastModify:",data)
+		#data_set[0]=null,data_set[1]=url
+		data_value_dic = [long(0)]*6
+		for x in xrange(1,len(feature_list)+1):
+			if(x==1):
+				if(term['not_null'](data_set[x])==True):
+					data_value_dic[0] = binascii.crc32(data_set[x])
+				else:
+					data_value_dic[0] = 0
+			elif(x==2):
+				if(term['not_null'](data_set[x])==True):
+					data_value_dic[1] = binascii.crc32(data_set[x])
+				else:
+					data_value_dic[1] = 0
+			elif(x==3):
+				data_value_dic[2] = long(data_set[x])
+			elif(x==4):
+				data_value_dic[3] = long(data_set[x])
+			elif(x==5):
+				if(term['not_null'](data_set[x])==True):
+					data_value_dic[4] = binascii.crc32(data_set[x])
+				else:
+					data_value_dic[4] = 0
+			elif(x==6):
+				if(term['not_null'](data_set[x])==True):
+					data_value_dic[5] = binascii.crc32(data_set[x])
+				else:
+					data_value_dic[5] = 0
+		return data_value_dic
+
+config = ConfigParser.RawConfigParser()
+config.read("cal_information.conf")
+
+raw_file_address = config.get("file","raw_file_address")
+ripe_file_address = config.get("file","ripe_file_address")
+feature_list =[i for i in config.get("feature","feature_name").split(",")]
+
+i=0
+with open(raw_file_address,'r') as infile:
+	for line in infile:
+		i+=1
+		if(i%10000==0):
+			print i
+		if(i==50000):
+			break
+		line_split = re.split(";",line)
+		data_value_temp = data_value.get_data_values(line_split[5])
+		data_value_temp.extend([binascii.crc32(line_split[j]) for j in range(6,19)])
+		data_value_temp.append(binascii.crc32(line_split[0]))
+		if(i==1):
+			a=numpy.array(data_value_temp)
+		else:
+			a=numpy.row_stack((a,numpy.array(data_value_temp)))
+	
+for i in range(20):
+	if(i==0):
+		print "URL:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==1):
+		print "ServerIP:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==2):
+		print "MediaType:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==3):
+		print "MediaLen:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==4):
+		print "Etag:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==5):
+		print "LastModify:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==6):
+		print "td_0k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==7):
+		print "td_data_md5_1k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==8):
+		print "td_1k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==9):
+		print "td_data_md5_2k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==10):
+		print "td_2k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==11):
+		print "td_data_md5_4k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==12):
+		print "td_4k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==13):
+		print "td_data_md5_8k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==14):
+		print "td_8k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==15):
+		print "td_data_md5_16k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==16):
+		print "td_16k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==17):
+		print "td_data_md5_32k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==18):
+		print "td_32k:"+str(calculation.cal_ent(a[:,i]))
+	elif(i==19):
+		print "id:"+str(calculation.cal_ent(a[:,i]))
+	
diff --git a/src/dataset_build/dataset_build.conf b/src/dataset_build/dataset_build.conf
new file mode 100644
index 0000000..400e160
--- /dev/null
+++ b/src/dataset_build/dataset_build.conf
@@ -0,0 +1,8 @@
+[file]
+raw_file_address = ../../data/ripe_data/td_data_20171207/all_av_digest_mistake_level
+ripe_file_address = ../../data/td_data_set/td_data_20171207/td_dataset
+base_sfh_sets = ../../data/td_data_set/td_data_20171207/base_sfh_set
+[output]
+breakpoints = 1048576,1310720,1572864,1835008,2097152,3145728,4194304
+[feature]
+feature_name = URL,ServerIP,MediaType,MediaLen,Etag,LastModify
\ No newline at end of file
diff --git a/src/dataset_build/dataset_build.py b/src/dataset_build/dataset_build.py
new file mode 100644
index 0000000..a832072
--- /dev/null
+++ b/src/dataset_build/dataset_build.py
@@ -0,0 +1,144 @@
+import re
+import ConfigParser
+import bisect
+import random
+import ctypes
+import hashlib
+import zlib
+import binascii
+import json
+import datetime
+import time
+
+term = {'td_len':(lambda x : len(x)==32),
+		'data_num':(lambda x : len(x)==21),
+		'url':(lambda x : x.find['NUll']),
+		'sfh_len':(lambda x : len(x)>20 and len(x)<(10*1024-100)),
+		'not_null':(lambda x : len(x)!=0),
+		'ysp_len':(lambda x : int(x)!=0),
+		'line_len':(lambda x: len(x)>20 and len(x)<(10*1024-100))} 
+	
+config = ConfigParser.RawConfigParser()
+config.read("dataset_build.conf")
+raw_file_address = config.get("file","raw_file_address")
+ripe_file_address = config.get("file","ripe_file_address")
+base_sfh_sets = config.get("file","base_sfh_sets")
+breakpoints = [int(i) for i in config.get("output","breakpoints").split(",")]
+feature_list =[i for i in config.get("feature","feature_name").split(",")]
+ll=ctypes.cdll.LoadLibrary
+lib = ll("libmaatframe.so")
+lost = dict()
+
+
+class data_value(object):
+	
+	@staticmethod
+	def get_feature(data):
+		return_data=list()
+		data_set = re.split(r"URL:|ServerIP:|MediaType:|MediaLen:|Etag:|LastModify:",data[5])
+		for x in xrange(1,21):
+			if(x==1):
+				if(term['not_null'](data_set[6])):
+					try:
+						time1=datetime.datetime.strptime(data[1],'%Y-%m-%d %H:%M:%S')+datetime.timedelta(hours=int(8))
+						data_set[6]=data_set[6][0:25]
+						time2=datetime.datetime.strptime(data_set[6],'%a, %d %b %Y %H:%M:%S')
+					except Exception, e:
+						return_data.append(-1)
+						return_data.append(-1)
+						return_data.append(-1)
+						return_data.append(-1)
+					else:
+						return_data.append(str((time1-time2).seconds))
+						return_data.append(((time1-time2).seconds)/60)
+						return_data.append(((time1-time2).seconds)/3600)
+						return_data.append((time1-time2).days)
+				else:
+					return_data.append(-1)
+					return_data.append(-1)
+					return_data.append(-1)
+					return_data.append(-1)
+			elif(x==2):
+				continue
+			elif(x==3):
+				continue
+			elif(x==4):
+				return_data.append(long(data[4]))
+			elif(x==5):
+				if(term['not_null'](data_set[1])):
+					return_data.append(len(data_set[1]))
+				else:
+					return_data.append(-1)
+				if(term['not_null'](data_set[2])):
+					ip_set=re.split(r'\.',data_set[2])
+					return_data.append(ip_set[0])
+					return_data.append(ip_set[1])
+					return_data.append(ip_set[2])
+					return_data.append(ip_set[3])
+				else:
+					return_data.append(-1)
+					return_data.append(-1)
+					return_data.append(-1)
+					return_data.append(-1)
+				if(term['not_null'](data_set[3])):
+					return_data.append(int(data_set[3]))
+				else:
+					return_data.append(-1)
+				if(term['not_null'](data_set[5])):
+					return_data.append(binascii.crc32(data_set[5]))
+				else:
+					return_data.append(-1)
+				if(term['not_null'](data_set[6])):
+					return_data.append(binascii.crc32(data_set[6]))
+				else:
+					return_data.append(-1)
+			elif(x==7):
+				return_data.append(binascii.crc32(data[7]))
+			elif(x==9):
+				return_data.append(binascii.crc32(data[9]))
+			elif(x==11):
+				return_data.append(binascii.crc32(data[11]))
+			elif(x==13):
+				return_data.append(binascii.crc32(data[13]))
+			elif(x==15):
+				return_data.append(binascii.crc32(data[15]))
+			elif(x==17):
+				return_data.append(binascii.crc32(data[17]))
+		return return_data
+		# data_set = re.split(r"URL:|ServerIP:|MediaType:|MediaLen:|Etag:|LastModify:",data)
+		# #data_set[0]=null,data_set[1]=url
+		# data_value_dic = dict()
+		# for x in xrange(1,len(feature_list)+1):
+		# 	if(x==1):
+		# 		data_value_dic[feature_list[x-1]] = binascii.crc32(data_set[x])
+		# 	elif(x==2):
+		# 		data_value_dic[feature_list[x-1]] = binascii.crc32(data_set[x])
+		# 	elif(x==3):
+		# 		data_value_dic[feature_list[x-1]] = data_set[x]
+		# 	elif(x==4):
+		# 		data_value_dic[feature_list[x-1]] = data_set[x]
+		# 	elif(x==5):
+		# 		data_value_dic[feature_list[x-1]] = binascii.crc32(data_set[x])
+		# 	elif(x==6):
+		# 		data_value_dic[feature_list[x-1]] = binascii.crc32(data_set[x])
+		# return data_value_dic
+			
+
+i=0
+sfh_set = list()
+with open(raw_file_address,'r') as infile:
+	with open(ripe_file_address,'w') as outfile:
+		for line in infile:
+			i+=1
+			if(i%10000==0):
+				print i
+			line_return = re.split(r';',line)
+			# if(int(line_return[0])==0):
+			# 	print 'td is right'
+			outfile.write(str(line_return[0])+',')
+			return_data=data_value.get_feature(line_return)
+			for x in range(19):
+				if(x==18):
+					outfile.write(str(return_data[18])+'\n')
+				else:
+					outfile.write(str(return_data[x])+',')
diff --git a/src/dataset_build/feature_statistics.conf b/src/dataset_build/feature_statistics.conf
new file mode 100644
index 0000000..12cf089
--- /dev/null
+++ b/src/dataset_build/feature_statistics.conf
@@ -0,0 +1,8 @@
+[file]
+raw_file_address = ../../data/td_data_set/td_data_20171207/td.txt
+ripe_file_address = ../../data/td_data_set/td_data_20171207/td_data_set_statistic
+[output]
+breakpoints = 1048576,1310720,1572864,1835008,2097152,3145728,44194304
+[feature]
+type = data_value_statistics
+feature_name = URL,ServerIP,MediaType,MediaLen,Etag,LastModify
\ No newline at end of file
diff --git a/src/dataset_build/feature_statistics.py b/src/dataset_build/feature_statistics.py
new file mode 100644
index 0000000..52ae8e0
--- /dev/null
+++ b/src/dataset_build/feature_statistics.py
@@ -0,0 +1,164 @@
+import re
+import ConfigParser
+import bisect
+import random
+import ctypes
+import hashlib
+import zlib
+import binascii
+
+term = {'td_len':(lambda x : len(x)==32),
+		'data_num':(lambda x : len(x)==4),
+		'url':(lambda x : x.find['NUll']),
+		'sfh_len':(lambda x : len(x)>20),
+		'not_null':(lambda x : len(x)!=0)} 
+
+class data_line(object):
+	"""docstring for ClassName"""
+	def __init__(self):
+		super(ClassName, self).__init__()
+	
+	@staticmethod
+	def  if_error(data_line_str):
+		data_line_val = re.split(r';',data_line_str)
+		hashed_len = sfh_fingerprint.get_hashed_len(data_line_val[19])
+		if(term['data_num'](data_line_val) and term['sfh_len'](data_line_val[19]) and term['td_len'](data_line_val[9])\
+		and term['td_len'](data_line_val[2]) and term['td_len'](data_line_val[13]) and term['td_len'](data_line_val[15])\
+		and term['td_len'](data_line_val[17]) and term['not_null'](data_line_val[18]) and term['not_null'](data_line_val[19])\
+		and hashed_len/float(data_line_val[3])>0.8):
+			return data_line_val
+		else:
+			return -1
+		
+
+class feature_statistics(object):
+	"""YSP feature_statistics"""
+	def __init__(self):
+		super(feature_statistics, self).__init__()
+		self.meida_len_statistics_set = [0,0,0,0,0,0,0]
+		self.lost_dict = dict()
+
+	def meida_len_statistics(meida_len):
+			j = bisect.bisect(breakpoints,meida_len)
+			self.meida_len_statistics_set[j-1]+=1
+
+	def data_value_statistics(data_value_dic,data_value):
+		data_value_str = str()
+		for x in xrange(0,len(feature_list)):
+			data_value_str = data_value_str+str(data_value_dic[feature_list[x]])+','
+
+		if(self.lost_dict.has_key(data_value_str)==False):
+				self.lost_dict[data_value_str]=[0,1,0.]
+		else:
+			if (int(result[3])==1):
+				self.lost_dict[data_value_str][0] += 1
+				self.lost_dict[data_value_str][1] += 1
+			else:
+				self.lost_dict[data_value_str][1] += 1
+		
+
+class sfh_fingerprint(object):
+	
+	def __init__(self,sfh):
+		self.sfh = sfh
+
+	@staticmethod
+	def get_hashed_len(sfh):
+		p = r"\[+\d+?:+\d+?\]"
+		pattern = re.compile(p)
+		hashed_len_set = pattern.findall(sfh)
+		if (term['not_null'](hashed_len_set)):
+			hashed_len = 0
+			for x in xrange(0,len(hashed_len_set)):
+				hashed_len_num = re.split(r"\[|\]|:",hashed_len_set[x])
+				hashed_len = hashed_len + int(hashed_len_num[2]) - int(hashed_len_num[1])
+			return hashed_len/len(hashed_len_set)
+		else :
+			return -1
+	
+	@staticmethod
+	def get_base_sfh(data_set):
+		base_sfh = list()
+		for x in xrange(0,10):
+			base_sfh.append(data_set[x])
+		return base_sfh
+			
+
+
+
+class data_value(object):
+	
+	@staticmethod
+	def get_data_values(data):
+		data_set = re.split(r"URL:|ServerIP:|MediaType:|MediaLen:|Etag:|LastModify:",data)
+		#data_set[0]=null,data_set[1]=url
+		data_value_dic = dict()
+		for x in xrange(1,len(feature_list)+1):
+			if(x==1):
+				data_value_dic[feature_list[x-1]] = 0 if(term['not_null']==False) else 1
+			elif(x==2):
+				data_value_dic[feature_list[x-1]] = 0 if(term['not_null']==False) else 1
+			elif(x==3):
+				data_value_dic[feature_list[x-1]] = data_set[x]
+			elif(x==4):
+				data_value_dic[feature_list[x-1]] = bisect.bisect(breakpoints,int(data_set[x]))
+			elif(x==5):
+				data_value_dic[feature_list[x-1]] = 0 if(term['not_null']==False) else 1
+			elif(x==6):
+				data_value_dic[feature_list[x-1]] = 0 if(term['not_null']==False) else 1
+		return data_value_dic
+				
+config = ConfigParser.RawConfigParser()
+config.read("feature_statistics.conf")
+
+feature_statistics_type = ("feature","type")
+raw_file_address = config.get("file","raw_file_address")
+ripe_file_address = config.get("file","ripe_file_address")
+
+if(feature_statistics_type=="meida_len_statistics"):
+	breakpoints = [int(i) for i in config.get("output","breakpoints").split(",")]
+elif(feature_statistics_type=="data_value_statistics"):
+	feature_list =[i for i in config.get("feature","feature_name").split(",")]
+# ll=ctypes.cdll.LoadLibrary
+# lib = ll("libmaatframe.so")
+
+i=0
+sfh_set = list()
+statistic = feature_statistics()
+with open(raw_file_address,'r') as infile:
+	for line in infile:
+		i+=1
+		
+
+
+		
+		line_return = data_line.if_error(line)
+		if(line_return != -1):
+			if(feature_statistics_type=="meida_len_statistics"):
+				statistic.meida_len_statistics(line_return[3])
+			elif(feature_statistics_type=="data_value_statistics"):
+				lost_list = list()
+				statistic.meida_len_statistics(line_return)
+				for i in statistic.lost:
+					(statistic.lost[i])[2] = float((statistic.lost[i])[0])/(statistic.lost[i])[1]
+					tmp = (i,int((statistic.lost[i])[0]),int((statistic.lost[i])[1]),float((statistic.lost[i])[2]))
+					lost_list.append(tmp)
+				print sorted(lost_list,cmp=lambda x,y:cmp(x[2],y[2]))
+						# if(x == len(feature_list)-1):
+						# 	outfile.write(data_value_dic[feature_list[x]]+'\n')
+						# else:
+			# print lost
+					# 	outfile.write(str(data_value_dic[feature_list[x]])+',') 
+					# outfile.write(result[3])
+						# sfh_dot=list()
+						# for x in xrange(0,10):
+						# 	#transform sfh to dot
+						# 	sfh_dot.append(lib.GIE_sfh_similiarity(result[19],len(result[19]),sfh_set[x],len(sfh_set[x])))
+						# if(len(data_set)==7):
+							# outfile.write(str(data_set[0])+','+str(data_set[1])+','+str(data_set[2])\
+							# +','+str(data_set[3])+','+str(data_set[4])+','+str(data_set[5])+','+result[5]\
+							# +','+result[7]+','+result[9]+','+result[11]+','+result[13]+','+result[15]+result[17]\
+							# +','+result[19]+'\n')
+
+# with open(ripe_file_address,'w') as outfile:
+# 	outfile.write(str(lost))
diff --git a/src/dataset_build/file_digest.py b/src/dataset_build/file_digest.py
new file mode 100644
index 0000000..590e059
--- /dev/null
+++ b/src/dataset_build/file_digest.py
@@ -0,0 +1,96 @@
+#-*-coding:utf-8-*-
+import re
+import random
+import ConfigParser
+import bisect
+import commands
+import os
+import hashlib
+
+class data_line(object):
+	"""docstring for ClassName"""
+	def __init__(self):
+		super(ClassName, self).__init__()
+	
+	@staticmethod
+	def  if_error(data_line_str):
+		data_line_val = re.split(r';',data_line_str)
+		hashed_len = sfh_fingerprint.get_hashed_len(data_line_val[19])
+		if(term['data_num'](data_line_val) and \
+		term['not_null'](data_line_val[0]) and \
+		term['ysp_len'](data_line_val[3]) and \
+		term['not_null'](data_line_val[4]) and \
+		term['td_len'](data_line_val[6]) and \
+		term['td_len'](data_line_val[8]) and \
+		term['td_len'](data_line_val[10]) and \
+		term['td_len'](data_line_val[12]) and \
+		term['td_len'](data_line_val[14]) and \
+		term['td_len'](data_line_val[16]) and \
+		term['not_null'](data_line_val[18]) and \
+		term['sfh_len'](data_line_val[19]) and \
+		term['not_null'](data_line_val[20]) and \
+		hashed_len/float(data_line_val[3])>=0.8):
+			return data_line_val
+		else:
+			return -1
+
+class TD_fingerprint(object):
+		def __init__():
+			self.td = td
+			self.td_string = td_string
+		@staticmethod
+		def td_generate(td_string):
+			td_val = hashlib.md5(td_string,encode('utf-8')).hexdigest()
+
+class sfh_fingerprint(object):
+	
+	def __init__(self,sfh):
+		self.sfh = sfh
+
+	@staticmethod
+	def get_hashed_len(sfh):
+		p = r"\[+\d+?:+\d+?\]"
+		pattern = re.compile(p)
+		hashed_len_set = pattern.findall(sfh)
+		if (term['not_null'](hashed_len_set)):
+			hashed_len = 0
+			for x in xrange(0,len(hashed_len_set)):
+				hashed_len_num = re.split(r"\[|\]|:",hashed_len_set[x])
+				hashed_len = hashed_len + int(hashed_len_num[2]) - int(hashed_len_num[1])
+			return hashed_len/len(hashed_len_set)
+		else :
+			return -1
+
+term = {'td_len':(lambda x : len(x)==32),
+		'data_num':(lambda x : len(x)==21),
+		'url':(lambda x : x.find['NUll']),
+		'sfh_len':(lambda x : len(x)>20 and len(x)<(10*1024-100)),
+		'not_null':(lambda x : len(x)!=0),
+		'ysp_len':(lambda x : int(x)!=0),
+		'line_len':(lambda x: len(x)>20 and len(x)<(10*1024-100))} 
+
+grain="./get_lost"
+ripe_files=[]
+config = ConfigParser.RawConfigParser()
+config.read("grain.conf")
+raw_file_address=config.get("file","raw_file_address")
+ripe_files_address=config.get("file","ripe_files_address")
+print ("%s %s" %(raw_file_address,ripe_files_address))
+num = [0,0,0,0,0,0,0]
+breakpoints = [int(i) for i in config.get("output","breakpoints").split(",")]
+# i=0
+# for i in xrange(0,ripe_file_num):
+# 	outfile=open(ripe_files_address+str(i)+'.txt','w')
+# 	ripe_files.append(outfile)
+
+i=0	
+with open(raw_file_address,'r') as infile:
+# with open('./ripe_data/mistake_td_sfh1_sfh2_sim_rate_len_url_unequal','r')as infile:
+	with open(ripe_files_address,'w')as outfile:
+		for line in infile:
+			i+=1
+			if(i%10000==0):
+				print i
+			line_return = data_line.if_error(line)
+			if(line_return != -1):
+				outfile.write(str(line))
\ No newline at end of file
diff --git a/src/dataset_build/get_lost.c b/src/dataset_build/get_lost.c
new file mode 100644
index 0000000..0e6c452
--- /dev/null
+++ b/src/dataset_build/get_lost.c
@@ -0,0 +1,116 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <MESA/MESA_htable.h>
+#include <assert.h>
+#include <ctype.h>
+#define HTABLE_SIZE 8*64*1024*1024
+#define SFH_PASS_RATE 0.8
+#define SIMILIAR 80
+
+typedef struct td
+{
+	char * tdstr;
+	unsigned int lost;
+}td;
+
+typedef struct file_sfh_data
+{
+	long id;
+	char * sfh;
+	td * td_value;
+	char * td_ori;
+}file_sfh_data;
+
+int main(int argc,char *argv[])
+{
+	FILE *fpread;//文件
+	FILE *fpwrite;//write file handle
+	int array_size = 1024;
+	file_sfh_data **file_data=(file_sfh_data **)malloc(sizeof(file_sfh_data)*array_size);
+	char* dirstr = "../../data/td_data_set/td_data_20171207/td_sfh_lost";
+	//char* dirstr = *++argv;
+	char* writestr = "../../data/td_data_set/td_data_20171207/td.txt";
+	int total_len = 0;
+	char TD_tmp[256], SFH_tmp[1024*300], TD_ORI[1024*10];
+	char buffer[1024*300+1];
+	int ret = 0;
+	int line = 0;
+	int thread_safe = 0;
+	int i;
+	int id;
+	int similiarity;
+	MESA_htable_handle htable = NULL;
+	fpread=fopen(dirstr,"rb");
+	fpwrite=fopen(writestr,"w");
+	printf("file str is %s\n",dirstr);
+	if(fpread==NULL)
+    {
+		printf("open file error\n");
+		return -1;
+	}
+	buffer[sizeof(buffer)]='\0';
+	while(feof(fpread)==0)
+	{
+		fgets(buffer,sizeof(buffer)-1,fpread);
+		ret=sscanf(buffer,"%d;%[^;];%[^;];%s",&total_len,TD_ORI,TD_tmp,SFH_tmp);
+		if(ret!=4)
+		{
+			continue;
+		}
+		file_data[line]=(file_sfh_data*)calloc(1,sizeof(file_sfh_data));
+		file_data[line]->id=line;
+		file_data[line]->sfh=strdup(SFH_tmp);
+		file_data[line]->td_value=(td*)calloc(1,sizeof(td));
+		file_data[line]->td_value->tdstr=strdup(TD_tmp);
+		file_data[line]->td_value->lost=0;
+		file_data[line]->td_ori=strdup(TD_ORI);
+		line++;
+		if(line==array_size)
+		{
+			array_size*=2;
+			file_data=realloc(file_data,sizeof(file_sfh_data)*array_size);
+		}
+	}
+	printf("read file success!\n");
+	htable = NULL;
+	htable=MESA_htable_born();
+	thread_safe = 0;
+	MESA_htable_set_opt(htable,MHO_SCREEN_PRINT_CTRL,&thread_safe,sizeof(unsigned int));
+	unsigned int slot_size=1024*1024*16;
+	MESA_htable_set_opt(htable,MHO_HASH_SLOT_SIZE,&slot_size,sizeof(slot_size));
+	MESA_htable_mature(htable);
+	for(i=0;i<line;i++)
+	{
+		if(MESA_htable_add(htable,(char*)(file_data[i]->td_value->tdstr),32,(void *)file_data[i]->id)<0)
+		{
+			id=(long)MESA_htable_search(htable,(char*)file_data[i]->td_value->tdstr,32);
+			similiarity=GIE_sfh_similiarity(file_data[id]->sfh,(int)strlen(file_data[id]->sfh),file_data[i]->sfh,(int)strlen(file_data[i]->sfh));
+			if(similiarity<SIMILIAR)
+			{
+				file_data[id]->td_value->lost = 1;
+				file_data[i]->td_value->lost = 1;
+			}
+		}
+	}
+	for(i=0;i<line;i++)
+	{
+		fprintf(fpwrite,"%s;%s;%s;%d\n",file_data[i]->td_value->tdstr,file_data[i]->sfh,file_data[i]->td_ori,file_data[i]->td_value->lost);
+	}
+	for(i=0;i<line;i++)
+	{
+		free(file_data[i]->sfh);
+		file_data[i]->sfh=NULL;
+		free(file_data[i]->td_value->tdstr);
+		file_data[i]->td_value->tdstr=NULL;
+		free(file_data[i]->td_value);
+		file_data[i]->td_value=NULL;
+		free(file_data[i]->td_ori);
+		file_data[i]->td_ori=NULL;
+		free(file_data[i]);
+		file_data[i]=NULL;
+	}
+	fclose(fpread);
+	fclose(fpwrite);
+	return 0;
+}
\ No newline at end of file
diff --git a/src/dataset_build/grain.conf b/src/dataset_build/grain.conf
new file mode 100644
index 0000000..944b337
--- /dev/null
+++ b/src/dataset_build/grain.conf
@@ -0,0 +1,5 @@
+[file]
+ripe_files_address = ../../data/td_data_set/td_data_20171207/get_lost_raw_data
+raw_file_address = ../../data/td_data_20171207/td_data/all_av_digest
+[output]
+breakpoints = 1048576,1310720,1572864,1835008,2097152,3145728,4194304
\ No newline at end of file
diff --git a/src/dataset_build/td_classification.py b/src/dataset_build/td_classification.py
new file mode 100644
index 0000000..8d4b97c
--- /dev/null
+++ b/src/dataset_build/td_classification.py
@@ -0,0 +1,5 @@
+from sklearn.datasets import load_iris
+from sklearn import tree
+
+with open() as infile:
+	
\ No newline at end of file
diff --git a/src/dataset_build/vedio_id_build.c b/src/dataset_build/vedio_id_build.c
new file mode 100644
index 0000000..9faaa64
--- /dev/null
+++ b/src/dataset_build/vedio_id_build.c
@@ -0,0 +1,171 @@
+/*
+gcc -g vedio_id_build.c -o vedio_id_build -lmaatframe -I../../inc 
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include "gram_index_engine.h"
+#include <MESA/MESA_htable.h>
+#include <assert.h>
+#include <ctype.h>
+#define BUFFER_LEN (10*1024)
+#define SFH_PASS_RATE 0.9
+#define SFH_LEN (10*1024)
+#define URL_LEN (10*1024)
+
+typedef struct video_id
+{
+	long id;
+	char *sfh;
+}video_id;
+
+typedef struct cache
+{
+	GIE_digest_t ** GIE_cache;
+	long cache_size;
+	long len;
+}cache;
+
+long get_hashed_len(const char* sfh)
+{
+	char *data=(char*)malloc(strlen(sfh)+1);
+	memcpy(data,sfh, strlen(sfh));
+	data[strlen(sfh)]='\0';
+	char *token=NULL,*sub_token=NULL,*saveptr;
+	long left_offset=0,right_offset=0,hashed_length=0;
+	int ret=0,first=0;
+	for (token = data; ;token= NULL)
+	{
+		sub_token= strtok_r(token,"[", &saveptr);
+		if (sub_token == NULL)
+		{
+			break;
+		}
+		if(first==0)//jump over the first sub string.
+		{
+			first=1;
+			continue;
+		}
+		ret=sscanf(sub_token,"%ld:%ld",&left_offset,&right_offset);
+		if(ret!=2)
+		{
+			return 0;
+		} 
+		assert(ret==2);
+		hashed_length+=right_offset-left_offset+1;
+	}
+	//printf("hashed length=%ld\n",hashed_length);
+	free(data);
+	return hashed_length/2;
+}
+
+int main(int argc,char *argv[])
+{
+	FILE *video_id_sets_file;
+	FILE *new_sfh_file;
+	const char *video_id_sets_file_dir="../../data/td_data_set/td_data_20171207/video_id_raw_data";
+	const char *new_sfh_file_dir="../../data/ripe_data/td_data_20171207/video_id.txt";
+	char *buffer=NULL;
+	int ret = 0,hashed_len = 0,total_len = 0,resultnum = 0,i = 0;
+	int update = 0,video_id = 0,j = 0;
+	int* temp_int = NULL;
+	float temp_sfh_pass = 0;
+	char *sfh_str,*url_str;
+	GIE_digest_t *sfh_video_id = NULL; 
+	GIE_result_t *query_result = NULL;
+	cache *GIE_digest_cache = NULL;
+	video_id_sets_file = fopen(video_id_sets_file_dir,"r+");
+	new_sfh_file = fopen(new_sfh_file_dir,"w");
+	if(video_id_sets_file == NULL)
+	{
+		printf("open video_id_sets_file error\n");
+		return -1;
+	}
+	if(new_sfh_file == NULL)
+	{
+		printf("open new_sfh_file error\n");
+		return -1;
+	}
+	buffer = (char*)calloc(BUFFER_LEN,sizeof(char));
+	GIE_create_para_t *query_para = (GIE_create_para_t*)calloc(1,sizeof(GIE_create_para_t));
+	query_result = (GIE_result_t*)calloc(1,sizeof(GIE_result_t));
+	GIE_handle_t *query_handle;
+	query_para->gram_value = 7;
+	query_para->position_accuracy = 5;
+	query_handle=GIE_create((const GIE_create_para_t *)query_para);
+	free(query_para);
+	if(query_handle==NULL)
+	{
+		printf("create GIE handle error\n");
+		return -1;
+	}
+	sfh_video_id = (GIE_digest_t*)calloc(1,sizeof(GIE_digest_t));
+	sfh_str = (char*)calloc(SFH_LEN,sizeof(char));
+	url_str = (char*)calloc(URL_LEN,sizeof(char));
+	i=0;
+	GIE_digest_cache =(cache*)calloc(1,sizeof(cache));
+	GIE_digest_cache->cache_size = 1000;
+	GIE_digest_cache->GIE_cache = (GIE_digest_t**)calloc(GIE_digest_cache->cache_size,sizeof(GIE_digest_t*));
+	GIE_digest_cache->len = 0;
+	while(feof(video_id_sets_file)==0)
+	{
+		i++;
+		if(i%10000==0)
+		{
+			printf("%d\n",i);
+		}
+		fgets(buffer,BUFFER_LEN-1,video_id_sets_file);
+		ret=sscanf(buffer,"%*[^;];%*[^;];%*[^;];%*[^;];%*[^;];%*[^;];%*[^;];\
+			%*[^;];%*[^;];%*[^;];%*[^;];%*[^;];%*[^;];%*[^;];%*[^;];%*[^;];\
+			%*[^;];%*[^;];%*[^;];%[^;];%[^;]",sfh_str,url_str);
+		if(ret!=2)
+		{
+			continue;
+		}
+		hashed_len = get_hashed_len((const char*)sfh_str);
+		temp_sfh_pass = (float)hashed_len/total_len;
+		if(temp_sfh_pass<SFH_PASS_RATE)
+		{
+			continue;
+		}
+		resultnum=GIE_query(query_handle,(const char *)sfh_str,(const long long)strlen(sfh_str),query_result,1);
+		if(resultnum == 0)
+		{
+			temp_int=(int*)calloc(1,sizeof(int));
+			*temp_int=i;
+			sfh_video_id = (GIE_digest_t*)calloc(1,sizeof(GIE_digest_t));
+			sfh_video_id->id=i;
+			sfh_video_id->sfh_length=strlen(sfh_str);
+			sfh_video_id->operation=GIE_INSERT_OPT;
+			sfh_video_id->cfds_lvl=5;
+			sfh_video_id->sfh=strdup(sfh_str);
+			sfh_video_id->tag=temp_int;
+			GIE_digest_cache->GIE_cache[GIE_digest_cache->len] = sfh_video_id;
+			GIE_digest_cache->len++;
+			if(GIE_digest_cache->len==GIE_digest_cache->cache_size)
+			{
+				update=GIE_update(query_handle,GIE_digest_cache->GIE_cache,GIE_digest_cache->cache_size);
+				GIE_digest_cache->len=0;
+				for(j=0;j<GIE_digest_cache->cache_size;j++)
+				{
+					free(GIE_digest_cache->GIE_cache[j]->sfh);
+					GIE_digest_cache->GIE_cache[j]->sfh=NULL;
+					free(GIE_digest_cache->GIE_cache[j]);
+					GIE_digest_cache->GIE_cache[j]=NULL;
+				}
+			}
+			fprintf(new_sfh_file,"%d,%s",i,buffer);
+		}
+		else
+		{
+			fprintf(new_sfh_file,"%d,%s",*((int*)query_result->tag),buffer);
+		}
+	}
+	free(buffer);
+	free(query_result);
+	free(sfh_video_id);
+	free(url_str);
+	free(sfh_str);
+	free(GIE_digest_cache);
+	return 0;
+}
\ No newline at end of file