Inverted-Indexer/task2.py at master · parshva45/Inverted-Indexer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
import os

# function to return index of key value pair if present, -1 if not present
def find(lst, key, value):
    for i, dic in enumerate(lst):
        if dic[key] == value:
            return i
    return -1

def indexer():
	path = r'Tokenization_Outputs'
	# dictionary to store unigrams' docid and tf
	tokens_dict = {}
	# dictionary to store bigrams' docid and tf
	bigram_tokens_dict = {}
	# dictionary to store trigrams' docid and tf
	trigram_tokens_dict = {}

	for file in os.listdir(path):
		current_file = os.path.join(path, file)
		print(current_file)
		content = open(current_file, 'r').read()
		# split content by space to get list of tokens
		tokens = content.split(' ')
		# remove all empty tokens
		tokens = [w for w in tokens if w != ""]
		# trim .txt from the end of file to get just the file_name
		file_name = file[:-4]

		for w in tokens:
			# if w already in dict
			if w in tokens_dict:
				# find index
				f_index = find(tokens_dict[w],"docid",file_name)
				if f_index == -1: # term present, key value pair not found
					tokens_dict[w].append({"docid":file_name,"tf":1})
				else: # key value pair already present, increment tf
					tokens_dict[w][f_index]['tf']+=1
			else:
				# create new term as key
				tokens_dict[w] = [{"docid":file_name,"tf":1}]

		bigrams = []
		# generate bigrams
		content_list = content.split()
		for i in range(len(content_list)-1):
			bigrams.append(content_list[i]+" "+content_list[i+1])

		for b in bigrams:
			# if b already in dict
			if b in bigram_tokens_dict:
				# find index
				f_index = find(bigram_tokens_dict[b],"docid",file_name)
				if f_index == -1: # term present, key value pair not found
					bigram_tokens_dict[b].append({"docid":file_name,"tf":1})
				else: # key value pair already present, increment tf
					bigram_tokens_dict[b][f_index]['tf']+=1
			else:
				# create new term as key
				bigram_tokens_dict[b] = [{"docid":file_name,"tf":1}]

		trigrams = []
		# generate trigrams
		for i in range(len(content_list)-2):
			trigrams.append(content_list[i]+" "+content_list[i+1]+" "+content_list[i+2])

		for t in trigrams:
			# if t already in dict
			if t in trigram_tokens_dict:
				# find index
				f_index = find(trigram_tokens_dict[t],"docid",file_name)
				if f_index == -1: # term present, key value pair not found
					trigram_tokens_dict[t].append({"docid":file_name,"tf":1})
				else: # key value pair already present, increment tf
					trigram_tokens_dict[t][f_index]['tf']+=1
			else:
				# create new term as key
				trigram_tokens_dict[t] = [{"docid":file_name,"tf":1}]

	# file for storing total number of unique unigrams, bigrams, trigrams
	global_name = open("Global_statistics.txt","w")

	# writing unigrams
	name = open("Indexing_Outputs/Unigrams.txt","w")
	for key,value in sorted(tokens_dict.items()):
		name.write(key+" ->")
		for dic in value:
			flag = True
			for k,v in dic.items():
				if flag:
					name.write(" ("+str(v)+",")
					flag = False
				else:
					name.write(str(v)+")")
					flag = True
		name.write("\n")
	name.close()
	# writing number of unigrams
	global_name.write("Total number of unique unigrams : "+str(len(tokens_dict)))

	# writing bigrams
	bi_name = open("Indexing_Outputs/Bigrams.txt","w")
	for key,value in sorted(bigram_tokens_dict.items()):
		bi_name.write(key+" ->")
		for dic in value:
			flag = True
			for k,v in dic.items():
				if flag:
					bi_name.write(" ("+str(v)+",")
					flag = False
				else:
					bi_name.write(str(v)+")")
					flag = True
		bi_name.write("\n")
	bi_name.close()
	# writing number of bigrams
	global_name.write("\nTotal number of unique bigrams : "+str(len(bigram_tokens_dict)))

	# for writing document frequency table for trigrams
	tri_dft = open("Frequency_Tables/Trigrams_Document_Frequency_Table.txt","w")
	tri_dft.write("Format : Term -> List of Document ID(s) -> Document Frequency\n")

	# writing trigrams
	tri_name = open("Indexing_Outputs/Trigrams.txt","w")
	for key,value in sorted(trigram_tokens_dict.items()):
		tri_name.write(key+" ->")
		rhs = []
		rhs.append([]) # docid(s)
		rhs.append(len(value))

		for dic in value:
			flag = True
			for k,v in dic.items():
				if flag:
					tri_name.write(" ("+str(v)+",")
					rhs[0].append(v)
					flag = False
				else:
					tri_name.write(str(v)+")")
					flag = True

		# write document frequency table for trigrams
		tri_dft.write("\n"+key+" -> "+str(rhs[0])+" -> "+str(rhs[1]))
		tri_name.write("\n")
	tri_dft.close()
	tri_name.close()
	# writing number of trigrams
	global_name.write("\nTotal number of unique trigrams : "+str(len(trigram_tokens_dict)))
	global_name.close()


# run the program as:
# python task2.py

newpath = r'Indexing_Outputs'
if not os.path.exists(newpath):
	os.makedirs(newpath)

newpath = r'Frequency_Tables'
if not os.path.exists(newpath):
	os.makedirs(newpath)

indexer()