Skip to content

Commit bb324fe

Browse files
anshikapacktanshikapackt
committed
Repository created
1 parent 8103225 commit bb324fe

25 files changed

+2122
-0
lines changed

‎Activity01/Activity01.ipynb

+353
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,353 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 10,
6+
"metadata": {},
7+
"outputs": [
8+
{
9+
"name": "stderr",
10+
"output_type": "stream",
11+
"text": [
12+
"[nltk_data] Downloading package stopwords to\n",
13+
"[nltk_data] /home/muzaffar/nltk_data...\n",
14+
"[nltk_data] Package stopwords is already up-to-date!\n",
15+
"[nltk_data] Downloading package wordnet to /home/muzaffar/nltk_data...\n",
16+
"[nltk_data] Package wordnet is already up-to-date!\n",
17+
"[nltk_data] Downloading package averaged_perceptron_tagger to\n",
18+
"[nltk_data] /home/muzaffar/nltk_data...\n",
19+
"[nltk_data] Package averaged_perceptron_tagger is already up-to-\n",
20+
"[nltk_data] date!\n"
21+
]
22+
}
23+
],
24+
"source": [
25+
"from nltk import download\n",
26+
"download('stopwords')\n",
27+
"download('wordnet')\n",
28+
"download('averaged_perceptron_tagger')\n",
29+
"from nltk import word_tokenize\n",
30+
"from nltk.stem.wordnet import WordNetLemmatizer\n",
31+
"from nltk.corpus import stopwords\n",
32+
"from autocorrect import Speller\n",
33+
"from autocorrect import spell\n",
34+
"from nltk.wsd import lesk\n",
35+
"from nltk.tokenize import sent_tokenize\n",
36+
"from nltk import stem, pos_tag\n",
37+
"import string\n"
38+
]
39+
},
40+
{
41+
"cell_type": "code",
42+
"execution_count": 14,
43+
"metadata": {},
44+
"outputs": [],
45+
"source": [
46+
"# load the text file into variable called sentence\n",
47+
"sentence = open(\"../data/file.txt\", 'r').read()"
48+
]
49+
},
50+
{
51+
"cell_type": "code",
52+
"execution_count": 15,
53+
"metadata": {},
54+
"outputs": [],
55+
"source": [
56+
"words = word_tokenize(sentence)"
57+
]
58+
},
59+
{
60+
"cell_type": "code",
61+
"execution_count": 16,
62+
"metadata": {},
63+
"outputs": [
64+
{
65+
"name": "stdout",
66+
"output_type": "stream",
67+
"text": [
68+
"['The', 'reader', 'of', 'this', 'course', 'should', 'have', 'a', 'basic', 'knowledge', 'of', 'the', 'Python', 'programming', 'lenguage', '.', 'He/she', 'must', 'have', 'knowldge']\n"
69+
]
70+
}
71+
],
72+
"source": [
73+
"print(words[0:20])"
74+
]
75+
},
76+
{
77+
"cell_type": "code",
78+
"execution_count": 8,
79+
"metadata": {},
80+
"outputs": [],
81+
"source": [
82+
"spell = Speller(lang='en')\n",
83+
"\n",
84+
"def correct_sentence(words):\n",
85+
" corrected_sentence = \"\"\n",
86+
" corrected_word_list = []\n",
87+
" for wd in words:\n",
88+
" if wd not in string.punctuation:\n",
89+
" wd_c = spell(wd)\n",
90+
" if wd_c != wd:\n",
91+
" print(wd+\" has been corrected to: \"+wd_c)\n",
92+
" corrected_sentence = corrected_sentence+\" \"+wd_c\n",
93+
" corrected_word_list.append(wd_c)\n",
94+
" else:\n",
95+
" corrected_sentence = corrected_sentence+\" \"+wd\n",
96+
" corrected_word_list.append(wd)\n",
97+
" else:\n",
98+
" corrected_sentence = corrected_sentence + wd\n",
99+
" corrected_word_list.append(wd)\n",
100+
" return corrected_sentence, corrected_word_list\n"
101+
]
102+
},
103+
{
104+
"cell_type": "code",
105+
"execution_count": null,
106+
"metadata": {},
107+
"outputs": [],
108+
"source": [
109+
"corrected_sentence, corrected_word_list = correct_sentence(words)"
110+
]
111+
},
112+
{
113+
"cell_type": "code",
114+
"execution_count": 18,
115+
"metadata": {},
116+
"outputs": [
117+
{
118+
"data": {
119+
"text/plain": [
120+
"\"'The reader of this course should have a basic knowledge of the Python programming language. He/she must have knowledge of data types in Python, should be able to write functions, and also have the ability to import and use libraries and packages in Python. familiarity with basic linguistics and probability is assumed although not required to fully complete this course.'\""
121+
]
122+
},
123+
"execution_count": 18,
124+
"metadata": {},
125+
"output_type": "execute_result"
126+
}
127+
],
128+
"source": [
129+
"corrected_sentence"
130+
]
131+
},
132+
{
133+
"cell_type": "code",
134+
"execution_count": 20,
135+
"metadata": {},
136+
"outputs": [
137+
{
138+
"name": "stdout",
139+
"output_type": "stream",
140+
"text": [
141+
"[\"'The\", 'reader', 'of', 'this', 'course', 'should', 'have', 'a', 'basic', 'knowledge', 'of', 'the', 'Python', 'programming', 'language', '.', 'He/she', 'must', 'have', 'knowledge']\n"
142+
]
143+
}
144+
],
145+
"source": [
146+
"print(corrected_word_list[0:20])"
147+
]
148+
},
149+
{
150+
"cell_type": "code",
151+
"execution_count": 21,
152+
"metadata": {},
153+
"outputs": [
154+
{
155+
"name": "stdout",
156+
"output_type": "stream",
157+
"text": [
158+
"[(\"'The\", 'CD'), ('reader', 'NN'), ('of', 'IN'), ('this', 'DT'), ('course', 'NN'), ('should', 'MD'), ('have', 'VB'), ('a', 'DT'), ('basic', 'JJ'), ('knowledge', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Python', 'NNP'), ('programming', 'NN'), ('language', 'NN'), ('.', '.'), ('He/she', 'NNP'), ('must', 'MD'), ('have', 'VB'), ('knowledge', 'NN'), ('of', 'IN'), ('data', 'NNS'), ('types', 'NNS'), ('in', 'IN'), ('Python', 'NNP'), (',', ','), ('should', 'MD'), ('be', 'VB'), ('able', 'JJ'), ('to', 'TO'), ('write', 'VB'), ('functions', 'NNS'), (',', ','), ('and', 'CC'), ('also', 'RB'), ('have', 'VBP'), ('the', 'DT'), ('ability', 'NN'), ('to', 'TO'), ('import', 'NN'), ('and', 'CC'), ('use', 'NN'), ('libraries', 'NNS'), ('and', 'CC'), ('packages', 'NNS'), ('in', 'IN'), ('Python', 'NNP'), ('.', '.'), ('familiarity', 'NN'), ('with', 'IN'), ('basic', 'JJ'), ('linguistics', 'NNS'), ('and', 'CC'), ('probability', 'NN'), ('is', 'VBZ'), ('assumed', 'VBN'), ('although', 'IN'), ('not', 'RB'), ('required', 'VBN'), ('to', 'TO'), ('fully', 'RB'), ('complete', 'VB'), ('this', 'DT'), ('course', 'NN'), ('.', '.'), (\"'\", \"''\")]\n"
159+
]
160+
}
161+
],
162+
"source": [
163+
"print(pos_tag(corrected_word_list))"
164+
]
165+
},
166+
{
167+
"cell_type": "code",
168+
"execution_count": 22,
169+
"metadata": {},
170+
"outputs": [
171+
{
172+
"data": {
173+
"text/plain": [
174+
"[\"'The\",\n",
175+
" 'reader',\n",
176+
" 'course',\n",
177+
" 'basic',\n",
178+
" 'knowledge',\n",
179+
" 'Python',\n",
180+
" 'programming',\n",
181+
" 'language',\n",
182+
" '.',\n",
183+
" 'He/she',\n",
184+
" 'must',\n",
185+
" 'knowledge',\n",
186+
" 'data',\n",
187+
" 'types',\n",
188+
" 'Python',\n",
189+
" ',',\n",
190+
" 'able',\n",
191+
" 'write',\n",
192+
" 'functions',\n",
193+
" ',']"
194+
]
195+
},
196+
"execution_count": 22,
197+
"metadata": {},
198+
"output_type": "execute_result"
199+
}
200+
],
201+
"source": [
202+
"stop_words = stopwords.words('english')\n",
203+
"def remove_stop_words(word_list):\n",
204+
" corrected_word_list_without_stopwords = []\n",
205+
" for wd in word_list:\n",
206+
" if wd not in stop_words:\n",
207+
" corrected_word_list_without_stopwords.append(wd)\n",
208+
" return corrected_word_list_without_stopwords\n",
209+
"\n",
210+
"corrected_word_list_without_stopwords = remove_stop_words(corrected_word_list)\n",
211+
"corrected_word_list_without_stopwords[:20]"
212+
]
213+
},
214+
{
215+
"cell_type": "code",
216+
"execution_count": 23,
217+
"metadata": {},
218+
"outputs": [
219+
{
220+
"data": {
221+
"text/plain": [
222+
"[\"'the\",\n",
223+
" 'reader',\n",
224+
" 'cours',\n",
225+
" 'basic',\n",
226+
" 'knowledg',\n",
227+
" 'python',\n",
228+
" 'program',\n",
229+
" 'languag',\n",
230+
" '.',\n",
231+
" 'he/sh',\n",
232+
" 'must',\n",
233+
" 'knowledg',\n",
234+
" 'data',\n",
235+
" 'type',\n",
236+
" 'python',\n",
237+
" ',',\n",
238+
" 'abl',\n",
239+
" 'write',\n",
240+
" 'function',\n",
241+
" ',']"
242+
]
243+
},
244+
"execution_count": 23,
245+
"metadata": {},
246+
"output_type": "execute_result"
247+
}
248+
],
249+
"source": [
250+
"stemmer = stem.PorterStemmer()\n",
251+
"def get_stems(word_list):\n",
252+
" corrected_word_list_without_stopwords_stemmed = []\n",
253+
" for wd in word_list:\n",
254+
" corrected_word_list_without_stopwords_stemmed.append(stemmer.stem(wd))\n",
255+
" return corrected_word_list_without_stopwords_stemmed\n",
256+
"\n",
257+
"corrected_word_list_without_stopwords_stemmed = get_stems(corrected_word_list_without_stopwords)\n",
258+
"corrected_word_list_without_stopwords_stemmed[:20]"
259+
]
260+
},
261+
{
262+
"cell_type": "code",
263+
"execution_count": 24,
264+
"metadata": {},
265+
"outputs": [
266+
{
267+
"data": {
268+
"text/plain": [
269+
"[\"'the\",\n",
270+
" 'reader',\n",
271+
" 'cours',\n",
272+
" 'basic',\n",
273+
" 'knowledg',\n",
274+
" 'python',\n",
275+
" 'program',\n",
276+
" 'languag',\n",
277+
" '.',\n",
278+
" 'he/sh',\n",
279+
" 'must',\n",
280+
" 'knowledg',\n",
281+
" 'data',\n",
282+
" 'type',\n",
283+
" 'python',\n",
284+
" ',',\n",
285+
" 'abl',\n",
286+
" 'write',\n",
287+
" 'function',\n",
288+
" ',']"
289+
]
290+
},
291+
"execution_count": 24,
292+
"metadata": {},
293+
"output_type": "execute_result"
294+
}
295+
],
296+
"source": [
297+
"lemmatizer = WordNetLemmatizer()\n",
298+
"def get_lemma(word_list):\n",
299+
" corrected_word_list_without_stopwords_lemmatized = []\n",
300+
" for wd in word_list:\n",
301+
" corrected_word_list_without_stopwords_lemmatized.append(lemmatizer.lemmatize(wd))\n",
302+
" return corrected_word_list_without_stopwords_lemmatized\n",
303+
"corrected_word_list_without_stopwords_lemmatized = get_lemma(corrected_word_list_without_stopwords_stemmed)\n",
304+
"corrected_word_list_without_stopwords_lemmatized[:20]"
305+
]
306+
},
307+
{
308+
"cell_type": "code",
309+
"execution_count": 25,
310+
"metadata": {},
311+
"outputs": [
312+
{
313+
"name": "stdout",
314+
"output_type": "stream",
315+
"text": [
316+
"[\"'The reader of this course should have a basic knowledge of the Python programming language.\", 'He/she must have knowledge of data types in Python, should be able to write functions, and also have the ability to import and use libraries and packages in Python.', \"familiarity with basic linguistics and probability is assumed although not required to fully complete this course.'\"]\n"
317+
]
318+
}
319+
],
320+
"source": [
321+
"print(sent_tokenize(corrected_sentence))"
322+
]
323+
},
324+
{
325+
"cell_type": "code",
326+
"execution_count": null,
327+
"metadata": {},
328+
"outputs": [],
329+
"source": []
330+
}
331+
],
332+
"metadata": {
333+
"kernelspec": {
334+
"display_name": "Python 3",
335+
"language": "python",
336+
"name": "python3"
337+
},
338+
"language_info": {
339+
"codemirror_mode": {
340+
"name": "ipython",
341+
"version": 3
342+
},
343+
"file_extension": ".py",
344+
"mimetype": "text/x-python",
345+
"name": "python",
346+
"nbconvert_exporter": "python",
347+
"pygments_lexer": "ipython3",
348+
"version": "3.5.2"
349+
}
350+
},
351+
"nbformat": 4,
352+
"nbformat_minor": 4
353+
}

0 commit comments

Comments
 (0)