fixed bug in search url

maskegger · Feb 17, 2016 · cf892b3 · cf892b3
1 parent d226102
commit cf892b3
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 1 deletion.
diff --git a/README.md~ b/README.md~
@@ -0,0 +1,18 @@
+# Academic word frequency extractor
+
+This script extracts the word frequency in papers of a search term. It writes the number of papers containing this
+word at a certain year to a CSV file:
+
+| year | results |
+|------+---------|
+| 2011 |    6320 |
+| 2012 |    7250 |
+| 2013 |    8170 |
+| 2014 |    8260 |
+| 2015 |    8150 |
+
+
+The script excludes patents and citations
+
+
+
diff --git a/extract_num_results.py~ b/extract_num_results.py~
@@ -0,0 +1,5 @@
+from bs4 import BeautifulSoup
+import urllib
+r = urllib.urlopen('https://scholar.google.nl/scholar?hl=en&as_sdt=1,5&q=self-disclosure').read()
+soup = BeautifulSoup(r)
+print type(soup)
diff --git a/extract_occurrences.py b/extract_occurrences.py
@@ -14,7 +14,7 @@ def get_num_results(search_term, start_date, end_date):
 
     # Open website and read html
     user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36'
-    url = "https://scholar.google.nl/scholar?as_vis=1&q=self-disclosure&hl=en&as_sdt=1,5&as_ylo={0}&as_yhi={1}".format(start_date, end_date)
+    url = "https://scholar.google.nl/scholar?as_vis=1&q={0}&hl=en&as_sdt=1,5&as_ylo={1}&as_yhi={2}".format(search_term, start_date, end_date)
     opener = build_opener()
     request = Request(url=url, headers={'User-Agent': user_agent})
     handler = opener.open(request)

diff --git a/out.csv b/out.csv
@@ -0,0 +1,2 @@
+year,results
+2014,8260
diff --git a/table.org b/table.org
@@ -0,0 +1,7 @@
+| year | results |
+|------+---------|
+| 2011 |    6320 |
+| 2012 |    7250 |
+| 2013 |    8170 |
+| 2014 |    8260 |
+| 2015 |    8150 |
diff --git a/table.org~ b/table.org~
@@ -0,0 +1,6 @@
+year,results
+2011,6320
+2012,7250
+2013,8170
+2014,8260
+2015,8150