Updated process_user_page to store data in Mongo, updated readme.

Pipfile

@@ -8,6 +8,7 @@ verify_ssl = true
 [packages]
 requests = "*"
 redis = "*"
+pymongo = "*"
 
 [requires]
 python_version = "3.7"

README.md

@@ -4,7 +4,7 @@ Software to visualize Last.FM data. Built with Python.
 
 # Requirements
 
-This project uses Python 3 and requires Redis to store all data.
+This project uses Python 3 and requires Redis to maintain a queue of users, as well as MongoDB to store all listening data.
 
 This project also requires a Last.FM API key. To get one, simply go to the Last.FM api website and create a new application. Create an `apiKey.py` file in the main directory with `apiKey` and `apiSecret` variables. Here's an example:
 

cron.py

@@ -16,8 +16,11 @@
 
         total_pages = int(current_page['recenttracks']['@attr']['totalPages'])
 
+        time.sleep(1)
+
         for i in range(2, total_pages):
             current_page = get_user_page(current_user, i + 1)
             process_user_page(current_page)
+            time.sleep(1)
 
     time.sleep(1)

main.py

@@ -1,9 +1,14 @@
 import requests
 import redis
+import pymongo
 from apiKey import api_key
 
 redis_client = redis.Redis()
 
+mongo_client = pymongo.MongoClient('mongodb://localhost:27017')
+mongo_database = mongo_client['main']
+mongo_collection = mongo_database['listeningData']
+
 # Adds user to the processing queue in Redis.
 # Raises a LookupError if the user was not found on LastFM, and raises RuntimeErrors if the user is already in the
 # queue or has already been processed.
@@ -31,7 +36,12 @@ def get_user_page(username, page):
 
 # Given a plain JSON object of the page, process and store the page's content in the database.
 def process_user_page(page_content):
-   tracks = page_content['recenttracks']['track']
+    user = page_content['recenttracks']['@attr']['user']
+    tracks = page_content['recenttracks']['track']
+
+    for track in tracks:
+        if 'date' in track:
+            to_insert = {'username': user, 'artist': track['artist']['#text'], 'name': track['name'], 'album': track['album']['#text'], 'imageUrl': track['image'][3]['#text'], 'date': track['date']['uts']}
+            mongo_collection.insert_one(to_insert)
 
-   for track in tracks:
-       print(track['name'])
+    print('Added page ', page_content['recenttracks']['@attr']['page'], 'out of ', page_content['recenttracks']['@attr']['totalPages'])