data generators

.gitignore

@@ -1,2 +1,3 @@
 .idea/
 *.iml
+cache/

generate_data/1.seed_zipcode_data/1.zipcodes-to-cassandra.py

@@ -27,7 +27,7 @@ def init_cassandra():
 
     # grab ip address information from application.cfg
     app = Flask(__name__)
-    app.config.from_pyfile('/cornerstone/web/datastax/cornerstone-python/Cornerstone/application.cfg')
+    app.config.from_pyfile('../../web/application.cfg')
     ip_addresses = app.config['DSE_CLUSTER'].split(',')
 
     # connect to Cassandra
@@ -109,7 +109,7 @@ def parse_zipcodes(futures, session):
             (?, ?, ?, ?, ?, ?, ?)''')
 
     # read from zipcodes from csv file
-    with open('/cornerstone/scripts/datastax/black-friday/1.seed_zipcode_data/free-zipcode-database.csv', 'rb') as csvfile:
+    with open('free-zipcode-database.csv', 'rb') as csvfile:
         csv_reader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
         for row in csv_reader:
             # create insert values object

generate_data/2.seed_retail_data/1.download-data.sh

@@ -2,5 +2,5 @@
 set -x # echo on
 
 # http://snap.stanford.edu/data/amazon/
-wget -c http://snap.stanford.edu/data/amazon/Electronics.txt.gz -P /cache/downloads
-wget -c http://snap.stanford.edu/data/amazon/brands.txt.gz -P /cache/downloads
+wget -c http://snap.stanford.edu/data/amazon/Electronics.txt.gz -P ../../cache/downloads
+wget -c http://snap.stanford.edu/data/amazon/brands.txt.gz -P ../../cache/downloads

generate_data/2.seed_retail_data/2.data-to-cassandra.py

@@ -27,7 +27,7 @@ def init_cassandra():
 
     # grab ip address information from application.cfg
     app = Flask(__name__)
-    app.config.from_pyfile('/cornerstone/web/datastax/cornerstone-python/Cornerstone/application.cfg')
+    app.config.from_pyfile('../../web/application.cfg')
     ip_addresses = app.config['DSE_CLUSTER'].split(',')
 
     # connect to Cassandra
@@ -104,7 +104,7 @@ def parse_products(futures, session):
         'INSERT INTO retail.products (product_id, title, price) '
         'VALUES (?, ?, ?)')
 
-    with gzip.open('/cache/downloads/Electronics.txt.gz') as f:
+    with gzip.open('../../cache/downloads/Electronics.txt.gz') as f:
         # compile regular expressions
         p = re.compile('^product')
         p_id = re.compile('^product/productId: (.*)')
@@ -172,7 +172,7 @@ def parse_brands(futures, session):
         'INSERT INTO retail.brands (brand) '
         'VALUES (?)')
 
-    with gzip.open('/cache/downloads/brands.txt.gz') as f:
+    with gzip.open('../../cache/downloads/brands.txt.gz') as f:
         # compile regular expressions
         p = re.compile('(\w*) (.*)')
 

generate_data/3.scan_data/1.extract-ids.py

@@ -8,7 +8,7 @@
 from cassandra.query import ordered_dict_factory
 
 app = Flask(__name__)
-app.config.from_pyfile('/cornerstone/web/datastax/cornerstone-python/Cornerstone/application.cfg')
+app.config.from_pyfile('../../web/application.cfg')
 ip_addresses = app.config['DSE_CLUSTER'].split(',')
 
 cluster = Cluster(ip_addresses)
@@ -17,6 +17,6 @@
 
 response = session.execute('SELECT product_id FROM retail.products')
 
-with open('/cache/product_ids.txt', 'w') as f:
+with open('../../cache/product_ids.txt', 'w') as f:
     for row in response:
         f.write('%s\n' % row['product_id'])

generate_data/3.scan_data/2.extract-zipcodes.py

@@ -6,7 +6,7 @@
 from cassandra.query import ordered_dict_factory
 
 app = Flask(__name__)
-app.config.from_pyfile('/cornerstone/web/datastax/cornerstone-python/Cornerstone/application.cfg')
+app.config.from_pyfile('../../web/application.cfg')
 ip_addresses = app.config['DSE_CLUSTER'].split(',')
 
 cluster = Cluster(ip_addresses)
@@ -15,6 +15,6 @@
 
 response = session.execute('SELECT zipcode FROM retail.zipcodes')
 
-with open('/cache/zipcodes.txt', 'w') as f:
+with open('../../cache/zipcodes.txt', 'w') as f:
     for row in response:
         f.write('%s\n' % row['zipcode'])