Skip to content

chore: improve api responses and clean up variable names #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 10, 2023
Merged

chore: improve api responses and clean up variable names #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 38 additions & 12 deletions app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from flask import Flask, json, request
from johnnydep import JohnnyDist
from utils import *

app = Flask(__name__)


Expand All @@ -10,29 +11,54 @@ def hello_world(): # put application's code here
return 'Hello World!'


@app.route('/packagename')
def package_to_import():
packageName = request.args.get('packageName')
@app.route('/library/<String:packages>')
def package_to_import(packages):
conn = get_db_connection()
cursor = conn.cursor()

packages_json_list = {"result": [], "error": []}
for package in packages.split(','):
package = package.split(':')
package_name = package[0]
package_version = ""
if package.len() > 1:
package_version = package[1]

does_the_table_contain_this_package = cursor.execute('SELECT * FROM packages WHERE package = ?', [package_name]).fetchone()
if does_the_table_contain_this_package is None:
packages_json_list["error"].append(package_name + " OptionalVersion(" + package_version +") isn't include in the dataset")
else:
bob = cursor.execute('SELECT * FROM importNames WHERE packageName = ?', (package_name,)).fetchall()
packages_json_list["result"].extend([{"import_name": i[1], "library_name": i[2], "version": i[3]} for i in bob])
conn.close()

importNames = get_import_name(packageName)
response = app.response_class(
response=json.dumps(importNames),
response=json.dumps(packages_json_list),
mimetype='application/json'
)
return response

@app.route('/importname')
def import_to_package():
importName = request.args.get('importName')

packageNames = []
@app.route('/importname<String:imports>')
def import_to_package(imports):
conn = get_db_connection()
cursor = conn.cursor()
bob = cursor.execute('SELECT * FROM importNames WHERE importName = ?', [importName]).fetchall()
packageNames = [{"packageName": i[2], "version": i[3]} for i in bob]

imports_json_list = {"result": [], "error": []}
for import_name in imports.split(','):
if ':' in import_name:
imports_json_list["error"].append("The import name provided: " + import_name + " contains a ':' which is invalid")
continue

does_the_table_contain_this_import = cursor.execute('SELECT * FROM importNames WHERE importName = ?', [import_name]).fetchall()
if does_the_table_contain_this_import is None:
imports_json_list["error"].append("The import name provided: " + import_name + " isn't contained in our database")
else:
imports_json_list["result"].extend([{"import_name": i[1], "library_name": i[2], "version": i[3]} for i in does_the_table_contain_this_import])
conn.close()

response = app.response_class(
response=json.dumps(packageNames),
response=json.dumps(imports_json_list),
mimetype='application/json'
)
return response
Expand Down
33 changes: 15 additions & 18 deletions utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
import pickle


# https://pypi.org/simple/

def get_db_connection():
conn = sqlite3.connect('/home/kolby/Documents/GitHub/Mining-Diamonds-I-mean-data/identifier.sqlite')
return conn


def check_if_we_care(packageName):
try:
response = requests.get("https://libraries.io/api/Pypi/" + str(packageName.split("=")[0]) +"?api_key=96f6c6227c05020af5b777f5f6e0134c").json()
response = requests.get("https://libraries.io/api/Pypi/" + str(
packageName.split("=")[0]) + "?api_key=96f6c6227c05020af5b777f5f6e0134c").json()
print(response)
if response["dependents_count"] > 0 and response["dependent_repos_count"] > 0:
return True, response["versions"]
Expand All @@ -30,19 +30,21 @@ def get_import_name(packageName):
importNames = []
conn = get_db_connection()
cursor = conn.cursor()
post = cursor.execute('SELECT * FROM packages WHERE package = ?', [packageName]).fetchone()
if post is None:
does_the_table_contain_this_package = cursor.execute('SELECT * FROM packages WHERE package = ?',
[packageName]).fetchone()
if does_the_table_contain_this_package is None:
do_we_care_boolean, versions_list = check_if_we_care(packageName)
if do_we_care_boolean:
posts = cursor.execute('INSERT INTO packages (package) VALUES (?);', [packageName]).fetchone()
cursor.execute('INSERT INTO packages (package) VALUES (?);', [packageName]).fetchone()
for version in versions_list:
version = version["number"]
try:
importNames = JohnnyDist(packageName + "==" + version).import_names
if len(importNames) != 0:
for i in importNames:
posts = cursor.execute('INSERT INTO importNames (importName, packageName, version) VALUES (?, ?, ?);',
(i, packageName, version)).fetchone()
cursor.execute(
'INSERT INTO importNames (importName, packageName, version) VALUES (?, ?, ?);',
(i, packageName, version)).fetchone()
conn.commit()
else:
print("error Library you requested doesn't exist")
Expand All @@ -51,18 +53,14 @@ def get_import_name(packageName):
(packageName, version, str(e))).fetchone()
conn.commit()
print("Error: python package: " + packageName + " failed on version: " + version + " error:", e)
else:
importNames = {"error": "Library you requested doesn't meet the requirements for our program to be considered a library"}
bob = cursor.execute('SELECT * FROM importNames WHERE packageName = ?', (packageName,)).fetchall()
importNames = [{"importName": i[1], "version": i[3]} for i in bob]
conn.close()
return importNames


def get_list_of_pypi_packages():
def get_yesterday_data():
url_set_old = set()
try:
with open('yesterday_set_data.yourmom', 'rb') as fp:
with open('yesterday_set.pickle', 'rb') as fp:
url_set_old = pickle.load(fp)
except:
pass
Expand All @@ -78,7 +76,7 @@ def get_today_data():
return url_set

def dump_todays_data(url_set):
with open('yesterday_set_data.yourmom', 'wb') as fp:
with open('yesterday_set.pickle', 'wb') as fp:
pickle.dump(url_set, fp)

url_yesterday_set = get_yesterday_data()
Expand All @@ -89,7 +87,6 @@ def dump_todays_data(url_set):


def update_package_dataset():

importNames = []
conn = get_db_connection()
cursor = conn.cursor()
Expand All @@ -98,12 +95,12 @@ def update_package_dataset():

for package in packages:
package = package[0]
# todo: add support for updating the list
do_we_care_boolean, versions_list = check_if_we_care(package)
if do_we_care_boolean:
for version in reversed(versions_list):
version = version["number"]
bob = cursor.execute('SELECT * FROM importNames WHERE packageName = ? AND version = ?', [package, version]).fetchone()
bob = cursor.execute('SELECT * FROM importNames WHERE packageName = ? AND version = ?',
[package, version]).fetchone()
if bob is None:
try:
importNames = JohnnyDist(package + "==" + version).import_names
Expand Down