Initial commit

Author: Ruturaj4

.gitignore

@@ -0,0 +1 @@
+packages/

all.json

@@ -0,0 +1,153 @@
+from __future__ import print_function, division
+
+import pandas as pd
+import networkx as nx
+from networkx.drawing.nx_pydot import write_dot
+import matplotlib.pyplot as plt
+from matplotlib import patches
+
+import seaborn as sns
+import operator
+import numpy as np
+sns.set_context('notebook', font_scale=1.5)
+sns.set_style('white')
+
+requirements = pd.read_csv('requirements.csv')
+
+def make_graph(df, min_edges=0):
+    DG = nx.DiGraph()
+    DG.add_nodes_from(df.package_name.unique())
+    edges = df.loc[df.requirement.notnull(), ['package_name', 'requirement']].values
+    DG.add_edges_from(edges)
+
+    # Remove bad nodes
+    DG.remove_nodes_from(['.', 'nan', np.nan])
+
+    deg = DG.degree()
+    #print(deg)
+    try:
+        to_remove = [n for n in deg if deg[n] <= min_edges]
+        DG.remove_nodes_from(to_remove)
+    except:
+        #print("key not present")
+        pass
+    return DG
+
+#DG = make_graph(requirements, min_edges=10)
+#write_dot(DG, 'requirements_graph.dot')
+
+#dep_graph = make_graph(requirements, min_edges=0)
+
+#print(len(dep_graph.node))
+G = make_graph(requirements)
+print(G.number_of_edges())
+
+def dependency_graph():
+    deplist = []
+
+    for node in G:
+        if len(G[node]) == 0:
+            continue
+        deplist.append(len(G.out_edges([node])))
+    x = zero_to_nan(deplist)
+    x = np.sort(x)
+    print(x)
+    print(len(x))
+    p = 1. * np.arange(len(x))/(len(x) - 1)
+    plt.plot(x, p, marker='.', linestyle='none')
+    _ = plt.xlabel('Dependencies')
+    _ = plt.ylabel('CDF')
+    plt.margins(0.02)
+    plt.show()
+
+def zero_to_nan(values):
+    return [float('nan') if x==0 else x for x in values ]
+
+def pageRank():
+    #Calculate the page rank
+    pr = {}
+    pr = nx.pagerank(G)
+    pr = sorted(pr.values(), reverse=True)
+    return pr
+
+max_d = []
+
+def dfs_depth(G, source=None, depth_limit=None):
+    if source is None:
+        nodes = G
+    else:
+        nodes = [source]
+    visited = set()
+    if depth_limit is None:
+        depth_limit = len(G)
+    for start in nodes:
+        print(start)
+        if start in visited:
+            continue
+        max_depth = 0
+        visited.add(start)
+        stack = [(start, depth_limit, iter(G[start]))]
+        while stack:
+            parent, depth_now, children = stack[-1]
+            try:
+                child = next(children)
+                if child not in visited:
+                    yield parent, child
+                    visited.add(child)
+                    if depth_now > 1:
+                        if((depth_limit - depth_now + 1)>max_depth):
+                            max_depth = depth_limit - depth_now + 1
+                        stack.append((child, depth_now - 1, iter(G[child])))
+            except StopIteration:
+                stack.pop()
+    global max_d
+    max_d.append(max_depth)
+
+def deplist(pr):
+    # Calculate all the dependencies, dependents
+    dcon = {}
+    list1 = []
+    list2 = []
+    list3 = []
+    list4 = []
+    for node in G:
+        print(node)
+        #temp = {node:len(G.out_edges(node))}
+        list1.append(node)
+        list2.append(len(G.out_edges(node)))
+        list3.append(len(G.in_edges(node)))
+        list4.append(len(list(nx.dfs_edges(G,node))))
+        list(dfs_depth(G, node))
+        #dcon.update(temp)
+    list2 = sorted(list2, reverse=True)
+    list3 = sorted(list3, reverse=True)
+    list4 = sorted(list4, reverse=True)
+    global max_d
+    max_d = sorted(max_d, reverse=True)
+    df = pd.DataFrame(data={"Dependencies":list2[:1000], "Dependents":list3[:1000], "DFS-Edges":list4[:1000], "Max-Depth":max_d[:1000], "Page Rank":pr[:1000]})
+    df.plot(kind="density", subplots=True, layout=(3,2), sharex=False)
+    plt.show()
+
+#pr = pageRank()
+#deplist(pr)
+
+#dependency_graph()
+
+print(G.in_edges())
+
+#sorted_dict = sorted(G.in_degree().items(), key=operator.itemgetter(1))[::-1]
+
+N = 10
+x = np.arange(N)
+y = np.array([d[1] for d in sorted_dict[:N]])
+xlabels = [d[0] for d in sorted_dict[:N]][::-1]
+fig, ax = plt.subplots(1, 1, figsize=(7, 7))
+
+ax.barh(x[::-1], y, height=1.0)
+ax.set_yticks(x + 0.5)
+_ = ax.set_yticklabels(xlabels)
+ax.set_xlabel('Number of Connections')
+ax.set_title('Graph Degree')
+fig.subplots_adjust(left=0.27, bottom=0.1, top=0.95)
+fig.show()
+

analysis.py

@@ -0,0 +1,58 @@
+import os, re
+
+def author():
+    os.chdir("packages")
+    print(os.getcwd())
+    main_dic = []
+    for f in os.listdir():
+        os.chdir(f)
+        print(os.listdir())
+        try:
+            with open("setup.py", "r") as f:
+                temp = f.readlines()
+        except:
+            os.chdir("../")
+            continue
+        dic = {}
+        for line in temp:
+            t = []
+            if 'name=' in line:
+                ln = "".join(line.split())
+                ln = re.search("name='(.+?)',", ln)
+                if ln:
+                    ln = ln.group(1)
+                    ln = ln.strip('\"')
+                    ln = ln.strip("\'")
+                    dic[ln] = []
+            if 'author=' in line:
+                la = "".join(line.split())
+                la = re.search("author='(.+?)',", la)
+                if la:
+                    la = la.group(1)
+                    la = la.strip('\"')
+                    la = la.strip("\'")
+                    t.append(la)
+                    dic[ln] = t
+            if 'author_email=' in line:
+                le = "".join(line.split())
+                le = re.search("author_email='(.+?)',", le)
+                if le:
+                    le = le.group(1)
+                    le = le.strip('\"')
+                    le = le.strip("\'")
+                    t.append(le)
+                    dic[ln] = t
+                    break
+        try:
+            print(dic)
+            main_dic.update(dic)
+        except:
+            pass
+        os.chdir("../")
+    print(main_dic)
+
+def main():
+    author()
+
+if __name__=="__main__":
+    main()

authors.py

@@ -0,0 +1,63 @@
+import pandas as pd
+from collections import defaultdict
+import numpy as np
+
+df = pd.read_csv("requirements.csv")
+
+class Tree(object):
+    def __init__(self, name):
+        self.name = name
+        self.children = []
+        return
+
+    def __contains__(self, obj):
+        return obj == self.name or any([obj in c for c in self.children])
+    
+    def add(self, obj):
+        if not self.__contains__(obj):
+            self.children.append(Tree(obj))
+            return True
+        return False
+    
+    def get_base_requirements(self):
+        base = []
+        for child in self.children:
+            if len(child.children) == 0:
+                base.append(child.name)
+            else:
+                for b in [c.get_base_requirements() for c in child.children()]:
+                    base.extend(b)
+        return np.unique(base)
+    
+
+def get_requirements(package):
+    return df.loc[(df.package_name == package) & (df.requirement.notnull()), 'requirement'].values
+
+
+def get_dependency_tree(package, tree):
+    reqs = get_requirements(package)
+    for req in reqs:
+        #print(req)
+        flg = tree.add(req)
+        if not flg:
+            continue
+        tree = get_dependency_tree(req, tree)
+    return tree
+
+datadict = defaultdict(list)
+for i, package in enumerate(df.package_name.unique()):
+    if i % 100 == 0:
+        print('Package {}: {}'.format(i+1, package))
+    try:
+        deptree = get_dependency_tree(package, Tree(package))
+    except:
+        print('Failure getting base dependencies for {}'.format(package))
+        raise ValueError
+    for dependency in deptree.get_base_requirements():
+        datadict['package_name'].append(package)
+        datadict['requirements'].append(dependency)
+
+base_df = pd.DataFrame(data=datadict)
+base_df.head()
+
+base_df.to_csv('base_requirements.csv', index=False)

base_dep.py

@@ -0,0 +1,4 @@
+import pandas as pd
+
+r = pd.read_csv("requirements.csv")
+print(r.to_json("all.json", orient='records'))

converter.py

@@ -0,0 +1,61 @@
+import pandas as pd
+from collections import defaultdict
+import os
+import requirements
+import numpy as np
+import xmlrpc.client as xc
+
+client = xc.ServerProxy('https://pypi.python.org/pypi')
+packages = client.list_packages()
+
+datadict = defaultdict(list)
+with open('requirements.txt', 'r') as infile:
+    new_package = True
+    for line in infile:
+        if line.strip() == '':
+            new_package = True
+            print(package_name)
+            if package_name not in datadict['package']:
+                datadict['package'].append(package_name)
+                datadict['requirement'].append(np.nan)
+            continue
+
+        if new_package:
+            # If this is the case, the current line gives the name of the package
+            package_name = os.path.basename(line).strip()
+            new_package = False
+        else:
+            # This line gives a requirement for the current package
+            try:
+                print(line)
+                for req in requirements.parse(line.strip()):
+                    datadict['package'].append(package_name)
+                    datadict['requirement'].append(req.name)
+            except ValueError:
+                pass
+
+
+# Convert to dataframe
+df = pd.DataFrame(data=datadict)
+df.head()
+
+df['package_name'] = np.nan
+df['package_version'] = np.nan
+for i, package in enumerate(packages):
+    try:
+        if i % 100 == 0:
+            print('Package {}: {}'.format(i+1, package))
+        for release in client.package_releases(package):
+            try:
+                pkg_str = '{}-{}'.format(package, release)
+                idx = df.loc[df.package == pkg_str].index
+                if len(idx) > 0:
+                    df.loc[idx, 'package_name'] = package
+                    df.loc[idx, 'package_version'] = release
+            except:
+                pass
+    except:
+        pass
+df.head()
+
+df.to_csv('requirements.csv', index=False)

dependecies.py

@@ -0,0 +1,43 @@
+import json
+import requests
+import bs4
+from collections import OrderedDict
+
+url = "https://pepy.tech/project/"
+
+def download_counts():
+    with open("all.json", "r") as f:
+        pack = json.load(f)
+
+    # A dictionary to store dowload counts for all the packages
+    dic = {}
+    #count = 0
+    packages = []
+    for item in pack:
+        packages.append(item["package_name"])
+    packages = list(OrderedDict.fromkeys(packages))
+    for item in packages:
+        #count += 1
+        print(item)
+        try:
+            res = requests.get(url + item)
+            soup = bs4.BeautifulSoup(res.text, 'lxml')
+            # td is the tag they used. And taking 1st element from that
+            td = soup.findAll("td")
+            print(td[0].getText())
+            temp = {}
+            temp[item] = td[0].getText()
+            dic.update(temp)
+        except:
+            print("couldn't find")
+            pass
+        #if count == 2:
+        #    break 
+    with open("downloads_counts.json", "w") as f:
+        json.dump(dic, f)
+
+def main():
+    download_counts()
+
+if __name__=="__main__":
+    main()