Materials from the recording.

Author: mikeckennedy

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,2 @@
+# jetbrains-webcast-build-with-mongodb
+Code and handouts for my JetBrains webcast recorded January 30, 2018

.idea/vcs.xml

@@ -0,0 +1,24 @@
+import datetime
+
+import mongoengine
+
+
+class Download(mongoengine.Document):
+    created = mongoengine.DateTimeField(default=datetime.datetime.now)
+    package_id = mongoengine.ObjectIdField()
+    release_id = mongoengine.ObjectIdField()
+    client_ip = mongoengine.StringField()
+    client_app = mongoengine.StringField()
+    python_version = mongoengine.StringField()
+
+    meta = {
+        'db_alias': 'core',
+        'collection': 'downloads',
+        'indexes': [
+            'created',
+            'package_id',
+            'release_id',
+            'python_version',
+        ],
+        'ordering': ['-created']
+    }

data/README.md

@@ -0,0 +1,5 @@
+import mongoengine
+
+
+def global_init(db_name: str):
+    mongoengine.register_connection(alias='core', name=db_name)

data/pypi_db.zip

@@ -0,0 +1,22 @@
+import datetime
+
+import mongoengine as mongoengine
+
+
+class Package(mongoengine.Document):
+    created = mongoengine.DateTimeField(default=datetime.datetime.now)
+    name = mongoengine.StringField(required=True)
+    maintainers = mongoengine.ListField(mongoengine.ObjectIdField())
+    total_downloads = mongoengine.LongField()
+
+    meta = {
+        'db_alias': 'core',
+        'collection': 'packages',
+        'indexes': [
+            'created',
+            'name',
+            'maintainers',
+            'total_downloads',
+        ],
+        'ordering': ['-created']
+    }

slides/mongo-mongengine-python-kennedy-webcast.pdf

@@ -0,0 +1,7 @@
+import mongoengine
+
+
+class ReleaseHealth(mongoengine.EmbeddedDocument):
+    ci = mongoengine.BooleanField()
+    coverage = mongoengine.FloatField()
+    health_index = mongoengine.FloatField()

src/data/downloads.py

@@ -0,0 +1,31 @@
+import datetime
+
+import mongoengine
+
+from data.release_health import ReleaseHealth
+
+
+class ReleaseHistory(mongoengine.Document):
+    created = mongoengine.DateTimeField(default=datetime.datetime.now)
+    package_id = mongoengine.ObjectIdField(required=True)
+    version_number = mongoengine.StringField(required=True)
+    description = mongoengine.StringField()
+
+    topics = mongoengine.ListField(mongoengine.StringField())
+    programming_languages = mongoengine.ListField(mongoengine.StringField())
+    dependencies = mongoengine.ListField(mongoengine.StringField())
+
+    health = mongoengine.EmbeddedDocumentField(ReleaseHealth)
+
+    meta = {
+        'db_alias': 'core',
+        'collection': 'releases',
+        'indexes': [
+            'package_id',
+            'version_number',
+            'topics',
+            'programming_languages',
+            'health.ci',
+        ],
+        'ordering': ['-version_number']
+    }

src/data/mongo_setup.py

@@ -0,0 +1,20 @@
+import datetime
+
+import mongoengine
+
+
+class User(mongoengine.Document):
+    created = mongoengine.DateTimeField(default=datetime.datetime.now)
+    name = mongoengine.StringField(required=True)
+    email = mongoengine.StringField(required=True)
+
+    meta = {
+        'db_alias': 'core',
+        'collection': 'users',
+        'indexes': [
+            'created',
+            'name',
+            'email',
+        ],
+        'ordering': ['name']
+    }

src/data/packages.py

@@ -0,0 +1,109 @@
+# noinspection PyPackageRequirements
+import time
+from data.mongo_setup import global_init
+from data.packages import Package
+from services.package_service import PackageService
+
+
+def main():
+    global_init('pypi')
+    print_header()
+    input_loop()
+
+
+def input_loop():
+    while True:
+
+        print("What do you want to do?")
+        val = input("[q]uery packages, view [d]ownloads, or e[x]it? ").lower().strip()
+        if val == 'q':
+            query_packages()
+        elif val == 'd':
+            view_downloads()
+        elif val == 'x':
+            print("Bye")
+            break
+        else:
+            print(f"Don't know what to do with '{val}'")
+        print()
+        print()
+
+
+def query_packages():
+    name = input("What package would you like details for? [hint: PackageNNNN]? ")
+
+    t0 = time.time()
+
+    package = PackageService.find_package_by_name(name)
+    if not package:
+        print(f"Sorry, no package with name '{name}'.")
+        return
+
+    r = PackageService.latest_release(package)
+    if not r:
+        print(f"Sorry the package {package.name} has no releases.")
+        return
+
+    maintainers = PackageService.find_maintainers(package)
+    t1 = time.time()
+
+    print("PACKAGE: " + package.name)
+    print(f"  Status: "
+          f"[CI: {'passing' if r.health.ci else 'failing'}] "
+          f"[Health: {r.health.health_index:.1f}]  "
+          f"[Coverage: {r.health.coverage:.1f}]")
+    print()
+    print(f"Current version: {r.version_number}")
+    print()
+    print(f"Description: {r.description[:100]} ...")
+    print()
+    print(f"Maintainers:")
+    for m in maintainers:
+        print(f"* {m.name} ({m.email})")
+    print()
+    print()
+    print(f"Topics:")
+    for t in r.topics:
+        print("* " + t)
+    print()
+    print(f"Supported languages:")
+    for lang in r.programming_languages:
+        print("* " + lang)
+    print()
+    print(f"Dependencies:")
+    for d in r.dependencies:
+        print("* " + d)
+    print()
+    print(f"Elapsed time: {(t1-t0)*1000:.3f} ms.")
+
+
+def view_downloads():
+    t0 = time.time()
+
+    tops = PackageService.popular_packages(limit=10)
+
+    dt = time.time() - t0
+    print("Top 10 packages by downloads")
+    print()
+    for idx, p in enumerate(tops):
+        print(f"{idx+1}. {p.total_downloads:,} {p.name}")
+    print()
+    print(f"Elapsed time: {dt*1000:,.03f} ms.")
+
+
+def print_header():
+    print('-------------------------------------')
+    print('       PYPI DATA EXPLORER')
+    print('-------------------------------------')
+    print()
+    print("Current stats: ")
+    # TODO: Show number of items
+    print(f"Packages: {PackageService.package_count():,}")
+    print(f"Releases: {PackageService.release_count():,}")
+    print(f"Users: {PackageService.user_count():,}")
+    print(f"Downloads: {PackageService.download_count():,}")
+    print()
+
+
+if __name__ == '__main__':
+    main()

src/data/release_health.py

@@ -0,0 +1,51 @@
+from typing import Optional, List
+
+from data.downloads import Download
+from data.packages import Package
+from data.release_history import ReleaseHistory
+from data.users import User
+
+
+class PackageService:
+    @classmethod
+    def package_count(cls):
+        return Package.objects().count()
+
+    @classmethod
+    def release_count(cls):
+        return ReleaseHistory.objects().count()
+
+    @classmethod
+    def user_count(cls):
+        return User.objects().count()
+
+    @classmethod
+    def download_count(cls):
+        return Download.objects().count()
+
+    @classmethod
+    def find_package_by_name(cls, name):
+        package = Package.objects(name=name).first()
+        return package
+
+    @classmethod
+    def latest_release(cls, package: Package) -> Optional[ReleaseHistory]:
+        release = ReleaseHistory \
+            .objects(package_id=package.id) \
+            .order_by('-created') \
+            .first()
+
+        return release
+
+    @classmethod
+    def find_maintainers(cls, package: Package) -> List[User]:
+        users = User.objects(id__in=package.maintainers)
+        return list(users)
+
+    @classmethod
+    def popular_packages(cls, limit: int) -> List[Package]:
+        packages = Package.objects()\
+            .order_by('-total_downloads')\
+            .limit(limit)
+
+        return list(packages)