Attempt 2 on publishing wheel (#1467)

* Attempt 2 on publishing wheel

* Star is not allowed, a bypass was found

* Want to try getting some script to push to pypi as well

* I think we should ideally have python packages installed from requirements.txt instead of from apt-get for easier version control

* Also, we should make sure setup.py relies on reading the contents of requirements.txt to avoid any drift. I do know this has caused me issues before that requirements.txt and install requirements did not match before

.github/workflows/publish_docker.yml

@@ -56,16 +56,23 @@ jobs:
       - name: Build package
         working-directory: panda/debian
         run: ./setup.sh Ubuntu ${{ matrix.ubuntu_version }}
+
+      - name: Get Name of Generated Wheel file
+        run: |
+          ARTIFACT_PATHNAME=$(ls panda/debian/*.whl | head -n 1)
+          ARTIFACT_NAME=$(basename $ARTIFACT_PATHNAME)
+          echo ::set-env name=ARTIFACT_NAME::${ARTIFACT_NAME}
+          echo ::set-env name=ARTIFACT_PATHNAME::${ARTIFACT_PATHNAME}
       
-      #- name: Upload whl to release
-      #  uses: actions/upload-release-asset@v1
-      #  env:
-      #    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #  with:
-      #    upload_url: ${{ needs.create_release.outputs.upload_url }}
-      #    asset_path: panda/debian/*.whl
-      #    asset_name: pandare-${{ matrix.ubuntu_version }}.whl
-      #    asset_content_type:  application/octet-stream
+      - name: Upload wheel to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.create_release.outputs.upload_url }}
+          asset_path: ${{ env.ARTIFACT_PATHNAME }}
+          asset_name: ${{ env.ARTIFACT_NAME }}
+          asset_content_type:  application/octet-stream
 
       - name: Upload deb to release
         uses: actions/upload-release-asset@v1
@@ -147,6 +154,28 @@ jobs:
              git commit -m "Documentation update for PANDA commit ${{ github.sha  }} branch dev" &&
              git push || true
 
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build_release_assets
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pandare  # Replace <package-name> with your PyPI project name
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: pandare
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        packages-dir: panda/debian/
+        verbose: true
+
   build_stable:
     if: github.repository  == 'panda-re/panda' && github.ref == 'refs/heads/stable'
     runs-on: panda-arc

panda/dependencies/ubuntu_18.04_base.txt

@@ -12,7 +12,6 @@ libwiretap-dev
 libxen-dev
 python3
 python3-pip
-python3-protobuf
 wget
 unzip
 
@@ -22,9 +21,6 @@ libpython3-dev
 # pypanda dependencies
 genisoimage
 libffi-dev
-python3-colorama
-python3-protobuf
-python3-pycparser
 
 
 # apt-rdepends qemu-system-common

panda/dependencies/ubuntu_20.04_base.txt

@@ -12,7 +12,6 @@ libxen-dev
 libz3-dev
 python3
 python3-pip
-python3-protobuf
 wget
 
 # pyperipheral (only needed for armel)
@@ -21,9 +20,6 @@ libpython3-dev
 # pypanda dependencies
 genisoimage
 libffi-dev
-python3-colorama
-python3-protobuf
-python3-pycparser
 
 # Not sure what this one is needed for
 liblzo2-2

panda/dependencies/ubuntu_22.04_base.txt

@@ -12,7 +12,6 @@ libxen-dev
 libz3-dev
 python3
 python3-pip
-python3-protobuf
 wget
 
 # pyperipheral (only needed for armel)
@@ -21,9 +20,6 @@ libpython3-dev
 # pypanda dependencies
 genisoimage
 libffi-dev
-python3-colorama
-python3-protobuf
-python3-pycparser
 
 # Not sure what this one is needed for
 liblzo2-2

panda/python/core/setup.py

@@ -8,17 +8,20 @@
 import shutil
 
 import os
+
 arches = ['arm', 'aarch64', 'i386', 'x86_64', 'ppc', 'mips', 'mipsel', 'mips64']
 
 # Check for PANDA binaries in /usr/local/bin/ or in our build directory
-#panda_binaries = ['/usr/local/bin/panda-system-{arch}' for arch in arches]
+# panda_binaries = ['/usr/local/bin/panda-system-{arch}' for arch in arches]
 # Do we actually care at this point?
 
-root_dir = os.path.join(*[os.path.dirname(__file__), "..", "..", ".."]) # panda-git/ root dir
+root_dir = os.path.join(*[os.path.dirname(__file__), "..", "..", ".."])  # panda-git/ root dir
 
-pypi_build = False # Set to true if trying to minimize size for pypi package upload. Note this disables some architectures
+pypi_build = False  # Set to true if trying to minimize size for pypi package upload. Note this disables some architectures
 
 lib_dir = os.path.join("pandare", "data")
+
+
 def copy_objs():
     '''
     Run to copy objects into a (local and temporary) python module before installing to the system.
@@ -65,40 +68,52 @@ def copy_objs():
             shutil.copy(    llvm2,      os.path.join(lib_dir, softmmu))
     '''
 
+
+def parse_requirements(filename):
+    with open(filename, 'r') as f:
+        return [line.strip() for line in f if line.strip() and not line.strip().startswith('#')]
+
+
 from setuptools.command.install import install as install_orig
 from setuptools.command.develop import develop as develop_orig
+
+
 class custom_develop(develop_orig):
     '''
     Install as a local module (not to system) by
         1) Creating datatype files for local-use
         2) Running regular setup tools logic
     '''
+
     def run(self):
         from create_panda_datatypes import main as create_datatypes
         create_datatypes(install=False)
         super().run()
 
+
 class custom_install(install_orig):
     '''
     We're going to install to the system. Two possible states to handle
     1) Running from within the panda repo with panda built - need to create_datatypes
     2) Running from a python sdist where all the files are already prepared
     '''
+
     def run(self):
         try:
             from create_panda_datatypes import main as create_datatypes
             # If we can do the import, we're in the panda repo
             create_datatypes(install=True)
             copy_objs()
-            
+
         except ImportError:
             # Import failed, we're either in a python sdist or something has gone very wrong
-            assert(os.path.isfile("pandare/include/panda_datatypes.h")), \
-                            "panda_datatypes.h missing and can't be generated"
-            assert(os.path.isfile("pandare/autogen/panda_datatypes.py")), \
-                            "panda_datatypes.py missing and can't be generated"
+            assert (os.path.isfile("pandare/include/panda_datatypes.h")), \
+                "panda_datatypes.h missing and can't be generated"
+            assert (os.path.isfile("pandare/autogen/panda_datatypes.py")), \
+                "panda_datatypes.py missing and can't be generated"
         super().run()
 
+
 # To build a package for pip:
 # python setup.py install
 # python setup.py sdist
@@ -115,13 +130,13 @@ def run(self):
       author_email='fasano@mit.edu',
       url='https://github.com/panda-re/panda/',
       packages=find_packages(),
-      package_data = { 'pandare': [ \
-          'data/*-softmmu/llvm-helpers*.bc*', # LLVM Helpers
-          'data/pypanda/include/*.h',         # Includes files
-          'data/pypanda/include/*.h',         # Includes files
-          'qcows.json'                        # Generic Images
-          ]},
-      install_requires=[ 'cffi>=1.14.3', 'colorama', 'protobuf>=4.25.1'],
+      package_data={'pandare': [
+          'data/*-softmmu/llvm-helpers*.bc*',  # LLVM Helpers
+          'data/pypanda/include/*.h',  # Includes files
+          'data/pypanda/include/*.h',  # Includes files
+          'qcows.json'  # Generic Images
+      ]},
+      install_requires=parse_requirements("requirements.txt"),
       python_requires='>=3.6',
       cmdclass={'install': custom_install, 'develop': custom_develop},
-     )
+      )