Document build process + reproducible builds

.github/workflows/test.yml

@@ -0,0 +1,97 @@
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    branches: ["**"]
+
+jobs:
+  test-compile:
+    runs-on: ubuntu-latest
+    name: "Test P4A Compiling"
+
+    steps:
+      - name: Clone LXMF
+        run: git clone https://github.com/markqvist/LXMF $GITHUB_WORKSPACE/LXMF
+
+      - name: Clone LXST
+        run: git clone https://github.com/markqvist/LXST $GITHUB_WORKSPACE/LXST
+
+      - name: Clone rnsh
+        run: git clone https://github.com/acehoss/rnsh $GITHUB_WORKSPACE/rnsh
+
+      - name: Clone Reticulum
+        run: git clone https://github.com/markqvist/Reticulum $GITHUB_WORKSPACE/Reticulum
+
+      - name: Clone rnode-flasher
+        run: git clone https://github.com/liamcottle/rnode-flasher $GITHUB_WORKSPACE/rnode-flasher
+
+      - name: Clone NomadNet
+        run: git clone https://github.com/markqvist/nomadnet $GITHUB_WORKSPACE/NomadNet
+
+      - name: Clone reticulum_website
+        run: git clone https://github.com/markqvist/reticulum_website $GITHUB_WORKSPACE/reticulum_website
+
+      - name: Checkout Sideband
+        uses: actions/checkout@v3
+        with:
+          path: Sideband/
+
+      # Set up caching for general cache directory
+      - name: Cache pip build cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache
+          key: ${{ runner.os }}-build-cache-${{ hashFiles('**/buildozer.spec') }}
+          restore-keys: |
+            ${{ runner.os }}-build-cache-
+
+      # Set up caching for buildozer directories
+      - name: Cache buildozer
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.buildozer
+            ${{ github.workspace }}/**/.buildozer
+            ${{ github.workspace }}/**/.buildozer-container
+          key: ${{ runner.os }}-buildozer-${{ hashFiles('**/buildozer.spec') }}
+          restore-keys: |
+            ${{ runner.os }}-buildozer-
+
+      # Set up caching for Gradle
+      - name: Cache Gradle files
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            ~/.gradle/daemon
+            ~/.gradle/native
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/buildozer.spec') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: Create Android SDK directory
+        run: mkdir -p ~/.android
+
+      - name: Generate debug keystore
+        run: |
+          keytool -genkeypair \
+            -keystore ~/.android/debug.keystore \
+            -storepass android \
+            -keypass android \
+            -alias androiddebugkey \
+            -keyalg RSA \
+            -keysize 2048 \
+            -validity 10000 \
+            -dname "CN=Android Debug,O=Android,C=US"
+
+      - name: Attempt to build Sideband for Android (Debug)
+        run: |
+          cd $GITHUB_WORKSPACE/Sideband
+          ./dmake devapk
+
+      - name: Upload built APK
+        uses: actions/upload-artifact@v4
+        with:
+          name: android-debug
+          path: ${{ github.workspace }}/Sideband/sbapp/bin/*.apk

.gitignore

@@ -33,3 +33,4 @@ dist
 docs/build
 sideband*.egg-info
 sbapp*.egg-info
+Sideband.iml

DEVELOPING.md

@@ -0,0 +1,145 @@
+# Establishing a Development environment
+
+Looking to contribute some code to Sideband? Awesome! Follow the guide below to get the repository building on your machine.
+
+## Creating folders
+
+Sideband relies on a certain folder structure to achieve a psuedo-monorepo structure with the other Reticulum projects.
+
+To make sure that the `getrns` target runs successfully, make sure your directory tree looks like this:
+
+```
+repositories/
+├─ LXMF/
+├─ LXST/
+├─ rnsh/
+├─ rnode-flasher/
+├─ Reticulum/
+├─ NomadNet/
+├─ reticulum_website/
+└─ Sideband/
+```
+
+Below are the git repositories for some of the above folders:
+
+- `LXMF`: https://github.com/markqvist/LXMF
+- `LXST`: https://github.com/markqvist/LXST
+- `rnsh`: https://github.com/acehoss/rnsh
+- `Reticulum`: https://github.com/markqvist/Reticulum
+- `rnode-flasher`: https://github.com/liamcottle/rnode-flasher
+- `NomadNet`: https://github.com/markqvist/nomadnet
+- `reticulum_website`: https://github.com/markqvist/reticulum_website
+
+> Please note: in order for the docker script and `createshare` make target to work correctly, your directory **must** be laid out like this.
+
+## Required dependencies for development (Docker)
+
+If you have a Fedora-based Linux system (see [Addendum: Fedora](#addendum-fedora)) or simply would not like to install all of P4A's dependencies manually, you may choose to use the containerized build.
+
+This method requires that you have Docker installed on your system: https://docs.docker.com/engine/install/
+
+Additionally, [rootless Docker](https://docs.docker.com/engine/install/) should be used to minimize any possible attack surface on your system. Never run a script you haven't vetted with sudo! The `./dmake.sh` script uses `set -ex` and is designed to be used with rootless docker.
+
+After configuring docker, you can replace any use of the `make` command with `dmake` (i.e. `./dmake devapk`) to run make commands in the container, building it on demand if needed. 
+
+Example:
+
+```
+./dmake devapk
+```
+
+(or if running in sbapp, it is smart enough to run itself in Sideband regardless of where it is called from)
+
+```
+../dmake devapk
+```
+
+## Required dependencies for development (Native)
+
+Until this repository has a `flake.nix` added, you will need to manually download the following dependencies using your Operating System's package manager.
+
+- `make`
+- `adb` (available as a part of the `android-tools` package on Fedora, `adb` on Debian/Ubuntu, `android-platform-tools` on Brew Casks)
+- `python3`/`python3-dev(el)` (must be available as `python`, on Ubuntu try `apt install python-is-python3`)
+- `patchelf`
+- `patch`
+- `perl`
+- `portaudio19-dev`
+- `libopus-dev`
+- `libogg-dev`
+- `buildozer` (see https://buildozer.readthedocs.io/en/latest/installation.html)
+    - buildozer's PyPI hosted version is very far behind, therefore you should install from source at https://github.com/kivy/buildozer.git; this is easy with pipx `pipx install git+https://github.com/kivy/buildozer.git`.
+    - buildozer needs `wheel` to run, but it is not currently marked as a dependency. If you are using pipx, you will need to inject it with `pipx inject buildozer wheel`.
+- all of buildozer's Android dependencies
+  - Ubuntu 22.04 LTS packages `git zip unzip openjdk-17-jdk python3-pip autoconf libtool pkg-config zlib1g-dev libncurses5-dev libncursesw5-dev libtinfo5 cmake libffi-dev libssl-dev`
+  - Ubuntu 24.04 LTS packages `git zip unzip openjdk-17-jdk python3-pip autoconf libtool pkg-config zlib1g-dev libncurses-dev libtinfo6 cmake libffi-dev libssl-dev`
+  - Fedora 41 `git zip unzip java-17-openjdk java-17-openjdk-devel python3-pip autoconf libtool pkgconf-pkg-config ghc-zlib-devel ncurses-devel ncurses-compat-libs cmake libffi-devel openssl-devel`
+
+In the root directory of the repository, use `pip install .` to install the package from `setup.py`. The use of a `venv` is strongly recommended.
+
+Make sure you manually install `Cython<3.0` into your Python install or `venv`, as buildozer will need it for Android.
+
+### Addendum: Fedora
+
+As many users of Kivy have noted before, some of Python4Android's recipes do not compile correctly on Fedora/RHEL. For this project, one package of interest is [`freetype-py` and its native dependency](https://github.com/kivy/python-for-android/blob/develop/pythonforandroid/recipes/freetype/__init__.py), which is a direct dependency of pillow, the ubiquitous Python image editing library.
+
+This is due to the fact that Fedora and several other distros include default versions of toolchains, which prompts python4android to abstain from downloading its own. [This issue has been encountered by many other users.](https://groups.google.com/g/kivy-users/c/z46lSJXgbjY/m/M1UoWwtWAgAJ)
+
+If you can't use Docker, use of Ubuntu 24.04 LTS is therefore recommended for developing this project. Ubuntu 22.04 LTS is not supported, as its `cmake` version (even with backports) is below the minimum 3.24.
+
+Sideband does run fine on Fedora, however.
+
+## Compiling and testing Sideband on an Android device
+
+With a correctly configured environment, run the following command to create a development APK.
+
+```
+make devapk
+```
+
+You can then install it to a connected device with
+
+```
+make devinstall
+```
+
+If you would like your release to be signed for release, you must configure the following four environment variables:
+
+- `P4A_RELEASE_KEYALIAS`
+- `P4A_RELEASE_KEYSTORE_PASSWD`
+- `P4A_RELEASE_KEYSTORE`
+- `P4A_RELEASE_KEYALIAS_PASSWD`
+
+With `./dmake`, omitting any of these values will cause it to default to the `debug.keystore` generated with each install of the Android SDK.
+
+After it is configured correctly, you may build it with
+
+```
+make apk
+```
+
+and install it to a connected device with
+
+```
+make install
+```
+
+The output will be placed in `./sbapp/bin/sideband-*.apk`.
+
+If you have multiple devices connected at once (for example, while developing the BLE interface between devices), you may use `devinstall-multi` or `install-multi` in place of `devinstall` and `install` respectively.
+
+If using an Android that provides the ability to toggle DCL via storage (like GrapheneOS), make sure to enable it for Sideband, as it must load its Cython executables.
+
+## Compiling and testing Sideband for other platforms
+
+For Windows, use the following command: (make sure you have `PyInstaller` in your venv, and you are on Windows, as PyInstaller removed cross-compilation)
+
+```
+make build_win_exe
+```
+
+Wheels for other platforms can be built with 
+
+```
+make build_wheel
+```

Dockerfile

@@ -0,0 +1,22 @@
+# exercise extreme caution when bumping this docker image to the newest ubuntu LTS version
+FROM ubuntu:24.04
+
+# for rationale behind each of these dependencies, consult the native section of DEVELOPING.md
+RUN DEBIAN_FRONTEND=noninteractive \
+  apt update \
+  && DEBIAN_FRONTEND=noninteractive apt install -y curl git libffi-dev python-is-python3 python3-dev python3-wheel python3-setuptools python3-virtualenv libssl-dev autoconf openjdk-17-jdk cmake libtool libssl-dev libncurses5-dev libsqlite3-dev libreadline-dev libtk8.6 libgdm-dev libpcap-dev unzip zip wget apksigner build-essential libopus-dev libogg-dev portaudio19-dev patchelf pipx \
+  && apt install --reinstall python3  \
+  && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+
+# need to run as root for a rootless docker runtime
+# the repository folders owned by 1000 on the host are mounted to 0 on the container; this is intentional and unchangable
+
+# add pipx 
+ENV PATH=$PATH:/root/.local/bin
+
+# install & inject wheel
+RUN pipx install git+https://github.com/kivy/buildozer.git@abc2d7e66c8abe096a95ed58befe6617f7efdad0
+RUN pipx inject buildozer wheel
+
+# needed for some transitive deps, like rnsh
+RUN pipx install poetry==2.1.1

Makefile

@@ -40,4 +40,4 @@ upload:
 	@echo Ready to publish release, hit enter to continue
 	@read VOID
 	@echo Uploading to PyPi...
-	twine upload dist/sbapp-*
+	twine upload dist/sbapp-*

dmake

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -ex
+
+# any of the tasks that require adb should be run on the host, not the guest docker container
+# ad-hoc docker usb passthrough sounds miserable...
+if [ $1 == "install" ] || [ $1 == "devinstall" ] || [ $1 == "install-multi" ] || [ $1 == "devinstall-multi" ]; then
+    echo "Command requiring adb detected, running on host..."
+    # shellcheck disable=SC2068 # goal is to re-split
+    make -C sbapp $@
+    exit 0
+fi 
+
+if [[ -z $P4A_RELEASE_KEYSTORE ]]; then 
+    echo "P4A_RELEASE_KEYSTORE is not set correctly! Using default debug keystore..."
+    P4A_RELEASE_KEYSTORE=$(realpath ~/.android/debug.keystore)
+    P4A_RELEASE_KEYALIAS=androiddebugkey
+    P4A_RELEASE_KEYSTORE_PASSWD=android
+    P4A_RELEASE_KEYALIAS_PASSWD=android
+fi
+
+# will hit caches automatically, unless the dockerfile was changed
+# buildozer writes to its own venv, so it can't be --read-only even though it really should be
+# /root/.buildozer has some build artifacts that are reused, and since they are dependent on the libraries available at compile-time, they must be isolated from ~/.buildozer as these may have different fingerprints
+# ~/.cache, on the other hand, may be reused because pip is smart enough to download/rebuild native dependencies if they wouldn't be compatible
+docker build --network=host -t sideband-dmake .
+docker run \
+    --rm \
+    --tty \
+    --network=host \
+    --tmpfs /tmp:rw,exec \
+    -e P4A_RELEASE_KEYALIAS=$P4A_RELEASE_KEYALIAS \
+    -e P4A_RELEASE_KEYSTORE_PASSWD=$P4A_RELEASE_KEYSTORE_PASSWD \
+    -e P4A_RELEASE_KEYSTORE="/keystore.jks" \
+    -e P4A_RELEASE_KEYALIAS_PASSWD=$P4A_RELEASE_KEYALIAS_PASSWD \
+    -v "$(realpath $(dirname "$0")/..):/repositories:rw" \
+    -v "$(realpath ./.buildozer-container):/root/.buildozer:rw" \
+    -v "$(realpath ./sbapp/.buildozer-container):/repositories/Sideband/sbapp/.buildozer:rw" \
+    -v "$(realpath ~/.cache):/root/.cache:rw" \
+    -v "$P4A_RELEASE_KEYSTORE:/keystore.jks:ro" \
+    -v "$(realpath ~/.android):/root/.android:rw" \
+    -v "$(realpath ~/.gradle):/root/.gradle:rw" \
+    sideband-dmake \
+    /bin/bash \
+    -c \
+    "python3 -m venv --system-site-packages /tmp/venv \
+    && source /tmp/venv/bin/activate \
+    && cd /repositories/Sideband \
+    && pip install -e . \"Cython<3.0\" \
+    && make -C sbapp $@"

recipes/codec2/__init__.py

@@ -1,13 +1,19 @@
 from os.path import join
+from tempfile import TemporaryDirectory
 from pythonforandroid.recipe import Recipe
 from pythonforandroid.toolchain import current_directory, shprint
+import os
 import sh
 
 # For debugging, clean with
 # buildozer android p4a -- clean_recipe_build codec2 --local-recipes ~/Information/Source/Sideband/recipes
 
 class Codec2Recipe(Recipe):
+    # recipe for building codec2 from https://github.com/markqvist/codec2
+
     url = "https://github.com/markqvist/codec2/archive/00e01c9d72d3b1607e165c71c4c9c942d277dfac.tar.gz"
+    sha512sum = "2f8db660592e19b7f853c146793ccbde90f1d505663084f055172c8e5088a9fc2ddb588cc014ed8dec46a678ec73aaf654bbe77ff29f21caa7c45fb121f2281f"
+
     built_libraries = {'libcodec2.so': 'build_android/src'}
 
     def include_flags(self, arch):
@@ -41,10 +47,18 @@ def build_arch(self, arch):
             # cd = sh.cd("build_android")
             os.chdir("build_android")
             cmake = sh.Command('cmake')
+            gcc = sh.Command("gcc")
 
             shprint(cmake, *flags, _env=env)
-            shprint(sh.make, _env=env)
-            sh.cp("../src/codec2.h", "./codec2/")
+
+            # before running the make, we need to compile `generate_codebook` from the codec2 repository for the architecture we are on
+            # allowing it to be compiled with the rest of the 
+            with TemporaryDirectory() as tmp:
+                shprint(gcc, "../src/generate_codebook.c", "-o" f"{tmp}{os.sep}generate_codebook", "-lm")
+
+                env_tmp = env | {"PATH": f"{env['PATH']}{os.pathsep}{tmp}"}
+                shprint(sh.make, _env=env_tmp)
+                sh.cp("../src/codec2.h", "./codec2/")
 
 
 recipe = Codec2Recipe()

recipes/opusfile/__init__.py

@@ -34,8 +34,6 @@ def build_arch(self, arch):
             # env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch)
             # env['LIBS'] = openssl_recipe.link_libs_flags()
 
-            from rich.pretty import pprint
-            pprint(env)
             time.sleep(5)
 
             configure = sh.Command('./configure')