Merge pull request KMKfw#138 from KMKfw/topic-prep-for-ckeys2019

Prep for ckeys2019: Round One

.circleci/config.yml

@@ -1,26 +1,24 @@
+---
 version: 2
 jobs:
-  test:
+  build:
     docker:
       - image: 'kmkfw/base'
 
     environment:
-        KMK_TEST: 1
-        PIPENV_VENV_IN_PROJECT: 1
+      KMK_TEST: 1
 
     steps:
       - checkout
-      - restore_cache:
-          keys:
-            - v2-kmk-venv-{{ checksum "Pipfile.lock" }}
-
       - run: make test
+      - run: make dist
+      - run: make dist-deploy
 
 workflows:
   version: 2
   build-deploy:
     jobs:
-      - test:
+      - build:
           filters:
             branches:
               only: /.*/

.gitignore

@@ -102,6 +102,7 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+.compiled/
 
 .ampy
 .submodules
@@ -119,3 +120,8 @@ venv.bak/
 # Mountpoints
 mnt/
 mnt2/
+
+# build artifacts
+kmk/release_info.py
+kmk/release_info.mpy
+*.mpy

.s3cfg

@@ -0,0 +1,81 @@
+[default]
+access_key = 
+access_token = 
+add_encoding_exts = 
+add_headers = 
+bucket_location = US
+ca_certs_file = 
+cache_file = 
+check_ssl_certificate = True
+check_ssl_hostname = True
+cloudfront_host = cloudfront.amazonaws.com
+content_disposition = 
+content_type = 
+default_mime_type = binary/octet-stream
+delay_updates = False
+delete_after = False
+delete_after_fetch = False
+delete_removed = False
+dry_run = False
+enable_multipart = True
+encoding = UTF-8
+encrypt = False
+expiry_date = 
+expiry_days = 
+expiry_prefix = 
+follow_symlinks = False
+force = False
+get_continue = False
+gpg_command = /usr/bin/gpg
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = 
+guess_mime_type = True
+host_base = sfo2.digitaloceanspaces.com
+host_bucket = %(bucket)s.sfo2.digitaloceanspaces.com
+human_readable_sizes = False
+invalidate_default_index_on_cf = False
+invalidate_default_index_root_on_cf = True
+invalidate_on_cf = False
+kms_key = 
+limit = -1
+limitrate = 0
+list_md5 = False
+log_target_prefix = 
+long_listing = False
+max_delete = -1
+mime_type = 
+multipart_chunk_size_mb = 15
+multipart_max_chunks = 10000
+preserve_attrs = True
+progress_meter = True
+proxy_host = 
+proxy_port = 0
+put_continue = False
+recursive = False
+recv_chunk = 65536
+reduced_redundancy = False
+requester_pays = False
+restore_days = 1
+restore_priority = Standard
+secret_key = 
+send_chunk = 65536
+server_side_encryption = False
+signature_v2 = False
+signurl_use_https = False
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+socket_timeout = 300
+stats = False
+stop_on_error = False
+storage_class = 
+throttle_max = 100
+upload_id = 
+urlencoding_mode = normal
+use_http_expect = False
+use_https = True
+use_mime_magic = True
+verbosity = WARNING
+website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
+website_error = 
+website_index = index.html

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.7-alpine
+
+RUN mkdir -p /app
+WORKDIR /app
+
+RUN apk update && apk add alpine-sdk coreutils curl gettext git git-lfs openssh rsync wget zip
+
+RUN pip install pipenv
+
+### Get a local copy of CircuitPython and its dependencies
+# Our absolute baseline is 4.0.0, which (as of writing) shares MPY compat
+# with all future versions. Our baseline will need to update as MPY compat
+# changes
+RUN git clone --branch 4.0.0 --depth 1 https://github.com/adafruit/CircuitPython /opt/circuitpython
+RUN git -C /opt/circuitpython submodule update --init
+
+### Build the MPY compiler
+RUN make -C /opt/circuitpython/mpy-cross
+
+ENV PATH=/opt/circuitpython/mpy-cross:${PATH}

Makefile

@@ -6,20 +6,73 @@
 
 .DEFAULT: all
 
+DIST_DESCRIBE_CMD = git describe --always --abbrev=0 --dirty --broken
+
 DOCKER_BASE_TAG ?= latest
 DOCKER_TAG ?= latest
 
 AMPY_PORT ?= /dev/ttyUSB0
 AMPY_BAUD ?= 115200
 AMPY_DELAY ?= 1.5
-ARDUINO ?= /usr/share/arduino
-PIPENV ?= $(shell which pipenv)
+PIPENV ?= $(shell which pipenv 2>/dev/null)
+
+MPY_CROSS ?= $(shell which mpy-cross 2>/dev/null)
+MPY_FLAGS ?= '-O2'
+MPY_SOURCES = 'kmk/'
+MPY_TARGET_DIR ?= .compiled
+PY_KMK_TREE = $(shell find $(MPY_SOURCES) -name "*.py")
+DIST_DESCRIBE = $(shell $(DIST_DESCRIBE_CMD))
 
 all: copy-kmk copy-bootpy copy-keymap
 
-.docker_base: Dockerfile_base
+compile: $(MPY_TARGET_DIR)/.mpy.compiled
+
+$(MPY_TARGET_DIR)/.mpy.compiled: $(PY_KMK_TREE)
+ifeq ($(MPY_CROSS),)
+	@echo "===> Could not find mpy-cross in PATH, exiting"
+	@false
+endif
+	@echo "===> Compiling all py files to mpy with flags $(MPY_FLAGS)"
+	@mkdir -p $(MPY_TARGET_DIR)
+	@echo "KMK_RELEASE = '$(DIST_DESCRIBE)'" > $(MPY_SOURCES)/release_info.py
+	@find $(MPY_SOURCES) -name "*.py" -exec sh -c 'mkdir -p $(MPY_TARGET_DIR)/$$(dirname {}) && mpy-cross $(MPY_FLAGS) {} -o $(MPY_TARGET_DIR)/$$(dirname {})/$$(basename -s .py {}).mpy' \;
+	@rm -rf $(MPY_SOURCES)/release_info.py
+	@touch $(MPY_TARGET_DIR)/.mpy.compiled
+
+dist: dist/kmk-latest.zip dist/kmk-latest.unoptimized.zip dist/$(DIST_DESCRIBE).zip dist/$(DIST_DESCRIBE).unoptimized.zip
+
+dist-deploy: devdeps dist
+	@echo "===> Uploading artifacts to https://cdn.kmkfw.io/"
+	@$(PIPENV) run s3cmd -c .s3cfg put -P dist/kmk-$(DIST_DESCRIBE).zip dist/kmk-$(DIST_DESCRIBE).unoptimized.zip s3://kmk-releases/ >/dev/null
+	@[[ "$${CIRCLE_BRANCH}" == "master" ]] && echo "====> Uploading artifacts as 'latest' to https://cdn.kmkfw.io/" || true
+	@[[ "$${CIRCLE_BRANCH}" == "master" ]] && $(PIPENV) run s3cmd -c .s3cfg put -P dist/kmk-latest.zip dist/kmk-latest.unoptimized.zip s3://kmk-releases/ >/dev/null || true
+	@[[ -n "$${CIRCLE_TAG}" ]] && echo "====> Uploading artifacts as '$${CIRCLE_TAG}' to https://cdn.kmkfw.io/" || true
+	@[[ -n "$${CIRCLE_TAG}" ]] && $(PIPENV) run s3cmd -c .s3cfg put -P dist/kmk-latest.zip s3://kmk-releases/$${CIRCLE_TAG}.zip >/dev/null || true
+	@[[ -n "$${CIRCLE_TAG}" ]] && $(PIPENV) run s3cmd -c .s3cfg put -P dist/kmk-latest.unoptimized.zip s3://kmk-releases/$${CIRCLE_TAG}.unoptimized.zip >/dev/null || true
+
+dist/kmk-latest.zip: compile
+	@echo "===> Building optimized ZIP"
+	@mkdir -p dist
+	@cd $(MPY_TARGET_DIR) && zip -r ../dist/kmk-latest.zip kmk 2>&1 >/dev/null
+
+dist/$(DIST_DESCRIBE).zip: dist/kmk-latest.zip
+	@echo "===> Aliasing optimized ZIP"
+	@cp dist/kmk-latest.zip dist/kmk-$(DIST_DESCRIBE).zip
+
+dist/kmk-latest.unoptimized.zip: $(PY_KMK_TREE)
+	@echo "===> Building unoptimized ZIP"
+	@mkdir -p dist
+	@echo "KMK_RELEASE = '$(DIST_DESCRIBE)'" > kmk/release_info.py
+	@zip -r dist/kmk-latest.unoptimized.zip kmk 2>&1 >/dev/null
+	@rm -rf kmk/release_info.py
+
+dist/$(DIST_DESCRIBE).unoptimized.zip: dist/kmk-latest.unoptimized.zip
+	@echo "===> Aliasing unoptimized ZIP"
+	@cp dist/kmk-latest.unoptimized.zip dist/kmk-$(DIST_DESCRIBE).unoptimized.zip
+
+.docker_base: Dockerfile
 	@echo "===> Building Docker base image kmkfw/base:${DOCKER_BASE_TAG}"
-	@docker build -f Dockerfile_base -t kmkfw/base:${DOCKER_BASE_TAG} .
+	@docker build -t kmkfw/base:${DOCKER_BASE_TAG} .
 	@touch .docker_base
 
 docker-base: .docker_base
@@ -30,7 +83,7 @@ docker-base-deploy: docker-base
 
 .devdeps: Pipfile.lock
 	@echo "===> Installing dependencies with pipenv"
-	@$(PIPENV) install --dev --ignore-pipfile
+	@$(PIPENV) sync --dev
 	@touch .devdeps
 
 devdeps: .devdeps
@@ -41,14 +94,14 @@ lint: devdeps
 fix-isort: devdeps
 	@find kmk/ tests/ user_keymaps/ -name "*.py" | xargs $(PIPENV) run isort
 
-clean: clean-build-log
+clean:
 	@echo "===> Cleaning build artifacts"
-	@rm -rf .devdeps build
-
-clean-build-log:
-	@echo "===> Clearing previous .build.log"
-	@rm -rf .build.log
+	@rm -rf .devdeps build dist $(MPY_TARGET_DIR)
 
+# This is mostly a leftover from the days we vendored stuff from
+# micropython-lib via submodules. Leaving this here mostly in case someone goes
+# exploring through the history of KMK's repo and manages to screw up their
+# repo state (those were glitchy times...)
 powerwash: clean
 	@echo "===> Removing vendor/ to force a re-pull"
 	@rm -rf vendor

Pipfile

@@ -4,19 +4,20 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pydux = "*"
 
 [dev-packages]
-adafruit-ampy = "*"
 "flake8" = "*"
+"flake8-commas" = "*"
 "flake8-comprehensions" = "*"
-ipython = "*"
+"flake8-isort" = "*"
+"flake8-per-file-ignores" = "*"
+"python-magic" = "*"
+adafruit-ampy = "*"
 ipdb = "*"
-"flake8-commas" = "*"
+ipython = "*"
 isort = "*"
-"flake8-isort" = "*"
 neovim = "*"
-"flake8-per-file-ignores" = "*"
+s3cmd = "*"
 
 [requires]
 python_version = "3.7"