diff --git a/.circleci/config.yml b/.circleci/config.yml index 2afef769..6151d3c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,17 +2,11 @@ version: 2 jobs: lint: - docker: - - image: debian:stretch + machine: true steps: - checkout - - setup_remote_docker - # Check version output for debugging - - run: apt-get update - - run: apt-get install -y python-pip make - - run: pip install flake8 - - run: python --version - - run: flake8 --version + - run: sudo apt-get update + - run: sudo apt-get install -qq make - run: make flake8 workflows: diff --git a/Makefile b/Makefile index 6b3de2af..0e73ac2c 100644 --- a/Makefile +++ b/Makefile @@ -119,13 +119,13 @@ validate: assert-dom0 ## Checks for local requirements in dev env { echo "ERROR: missing 'config.json'!" && \ echo "Create from 'config.json.example'." && exit 1 ; } +.PHONY: flake8 flake8: ## Lints all Python files with flake8 # Not requiring dom0 since linting requires extra packages, # available only in the developer environment, i.e. Work VM. - @flake8 . - @find -type f -exec file -i {} + \ - | perl -F':\s+' -nE '$$F[1] =~ m/text\/x-python/ and say $$F[0]' \ - | xargs flake8 + @docker run -v $(PWD):/code -w /code --name sdw_flake8 --rm \ + --entrypoint /code/scripts/flake8-linting \ + quay.io/freedomofpress/ci-python \ update-fedora-templates: assert-dom0 ## Upgrade to Fedora 28 templates @./scripts/update-fedora-templates diff --git a/scripts/flake8-linting b/scripts/flake8-linting new file mode 100755 index 00000000..5fbd3e3c --- /dev/null +++ b/scripts/flake8-linting @@ -0,0 +1,26 @@ +#!/bin/bash +set -e +set -u +set -o pipefail + + +# Install 'file', used for discovering all Python scripts. +# This is a hack, we should fold in the 'file' package as a dependecy +# in the underlying container. +printf "Updating apt lists..." +apt-get update > /dev/null +printf " done.\n" + +apt-get install -qq file + +# Install flake8 inside container +pip install -q flake8 + +# First pass with flake8 autodiscovery +flake8 + +# Second pass to make sure we've found all Python scripts, +# since flake8's autodiscovery option can miss some. +find -type f -exec file -i {} + \ + | perl -F':\s+' -nE '$F[1] =~ m/text\/x-python/ and say $F[0]' \ + | xargs flake8 diff --git a/sd-journalist/dev-monitor.py b/sd-journalist/dev-monitor.py index 04070c20..25ec6848 100755 --- a/sd-journalist/dev-monitor.py +++ b/sd-journalist/dev-monitor.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import datetime import pipereader @@ -12,7 +12,7 @@ def poller_cb(poller, msg, err): if msg in sd_process_display.messages: longer = sd_process_display.messages[msg] - print "[{}] {}: {}".format(datetime.datetime.now(), msg, longer) + print("[{}] {}: {}".format(datetime.datetime.now(), msg, longer)) reader = pipereader.PipeReader("sdfifo", poller_cb) diff --git a/sd-journalist/do-not-open-here b/sd-journalist/do-not-open-here index 2135c968..39096d65 100755 --- a/sd-journalist/do-not-open-here +++ b/sd-journalist/do-not-open-here @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys from PyQt4 import Qt diff --git a/sd-journalist/move-to-svs b/sd-journalist/move-to-svs index a4447796..edbc50a9 100755 --- a/sd-journalist/move-to-svs +++ b/sd-journalist/move-to-svs @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import glob import re @@ -19,7 +19,7 @@ probable_sd_downloads = [z.group(0) for z in matched_sd_pattern if z] # make a tarball of these zips fh = tempfile.NamedTemporaryFile(suffix=".sd-xfer", delete=False) -print "fh name " + fh.name +print("fh name " + fh.name) out_tar = tarfile.open(mode='w', fileobj=fh) diff --git a/sd-journalist/pipereader.py b/sd-journalist/pipereader.py index d7322d0d..9fbdc93a 100755 --- a/sd-journalist/pipereader.py +++ b/sd-journalist/pipereader.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import os import select @@ -60,7 +60,7 @@ def read(self): poller.register(fifo) elif event & select.EPOLLERR: - print "Error while polling." + print("Error while polling.") cb(None, "POLLING_ERROR") poller.unregister(fileno) os.close(fileno) @@ -68,7 +68,7 @@ def read(self): print("FIFO opened {}".format(fifo)) poller.register(fifo) elif event: - print "Totally unhandled event: {}".format(event) + print("Totally unhandled event: {}".format(event)) cb(None, "POLLING_ERROR") poller.unregister(fileno) os.close(fileno) @@ -77,7 +77,7 @@ def read(self): def reporter(poller, msg, err): - print "Got a message: {} (error: {})".format(msg.rstrip(), err) + print("Got a message: {} (error: {})".format(msg.rstrip(), err)) if msg.rstrip() == "quit": poller.quit() diff --git a/sd-journalist/sd-process-display b/sd-journalist/sd-process-display index e2c64751..75915485 100644 --- a/sd-journalist/sd-process-display +++ b/sd-journalist/sd-process-display @@ -35,6 +35,7 @@ class SDDialog(QDialog): self.buttons.accepted.connect(self.accept) self.buttons.rejected.connect(self.reject) + messages = { # bootstrapping 'EXISTING_SIGFILE': ( diff --git a/sd-journalist/sd-process-download b/sd-journalist/sd-process-download index 122632d5..a6adccbb 100755 --- a/sd-journalist/sd-process-download +++ b/sd-journalist/sd-process-download @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys import time @@ -20,7 +20,8 @@ def spawn_monitor_process(sigfile): # congrats, new parent! return except OSError as e: - print >>sys.stderr, "fork #1 failed: %d (%s)" % (e.errno, e.strerror) + print("fork #1 failed: %d (%s)" % (e.errno, e.strerror), + file=sys.stderr) sys.exit(1) os.setsid() @@ -30,8 +31,9 @@ def spawn_monitor_process(sigfile): if pid > 0: # exit from second parent. init process inherits child sys.exit(0) - except OSError, e: - print >>sys.stderr, "fork #2 failed: %d (%s)" % (e.errno, e.strerror) + except OSError as e: + print("fork #2 failed: %d (%s)" % (e.errno, e.strerror), + file=sys.stderr) sys.exit(1) subprocess.Popen(["python", "/usr/local/bin/sd-process-display", sigfile]) @@ -43,11 +45,11 @@ def wait_for_sigfile(sigfile): while os.path.isfile(sigfile) is False: if int(time.time() - started > 30): - print "Waited too long for monitoring process!" + print("Waited too long for monitoring process!") sys.exit(1) time.sleep(0.5) - print "{} is still not a file".format(sigfile) + print("{} is still not a file".format(sigfile)) os.remove(sigfile) diff --git a/sd-journalist/sd-process-feedback b/sd-journalist/sd-process-feedback index 44744a41..314135a5 100755 --- a/sd-journalist/sd-process-feedback +++ b/sd-journalist/sd-process-feedback @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import errno import posix diff --git a/tests/base.py b/tests/base.py index 4b4c97bd..4aab7f66 100644 --- a/tests/base.py +++ b/tests/base.py @@ -29,7 +29,7 @@ def _reboot(self): # * CalledProcessorError # * QubesVMError (from qubesadmin.base) # * QubesVMNotStartedError (from qubesadmin.base) - for v in self.vm.connected_vms.values(): + for v in list(self.vm.connected_vms.values()): if v.is_running(): msg = ("Need to halt connected VM {}" " before testing".format(v)) @@ -58,7 +58,7 @@ def assertFilesMatch(self, remote_path, local_path): with open(local_path) as f: content = f.read() import difflib - print "".join(difflib.unified_diff(remote_content, content)) + print("".join(difflib.unified_diff(remote_content, content))) self.assertTrue(remote_content == content) def assertFileHasLine(self, remote_path, wanted_line): diff --git a/tests/integration/test_integration b/tests/integration/test_integration index c86626b0..83050120 100755 --- a/tests/integration/test_integration +++ b/tests/integration/test_integration @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys import pipereader