Merge pull request #144 from freedomofpress/128_2to3

Migrate all scripts to Python 3

.circleci/config.yml

@@ -2,17 +2,11 @@
 version: 2
 jobs:
   lint:
-    docker:
-      - image: debian:stretch
+    machine: true
     steps:
       - checkout
-      - setup_remote_docker
-      # Check version output for debugging
-      - run: apt-get update
-      - run: apt-get install -y python-pip make
-      - run: pip install flake8
-      - run: python --version
-      - run: flake8 --version
+      - run: sudo apt-get update
+      - run: sudo apt-get install -qq make
       - run: make flake8
 
 workflows:

Makefile

@@ -119,13 +119,13 @@ validate: assert-dom0 ## Checks for local requirements in dev env
 		{ echo "ERROR: missing 'config.json'!" && \
 		echo "Create from 'config.json.example'." && exit 1 ; }
 
+.PHONY: flake8
 flake8: ## Lints all Python files with flake8
 # Not requiring dom0 since linting requires extra packages,
 # available only in the developer environment, i.e. Work VM.
-	@flake8 .
-	@find -type f -exec file -i {} + \
-		| perl -F':\s+' -nE '$$F[1] =~ m/text\/x-python/ and say $$F[0]' \
-		| xargs flake8
+	@docker run -v $(PWD):/code -w /code --name sdw_flake8 --rm \
+		--entrypoint /code/scripts/flake8-linting \
+		quay.io/freedomofpress/ci-python \
 
 update-fedora-templates: assert-dom0 ## Upgrade to Fedora 28 templates
 	@./scripts/update-fedora-templates

scripts/flake8-linting

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+set -u
+set -o pipefail
+
+
+# Install 'file', used for discovering all Python scripts.
+# This is a hack, we should fold in the 'file' package as a dependecy
+# in the underlying container.
+printf "Updating apt lists..."
+apt-get update > /dev/null
+printf " done.\n"
+
+apt-get install -qq file
+
+# Install flake8 inside container
+pip install -q flake8
+
+# First pass with flake8 autodiscovery
+flake8
+
+# Second pass to make sure we've found all Python scripts,
+# since flake8's autodiscovery option can miss some.
+find -type f -exec file -i {} + \
+  | perl -F':\s+' -nE '$F[1] =~ m/text\/x-python/ and say $F[0]' \
+  | xargs flake8

sd-journalist/dev-monitor.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import datetime
 import pipereader
@@ -12,7 +12,7 @@ def poller_cb(poller, msg, err):
     if msg in sd_process_display.messages:
         longer = sd_process_display.messages[msg]
 
-    print "[{}] {}: {}".format(datetime.datetime.now(), msg, longer)
+    print("[{}] {}: {}".format(datetime.datetime.now(), msg, longer))
 
 
 reader = pipereader.PipeReader("sdfifo", poller_cb)

sd-journalist/do-not-open-here

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import sys
 from PyQt4 import Qt

sd-journalist/move-to-svs

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import glob
 import re
@@ -19,7 +19,7 @@ probable_sd_downloads = [z.group(0) for z in matched_sd_pattern if z]
 # make a tarball of these zips
 fh = tempfile.NamedTemporaryFile(suffix=".sd-xfer", delete=False)
 
-print "fh name " + fh.name
+print("fh name " + fh.name)
 
 out_tar = tarfile.open(mode='w', fileobj=fh)
 

sd-journalist/pipereader.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import os
 import select
@@ -60,15 +60,15 @@ def read(self):
                     poller.register(fifo)
 
                 elif event & select.EPOLLERR:
-                    print "Error while polling."
+                    print("Error while polling.")
                     cb(None, "POLLING_ERROR")
                     poller.unregister(fileno)
                     os.close(fileno)
                     fifo = os.open(pipe, os.O_RDONLY | os.O_NONBLOCK)
                     print("FIFO opened {}".format(fifo))
                     poller.register(fifo)
                 elif event:
-                    print "Totally unhandled event: {}".format(event)
+                    print("Totally unhandled event: {}".format(event))
                     cb(None, "POLLING_ERROR")
                     poller.unregister(fileno)
                     os.close(fileno)
@@ -77,7 +77,7 @@ def read(self):
 
 
 def reporter(poller, msg, err):
-    print "Got a message: {} (error: {})".format(msg.rstrip(), err)
+    print("Got a message: {} (error: {})".format(msg.rstrip(), err))
     if msg.rstrip() == "quit":
         poller.quit()
 

sd-journalist/sd-process-display

@@ -35,6 +35,7 @@ class SDDialog(QDialog):
         self.buttons.accepted.connect(self.accept)
         self.buttons.rejected.connect(self.reject)
 
+
 messages = {
     # bootstrapping
     'EXISTING_SIGFILE': (

sd-journalist/sd-process-download

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys
 import time
@@ -20,7 +20,8 @@ def spawn_monitor_process(sigfile):
             # congrats, new parent!
             return
     except OSError as e:
-        print >>sys.stderr, "fork #1 failed: %d (%s)" % (e.errno, e.strerror)
+        print("fork #1 failed: %d (%s)" % (e.errno, e.strerror),
+              file=sys.stderr)
         sys.exit(1)
 
     os.setsid()
@@ -30,8 +31,9 @@ def spawn_monitor_process(sigfile):
         if pid > 0:
             # exit from second parent. init process inherits child
             sys.exit(0)
-    except OSError, e:
-        print >>sys.stderr, "fork #2 failed: %d (%s)" % (e.errno, e.strerror)
+    except OSError as e:
+        print("fork #2 failed: %d (%s)" % (e.errno, e.strerror),
+              file=sys.stderr)
         sys.exit(1)
 
     subprocess.Popen(["python", "/usr/local/bin/sd-process-display", sigfile])
@@ -43,11 +45,11 @@ def wait_for_sigfile(sigfile):
 
     while os.path.isfile(sigfile) is False:
         if int(time.time() - started > 30):
-            print "Waited too long for monitoring process!"
+            print("Waited too long for monitoring process!")
             sys.exit(1)
 
         time.sleep(0.5)
-        print "{} is still not a file".format(sigfile)
+        print("{} is still not a file".format(sigfile))
 
     os.remove(sigfile)
 

sd-journalist/sd-process-feedback

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import errno
 import posix

tests/base.py

@@ -29,7 +29,7 @@ def _reboot(self):
         #   * CalledProcessorError
         #   * QubesVMError (from qubesadmin.base)
         #   * QubesVMNotStartedError (from qubesadmin.base)
-        for v in self.vm.connected_vms.values():
+        for v in list(self.vm.connected_vms.values()):
             if v.is_running():
                 msg = ("Need to halt connected VM {}"
                        " before testing".format(v))
@@ -58,7 +58,7 @@ def assertFilesMatch(self, remote_path, local_path):
         with open(local_path) as f:
             content = f.read()
         import difflib
-        print "".join(difflib.unified_diff(remote_content, content))
+        print("".join(difflib.unified_diff(remote_content, content)))
         self.assertTrue(remote_content == content)
 
     def assertFileHasLine(self, remote_path, wanted_line):

tests/integration/test_integration

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import sys
 import pipereader