From 9cc43549618447e7c44094593512f992fde19cdb Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Fri, 8 Jan 2021 16:35:36 +0100
Subject: [PATCH 01/16] testing: Add a simple test to run each plugin in
 isolation

---
 .ci/test.py | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)
 create mode 100644 .ci/test.py

diff --git a/.ci/test.py b/.ci/test.py
new file mode 100644
index 000000000..4b20119a7
--- /dev/null
+++ b/.ci/test.py
@@ -0,0 +1,137 @@
+from pathlib import Path
+import subprocess
+from pprint import pprint
+from collections import namedtuple
+from typing import Generator
+import sys
+import tempfile
+import shlex
+import os
+
+
+# Directories that are not plugins
+exclude = ['.ci', '.git', '.github']
+global_dependencies = [
+    'pytest==5.*',
+    'pytest-xdist==1.34.0',
+    'pytest-timeout',
+    'pyln-testing'
+]
+
+Plugin = namedtuple('Plugin', ['name', 'path', 'requirements', 'devrequirements'])
+
+
+def enumerate_plugins(basedir: Path) -> Generator[Plugin, None, None]:
+    plugins = [x for x in basedir.iterdir() if x.is_dir() and x.name not in exclude]
+    pytests = [x for x in plugins if (x / Path('requirements.txt')).exists()]
+
+    for p in sorted(pytests):
+        yield Plugin(
+            name=p.name,
+            path=p,
+            requirements=p/Path('requirements.txt'),
+            devrequirements=p/Path('requirements-dev.txt'),
+        )
+
+
+def run_one(p: Plugin) -> bool:
+    print("Running tests on plugin {p.name}".format(p=p))
+
+    testfiles = [
+        x for x in p.path.iterdir()
+        if (x.is_dir() and x.name == 'tests')
+        or (x.name.startswith("test_") and x.name.endswith('.py'))
+    ]
+
+    if len(testfiles) == 0:
+        print("No test files found, skipping plugin {p.name}".format(p=p))
+        return True
+
+    print("Found {ctestfiles} test files, creating virtualenv and running tests".format(ctestfiles=len(testfiles)))
+    print("##[group]{p.name}".format(p=p))
+
+    # Create a virtual env
+    vdir = tempfile.TemporaryDirectory()
+    vpath = Path(vdir.name)
+
+    subprocess.check_call(['virtualenv', '--clear', '-q', vpath])
+    bin_path = vpath / 'bin'
+    pip_path = vpath / 'bin' / 'pip3'
+    python_path = vpath / 'bin' / 'python'
+    pytest_path = vpath / 'bin' / 'pytest'
+    pip_opts = ['--use-feature=2020-resolver', '-q']
+
+    # Install pytest (eventually we'd want plugin authors to include
+    # it in their requirements-dev.txt, but for now let's help them a
+    # bit).
+    subprocess.check_call(
+        [pip_path, 'install', *pip_opts, *global_dependencies],
+        stderr=subprocess.STDOUT,
+    )
+
+    # Now install all the requirements
+    print("Installing requirements from {p.requirements}".format(p=p))
+    subprocess.check_call(
+        [pip_path, 'install', '-U', *pip_opts, '-r', p.requirements],
+        stderr=subprocess.STDOUT,
+    )
+
+    if p.devrequirements.exists():
+        print("Installing requirements from {p.devrequirements}".format(p=p))
+        subprocess.check_call(
+            [pip_path, 'install', '-U', *pip_opts, '-r', p.devrequirements],
+            stderr=subprocess.STDOUT,
+        )
+
+    if os.environ.get("PYLN_MASTER", "0") == "1":
+        pass
+
+    assert pytest_path.exists()
+
+    print("Current environment packages:")
+    subprocess.check_call(
+        [pip_path, 'freeze'],
+        stderr=subprocess.STDOUT,
+    )
+
+    print("Running tests")
+    try:
+        env = env={
+            # Need to customize PATH so lightningd can find the correct python3
+            'PATH': "{}:{}".format(bin_path, os.environ['PATH']),
+        }
+        subprocess.check_call(
+            [pytest_path, p.path, '-vvv', '-n 5', '--timeout=90'],
+            stderr=subprocess.STDOUT,
+            env=env,
+        )
+        return True
+    except:
+        return False
+    finally:
+        print("##[endgroup]")
+
+
+def run_all(args):
+    root_path = subprocess.check_output([
+        'git',
+        'rev-parse',
+        '--show-toplevel'
+    ]).decode('ASCII').strip()
+
+    root = Path(root_path)
+
+    print("Testing all plugins in {root}".format(root=root))
+
+
+    results = [(p, run_one(p)) for p in enumerate_plugins(root)]
+    success = all([t[1] for t in results])
+
+    if not success:
+        print("The following tests failed:")
+        for t in filter(lambda t: not t[1], results):
+            print(" - {p.name} ({p.path})".format(p=t[0]))
+        sys.exit(1)
+
+if __name__ == "__main__":
+    run_all(sys.argv[1:])

From 85a1c4f042dc57ca10c1ca27111f771e4a69ab3f Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Fri, 8 Jan 2021 17:49:16 +0100
Subject: [PATCH 02/16] meta: Add development requirements where missing

We're about to start testing in isolation, so better make sure that
each plugin can be tested in isolation.
---
 backup/requirements-dev.txt      | 1 +
 backup/requirements.txt          | 1 +
 donations/requirements-dev.txt   | 1 +
 drain/requirements-dev.txt       | 1 +
 helpme/requirements-dev.txt      | 1 +
 noise/requirements-dev.txt       | 1 +
 request-invoice/requirements.txt | 2 +-
 7 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 backup/requirements-dev.txt
 create mode 100644 donations/requirements-dev.txt
 create mode 100644 drain/requirements-dev.txt
 create mode 100644 helpme/requirements-dev.txt
 create mode 100644 noise/requirements-dev.txt

diff --git a/backup/requirements-dev.txt b/backup/requirements-dev.txt
new file mode 100644
index 000000000..d2548c475
--- /dev/null
+++ b/backup/requirements-dev.txt
@@ -0,0 +1 @@
+pyln-testing ~= 0.9.2
diff --git a/backup/requirements.txt b/backup/requirements.txt
index 80e941da1..9c00eb1b2 100644
--- a/backup/requirements.txt
+++ b/backup/requirements.txt
@@ -2,3 +2,4 @@ Click==7.0
 flaky>=3.6.1
 tqdm>=4.45
 psutil>=5
+pyln-client ~= 0.9.2
diff --git a/donations/requirements-dev.txt b/donations/requirements-dev.txt
new file mode 100644
index 000000000..d2548c475
--- /dev/null
+++ b/donations/requirements-dev.txt
@@ -0,0 +1 @@
+pyln-testing ~= 0.9.2
diff --git a/drain/requirements-dev.txt b/drain/requirements-dev.txt
new file mode 100644
index 000000000..40f828434
--- /dev/null
+++ b/drain/requirements-dev.txt
@@ -0,0 +1 @@
+flaky==3.7.0
diff --git a/helpme/requirements-dev.txt b/helpme/requirements-dev.txt
new file mode 100644
index 000000000..d2548c475
--- /dev/null
+++ b/helpme/requirements-dev.txt
@@ -0,0 +1 @@
+pyln-testing ~= 0.9.2
diff --git a/noise/requirements-dev.txt b/noise/requirements-dev.txt
new file mode 100644
index 000000000..dd26539d5
--- /dev/null
+++ b/noise/requirements-dev.txt
@@ -0,0 +1 @@
+flaky ~= 3.7.0
diff --git a/request-invoice/requirements.txt b/request-invoice/requirements.txt
index 03da153ca..1b0b2cea4 100644
--- a/request-invoice/requirements.txt
+++ b/request-invoice/requirements.txt
@@ -1,5 +1,5 @@
 python-dotenv>=0.13.0
-flask
+flask==1.1.1
 flask-limiter
 tornado
 pyln-client

From f373994f0352a58986eb2fc21c5d9477a888ae5f Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Fri, 8 Jan 2021 16:36:03 +0100
Subject: [PATCH 03/16] gci: Use the new script to run the tests

---
 .github/workflows/main.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0690fa15e..3ead263a9 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -67,4 +67,5 @@ jobs:
         export TEST_DEBUG=1
         export TRAVIS=1
         export DEVELOPER=${{ matrix.developer }}
-        pytest --ignore=lightning -n 4 -vvv
+        pip3 install -U virtualenv pip
+        python3 .ci/test.py

From b327cbf2aaba5d274bf19c6bc5998f0262f7146a Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Fri, 8 Jan 2021 17:00:12 +0100
Subject: [PATCH 04/16] gci: Remove pointless dependencies stage

---
 .github/workflows/main.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3ead263a9..22f862734 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -54,13 +54,6 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
         
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        pip install -U pyln-client pyln-testing pyln-proto
-        find . -name requirements.txt | xargs -L 1 pip install -r
-        
     - name: Test with pytest
       run: |
         export SLOW_MACHINE=1

From 9ad09577c6b4bd0bde59dae3809dd463f5b88051 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Fri, 8 Jan 2021 17:33:44 +0100
Subject: [PATCH 05/16] gci: Run on all pushes

---
 .github/workflows/main.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 22f862734..b84f99b0c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -2,8 +2,6 @@ name: Integration Tests
 
 on:
   push:
-    branches: [ master ]
-
   pull_request:
     branches: [ master ]
 

From 7c7ebfb8bd963e6ef4b250cbcdb2c061a45eb3f4 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 11:39:16 +0100
Subject: [PATCH 06/16] gci: Add Junit test reporting

---
 .ci/test.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.ci/test.py b/.ci/test.py
index 4b20119a7..4184ad482 100644
--- a/.ci/test.py
+++ b/.ci/test.py
@@ -101,7 +101,14 @@ def run_one(p: Plugin) -> bool:
             'PATH': "{}:{}".format(bin_path, os.environ['PATH']),
         }
         subprocess.check_call(
-            [pytest_path, p.path, '-vvv', '-n 5', '--timeout=90'],
+            [
+                pytest_path,
+                p.path,
+                '-vvv',
+                '-n 5',
+                '--timeout=300',
+                '--junitxml=/tmp/report-{}.xml'.format(p.name)
+            ],
             stderr=subprocess.STDOUT,
             env=env,
         )

From ec1e3d1ef9dee1e5e868e7b22a9d6667f6c405d5 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 12:00:19 +0100
Subject: [PATCH 07/16] meta: Exclude lightning directory from tests

On CI we clone into the CWD, which results in it being picked up by
our runner. Exclude it.
---
 .ci/test.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.ci/test.py b/.ci/test.py
index 4184ad482..48b63a597 100644
--- a/.ci/test.py
+++ b/.ci/test.py
@@ -10,7 +10,12 @@
 
 
 # Directories that are not plugins
-exclude = ['.ci', '.git', '.github']
+exclude = [
+    '.ci',
+    '.git',
+    '.github',
+    'lightning',
+]
 global_dependencies = [
     'pytest==5.*',
     'pytest-xdist==1.34.0',
@@ -59,7 +64,7 @@ def run_one(p: Plugin) -> bool:
     pip_path = vpath / 'bin' / 'pip3'
     python_path = vpath / 'bin' / 'python'
     pytest_path = vpath / 'bin' / 'pytest'
-    pip_opts = ['--use-feature=2020-resolver', '-q']
+    pip_opts = ['-q']
 
     # Install pytest (eventually we'd want plugin authors to include
     # it in their requirements-dev.txt, but for now let's help them a

From 8f69b6f98ff082469eafc8d2de5c29e4501a8aa7 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 12:01:09 +0100
Subject: [PATCH 08/16] meta: Add locale to the environment when running tests

Click seems particularly upset if the locale is not UTF-8
---
 .ci/test.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.ci/test.py b/.ci/test.py
index 48b63a597..26f74fc46 100644
--- a/.ci/test.py
+++ b/.ci/test.py
@@ -101,10 +101,14 @@ def run_one(p: Plugin) -> bool:
 
     print("Running tests")
     try:
-        env = env={
+        env = os.environ.copy()
+        env.update({
             # Need to customize PATH so lightningd can find the correct python3
             'PATH': "{}:{}".format(bin_path, os.environ['PATH']),
-        }
+            # Some plugins require a valid locale to be set
+            'LC_ALL': 'C.UTF-8',
+            'LANG': 'C.UTF-8',
+        })
         subprocess.check_call(
             [
                 pytest_path,

From b3db4b93b492a4ed2e6824a64cc77e710453c731 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 12:02:07 +0100
Subject: [PATCH 09/16] meta: Temporarily disable feeadjuster tests

An msat conversion seems to fail atm.
---
 .ci/test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/test.py b/.ci/test.py
index 26f74fc46..5d2d115f9 100644
--- a/.ci/test.py
+++ b/.ci/test.py
@@ -15,6 +15,7 @@
     '.git',
     '.github',
     'lightning',
+    'feeadjuster'
 ]
 global_dependencies = [
     'pytest==5.*',

From 88432e76cada2666aeb05dbc56989218f29a0ede Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 12:09:45 +0100
Subject: [PATCH 10/16] gci: Disable DEVELOPER=0 test configs temporarily

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b84f99b0c..95d95401a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8]
-        developer: [0, 1]
+        developer: [1]
 
     steps:
     - uses: actions/checkout@v2

From 17f2b1c85212995310032815ff4ddddfbe1589db Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 16:13:59 +0100
Subject: [PATCH 11/16] meta: JUnit reports as artifacts

---
 .ci/test.py                |  2 +-
 .github/workflows/main.yml | 27 ++++++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/.ci/test.py b/.ci/test.py
index 5d2d115f9..69c3aea96 100644
--- a/.ci/test.py
+++ b/.ci/test.py
@@ -117,7 +117,7 @@ def run_one(p: Plugin) -> bool:
                 '-vvv',
                 '-n 5',
                 '--timeout=300',
-                '--junitxml=/tmp/report-{}.xml'.format(p.name)
+                '--junitxml=report-{}.xml'.format(p.name)
             ],
             stderr=subprocess.STDOUT,
             env=env,
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 95d95401a..74b6a27ec 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -12,8 +12,8 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
+    timeout-minutes: 45
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8]
@@ -60,3 +60,28 @@ jobs:
         export DEVELOPER=${{ matrix.developer }}
         pip3 install -U virtualenv pip
         python3 .ci/test.py
+    - name: Upload Unit Test Results
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: Unit Test Results (Python ${{ matrix.python-version }})
+        path: report-*.xml
+  report:
+    - name: "Publish Unit Tests Results"
+      needs: build-and-test
+      runs-on: ubuntu-latest
+      # the build-and-test job might be skipped, we don't need to run this job then
+      if: success() || failure()
+
+      steps:
+        - name: Download Artifacts
+          uses: actions/download-artifact@v2
+          with:
+            path: artifacts
+
+        - name: Publish Unit Test Results
+          uses: EnricoMi/publish-unit-test-result-action@v1.6
+          with:
+            check_name: Unit Test Results
+            github_token: ${{ secrets.GITHUB_TOKEN }}
+            files: report-*.xml

From b393e32b490b28dc3578daf528fcd8fcae7b4771 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 16:15:13 +0100
Subject: [PATCH 12/16] fixup! meta: JUnit reports as artifacts

---
 .github/workflows/main.yml | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 74b6a27ec..974838dad 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -67,21 +67,21 @@ jobs:
         name: Unit Test Results (Python ${{ matrix.python-version }})
         path: report-*.xml
   report:
-    - name: "Publish Unit Tests Results"
-      needs: build-and-test
-      runs-on: ubuntu-latest
-      # the build-and-test job might be skipped, we don't need to run this job then
-      if: success() || failure()
+    name: "Publish Unit Tests Results"
+    needs: build-and-test
+    runs-on: ubuntu-latest
+    # the build-and-test job might be skipped, we don't need to run this job then
+    if: success() || failure()
 
-      steps:
-        - name: Download Artifacts
-          uses: actions/download-artifact@v2
-          with:
-            path: artifacts
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: artifacts
 
-        - name: Publish Unit Test Results
-          uses: EnricoMi/publish-unit-test-result-action@v1.6
-          with:
-            check_name: Unit Test Results
-            github_token: ${{ secrets.GITHUB_TOKEN }}
-            files: report-*.xml
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1.6
+        with:
+          check_name: Unit Test Results
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          files: report-*.xml

From 15759e58d7a8c7cdfd999dd2eb61a8e44412a9fa Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 16:15:54 +0100
Subject: [PATCH 13/16] fixup! fixup! meta: JUnit reports as artifacts

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 974838dad..42a7e644a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,7 +11,7 @@ on:
     - cron:  '0 17 * * *'
 
 jobs:
-  build:
+  build-and-test:
     runs-on: ubuntu-latest
     timeout-minutes: 45
     strategy:

From da9fc562cdd06a367b098aeb28a9e00175843a8e Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sat, 9 Jan 2021 16:29:27 +0100
Subject: [PATCH 14/16] fixup! testing: Add a simple test to run each plugin in
 isolation

---
 .github/workflows/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 42a7e644a..41f58b706 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -15,6 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 45
     strategy:
+      fail-fast: false
       matrix:
         python-version: [3.6, 3.7, 3.8]
         developer: [1]

From 4831fadee1148e964ebd9fb0ca6a28803b12256f Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sun, 10 Jan 2021 14:50:13 +0100
Subject: [PATCH 15/16] fixup! fixup! fixup! meta: JUnit reports as artifacts

---
 .github/workflows/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 41f58b706..add5d2017 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -65,7 +65,7 @@ jobs:
       if: always()
       uses: actions/upload-artifact@v2
       with:
-        name: Unit Test Results (Python ${{ matrix.python-version }})
+        name: reports-${{ matrix.python-version }}
         path: report-*.xml
   report:
     name: "Publish Unit Tests Results"
@@ -85,4 +85,4 @@ jobs:
         with:
           check_name: Unit Test Results
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          files: report-*.xml
+          files: reports/**/report-*.xml

From 98cb721e5358f3603cbcf1f8d376c399a01d564f Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Sun, 10 Jan 2021 15:03:07 +0100
Subject: [PATCH 16/16] fixup! fixup! fixup! fixup! meta: JUnit reports as
 artifacts

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index add5d2017..0f277fbb1 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -65,7 +65,7 @@ jobs:
       if: always()
       uses: actions/upload-artifact@v2
       with:
-        name: reports-${{ matrix.python-version }}
+        name: reports/${{ matrix.python-version }}
         path: report-*.xml
   report:
     name: "Publish Unit Tests Results"