pytest: Remove all trace of python's "flaky" module.

Over time, it has cost us more developer cycles than it has gained.
It has hidden intermittant bugs, and allowed cruft to accumulate:
when we eventually tried to figure out what was going wrong, the
actual change which caused it was now stale and forgotten.

This was a particular bane during the connectd rewrite, and I
worked through some issues which had occurred before, but were not
more likely.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

.github/scripts/build.sh

@@ -33,7 +33,7 @@ cat config.vars
 
 cat << EOF > pytest.ini
 [pytest]
-addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42
 markers =
     slow_test: marks tests as slow (deselect with '-m "not slow_test"')
 EOF

.github/workflows/bsd.yml

@@ -79,7 +79,7 @@ jobs:
 
           cat << EOF > pytest.ini
           [pytest]
-          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
           markers =
               slow_test: marks tests as slow (deselect with '-m "not slow_test"')
           EOF

.github/workflows/macos.yaml

@@ -59,7 +59,7 @@ jobs:
 
           cat << EOF > pytest.ini
           [pytest]
-          addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
           markers =
               slow_test: marks tests as slow (deselect with '-m "not slow_test"')
           EOF

contrib/docker/Dockerfile.builder

@@ -56,7 +56,6 @@ RUN pip3 install --upgrade pip && \
 	Flask==1.0.2 \
 	cheroot==8.2.1 \
 	ephemeral-port-reserve==1.1.0 \
-	flaky==3.4.0 \
 	pytest-benchmark==3.1.1 \
         pytest-forked==0.2 \
 	pytest-timeout==1.3.3 \

contrib/docker/scripts/build.sh

@@ -34,7 +34,7 @@ cat config.vars
 
 cat << EOF > pytest.ini
 [pytest]
-addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42
 markers =
     slow_test: marks tests as slow (deselect with '-m "not slow_test"')
 EOF

contrib/pyln-client/Makefile

@@ -46,7 +46,7 @@ test-release: check $(ARTEFACTS) pyproject.toml
 	# tests against it (make sure not to use any virtualenv that may have
 	# pyln-${PKG} already installed).
 	virtualenv testpypi --python=/usr/bin/python3 --download --always-copy --clear
-	testpypi/bin/python3 -m pip install -r requirements.txt flaky pytest-timeout
+	testpypi/bin/python3 -m pip install -r requirements.txt pytest-timeout
 	testpypi/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-${PKG}==${VERSION}
 	testpypi/bin/python3 -c "from pyln import ${PKG};assert(${PKG}.__version__ == '$(VERSION)')"
 	testpypi/bin/pytest tests

contrib/pyln-proto/Makefile

@@ -46,7 +46,7 @@ test-release: check $(ARTEFACTS) pyproject.toml
 	# tests against it (make sure not to use any virtualenv that may have
 	# pyln-${PKG} already installed).
 	virtualenv testpypi --python=/usr/bin/python3 --download --always-copy --clear
-	testpypi/bin/python3 -m pip install -r requirements.txt flaky pytest-timeout
+	testpypi/bin/python3 -m pip install -r requirements.txt pytest-timeout
 	testpypi/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-${PKG}==${VERSION}
 	testpypi/bin/python3 -c "from pyln import ${PKG};assert(${PKG}.__version__ == '$(VERSION)')"
 	testpypi/bin/pytest tests

contrib/pyln-spec/Makefile

@@ -58,7 +58,7 @@ test-release-bolt%: $(ARTEFACTS)
 	# pyln-proto already installed).
 	virtualenv testpypi-$* --python=/usr/bin/python3 --download --always-copy --clear
 	# Install the requirements from the prod repo, they are not being kept up to date on the test repo
-	testpypi-$*/bin/python3 -m pip install -r requirements.txt pytest flaky pytest-timeout
+	testpypi-$*/bin/python3 -m pip install -r requirements.txt pytest pytest-timeout
 	testpypi-$*/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-bolt$*
 	testpypi-$*/bin/python3 -c "from pyln.spec import bolt$* as bolt;assert(bolt.__version__ == '$(call version,$*)')"
 	testpypi-$*/bin/pytest bolt$*/tests

contrib/pyln-testing/Makefile

@@ -46,7 +46,7 @@ test-release: check $(ARTEFACTS) pyproject.toml
 	# tests against it (make sure not to use any virtualenv that may have
 	# pyln-${PKG} already installed).
 	virtualenv testpypi --python=/usr/bin/python3 --download --always-copy --clear
-	testpypi/bin/python3 -m pip install -r requirements.txt flaky pytest-timeout
+	testpypi/bin/python3 -m pip install -r requirements.txt pytest-timeout
 	testpypi/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-${PKG}==${VERSION}
 	testpypi/bin/python3 -c "from pyln import ${PKG};assert(${PKG}.__version__ == '$(VERSION)')"
 	testpypi/bin/pytest tests

pyproject.toml

@@ -24,7 +24,6 @@ flake8 = "^4.0.1"
 mypy = "^0.931"
 pytest-custom-exit-code = "0.3.0"
 pyln-testing = { path = "./contrib/pyln-testing", develop = true }
-flaky = "^3.7.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

tests/test_connection.py

@@ -1,6 +1,5 @@
 from fixtures import *  # noqa: F401,F403
 from fixtures import TEST_NETWORK
-from flaky import flaky  # noqa: F401
 from ephemeral_port_reserve import reserve  # type: ignore
 from pyln.client import RpcError, Millisatoshi
 import pyln.proto.wire as wire
@@ -658,7 +657,6 @@ def test_reconnect_gossiping(node_factory):
     l2.daemon.wait_for_log('processing now old peer gone')
 
 
-@flaky
 @pytest.mark.developer("needs dev-disconnect")
 @pytest.mark.openchannel('v1')
 @pytest.mark.openchannel('v2')
@@ -853,7 +851,6 @@ def test_reconnect_receiver_fulfill(node_factory):
     assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'paid'
 
 
-@flaky
 @pytest.mark.developer
 @pytest.mark.openchannel('v1')
 @pytest.mark.openchannel('v2')
@@ -886,7 +883,6 @@ def test_shutdown_reconnect(node_factory):
     assert l1.bitcoin.rpc.getmempoolinfo()['size'] == 1
 
 
-@flaky
 @pytest.mark.developer
 def test_reconnect_remote_sends_no_sigs(node_factory):
     """We re-announce, even when remote node doesn't send its announcement_signatures on reconnect.

tests/test_misc.py

@@ -2,7 +2,6 @@
 from decimal import Decimal
 from fixtures import *  # noqa: F401,F403
 from fixtures import LightningNode, TEST_NETWORK
-from flaky import flaky  # noqa: F401
 from pyln.client import RpcError
 from threading import Event
 from pyln.testing.utils import (
@@ -1031,7 +1030,6 @@ def test_daemon_option(node_factory):
         assert 'No child process' not in f.read()
 
 
-@flaky
 @pytest.mark.developer("needs DEVELOPER=1")
 def test_blockchaintrack(node_factory, bitcoind):
     """Check that we track the blockchain correctly across reorgs
@@ -1276,7 +1274,6 @@ def test_bitcoind_goes_backwards(node_factory, bitcoind):
     l1.daemon.wait_for_log('Adding block 111')
 
 
-@flaky
 @pytest.mark.openchannel('v1')
 @pytest.mark.openchannel('v2')
 def test_reserve_enforcement(node_factory, executor):

tests/test_pay.py

@@ -1,6 +1,5 @@
 from fixtures import *  # noqa: F401,F403
 from fixtures import TEST_NETWORK
-from flaky import flaky  # noqa: F401
 from io import BytesIO
 from pyln.client import RpcError, Millisatoshi
 from pyln.proto.onion import TlvPayload
@@ -2765,13 +2764,11 @@ def test_pay_no_secret(node_factory, bitcoind):
         l1.rpc.pay(inv_nosecret)
 
 
-@flaky
 def test_shadow_routing(node_factory):
     """
     Test the value randomization through shadow routing
 
-    Since there is a very low (0.5**10) probability that it fails we mark it
-    as flaky.
+    Note there is a very low (0.5**10) probability that it fails.
     """
     # We need l3 for random walk
     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)

tests/test_plugin.py

@@ -1,7 +1,6 @@
 from collections import OrderedDict
 from datetime import datetime
 from fixtures import *  # noqa: F401,F403
-from flaky import flaky  # noqa: F401
 from hashlib import sha256
 from pyln.client import RpcError, Millisatoshi
 from pyln.proto import Invoice

tests/test_wallet.py

@@ -2,7 +2,6 @@
 from decimal import Decimal
 from fixtures import *  # noqa: F401,F403
 from fixtures import TEST_NETWORK
-from flaky import flaky  # noqa: F401
 from pyln.client import RpcError, Millisatoshi
 from utils import (
     only_one, wait_for, sync_blockheight, EXPERIMENTAL_FEATURES,
@@ -1295,13 +1294,10 @@ def test_withdraw_nlocktime(node_factory):
     assert nlocktime > 0 and nlocktime <= tip
 
 
-@flaky
 @unittest.skipIf(VALGRIND, "A big loop is used to check fuzz.")
 def test_withdraw_nlocktime_fuzz(node_factory, bitcoind):
     """
     Test that we eventually fuzz nLockTime for withdrawal transactions.
-    Marked flaky "just in case" as we fuzz from 0 to 100 with a 10%
-    probability.
     """
     l1 = node_factory.get_node(1)
     l1.fundwallet(10**8)