Merge remote tracking branch & minor improvements

* Merge branch 'master' into electrum-satochip-v4.0.2-0.11
* Bump pysatochip version to v0.11.4 from v0.11.3
* Add 'enable 2FA' and 'disable 2FA' buttons in Satochip plugin options
* In has_usable_connection_with_device(), replace card_select() with card_get_ATR() for efficiency

# Conflicts:
#	contrib/deterministic-build/requirements-hw.txt
#	contrib/requirements/requirements-hw.txt

.gitignore

@@ -32,9 +32,13 @@ electrum/gui/kivy/theming/light.atlas
 # build workspaces
 contrib/build-wine/tmp/
 contrib/build-wine/fresh_clone/
+contrib/build-linux/sdist/fresh_clone/
 contrib/build-linux/appimage/build/
 contrib/build-linux/appimage/.cache/
 contrib/android_debug.keystore
+contrib/android/android_debug.keystore
+contrib/secp256k1/
+contrib/zbar/
 
 # shared objects
 electrum/*.so

.travis.yml

@@ -76,7 +76,8 @@ jobs:
         - ls -la bin
         - if [ $(ls bin | grep -c Electrum-*) -eq 0 ]; then exit 1; fi
       after_success: true
-    - if: (branch = master) OR (tag IS present)
+      # disabled for now as travis started to always time out:
+    - if: false AND ((branch = master) OR (tag IS present))
       name: "MacOS build"
       os: osx
       language: c

README.rst

@@ -55,9 +55,8 @@ libsecp256k1 yourself::
     sudo apt-get install automake libtool
     ./contrib/make_libsecp256k1.sh
 
-Due to the need for fast symmetric ciphers, either one of `pycryptodomex`_
-or `cryptography`_ is required. Install from your package manager
-(or from pip)::
+Due to the need for fast symmetric ciphers, `cryptography`_ is required.
+Install from your package manager (or from pip)::
 
     sudo apt-get install python3-cryptography
 

RELEASE-NOTES

@@ -1,3 +1,57 @@
+# Release 4.0.9 - (Dec 18, 2020)
+ * fixes a regression introduced in 4.0.8, that prevents from
+   paying BIP70 invoices (#6859)
+ * reflect frozen channels and disconnected peers in the displayed
+   'can send/can receive' amounts.
+
+# Release 4.0.8 - (Dec 17, 2020)
+ * fix decoding BIP21 URIs with uppercase schema (d40bedb2)
+ * psbt: put full derivation paths into PSBT by default (c8155129)
+ * invoices: allow address-reuse (#6609, #6852)
+ * A few other minor bugfixes.
+
+# Release 4.0.7 - (Dec 9, 2020)
+ * kivy: fix open channel with 'max' amount
+ * kivy: fix regression introduced in last release (a9fc440)
+ * other minor GUI fixes
+ * Dependencies: as part of adapting to new dnspython (#6828),
+   - python-ecdsa is no longer needed at all,
+   - cryptography is now required (min 2.6), the user can no
+     longer choose between cryptography and pycryptodomex
+
+# Release 4.0.6 - (Dec 4, 2020)
+ * Fix 'Max' button issue for submarine swaps button (#6770)
+ * Fix 'Max' button in kivy (#6169)
+ * Various fixes for Kivy/Android install wizard
+ * More robust account keypath for BitBox02 (#6766)
+
+# Release 4.0.5 - (Nov 18, 2020)
+ * Fix .dmg binary hanging on recently released macOS 11 Big Sur (#6461)
+ * Lightning:
+   - bugfix: during LN channel opening, if the client crashed at the
+     wrong moment, the channel might not get fully persisted to disk,
+     and would need manual console-tinkering to recover (#6656)
+   - Lightning is enabled by default. Electrum will not connect to
+     the Lightning Network until the user opens a channel. (#6639)
+   - smarter node recommendation (to open channels with) (#6705)
+ * user interface: some minor changes that aim to improve usability
+ * Ledger:
+   - fix enumerating devices with new bitcoin app (1.5.1) (b78cbcff)
+   - fix compat with HW.1 (200f547a)
+ * A few other minor bugfixes.
+
+# Release 4.0.4 - (Oct 15, 2020)
+ * PSBT: fix regression in 4.0.3 where UTXO data was not included in
+   QR codes (#6600)
+ * new feature: "Cancel tx" (#6641). The Qt/kivy GUI allows cancelling
+   an unconfirmed RBF tx by double-spending its inputs to self.
+ * Windows binary:
+   - fix some issues with QR scanning by building zbar ourselves (#6593)
+   - when using setup exe, also install a debug binary (#6603)
+ * Ledger: fix "The derivation path is unusual" warnings (#6512)
+   (needs Bitcoin app 1.4.8+ installed on device)
+ * A few other minor bugfixes and usability improvements.
+
 # Release 4.0.3 - (Sep 11, 2020)
  * PSBT: restore compatibility with Bitcoin Core following CVE-2020-14199:
    we now allow a PSBT input to have both UTXO and WITNESS_UTXO (#6429).

contrib/android/Dockerfile

@@ -67,7 +67,7 @@ RUN yes | ${ANDROID_SDK_MANAGER} --licenses > /dev/null
 
 # download platforms, API, build tools
 RUN ${ANDROID_SDK_MANAGER} "platforms;android-24" > /dev/null && \
-    ${ANDROID_SDK_MANAGER} "platforms;android-28" > /dev/null && \
+    ${ANDROID_SDK_MANAGER} "platforms;android-29" > /dev/null && \
     ${ANDROID_SDK_MANAGER} "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null && \
     ${ANDROID_SDK_MANAGER} "extras;android;m2repository" > /dev/null && \
     chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager"
@@ -92,29 +92,38 @@ ENV HOME_DIR="/home/${USER}"
 ENV WORK_DIR="${HOME_DIR}/wspace" \
     PATH="${HOME_DIR}/.local/bin:${PATH}"
 
-# install system dependencies
+# install system/build dependencies
+# https://github.com/kivy/buildozer/blob/master/docs/source/installation.rst#android-on-ubuntu-2004-64bit
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        python3 python3-pip python3-setuptools git wget lbzip2 patch sudo \
-        software-properties-common libssl-dev \
-    && apt -y autoremove
-
-# build dependencies
-# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
-RUN dpkg --add-architecture i386 \
-    && apt -y update -qq \
-    && apt -y install -qq --no-install-recommends \
-        build-essential ccache git python3 python3-dev \
-        libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
-        libidn11:i386 \
-        zip zlib1g-dev zlib1g:i386 \
-    && apt -y autoremove \
-    && apt -y clean
-
-# specific recipes dependencies (e.g. libffi requires autoreconf binary)
-RUN apt -y update -qq \
-    && apt -y install -qq --no-install-recommends \
-        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        wget \
+        lbzip2 \
+        patch \
+        sudo \
+        software-properties-common \
+        git \
+        zip \
+        unzip \
+        build-essential \
+        ccache \
+        openjdk-13-jdk \
+        autoconf \
+        libtool \
+        pkg-config \
+        zlib1g-dev \
+        libncurses5-dev \
+        libncursesw5-dev \
+        libtinfo5 \
+        cmake \
+        libffi-dev \
+        libssl-dev \
+        automake \
+        gettext \
+        libltdl-dev \
     && apt -y autoremove \
     && apt -y clean
 

contrib/android/buildozer.spec

@@ -70,8 +70,9 @@ fullscreen = False
 # (list) Permissions
 android.permissions = INTERNET, CAMERA, WRITE_EXTERNAL_STORAGE
 
-# (int) Android API to use
-android.api = 28
+# (int) Android API to use  (targetSdkVersion AND compileSdkVersion)
+# note: when changing, Dockerfile also needs to be changed to install corresponding build tools
+android.api = 29
 
 # (int) Minimum API required. You will need to set the android.ndk_api to be as low as this value.
 android.minapi = 21

contrib/build-linux/appimage/Dockerfile

@@ -11,9 +11,9 @@ RUN apt-get update -q && \
         autoconf=2.69-9 \
         libtool=2.4.6-0.1 \
         xz-utils=5.1.1alpha+20120614-2ubuntu2 \
-        libssl-dev=1.0.2g-1ubuntu4.17 \
-        libssl1.0.0=1.0.2g-1ubuntu4.17 \
-        openssl=1.0.2g-1ubuntu4.17 \
+        libssl-dev=1.0.2g-1ubuntu4.18 \
+        libssl1.0.0=1.0.2g-1ubuntu4.18 \
+        openssl=1.0.2g-1ubuntu4.18 \
         zlib1g-dev=1:1.2.8.dfsg-2ubuntu4.3 \
         libffi-dev=3.2.1-4 \
         libncurses5-dev=6.0+20160213-1ubuntu1 \

contrib/build-linux/appimage/build.sh

@@ -94,6 +94,8 @@ python='appdir_python'
 info "installing pip."
 "$python" -m ensurepip
 
+break_legacy_easy_install
+
 
 info "preparing electrum-locale."
 (
@@ -115,13 +117,23 @@ info "preparing electrum-locale."
 
 info "Installing build dependencies."
 mkdir -p "$CACHEDIR/pip_cache"
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-appimage-build.txt"
+"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-build-appimage.txt"
 
 info "installing electrum and its dependencies."
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
-"$python" -m pip install --no-dependencies --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" "$PROJECT_ROOT"
+# note: we prefer compiling C extensions ourselves, instead of using binary wheels,
+#       hence "--no-binary :all:" flags. However, we specifically allow
+#       - PyQt5, as it's harder to build from source
+#       - cryptography, as building it would need openssl 1.1, not available on ubuntu 16.04
+"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
+"$python" -m pip install --no-dependencies --no-binary :all: --only-binary pyqt5,cryptography --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
+"$python" -m pip install --no-dependencies --no-binary :all: --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
+
+"$python" -m pip install --no-dependencies --no-warn-script-location \
+    --cache-dir "$CACHEDIR/pip_cache" "$PROJECT_ROOT"
 
 # was only needed during build time, not runtime
 "$python" -m pip uninstall -y Cython

contrib/build-linux/sdist/build.sh

@@ -12,12 +12,14 @@ DISTDIR="$PROJECT_ROOT/dist"
 # note that at least py3.7 is needed, to have https://bugs.python.org/issue30693
 python3 --version || fail "python interpreter not found"
 
+break_legacy_easy_install
+
 # upgrade to modern pip so that it knows the flags we need.
-# we will then install a pinned version of pip as part of requirements-sdist-build
+# we will then install a pinned version of pip as part of requirements-build-sdist
 python3 -m pip install --upgrade pip
 
 info "Installing pinned requirements."
-python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-sdist-build.txt
+python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-sdist.txt
 
 
 "$CONTRIB"/make_packages || fail "make_packages failed"

contrib/build-wine/deterministic.spec

@@ -145,7 +145,7 @@ exe_portable = EXE(
 #####
 # exe and separate files that NSIS uses to build installer "setup" exe
 
-exe_dependent = EXE(
+exe_inside_setup_noconsole = EXE(
     pyz,
     a.scripts,
     exclude_binaries=True,
@@ -156,8 +156,20 @@ exe_dependent = EXE(
     icon=home+'electrum/gui/icons/electrum.ico',
     console=False)
 
+exe_inside_setup_console = EXE(
+    pyz,
+    a.scripts,
+    exclude_binaries=True,
+    name=os.path.join('build\\pyi.win32\\electrum', cmdline_name+"-debug"),
+    debug=False,
+    strip=None,
+    upx=False,
+    icon=home+'electrum/gui/icons/electrum.ico',
+    console=True)
+
 coll = COLLECT(
-    exe_dependent,
+    exe_inside_setup_noconsole,
+    exe_inside_setup_console,
     a.binaries,
     a.zipfiles,
     a.datas,

contrib/build-wine/prepare-wine.sh

@@ -68,8 +68,10 @@ for msifile in core dev exe lib pip tools; do
     wine msiexec /i "$PYTHON_DOWNLOADS/${msifile}.msi" /qb TARGETDIR=$PYHOME
 done
 
+break_legacy_easy_install
+
 info "Installing build dependencies."
-$PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-wine-build.txt
+$PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-build-wine.txt
 
 info "Installing dependencies specific to binaries."
 $PYTHON -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-binaries.txt

contrib/build_tools_util.sh

@@ -119,8 +119,6 @@ export SOURCE_DATE_EPOCH=1530212462
 export PYTHONHASHSEED=22
 # Set the build type, overridden by wine build
 export BUILD_TYPE="${BUILD_TYPE:-$(uname | tr '[:upper:]' '[:lower:]')}"
-# No additional autoconf flags by default
-export AUTOCONF_FLAGS=""
 # Add host / build flags if the triplets are set
 if [ -n "$GCC_TRIPLET_HOST" ] ; then
     export AUTOCONF_FLAGS="$AUTOCONF_FLAGS --host=$GCC_TRIPLET_HOST"
@@ -131,3 +129,25 @@ fi
 
 export GCC_STRIP_BINARIES="${GCC_STRIP_BINARIES:-0}"
 
+
+function break_legacy_easy_install() {
+    # We don't want setuptools sneakily installing dependencies, invisible to pip.
+    # This ensures that if setuptools calls distutils which then calls easy_install,
+    # easy_install will not download packages over the network.
+    # see https://pip.pypa.io/en/stable/reference/pip_install/#controlling-setup-requires
+    # see https://github.com/pypa/setuptools/issues/1916#issuecomment-743350566
+    info "Intentionally breaking legacy easy_install."
+    DISTUTILS_CFG="${HOME}/.pydistutils.cfg"
+    DISTUTILS_CFG_BAK="${HOME}/.pydistutils.cfg.orig"
+    # If we are not inside docker, we might be overwriting a config file on the user's system...
+    if [ -e "$DISTUTILS_CFG" ] && [ ! -e "$DISTUTILS_CFG_BAK" ]; then
+        warn "Overwriting python distutils config file at '$DISTUTILS_CFG'. A copy will be saved at '$DISTUTILS_CFG_BAK'."
+        mv "$DISTUTILS_CFG" "$DISTUTILS_CFG_BAK"
+    fi
+    cat <<EOF > "$DISTUTILS_CFG"
+[easy_install]
+index_url = ''
+find_links = ''
+EOF
+}
+