Updates for Ubuntu 20.04 (p4lang#3074)

* Updates for Ubuntu 20.04

Some changes are required for the Docker build now that the base image
is Ubuntu 20.04.

* Use default Python 3 version (3.8) for Gauntlet

.github/.travis.yml.disabled

@@ -59,4 +59,4 @@ jobs:
         - tools/start_ccache
         - docker build --network ccache_network -t p4c --build-arg IMAGE_TYPE=test --build-arg ENABLE_UNIFIED_COMPILATION=$UNIFIED --build-arg ENABLE_GMP=$ENABLE_GMP --build-arg VALIDATION=ON . || { echo "Building Gauntlet failed." ; travis_terminate 0; }
       script:
-        - docker run -w /gauntlet p4c python3.6 -m pytest test.py -vrf -k "test_p4c" -n $CTEST_PARALLEL_LEVEL --suppress-crashes
+        - docker run -w /gauntlet p4c python3 -m pytest test.py -vrf -k "test_p4c" -n $CTEST_PARALLEL_LEVEL --suppress-crashes

.github/workflows/ci-static-build-test.yml

@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 jobs:
-  # Build a p4c release on Ubuntu 16.04
+  # Build a p4c release on Ubuntu 20.04
   build-linux:
     strategy:
       fail-fast: false

.github/workflows/ci-test.yml

@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 jobs:
-  # Build and test p4c on Ubuntu 16.04
+  # Build and test p4c on Ubuntu 20.04
   build-linux:
     strategy:
       fail-fast: false

.github/workflows/ci-validation.yml

@@ -30,7 +30,7 @@ jobs:
     - name: Validate
       if: env.BUILD_SUCCESS == 'true'
       run: |
-          docker run -w /gauntlet p4c python3.6 -m pytest test.py -vrf -k "test_p4c" -n $CTEST_PARALLEL_LEVEL --suppress-crashes
+          docker run -w /gauntlet p4c python3 -m pytest test.py -vrf -k "test_p4c" -n $CTEST_PARALLEL_LEVEL --suppress-crashes
 
     - name: Build Failed
       if: env.BUILD_SUCCESS == 'false'

CMakeLists.txt

@@ -92,6 +92,22 @@ if (BUILD_STATIC_RELEASE)
   # Link Boost statically
   set(Boost_USE_STATIC_LIBS ON)
   set(Boost_USE_STATIC_RUNTIME ON)
+  # After updating to the Ubuntu 20.04 base OS and to Boost 1.71 for CI builds,
+  # the cmake invocation for static builds started failing, with no suitable
+  # iostream library being found:
+  # ***************************
+  # The following variants have been tried and rejected:
+  # * libboost_iostreams.so.1.71.0 (shared, Boost_USE_STATIC_LIBS=ON)
+  # * libboost_iostreams.a (shared runtime, Boost_USE_STATIC_RUNTIME=ON)
+  # ***************************
+  # It was suggested online to try building with -DBoost_NO_BOOST_CMAKE=ON,
+  # which ignores some cmake files shipping with Boost (from Boost 1.70 on).
+  # It seems that setting Boost_USE_STATIC_RUNTIME to OFF may also resolve the
+  # issue. It is unclear why we set it to ON in the first place, since the Boost
+  # cmake documentation includes an example to build against Boost statically,
+  # and they explictly set Boost_USE_STATIC_RUNTIME to OFF.
+  # See https://cmake.org/cmake/help/latest/module/FindBoost.html
+  set(Boost_NO_BOOST_CMAKE ON)
   # Set the static variable
   set(P4C_STATIC_BUILD STATIC)
   # Do not bring in dynamic libstdcc and libgcc

Dockerfile

@@ -1,4 +1,5 @@
 FROM p4lang/behavioral-model:latest
+LABEL maintainer="P4 Developers <p4-dev@lists.p4.org>"
 
 # Default to using 2 make jobs, which is a good default for CI. If you're
 # building locally or you know there are more cores available, you may want to
@@ -17,6 +18,8 @@ ARG VALIDATION=OFF
 # This creates a release build that includes link time optimization and links
 # all libraries statically.
 ARG BUILD_STATIC_RELEASE=OFF
+# No questions asked during package installation.
+ARG DEBIAN_FRONTEND=noninteractive
 
 # Delegate the build to tools/ci-build.
 COPY . /p4c/

README.md

@@ -153,7 +153,7 @@ sudo dpkg -i /path/to/package.deb
     ```
 
 2.  Install [dependencies](#dependencies). You can find specific instructions
-    for Ubuntu 16.04 [here](#ubuntu-dependencies) and for macOS 10.12
+    for Ubuntu 20.04 [here](#ubuntu-dependencies) and for macOS 10.12
     [here](#macos-dependencies).  You can also look at the
     [CI installation script](tools/ci-build.sh).
 
@@ -217,7 +217,7 @@ If you plan to contribute to p4c, you'll find more useful information
 
 # Dependencies
 
-Ubuntu 16.04 is the officially supported platform for p4c. There's also
+Ubuntu 20.04 is the officially supported platform for p4c. There's also
 unofficial support for macOS 10.12. Other platforms are untested; you can try to
 use them, but YMMV.
 
@@ -239,7 +239,7 @@ use them, but YMMV.
 
 - C++ boost library (minimally used)
 
-- Python 2.7 for scripting and running tests
+- Python 3 for scripting and running tests
 
 - Optional: Documentation generation (enabled when configuring with
   --enable-doxygen-doc) requires Doxygen (1.8.10 or higher) and Graphviz
@@ -258,32 +258,29 @@ Most dependencies can be installed using `apt-get install`:
 ```bash
 sudo apt-get install cmake g++ git automake libtool libgc-dev bison flex \
 libfl-dev libgmp-dev libboost-dev libboost-iostreams-dev \
-libboost-graph-dev llvm pkg-config python python-scapy python-ipaddr python-ply python3-pip \
+libboost-graph-dev llvm pkg-config python3 python3-pip \
 tcpdump
 
-pip3 install scapy ply
+pip3 install ipaddr scapy ply
 ```
 
 For documentation building:
 `sudo apt-get install -y doxygen graphviz texlive-full`
 
-An exception is Google Protocol Buffers; `p4c` depends on version 3.0 or higher,
-which is not available until Ubuntu 16.10. For earlier releases of Ubuntu,
-you'll need to install from source. You can find instructions
-[here](https://github.com/google/protobuf/blob/master/src/README.md). **We
-recommend that you use version
-[3.6.1](https://github.com/google/protobuf/releases/tag/v3.6.1)**. Earlier
-versions in the 3 series may not be supported by other p4lang projects, such as
-[p4lang/PI](https://github.com/p4lang/PI). More recent versions may work as
-well, but all our CI testing is done with version 3.6.1. After cloning protobuf
-and before you build, check-out version 3.6.1:
+`p4c` also depends on Google Protocol Buffers (Protobuf). `p4c` requires version
+3.0 or higher, so the packaged version provided in Ubuntu 20.04 **should**
+work. However, all our CI testing is done with a more recent version of Protobuf
+(at the moment, 3.18.1), which we install from source. If you are experiencing
+issues with the Protobuf version shipped with your OS distribution, we recommend
+that we install Protobuf 3.18.1 from source. You can find instructions
+[here](https://github.com/google/protobuf/blob/master/src/README.md). After
+cloning Protobuf and before you build, check-out version 3.18.1:
 
-`git checkout v3.6.1`
+`git checkout v3.18.1`
 
-Please note that while all protobuf versions newer than 3.0 should work for
+Please note that while all Protobuf versions newer than 3.0 should work for
 `p4c` itself, you may run into trouble with some extensions and other p4lang
-projects unless you install version 3.6.1, so you may want to install from
-source even on newer releases of Ubuntu.
+projects unless you install version 3.18.1.
 
 ## Fedora dependencies
 

tools/ci-build.sh

@@ -5,9 +5,6 @@
 set -e  # Exit on error.
 set -x  # Make command execution verbose
 
-# Python3 is required for p4c to run, P4C_DEPS would otherwise uninstall it
-export P4C_PYTHON3="python3"
-
 export P4C_DEPS="bison \
              build-essential \
              cmake \
@@ -16,11 +13,12 @@ export P4C_DEPS="bison \
              g++ \
              libboost-dev \
              libboost-graph-dev \
-             libboost-iostreams1.58-dev \
+             libboost-iostreams1.71-dev \
              libfl-dev \
              libgc-dev \
              libgmp-dev \
              pkg-config \
+             python3 \
              python3-pip \
              python3-setuptools \
              tcpdump"
@@ -30,12 +28,13 @@ export P4C_EBPF_DEPS="libpcap-dev \
              zlib1g-dev \
              llvm \
              clang \
+             iproute2 \
              iptables \
              net-tools"
 
 export P4C_RUNTIME_DEPS="cpp \
-                     libboost-graph1.58.0 \
-                     libboost-iostreams1.58.0 \
+                     libboost-graph1.71.0 \
+                     libboost-iostreams1.71.0 \
                      libgc1c2 \
                      libgmp10 \
                      libgmpxx4ldbl \
@@ -51,7 +50,6 @@ export P4C_PIP_PACKAGES="ipaddr \
 apt-get update
 apt-get install -y --no-install-recommends \
   ${P4C_DEPS} \
-  ${P4C_PYTHON3} \
   ${P4C_EBPF_DEPS} \
   ${P4C_RUNTIME_DEPS} \
   git
@@ -68,17 +66,6 @@ cd /p4c
 backends/ebpf/build_libbpf
 cd -
 
-# We also need to build iproute2 from source to support Ubuntu 16.04 eBPF.
-cd /tmp
-git clone -b v5.0.0 git://git.kernel.org/pub/scm/linux/kernel/git/shemminger/iproute2.git
-cd /tmp/iproute2
-./configure
-make -j `getconf _NPROCESSORS_ONLN` && \
-make install
-cd /p4c
-rm -rf /tmp/pip
-# iproute2-end
-
 # ! ------  BEGIN VALIDATION -----------------------------------------------
 
 function build_gauntlet() {
@@ -98,21 +85,9 @@ function build_gauntlet() {
   ln -sf /gauntlet/modules/toz3 /p4c/extensions/toz3
 
   # Install Gauntlet Python dependencies locally.
-  # Unfortunately we need Python 3.6, which Xenial (Ubuntu 16.04) does not have.
-  add-apt-repository ppa:deadsnakes/ppa
-  # The interpreter requires a more modern version of GCC.
-  # We install gcc-8 and set it as default. This is also because of Xenial.
-  add-apt-repository ppa:ubuntu-toolchain-r/test
-  apt update
-  apt install -y python3.6
-  apt install -y g++-8
-  apt install -y gcc-8
-  update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 10
-  update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 10
-  # The interpreter requires boost filesystem for file management.
   apt install -y libboost-filesystem-dev
   # Install pytest and pytest-xdist to parallelize tests.
-  python3.6 -m pip install pytest pytest-xdist==1.34.0
+  python -m pip install pytest pytest-xdist==1.34.0
 }
 
 # These steps are necessary to validate the correct compilation of the P4C test
@@ -134,15 +109,6 @@ function build() {
 
 # Strong optimization.
 export CXXFLAGS="${CXXFLAGS} -O3"
-# Add the gold linker early to allow sanitization in Ubuntu 16.04
-# Context: https://stackoverflow.com/a/50357910
-export CXXFLAGS="${CXXFLAGS} -fuse-ld=gold"
-# Because of a bug with Ubuntu16.04 and static linking combined with the
-# sanitize library, we have to disable this check for static builds.
-if [ "$ENABLE_UNIFIED_COMPILATION" != "ON" ]; then
- # Catch null pointer dereferencing.
-  export CXXFLAGS="${CXXFLAGS} -fsanitize=null"
-fi
 # Toggle unified compilation.
 CMAKE_FLAGS+="-DENABLE_UNIFIED_COMPILATION=${ENABLE_UNIFIED_COMPILATION} "
 # Toggle static builds.