From 34096fc381b8bbe5cc6f9301ed8a742d666f7829 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Wed, 22 Sep 2021 18:07:41 -0700 Subject: [PATCH 01/54] Add dockerfiles to build MoveIt2 and associated tutorials using Space ROS (#15) * Add system dependencies directly Signed-off-by: Michael Jeronimo * Create Dockerfiles for MoveIt2 and the MoveIt2 tutorials Signed-off-by: Michael Jeronimo * Add a few more missing dependencies Signed-off-by: Michael Jeronimo * Add the clang-tidy tool Signed-off-by: Michael Jeronimo * Remove a stray .vscode directory and add a firefox dependency (included for testing GUIs from a docker image) Signed-off-by: Michael Jeronimo * Add a few more missing libraries found when running tests Signed-off-by: Michael Jeronimo * Add xterm to the MoveIt2 install since it's needed by the demos * Fetch the ros2.repos file from ros2/ros2#spaceros Signed-off-by: Michael Jeronimo * Add content to the docker README files Signed-off-by: Michael Jeronimo * Use rosdep for Space ROS and MoveIt2 system dependencies Signed-off-by: Michael Jeronimo * Remove the 'joy' repos to avoid a compile-time error with the latest source. The repo isn't needed currently to run the MoveIt2 MoveGroup demo. Signed-off-by: Michael Jeronimo * Fix typo in README Signed-off-by: Michael Jeronimo * Build IKOS in the spaceros docker image and add info to the README Signed-off-by: Michael Jeronimo * Address review feedback * Remove extra WORKDIR statements * Remove --symlink-install Also, * Add new dependencies because of recent source changes * Remove ros_testing as it is now in ros2.repos for spaceros Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 136 ++++++++++++++++++++++++ spaceros/README.md | 207 +++++++++++++++++++++++++++++++++++++ spaceros/build-image.sh | 20 ++++ spaceros/ros_entrypoint.sh | 6 ++ 4 files changed, 369 insertions(+) create mode 100644 spaceros/Dockerfile create mode 100644 spaceros/README.md create mode 100755 spaceros/build-image.sh create mode 100755 spaceros/ros_entrypoint.sh diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile new file mode 100644 index 00000000..8c32f615 --- /dev/null +++ b/spaceros/Dockerfile @@ -0,0 +1,136 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A Docker configuration script to build the Space ROS image. +# +# The script provides the following build arguments: +# +# VCS_REF - The git revision of the Space ROS source code (no default value). +# VERSION - The version of Space ROS (default: "preview") +# WORKSPACE - The location for the Space ROS source code in the container (default: /usr/local/src/spaceros_ws) + +FROM nvidia/cudagl:11.4.1-devel-ubuntu20.04 + +ARG VCS_REF +ARG VERSION="preview" +ARG WORKSPACE=/usr/local/src/spaceros_ws + +# LABEL the image +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.name="Space ROS" +LABEL org.label-schema.description="Preview version of the Space ROS platform" +LABEL org.label-schema.vendor="Open Robotics" +LABEL org.label-schema.version=${VERSION} +LABEL org.label-schema.url="https://openrobotics.org" +LABEL org.label-schema.vcs-url="https://github.com/ros2/ros2/tree/spaceros" +LABEL org.label-schema.vcs-ref=${VCS_REF} + +# Disable prompting during package installation +ARG DEBIAN_FRONTEND=noninteractive + +# The following commands are based on the source install for ROS 2 Rolling Ridley. +# See: https://docs.ros.org/en/ros2_documentation/rolling/Installation/Ubuntu-Development-Setup.html +# The main variation is getting Space ROS sources instead of the Rolling sources. + +# Update the Ubuntu software repository +RUN apt-get update + +# Set the locale +RUN apt-get install -y locales +RUN locale-gen en_US en_US.UTF-8 +RUN update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 +ENV LANG=en_US.UTF-8 + +# Add the ROS 2 apt repository +RUN apt-get install -y software-properties-common +RUN add-apt-repository universe +RUN apt-get update && apt-get install -y curl gnupg lsb-release +RUN curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg +RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/ros2.list > /dev/null + +# Install required software development tools and ROS tools (and vim included for convenience) +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + python3-colcon-common-extensions \ + python3-flake8 \ + python3-pip \ + python3-pytest-cov \ + python3-rosdep \ + python3-setuptools \ + python3-vcstool \ + wget \ + vim + +# Install the required pip packages +RUN python3 -m pip install -U \ + flake8-blind-except \ + flake8-builtins \ + flake8-class-newline \ + flake8-comprehensions \ + flake8-deprecated \ + flake8-docstrings \ + flake8-import-order \ + flake8-quotes \ + pytest-repeat \ + pytest-rerunfailures \ + pytest \ + setuptools + +# Get the Space ROS source code +RUN mkdir -p ${WORKSPACE}/src +WORKDIR ${WORKSPACE} +RUN wget https://raw.githubusercontent.com/ros2/ros2/spaceros/ros2.repos +RUN vcs import src < ros2.repos + +# Install system dependencies +RUN rosdep init && rosdep update +RUN rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_c ikos" + +# Build the code in the workspace +RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + +# Install dependencies needed by IKOS +RUN apt-get install -y \ + clang-9 \ + cmake \ + g++ \ + gcc \ + libboost-dev \ + libboost-filesystem-dev \ + libboost-test-dev \ + libboost-thread-dev \ + libedit-dev \ + libgmp-dev \ + libsqlite3-dev \ + libtbb-dev \ + llvm-9 \ + llvm-9-dev \ + llvm-9-tools \ + python3 \ + python3-pygments \ + zlib1g-dev + +# Get the IKOS source and build it +WORKDIR /usr/local/src +RUN git clone https://github.com/nuclearsandwich/ikos && cd ikos && git checkout nuclearsandwich/log-executables-to-file +RUN mkdir ikos/build && cd ikos/build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-9 --no-warn-unused-cli && make && make install +ENV PATH "$PATH:/usr/local/src/ikos/install/bin" +ENV IKOS_SCAN_NOTIFIER_FILES "yes" + +# Set up the entrypoint +COPY ./ros_entrypoint.sh / +ENTRYPOINT ["/ros_entrypoint.sh"] +CMD ["bash"] diff --git a/spaceros/README.md b/spaceros/README.md new file mode 100644 index 00000000..f80da532 --- /dev/null +++ b/spaceros/README.md @@ -0,0 +1,207 @@ +# Space ROS Docker Image + +The Dockerfile in this directory builds Space ROS from source code. + +## Building the Docker Image + +To include drivers for NVIDIA GPUs, the Space ROS docker image is based on the NVIDIA CudaGL development image, version 11.4.1, which is, in turn, based on Ubuntu 20.04. + +To build the docker image, run: + +``` +$ ./build-image.sh +``` + +The build process will take about 20 or 30 minutes, depending on the host computer. + +## Running the Space ROS Docker Image in a Container + +After building the image, you can see the newly-built image by running: + +``` +$ docker image list +``` + +The output will look something like this: + +``` +$ docker image list +REPOSITORY TAG IMAGE ID CREATED SIZE +openrobotics/spaceros latest 629b13cf7b74 1 hour ago 7.8GB +nvidia/cudagl 11.4.1-devel-ubuntu20.04 336416dfcbba 1 day ago 5.35GB +``` + +The new image is named **openrobotics/spaceros:latest**. + +To run the resulting image in a container, run: + +``` +$ docker run -it openrobotics/spaceros +``` + +You can also run using the image ID. In this case, the image ID is *629b13cf7b74* (each build will result in a different image ID), so the following command will run the same image: + +``` +$ docker run -it 629b13cf7b74 +``` + +Upon startup, the container automatically runs the ros_entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: + +``` +root@d10d85c68f0e:/usr/local/src/spaceros_ws# +``` + +At this point, you can run the 'ros2' command line utility to make sure everything is working OK: + +``` +root@d10d85c68f0e:/usr/local/src/spaceros_ws# ros2 +usage: ros2 [-h] [--use-python-default-buffering] Call `ros2 -h` for more detailed usage. ... + +ros2 is an extensible command-line tool for ROS 2. + +optional arguments: + -h, --help show this help message and exit + --use-python-default-buffering + Do not force line buffering in stdout and instead use the python default buffering, which might be affected by PYTHONUNBUFFERED/-u and depends on whatever stdout is interactive or not + +Commands: + action Various action related sub-commands + component Various component related sub-commands + daemon Various daemon related sub-commands + doctor Check ROS setup and other potential issues + interface Show information about ROS interfaces + launch Run a launch file + lifecycle Various lifecycle related sub-commands + multicast Various multicast related sub-commands + node Various node related sub-commands + param Various param related sub-commands + pkg Various package related sub-commands + run Run a package specific executable + service Various service related sub-commands + topic Various topic related sub-commands + trace Trace ROS nodes to get information on their execution + wtf Use `wtf` as alias to `doctor` + + Call `ros2 -h` for more detailed usage. +``` + +## Running the Space ROS Unit Tests + +The Space ROS unit tests are not built as part of the Docker image build. To run the unit tests in the container, execute the following command from the top of the Space ROS workspace: + +``` +root@d10d85c68f0e:/usr/local/src/spaceros_ws# colcon test +``` + +The tests include running the static analysis tools clang_tidy and cppcheck (which has the MISRA 2012 add-on enabled). + +You can use colcon's --packages-select option to run a subset of packages. For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: + +``` +root@d10d85c68f0e:/usr/local/src/spaceros_ws# colcon test --event-handlers console_direct+ --packages-select rcpputils +``` + +## Viewing Test Output + + The output from the tests are stored in XUnit XML files, named *\*.xunit.xml. After running the unit tests, you can scan the build directory for the various *\*.xunit.xml* files. + + For example, a clang_tidy.xunit.xml file looks like this: + +```xml + + + + + + + + + + + + + + +... + +``` + +## Connecting Another Terminal to a Running Docker Container + +Sometimes it may be convenient to attach additional terminals to a running Docker container. + +With the Space ROS Docker container running, open a second host terminal and then run the following command to determine the container ID: + +``` +$ docker container list +``` + +The output will look something like this: + +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +d10d85c68f0e openrobotics/spaceros "/ros_entrypoint.sh …" 28 minutes ago Up 28 minutes inspiring_moser +``` + +The container ID in this case, is *d10d85c68f0e*. So, run the following command in the host terminal: + +``` +docker exec -it d10d85c68f0e /bin/bash /ros_entrypoint.sh +``` + +You will then be at a prompt in the same running container. + +In place of the container ID, you can also use the automatically-generated container name ("inspiring_moser" in this case). + +## Running an IKOS Scan + +In the docker container, the IKOS executables are provided on the PATH at /usr/local/src/ikos/install/bin. +To run an IKOS scan on all of the Space ROS source (which will take a very long time), run the following command at the root of the Space ROS workspace: + +``` +root@d10d85c68f0e:/usr/local/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +``` + +To run an IKOS scan on a specific package, such as rcpputils in this case, use the *--packages-select* option, as follows: + +``` +root@d10d85c68f0e:/usr/local/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +``` + +The build output is put in the **build_ikos** subdirectory, per the *--build-base* option. + +## Generating IKOS Results + +To generate results for IKOS you can use the **ament_ikos** utility, which performs an IKOS analysis on the files previously compiled in the previous step. +The utility takes one argument which is the directory to scan recursively for the generated IKOS marker files. +For example, to generate results for all of the IKOS analysis files in the build_ikos subdirectory, execute the following command: + +``` +root@d10d85c68f0e:/usr/local/src/spaceros_ws# ament_ikos build_ikos +``` diff --git a/spaceros/build-image.sh b/spaceros/build-image.sh new file mode 100755 index 00000000..e638db2b --- /dev/null +++ b/spaceros/build-image.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +ORG=openrobotics +IMAGE=spaceros +TAG=latest + +VCS_REF="" +VERSION=preview + +echo "" +echo "##### Building Space ROS Docker Image #####" +echo "" + +docker build -t $ORG/$IMAGE:$TAG \ + --build-arg VCS_REF="$VCS_REF" \ + --build-arg VERSION="$VERSION" . + +echo "" +echo "##### Done! #####" + diff --git a/spaceros/ros_entrypoint.sh b/spaceros/ros_entrypoint.sh new file mode 100755 index 00000000..2b05d294 --- /dev/null +++ b/spaceros/ros_entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +# Setup the Space ROS environment +source "/usr/local/src/spaceros_ws/install/setup.bash" +exec "$@" From 552a7a11c01be96a215d94bece1eb5b96561a0fc Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Fri, 22 Oct 2021 08:20:02 -0700 Subject: [PATCH 02/54] Get the Cobra source and build it. (#16) Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 8c32f615..d3f06053 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -130,6 +130,12 @@ RUN mkdir ikos/build && cd ikos/build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/ ENV PATH "$PATH:/usr/local/src/ikos/install/bin" ENV IKOS_SCAN_NOTIFIER_FILES "yes" +# Get the Cobra source and build it +RUN git clone https://github.com/nimble-code/Cobra.git +RUN cd Cobra/src && make install_linux +RUN /usr/local/src/Cobra/bin_linux/cobra -configure /usr/local/src/Cobra/rules +ENV PATH "$PATH:/usr/local/src/Cobra/bin_linux" + # Set up the entrypoint COPY ./ros_entrypoint.sh / ENTRYPOINT ["/ros_entrypoint.sh"] From 07b476c6a33b94c256c3edb18e40f62e1e01becd Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Tue, 2 Nov 2021 11:02:22 -0700 Subject: [PATCH 03/54] Change the Space ROS installation directory to accomodate the Cobra tool (#18) * Change the installation directory The Cobra tool won't process files in /usr. It's probably better to put the source code in the home directory anyway. Signed-off-by: Michael Jeronimo * Update the target installation director for MoveIt2 as well Signed-off-by: Michael Jeronimo * Remove a .repos file that is no longer in the MoveIt2 repo Signed-off-by: Michael Jeronimo * Use the spaceros repo for the ros.repos list now that changes have been integrated Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 14 ++++---------- spaceros/README.md | 36 ++++++++++++++++++------------------ spaceros/ros_entrypoint.sh | 2 +- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index d3f06053..6fa0cb9e 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -18,13 +18,13 @@ # # VCS_REF - The git revision of the Space ROS source code (no default value). # VERSION - The version of Space ROS (default: "preview") -# WORKSPACE - The location for the Space ROS source code in the container (default: /usr/local/src/spaceros_ws) +# WORKSPACE - The location for the Space ROS source code in the container (default: /root/src/spaceros_ws) FROM nvidia/cudagl:11.4.1-devel-ubuntu20.04 ARG VCS_REF ARG VERSION="preview" -ARG WORKSPACE=/usr/local/src/spaceros_ws +ARG WORKSPACE=/root/src/spaceros_ws # LABEL the image LABEL org.label-schema.schema-version="1.0" @@ -124,18 +124,12 @@ RUN apt-get install -y \ zlib1g-dev # Get the IKOS source and build it -WORKDIR /usr/local/src +WORKDIR /root/src RUN git clone https://github.com/nuclearsandwich/ikos && cd ikos && git checkout nuclearsandwich/log-executables-to-file RUN mkdir ikos/build && cd ikos/build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-9 --no-warn-unused-cli && make && make install -ENV PATH "$PATH:/usr/local/src/ikos/install/bin" +ENV PATH "$PATH:/root/src/ikos/install/bin" ENV IKOS_SCAN_NOTIFIER_FILES "yes" -# Get the Cobra source and build it -RUN git clone https://github.com/nimble-code/Cobra.git -RUN cd Cobra/src && make install_linux -RUN /usr/local/src/Cobra/bin_linux/cobra -configure /usr/local/src/Cobra/rules -ENV PATH "$PATH:/usr/local/src/Cobra/bin_linux" - # Set up the entrypoint COPY ./ros_entrypoint.sh / ENTRYPOINT ["/ros_entrypoint.sh"] diff --git a/spaceros/README.md b/spaceros/README.md index f80da532..6ba5f717 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -48,13 +48,13 @@ $ docker run -it 629b13cf7b74 Upon startup, the container automatically runs the ros_entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: ``` -root@d10d85c68f0e:/usr/local/src/spaceros_ws# +root@d10d85c68f0e:/root/src/spaceros_ws# ``` At this point, you can run the 'ros2' command line utility to make sure everything is working OK: ``` -root@d10d85c68f0e:/usr/local/src/spaceros_ws# ros2 +root@d10d85c68f0e:/root/src/spaceros_ws# ros2 usage: ros2 [-h] [--use-python-default-buffering] Call `ros2 -h` for more detailed usage. ... ros2 is an extensible command-line tool for ROS 2. @@ -90,7 +90,7 @@ Commands: The Space ROS unit tests are not built as part of the Docker image build. To run the unit tests in the container, execute the following command from the top of the Space ROS workspace: ``` -root@d10d85c68f0e:/usr/local/src/spaceros_ws# colcon test +root@d10d85c68f0e:/root/src/spaceros_ws# colcon test ``` The tests include running the static analysis tools clang_tidy and cppcheck (which has the MISRA 2012 add-on enabled). @@ -98,7 +98,7 @@ The tests include running the static analysis tools clang_tidy and cppcheck (whi You can use colcon's --packages-select option to run a subset of packages. For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: ``` -root@d10d85c68f0e:/usr/local/src/spaceros_ws# colcon test --event-handlers console_direct+ --packages-select rcpputils +root@d10d85c68f0e:/root/src/spaceros_ws# colcon test --event-handlers console_direct+ --packages-select rcpputils ``` ## Viewing Test Output @@ -117,34 +117,34 @@ root@d10d85c68f0e:/usr/local/src/spaceros_ws# colcon test --event-handlers conso time="1.248" > @@ -181,17 +181,17 @@ In place of the container ID, you can also use the automatically-generated conta ## Running an IKOS Scan -In the docker container, the IKOS executables are provided on the PATH at /usr/local/src/ikos/install/bin. +In the docker container, the IKOS executables are provided on the PATH at /root/src/ikos/install/bin. To run an IKOS scan on all of the Space ROS source (which will take a very long time), run the following command at the root of the Space ROS workspace: ``` -root@d10d85c68f0e:/usr/local/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +root@d10d85c68f0e:/root/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` To run an IKOS scan on a specific package, such as rcpputils in this case, use the *--packages-select* option, as follows: ``` -root@d10d85c68f0e:/usr/local/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +root@d10d85c68f0e:/root/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` The build output is put in the **build_ikos** subdirectory, per the *--build-base* option. @@ -203,5 +203,5 @@ The utility takes one argument which is the directory to scan recursively for th For example, to generate results for all of the IKOS analysis files in the build_ikos subdirectory, execute the following command: ``` -root@d10d85c68f0e:/usr/local/src/spaceros_ws# ament_ikos build_ikos +root@d10d85c68f0e:/root/src/spaceros_ws# ament_ikos build_ikos ``` diff --git a/spaceros/ros_entrypoint.sh b/spaceros/ros_entrypoint.sh index 2b05d294..355d0e3a 100755 --- a/spaceros/ros_entrypoint.sh +++ b/spaceros/ros_entrypoint.sh @@ -2,5 +2,5 @@ set -e # Setup the Space ROS environment -source "/usr/local/src/spaceros_ws/install/setup.bash" +source "/root/src/spaceros_ws/install/setup.bash" exec "$@" From 8916af8476fdd95b11218caf7772800f77c5e0d6 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Fri, 3 Dec 2021 13:12:57 -0800 Subject: [PATCH 04/54] Use specific repository versions instead of branch names (#19) * Use specific repository versions instead of branch names to ensure the docker builds don't break Signed-off-by: Michael Jeronimo * Update the IKOS repo information now that Steven's changes have been integrated Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 6fa0cb9e..1d6a3d78 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -125,7 +125,8 @@ RUN apt-get install -y \ # Get the IKOS source and build it WORKDIR /root/src -RUN git clone https://github.com/nuclearsandwich/ikos && cd ikos && git checkout nuclearsandwich/log-executables-to-file +RUN git clone https://github.com/NASA-SW-VnV/ikos.git && cd ikos && git checkout 8b0bab34846b72777cca2a60d473282971806cfc + RUN mkdir ikos/build && cd ikos/build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-9 --no-warn-unused-cli && make && make install ENV PATH "$PATH:/root/src/ikos/install/bin" ENV IKOS_SCAN_NOTIFIER_FILES "yes" From 88e13d1549ce8aa9865e939b6f7ea47fa8f08902 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Wed, 26 Jan 2022 09:21:38 -0800 Subject: [PATCH 05/54] Update the repository version in the Space ROS Dockerfiles (#1) Adapt to changes in various repos, bringing things up to date. The MoveGroup demo has also moved from the MoveIt2 repository to the MoveIt2 tutorials repository, so also update the READMEs accordingly as well. Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 12 ++++++++---- spaceros/README.md | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 1d6a3d78..fe738aca 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -32,8 +32,8 @@ LABEL org.label-schema.name="Space ROS" LABEL org.label-schema.description="Preview version of the Space ROS platform" LABEL org.label-schema.vendor="Open Robotics" LABEL org.label-schema.version=${VERSION} -LABEL org.label-schema.url="https://openrobotics.org" -LABEL org.label-schema.vcs-url="https://github.com/ros2/ros2/tree/spaceros" +LABEL org.label-schema.url="https://github.com/space-ros" +LABEL org.label-schema.vcs-url="https://github.com/space-ros/docker-images" LABEL org.label-schema.vcs-ref=${VCS_REF} # Disable prompting during package installation @@ -92,7 +92,7 @@ RUN python3 -m pip install -U \ # Get the Space ROS source code RUN mkdir -p ${WORKSPACE}/src WORKDIR ${WORKSPACE} -RUN wget https://raw.githubusercontent.com/ros2/ros2/spaceros/ros2.repos +RUN wget https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos RUN vcs import src < ros2.repos # Install system dependencies @@ -126,12 +126,16 @@ RUN apt-get install -y \ # Get the IKOS source and build it WORKDIR /root/src RUN git clone https://github.com/NASA-SW-VnV/ikos.git && cd ikos && git checkout 8b0bab34846b72777cca2a60d473282971806cfc - +ENV PYTHONPATH "$PYTHONPATH:/root/src/ikos/install/lib/python3.8/site-packages" RUN mkdir ikos/build && cd ikos/build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-9 --no-warn-unused-cli && make && make install ENV PATH "$PATH:/root/src/ikos/install/bin" ENV IKOS_SCAN_NOTIFIER_FILES "yes" +# The above installation doesn't properly install the IKOS egg file, so do it manually +RUN cd /root/src/ikos/install/lib/python3.8/site-packages && python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg + # Set up the entrypoint +WORKDIR ${WORKSPACE} COPY ./ros_entrypoint.sh / ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] diff --git a/spaceros/README.md b/spaceros/README.md index 6ba5f717..fb3e24d7 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -172,7 +172,7 @@ d10d85c68f0e openrobotics/spaceros "/ros_entrypoint.sh …" 28 minutes ago The container ID in this case, is *d10d85c68f0e*. So, run the following command in the host terminal: ``` -docker exec -it d10d85c68f0e /bin/bash /ros_entrypoint.sh +docker exec -it d10d85c68f0e /bin/bash --init-file "install/setup.bash" ``` You will then be at a prompt in the same running container. From d935d584602b797c93a2453bcf82539255932f41 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Wed, 30 Mar 2022 15:35:34 -0700 Subject: [PATCH 06/54] Fix spaceros base docker image build (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix the IKOS build Signed-off-by: Michael Jeronimo * Reference the issue in the Dockerfile comment. Co-authored-by: Steven! Ragnarök --- spaceros/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index fe738aca..b16a723d 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -61,6 +61,7 @@ RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/r # Install required software development tools and ROS tools (and vim included for convenience) RUN apt-get update && apt-get install -y \ + bison \ build-essential \ cmake \ git \ @@ -121,6 +122,7 @@ RUN apt-get install -y \ llvm-9-tools \ python3 \ python3-pygments \ + zip \ zlib1g-dev # Get the IKOS source and build it @@ -131,8 +133,8 @@ RUN mkdir ikos/build && cd ikos/build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/ ENV PATH "$PATH:/root/src/ikos/install/bin" ENV IKOS_SCAN_NOTIFIER_FILES "yes" -# The above installation doesn't properly install the IKOS egg file, so do it manually -RUN cd /root/src/ikos/install/lib/python3.8/site-packages && python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg +# The above installation doesn't properly install the IKOS egg file, so do it manually https://github.com/NASA-SW-VnV/ikos/issues/185 +RUN cd /root/src/ikos/install/lib/python3.8/site-packages && zip ikos-3.0-py3.8.egg ikos-3.0-py3.8.egg-info/* && python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg # Set up the entrypoint WORKDIR ${WORKSPACE} From 1a01f8356f43d21d9362de0bf6e441ea567fa010 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Tue, 5 Apr 2022 07:44:24 -0700 Subject: [PATCH 07/54] Update the docker scripts and the versions of dependent packages (#4) * Update the docker scripts and the versions of dependent packages The package versions now match Rolling (intend to keep in sync). This will help to keep compiles working as the latest MoveIt2 code should be building against Rolling. Also, there now seems to be no need to manually zip the .egg file from the egg-info directory, so removed that. Signed-off-by: Michael Jeronimo * Restore prefix to ignition packages; add rosidl_typesupport_fastrtps_cpp to ignore list Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index b16a723d..8b1410a8 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -98,7 +98,7 @@ RUN vcs import src < ros2.repos # Install system dependencies RUN rosdep init && rosdep update -RUN rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_c ikos" +RUN rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" # Build the code in the workspace RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON @@ -122,7 +122,6 @@ RUN apt-get install -y \ llvm-9-tools \ python3 \ python3-pygments \ - zip \ zlib1g-dev # Get the IKOS source and build it @@ -133,8 +132,8 @@ RUN mkdir ikos/build && cd ikos/build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/ ENV PATH "$PATH:/root/src/ikos/install/bin" ENV IKOS_SCAN_NOTIFIER_FILES "yes" -# The above installation doesn't properly install the IKOS egg file, so do it manually https://github.com/NASA-SW-VnV/ikos/issues/185 -RUN cd /root/src/ikos/install/lib/python3.8/site-packages && zip ikos-3.0-py3.8.egg ikos-3.0-py3.8.egg-info/* && python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg +# The above installation doesn't install the IKOS egg file, so do it manually (https://github.com/NASA-SW-VnV/ikos/issues/185) +RUN cd /root/src/ikos/install/lib/python3.8/site-packages && python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg # Set up the entrypoint WORKDIR ${WORKSPACE} From ef16beec68da8dd5e76b2ec15431add8c4beeb3c Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Fri, 8 Apr 2022 17:23:52 -0700 Subject: [PATCH 08/54] Update the Space ROS README to describe how to generate and view IKOS reports. (#7) Signed-off-by: Michael Jeronimo --- spaceros/README.md | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/spaceros/README.md b/spaceros/README.md index fb3e24d7..585435eb 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -48,7 +48,7 @@ $ docker run -it 629b13cf7b74 Upon startup, the container automatically runs the ros_entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# +root@d10d85c68f0e:/root/src/spaceros_ws# ``` At this point, you can run the 'ros2' command line utility to make sure everything is working OK: @@ -147,7 +147,7 @@ root@d10d85c68f0e:/root/src/spaceros_ws# colcon test --event-handlers console_di name="/root/src/spaceros_ws/src/rmw/rmw/src/publisher_options.c" classname="rmw.clang_tidy"/> - + ... ``` @@ -181,27 +181,40 @@ In place of the container ID, you can also use the automatically-generated conta ## Running an IKOS Scan -In the docker container, the IKOS executables are provided on the PATH at /root/src/ikos/install/bin. -To run an IKOS scan on all of the Space ROS source (which will take a very long time), run the following command at the root of the Space ROS workspace: +IKOS uses special compiler and linker settings in order to instrument and analyze binaries. +To run an IKOS scan on all of the Space ROS test binaries (which will take a very long time), run the following command at the root of the Space ROS workspace: ``` root@d10d85c68f0e:/root/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` +The previous command generates the instrumented binaries and the associated output in a separate directory from the normal Space ROS build; the command uses *--build-base* option to specify **build_ikos** as the build output directory instead of the default **build** directory. + To run an IKOS scan on a specific package, such as rcpputils in this case, use the *--packages-select* option, as follows: ``` root@d10d85c68f0e:/root/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` -The build output is put in the **build_ikos** subdirectory, per the *--build-base* option. - ## Generating IKOS Results -To generate results for IKOS you can use the **ament_ikos** utility, which performs an IKOS analysis on the files previously compiled in the previous step. -The utility takes one argument which is the directory to scan recursively for the generated IKOS marker files. -For example, to generate results for all of the IKOS analysis files in the build_ikos subdirectory, execute the following command: +To generate JUnit XML files for all of the binaries resulting from the build command in the previous step, you can use **colcon test**, as follows: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# ament_ikos build_ikos +root@d10d85c68f0e:/root/src/spaceros_ws# colcon test --build-base build_ikos --install-base install_ikos ``` + +To generate a JUnit XML file for a specific package only, you can add the *--packages-select* option, as follows: + +``` +root@d10d85c68f0e:/root/src/spaceros_ws# colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils +``` + +The 'colcon test' command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. After running 'colcon test', you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: + +``` +root@d10d85c68f0e:/root/src/spaceros_ws# more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml + +``` + + From dd8947abd02a718e9184d7e05e58d6ba32f38e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Tue, 12 Apr 2022 14:52:11 -0400 Subject: [PATCH 09/54] Use ADD to fetch an updated repos file on each build. (#9) --- spaceros/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 8b1410a8..c88033a0 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -25,6 +25,7 @@ FROM nvidia/cudagl:11.4.1-devel-ubuntu20.04 ARG VCS_REF ARG VERSION="preview" ARG WORKSPACE=/root/src/spaceros_ws +ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" # LABEL the image LABEL org.label-schema.schema-version="1.0" @@ -93,7 +94,7 @@ RUN python3 -m pip install -U \ # Get the Space ROS source code RUN mkdir -p ${WORKSPACE}/src WORKDIR ${WORKSPACE} -RUN wget https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos +ADD ${REPOS_FILE_URL} ros2.repos RUN vcs import src < ros2.repos # Install system dependencies From 6c6cd62451af23f92ecaa6ed45a81dd1638dfe8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Thu, 28 Apr 2022 16:22:48 -0700 Subject: [PATCH 10/54] Add key update to workaround outdated base image. (#12) Key rotation in the nvidia repositories has not yet been propagated to the baseimage used by this project. For now, update the key ourselves, we should be able to periodically check the base image for updates that may allow us to revert this change. If we're "lucky" the build will fail when apt-key del has no key to remove. --- spaceros/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index c88033a0..58388de7 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -21,6 +21,10 @@ # WORKSPACE - The location for the Space ROS source code in the container (default: /root/src/spaceros_ws) FROM nvidia/cudagl:11.4.1-devel-ubuntu20.04 +# https://github.com/NVIDIA/cuda-repo-management/issues/1 +# Use new GPG keys for nvidia apt repositories that have not yet propagated to base images. +RUN apt-key del 7fa2af80 +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub ARG VCS_REF ARG VERSION="preview" From c9620518438ff0fb9adc7f6091d51fbab0c09ab7 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Thu, 18 Aug 2022 07:57:08 -0700 Subject: [PATCH 11/54] Introduce a consistent set of run scripts (#22) Signed-off-by: Michael Jeronimo Signed-off-by: Michael Jeronimo --- spaceros/README.md | 15 ++++----------- spaceros/run.sh | 2 ++ 2 files changed, 6 insertions(+), 11 deletions(-) create mode 100755 spaceros/run.sh diff --git a/spaceros/README.md b/spaceros/README.md index 585435eb..164f0fd3 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -1,11 +1,10 @@ # Space ROS Docker Image -The Dockerfile in this directory builds Space ROS from source code. +The Dockerfile in this directory builds Space ROS from source code. +To include drivers for NVIDIA GPUs, the Space ROS docker image is based on the NVIDIA CudaGL development image, version 11.4.1, which is, in turn, based on Ubuntu 20.04. ## Building the Docker Image -To include drivers for NVIDIA GPUs, the Space ROS docker image is based on the NVIDIA CudaGL development image, version 11.4.1, which is, in turn, based on Ubuntu 20.04. - To build the docker image, run: ``` @@ -33,16 +32,10 @@ nvidia/cudagl 11.4.1-devel-ubuntu20.04 336416dfcbba 1 day ago The new image is named **openrobotics/spaceros:latest**. -To run the resulting image in a container, run: - -``` -$ docker run -it openrobotics/spaceros -``` - -You can also run using the image ID. In this case, the image ID is *629b13cf7b74* (each build will result in a different image ID), so the following command will run the same image: +There is a run.sh script provided for convenience that will run the spaceros image in a container. ``` -$ docker run -it 629b13cf7b74 +$ ./run.sh ``` Upon startup, the container automatically runs the ros_entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: diff --git a/spaceros/run.sh b/spaceros/run.sh new file mode 100755 index 00000000..6a868d2a --- /dev/null +++ b/spaceros/run.sh @@ -0,0 +1,2 @@ +# Start the container +docker run --rm --gpus all --net=host -e DISPLAY=$DISPLAY --device=/dev/dri:/dev/dri --volume="$HOME/.Xauthority:/home/spaceros-user/.Xauthority:rw" -it openrobotics/spaceros From a593ec9068dcb1f2ec88d5cf60493908575628af Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Thu, 18 Aug 2022 07:57:18 -0700 Subject: [PATCH 12/54] Update the docker image metadata (#23) Signed-off-by: Michael Jeronimo Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 58388de7..e6e3f79b 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -18,20 +18,14 @@ # # VCS_REF - The git revision of the Space ROS source code (no default value). # VERSION - The version of Space ROS (default: "preview") -# WORKSPACE - The location for the Space ROS source code in the container (default: /root/src/spaceros_ws) FROM nvidia/cudagl:11.4.1-devel-ubuntu20.04 -# https://github.com/NVIDIA/cuda-repo-management/issues/1 -# Use new GPG keys for nvidia apt repositories that have not yet propagated to base images. -RUN apt-key del 7fa2af80 -RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub +# Define arguments used in the metadata definition ARG VCS_REF ARG VERSION="preview" -ARG WORKSPACE=/root/src/spaceros_ws -ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" -# LABEL the image +# Specify the docker image metadata LABEL org.label-schema.schema-version="1.0" LABEL org.label-schema.name="Space ROS" LABEL org.label-schema.description="Preview version of the Space ROS platform" @@ -44,6 +38,15 @@ LABEL org.label-schema.vcs-ref=${VCS_REF} # Disable prompting during package installation ARG DEBIAN_FRONTEND=noninteractive +# Define a few key variables +ARG WORKSPACE=/root/src/spaceros_ws +ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" + +# https://github.com/NVIDIA/cuda-repo-management/issues/1 +# Use new GPG keys for nvidia apt repositories that have not yet propagated to base images. +RUN apt-key del 7fa2af80 +RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub + # The following commands are based on the source install for ROS 2 Rolling Ridley. # See: https://docs.ros.org/en/ros2_documentation/rolling/Installation/Ubuntu-Development-Setup.html # The main variation is getting Space ROS sources instead of the Rolling sources. From 70a328acfcfbfdd281211f6afda672d02582ef7d Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Thu, 18 Aug 2022 08:09:40 -0700 Subject: [PATCH 13/54] Be consistent with script names (#24) * Be consistent with script names * Use build.sh and run.sh (instead of build-image.sh and run.sh) * Rename entrypoint files to be consistent (no need to be different) Signed-off-by: Michael Jeronimo * Update READMEs too Signed-off-by: Michael Jeronimo Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 4 ++-- spaceros/README.md | 2 +- spaceros/{build-image.sh => build.sh} | 0 spaceros/{ros_entrypoint.sh => entrypoint.sh} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename spaceros/{build-image.sh => build.sh} (100%) rename spaceros/{ros_entrypoint.sh => entrypoint.sh} (100%) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index e6e3f79b..4e28ea2c 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -145,6 +145,6 @@ RUN cd /root/src/ikos/install/lib/python3.8/site-packages && python3 /usr/lib/py # Set up the entrypoint WORKDIR ${WORKSPACE} -COPY ./ros_entrypoint.sh / -ENTRYPOINT ["/ros_entrypoint.sh"] +COPY ./entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] CMD ["bash"] diff --git a/spaceros/README.md b/spaceros/README.md index 164f0fd3..91de9d5b 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -8,7 +8,7 @@ To include drivers for NVIDIA GPUs, the Space ROS docker image is based on the N To build the docker image, run: ``` -$ ./build-image.sh +$ ./build.sh ``` The build process will take about 20 or 30 minutes, depending on the host computer. diff --git a/spaceros/build-image.sh b/spaceros/build.sh similarity index 100% rename from spaceros/build-image.sh rename to spaceros/build.sh diff --git a/spaceros/ros_entrypoint.sh b/spaceros/entrypoint.sh similarity index 100% rename from spaceros/ros_entrypoint.sh rename to spaceros/entrypoint.sh From 33889fb6e77ace2e6249dd513463caa61f27ce1e Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Fri, 19 Aug 2022 08:34:48 -0700 Subject: [PATCH 14/54] Fix a couple missed spots in the READMEs; fix a WORKDIR statement (#25) Signed-off-by: Michael Jeronimo Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 2 +- spaceros/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 4e28ea2c..173d9d21 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -144,7 +144,7 @@ ENV IKOS_SCAN_NOTIFIER_FILES "yes" RUN cd /root/src/ikos/install/lib/python3.8/site-packages && python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg # Set up the entrypoint -WORKDIR ${WORKSPACE} +WORKDIR ${SPACEROS_DIR} COPY ./entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] CMD ["bash"] diff --git a/spaceros/README.md b/spaceros/README.md index 91de9d5b..d0c70c9d 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -38,7 +38,7 @@ There is a run.sh script provided for convenience that will run the spaceros ima $ ./run.sh ``` -Upon startup, the container automatically runs the ros_entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: +Upon startup, the container automatically runs the entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: ``` root@d10d85c68f0e:/root/src/spaceros_ws# @@ -159,7 +159,7 @@ The output will look something like this: ``` CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -d10d85c68f0e openrobotics/spaceros "/ros_entrypoint.sh …" 28 minutes ago Up 28 minutes inspiring_moser +d10d85c68f0e openrobotics/spaceros "/entrypoint.sh …" 28 minutes ago Up 28 minutes inspiring_moser ``` The container ID in this case, is *d10d85c68f0e*. So, run the following command in the host terminal: From c9ac87e4a60fa3f3402057fafba2cc9a6b66f771 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Fri, 19 Aug 2022 08:37:25 -0700 Subject: [PATCH 15/54] Update the workspace names (#26) Signed-off-by: Michael Jeronimo Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 2 +- spaceros/README.md | 40 ++++++++++++++++++++-------------------- spaceros/entrypoint.sh | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 173d9d21..58ee6fc6 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -39,7 +39,7 @@ LABEL org.label-schema.vcs-ref=${VCS_REF} ARG DEBIAN_FRONTEND=noninteractive # Define a few key variables -ARG WORKSPACE=/root/src/spaceros_ws +ARG WORKSPACE=/root/src/spaceros ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" # https://github.com/NVIDIA/cuda-repo-management/issues/1 diff --git a/spaceros/README.md b/spaceros/README.md index d0c70c9d..908cd73e 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -1,6 +1,6 @@ # Space ROS Docker Image -The Dockerfile in this directory builds Space ROS from source code. +The Dockerfile in this directory builds Space ROS from source code. To include drivers for NVIDIA GPUs, the Space ROS docker image is based on the NVIDIA CudaGL development image, version 11.4.1, which is, in turn, based on Ubuntu 20.04. ## Building the Docker Image @@ -41,13 +41,13 @@ $ ./run.sh Upon startup, the container automatically runs the entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# +root@d10d85c68f0e:/root/src/spaceros# ``` At this point, you can run the 'ros2' command line utility to make sure everything is working OK: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# ros2 +root@d10d85c68f0e:/root/src/spaceros# ros2 usage: ros2 [-h] [--use-python-default-buffering] Call `ros2 -h` for more detailed usage. ... ros2 is an extensible command-line tool for ROS 2. @@ -83,7 +83,7 @@ Commands: The Space ROS unit tests are not built as part of the Docker image build. To run the unit tests in the container, execute the following command from the top of the Space ROS workspace: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# colcon test +root@d10d85c68f0e:/root/src/spaceros# colcon test ``` The tests include running the static analysis tools clang_tidy and cppcheck (which has the MISRA 2012 add-on enabled). @@ -91,7 +91,7 @@ The tests include running the static analysis tools clang_tidy and cppcheck (whi You can use colcon's --packages-select option to run a subset of packages. For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# colcon test --event-handlers console_direct+ --packages-select rcpputils +root@d10d85c68f0e:/root/src/spaceros# colcon test --event-handlers console_direct+ --packages-select rcpputils ``` ## Viewing Test Output @@ -110,34 +110,34 @@ root@d10d85c68f0e:/root/src/spaceros_ws# colcon test --event-handlers console_di time="1.248" > @@ -178,7 +178,7 @@ IKOS uses special compiler and linker settings in order to instrument and analyz To run an IKOS scan on all of the Space ROS test binaries (which will take a very long time), run the following command at the root of the Space ROS workspace: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +root@d10d85c68f0e:/root/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` The previous command generates the instrumented binaries and the associated output in a separate directory from the normal Space ROS build; the command uses *--build-base* option to specify **build_ikos** as the build output directory instead of the default **build** directory. @@ -186,7 +186,7 @@ The previous command generates the instrumented binaries and the associated outp To run an IKOS scan on a specific package, such as rcpputils in this case, use the *--packages-select* option, as follows: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +root@d10d85c68f0e:/root/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` ## Generating IKOS Results @@ -194,19 +194,19 @@ root@d10d85c68f0e:/root/src/spaceros_ws# CC="ikos-scan-cc" CXX="ikos-scan-c++" L To generate JUnit XML files for all of the binaries resulting from the build command in the previous step, you can use **colcon test**, as follows: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# colcon test --build-base build_ikos --install-base install_ikos +root@d10d85c68f0e:/root/src/spaceros# colcon test --build-base build_ikos --install-base install_ikos ``` To generate a JUnit XML file for a specific package only, you can add the *--packages-select* option, as follows: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils +root@d10d85c68f0e:/root/src/spaceros# colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils ``` The 'colcon test' command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. After running 'colcon test', you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: ``` -root@d10d85c68f0e:/root/src/spaceros_ws# more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml +root@d10d85c68f0e:/root/src/spaceros# more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml ``` diff --git a/spaceros/entrypoint.sh b/spaceros/entrypoint.sh index 355d0e3a..6f5216a1 100755 --- a/spaceros/entrypoint.sh +++ b/spaceros/entrypoint.sh @@ -2,5 +2,5 @@ set -e # Setup the Space ROS environment -source "/root/src/spaceros_ws/install/setup.bash" +source "/root/src/spaceros/install/setup.bash" exec "$@" From bac79f61ca938d730a9752a617cbfc80466f979e Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Mon, 22 Aug 2022 14:56:21 -0700 Subject: [PATCH 16/54] Run docker images as non-root user ('spaceros-user') (#20) * Create a space-ros user Signed-off-by: Michael Jeronimo * Update MoveIt2 and the MoveIt2 Tutorials also Signed-off-by: Michael Jeronimo * Add non-root user support to the demo docker script Signed-off-by: Michael Jeronimo * Use repo with updated paths Signed-off-by: Michael Jeronimo * Update the run.sh scripts to make them consistent Signed-off-by: Michael Jeronimo * Update READMEs and run scripts Signed-off-by: Michael Jeronimo * Move adding the space-ros user to the render group to the demo dockerfile (when the group exists) Signed-off-by: Michael Jeronimo * Remove a directory accidentally committed Signed-off-by: Michael Jeronimo * Fix a merge error Signed-off-by: Michael Jeronimo * Minor fixes found during testing Signed-off-by: Michael Jeronimo * Don't use upstream-development when getting dependencies; work around a gazebo crash Signed-off-by: Michael Jeronimo Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 57 +++++++++++++++++++++++++----------------- spaceros/README.md | 39 ++++++++++++++--------------- spaceros/entrypoint.sh | 2 +- 3 files changed, 54 insertions(+), 44 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 58ee6fc6..ee753537 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -38,8 +38,11 @@ LABEL org.label-schema.vcs-ref=${VCS_REF} # Disable prompting during package installation ARG DEBIAN_FRONTEND=noninteractive -# Define a few key variables -ARG WORKSPACE=/root/src/spaceros +# Define the username and key locations +ENV USERNAME spaceros-user +ENV SRC_DIR=/home/${USERNAME}/src +ENV SPACEROS_DIR=${SRC_DIR}/spaceros +ARG IKOS_DIR=${SRC_DIR}/ikos ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" # https://github.com/NVIDIA/cuda-repo-management/issues/1 @@ -47,10 +50,6 @@ ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/r RUN apt-key del 7fa2af80 RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub -# The following commands are based on the source install for ROS 2 Rolling Ridley. -# See: https://docs.ros.org/en/ros2_documentation/rolling/Installation/Ubuntu-Development-Setup.html -# The main variation is getting Space ROS sources instead of the Rolling sources. - # Update the Ubuntu software repository RUN apt-get update @@ -67,7 +66,7 @@ RUN apt-get update && apt-get install -y curl gnupg lsb-release RUN curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/ros2.list > /dev/null -# Install required software development tools and ROS tools (and vim included for convenience) +# Install required software development tools and ROS tools (and vim, included for convenience) RUN apt-get update && apt-get install -y \ bison \ build-essential \ @@ -98,21 +97,32 @@ RUN python3 -m pip install -U \ pytest \ setuptools -# Get the Space ROS source code -RUN mkdir -p ${WORKSPACE}/src -WORKDIR ${WORKSPACE} -ADD ${REPOS_FILE_URL} ros2.repos +# Create a spaceros user +RUN useradd -m $USERNAME && \ + echo "$USERNAME:$USERNAME" | chpasswd && \ + usermod --shell /bin/bash $USERNAME && \ + usermod -aG sudo $USERNAME && \ + mkdir -p /etc/sudoers.d && \ + echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/$USERNAME && \ + chmod 0440 /etc/sudoers.d/$USERNAME +USER ${USERNAME} + +## Get the Space ROS source code +RUN mkdir -p ${SPACEROS_DIR}/src +WORKDIR ${SPACEROS_DIR} +ADD --chown=${USERNAME}:${USERNAME} ${REPOS_FILE_URL} ros2.repos RUN vcs import src < ros2.repos # Install system dependencies -RUN rosdep init && rosdep update -RUN rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" +RUN sudo rosdep init && \ + rosdep update && \ + rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" -# Build the code in the workspace -RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +# Build the Space ROS source code +RUN colcon build --symlink-install --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -# Install dependencies needed by IKOS -RUN apt-get install -y \ +# Install dependencies needed by the IKOS static analyzer +RUN sudo apt-get install -y \ clang-9 \ cmake \ g++ \ @@ -133,15 +143,16 @@ RUN apt-get install -y \ zlib1g-dev # Get the IKOS source and build it -WORKDIR /root/src +WORKDIR ${SRC_DIR} RUN git clone https://github.com/NASA-SW-VnV/ikos.git && cd ikos && git checkout 8b0bab34846b72777cca2a60d473282971806cfc -ENV PYTHONPATH "$PYTHONPATH:/root/src/ikos/install/lib/python3.8/site-packages" -RUN mkdir ikos/build && cd ikos/build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-9 --no-warn-unused-cli && make && make install -ENV PATH "$PATH:/root/src/ikos/install/bin" +ENV PYTHONPATH "$PYTHONPATH:${IKOS_DIR}/install/lib/python3.8/site-packages" +WORKDIR ${IKOS_DIR} +RUN mkdir build && cd build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-9 --no-warn-unused-cli && make && sudo make install +ENV PATH "$PATH:${IKOS_DIR}/install/bin" ENV IKOS_SCAN_NOTIFIER_FILES "yes" -# The above installation doesn't install the IKOS egg file, so do it manually (https://github.com/NASA-SW-VnV/ikos/issues/185) -RUN cd /root/src/ikos/install/lib/python3.8/site-packages && python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg +# The IKOS installation doesn't install the IKOS egg file, so do it manually (https://github.com/NASA-SW-VnV/ikos/issues/185) +RUN cd ${IKOS_DIR}/install/lib/python3.8/site-packages && sudo python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg # Set up the entrypoint WORKDIR ${SPACEROS_DIR} diff --git a/spaceros/README.md b/spaceros/README.md index 908cd73e..b9f3dba7 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -41,13 +41,13 @@ $ ./run.sh Upon startup, the container automatically runs the entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: ``` -root@d10d85c68f0e:/root/src/spaceros# +spaceros-user@d10d85c68f0e:~/src/spaceros# ``` At this point, you can run the 'ros2' command line utility to make sure everything is working OK: ``` -root@d10d85c68f0e:/root/src/spaceros# ros2 +spaceros-user@d10d85c68f0e:~/src/spaceros# ros2 usage: ros2 [-h] [--use-python-default-buffering] Call `ros2 -h` for more detailed usage. ... ros2 is an extensible command-line tool for ROS 2. @@ -83,7 +83,7 @@ Commands: The Space ROS unit tests are not built as part of the Docker image build. To run the unit tests in the container, execute the following command from the top of the Space ROS workspace: ``` -root@d10d85c68f0e:/root/src/spaceros# colcon test +spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test ``` The tests include running the static analysis tools clang_tidy and cppcheck (which has the MISRA 2012 add-on enabled). @@ -91,7 +91,7 @@ The tests include running the static analysis tools clang_tidy and cppcheck (whi You can use colcon's --packages-select option to run a subset of packages. For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: ``` -root@d10d85c68f0e:/root/src/spaceros# colcon test --event-handlers console_direct+ --packages-select rcpputils +spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test --event-handlers console_direct+ --packages-select rcpputils ``` ## Viewing Test Output @@ -110,34 +110,34 @@ root@d10d85c68f0e:/root/src/spaceros# colcon test --event-handlers console_direc time="1.248" > @@ -178,7 +178,7 @@ IKOS uses special compiler and linker settings in order to instrument and analyz To run an IKOS scan on all of the Space ROS test binaries (which will take a very long time), run the following command at the root of the Space ROS workspace: ``` -root@d10d85c68f0e:/root/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +spaceros-user@d10d85c68f0e:~/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` The previous command generates the instrumented binaries and the associated output in a separate directory from the normal Space ROS build; the command uses *--build-base* option to specify **build_ikos** as the build output directory instead of the default **build** directory. @@ -186,7 +186,7 @@ The previous command generates the instrumented binaries and the associated outp To run an IKOS scan on a specific package, such as rcpputils in this case, use the *--packages-select* option, as follows: ``` -root@d10d85c68f0e:/root/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +spaceros-user@d10d85c68f0e:~/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` ## Generating IKOS Results @@ -194,20 +194,19 @@ root@d10d85c68f0e:/root/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD=" To generate JUnit XML files for all of the binaries resulting from the build command in the previous step, you can use **colcon test**, as follows: ``` -root@d10d85c68f0e:/root/src/spaceros# colcon test --build-base build_ikos --install-base install_ikos +spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test --build-base build_ikos --install-base install_ikos ``` To generate a JUnit XML file for a specific package only, you can add the *--packages-select* option, as follows: ``` -root@d10d85c68f0e:/root/src/spaceros# colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils +spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils ``` The 'colcon test' command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. After running 'colcon test', you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: ``` -root@d10d85c68f0e:/root/src/spaceros# more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml +spaceros-user@d10d85c68f0e:~/src/spaceros# more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml ``` - diff --git a/spaceros/entrypoint.sh b/spaceros/entrypoint.sh index 6f5216a1..a87275ca 100755 --- a/spaceros/entrypoint.sh +++ b/spaceros/entrypoint.sh @@ -2,5 +2,5 @@ set -e # Setup the Space ROS environment -source "/root/src/spaceros/install/setup.bash" +source "/home/spaceros-user/src/spaceros/install/setup.bash" exec "$@" From 83854e3557a2eb731955e815e7ed0e2c984591a1 Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Thu, 1 Sep 2022 16:30:56 -0700 Subject: [PATCH 17/54] Update for Humble (#28) Signed-off-by: Michael Jeronimo Signed-off-by: Michael Jeronimo --- spaceros/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index ee753537..3e9400af 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -38,12 +38,13 @@ LABEL org.label-schema.vcs-ref=${VCS_REF} # Disable prompting during package installation ARG DEBIAN_FRONTEND=noninteractive -# Define the username and key locations +# Define the username and key variables ENV USERNAME spaceros-user ENV SRC_DIR=/home/${USERNAME}/src ENV SPACEROS_DIR=${SRC_DIR}/spaceros ARG IKOS_DIR=${SRC_DIR}/ikos ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" +ENV ROSDISTRO=humble # https://github.com/NVIDIA/cuda-repo-management/issues/1 # Use new GPG keys for nvidia apt repositories that have not yet propagated to base images. @@ -116,7 +117,7 @@ RUN vcs import src < ros2.repos # Install system dependencies RUN sudo rosdep init && \ rosdep update && \ - rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" + rosdep install --from-paths src --ignore-src --rosdistro ${ROSDISTRO} -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" # Build the Space ROS source code RUN colcon build --symlink-install --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON From 7dc3c66b16cba272521c1e5042cebf65b46dde74 Mon Sep 17 00:00:00 2001 From: Dharini Dutia Date: Thu, 3 Nov 2022 17:14:12 -0700 Subject: [PATCH 18/54] Updating demos structure (#29) * updated names and paths Signed-off-by: Dharini Dutia * updates Signed-off-by: Dharini Dutia * manually get deps Signed-off-by: Dharini Dutia * updates Signed-off-by: Dharini Dutia * source space ros Signed-off-by: Dharini Dutia * cleaning Signed-off-by: Dharini Dutia * names in sh Signed-off-by: Dharini Dutia * updated all affected dockerfiles and readmes Signed-off-by: Dharini Dutia * removing apt-utils Signed-off-by: Dharini Dutia Signed-off-by: Dharini Dutia --- spaceros/Dockerfile | 8 ++++---- spaceros/README.md | 38 +++++++++++++++++++------------------- spaceros/entrypoint.sh | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile index 3e9400af..1052288f 100644 --- a/spaceros/Dockerfile +++ b/spaceros/Dockerfile @@ -40,9 +40,9 @@ ARG DEBIAN_FRONTEND=noninteractive # Define the username and key variables ENV USERNAME spaceros-user -ENV SRC_DIR=/home/${USERNAME}/src -ENV SPACEROS_DIR=${SRC_DIR}/spaceros -ARG IKOS_DIR=${SRC_DIR}/ikos +ENV HOME_DIR=/home/${USERNAME} +ENV SPACEROS_DIR=${HOME_DIR}/spaceros +ARG IKOS_DIR=${HOME_DIR}/ikos ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" ENV ROSDISTRO=humble @@ -144,7 +144,7 @@ RUN sudo apt-get install -y \ zlib1g-dev # Get the IKOS source and build it -WORKDIR ${SRC_DIR} +WORKDIR ${HOME_DIR} RUN git clone https://github.com/NASA-SW-VnV/ikos.git && cd ikos && git checkout 8b0bab34846b72777cca2a60d473282971806cfc ENV PYTHONPATH "$PYTHONPATH:${IKOS_DIR}/install/lib/python3.8/site-packages" WORKDIR ${IKOS_DIR} diff --git a/spaceros/README.md b/spaceros/README.md index b9f3dba7..4b5ce9a7 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -41,13 +41,13 @@ $ ./run.sh Upon startup, the container automatically runs the entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# +spaceros-user@d10d85c68f0e:~/spaceros$ ``` At this point, you can run the 'ros2' command line utility to make sure everything is working OK: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# ros2 +spaceros-user@d10d85c68f0e:~/spaceros$ ros2 usage: ros2 [-h] [--use-python-default-buffering] Call `ros2 -h` for more detailed usage. ... ros2 is an extensible command-line tool for ROS 2. @@ -83,7 +83,7 @@ Commands: The Space ROS unit tests are not built as part of the Docker image build. To run the unit tests in the container, execute the following command from the top of the Space ROS workspace: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test +spaceros-user@d10d85c68f0e:~/spaceros$ colcon test ``` The tests include running the static analysis tools clang_tidy and cppcheck (which has the MISRA 2012 add-on enabled). @@ -91,7 +91,7 @@ The tests include running the static analysis tools clang_tidy and cppcheck (whi You can use colcon's --packages-select option to run a subset of packages. For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test --event-handlers console_direct+ --packages-select rcpputils +spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --event-handlers console_direct+ --packages-select rcpputils ``` ## Viewing Test Output @@ -110,34 +110,34 @@ spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test --event-handlers console_ time="1.248" > @@ -178,7 +178,7 @@ IKOS uses special compiler and linker settings in order to instrument and analyz To run an IKOS scan on all of the Space ROS test binaries (which will take a very long time), run the following command at the root of the Space ROS workspace: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +spaceros-user@d10d85c68f0e:~/spaceros$ CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` The previous command generates the instrumented binaries and the associated output in a separate directory from the normal Space ROS build; the command uses *--build-base* option to specify **build_ikos** as the build output directory instead of the default **build** directory. @@ -186,7 +186,7 @@ The previous command generates the instrumented binaries and the associated outp To run an IKOS scan on a specific package, such as rcpputils in this case, use the *--packages-select* option, as follows: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +spaceros-user@d10d85c68f0e:~/spaceros$ CC="ikos-scan-cc" CXX="ikos-scan-c++" LD="ikos-scan-cc" colcon build --build-base build_ikos --install-base install_ikos --packages-select rcpputils --cmake-args -DSECURITY=ON -DINSTALL_EXAMPLES=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli ``` ## Generating IKOS Results @@ -194,19 +194,19 @@ spaceros-user@d10d85c68f0e:~/src/spaceros# CC="ikos-scan-cc" CXX="ikos-scan-c++" To generate JUnit XML files for all of the binaries resulting from the build command in the previous step, you can use **colcon test**, as follows: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test --build-base build_ikos --install-base install_ikos +spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --install-base install_ikos ``` To generate a JUnit XML file for a specific package only, you can add the *--packages-select* option, as follows: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils +spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils ``` The 'colcon test' command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. After running 'colcon test', you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: ``` -spaceros-user@d10d85c68f0e:~/src/spaceros# more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml +spaceros-user@d10d85c68f0e:~/spaceros$ more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml ``` diff --git a/spaceros/entrypoint.sh b/spaceros/entrypoint.sh index a87275ca..4349ddba 100755 --- a/spaceros/entrypoint.sh +++ b/spaceros/entrypoint.sh @@ -2,5 +2,5 @@ set -e # Setup the Space ROS environment -source "/home/spaceros-user/src/spaceros/install/setup.bash" +source "${SPACEROS_DIR}/install/setup.bash" exec "$@" From 8def497297467c90a075bc37ca677b90bfddf282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Wed, 9 Nov 2022 10:28:10 -0800 Subject: [PATCH 19/54] Add Earthly configuration and use it to build the spaceros base image. (#15) * Add Earthfile for building Space ROS base image. Converting the base docker image (sans Ikos for now) to Earthly targets was fairly straightforward. The targets are chosen someone arbitrarily based on both expected semantics (i.e. `+build` for "build" and `+image` for "install" (by registering a Docker image). * Add build-testing target to build and run colcon tests. Does not yet produce test results artifact. * Use ubuntu:focal as default base image. The nvidia:cudagl images are required for doing some GPU work in builds beyond the main build of Space ROS. In the usual layered Docker approach we needed to use this as the base base image for each build but with Earthly it is even easier to move to a multi-stage build and we'll be able to pull in the install directory from this Space ROS build in other later builds which will use the cudagl base image. * Use a remote artifact for the repos file. There's a branch of the space-ros/space-ros repository which will regenerate and store the ros2.repos file. We can use directly asa remote earthly artifact. * Don't change artifact paths. * Switch base image to Ubuntu Jammy. * Save artifacts directly to current directory instead of subdirectory. New tutorials will make use of these earthly targets in a workspace that is already created rather than requiring a superdirectory containing the earthly configuration file. This may end up conflicting with building additional docker images with more contents and require some finessing but it's better for the current workflows. * Add trailing slash to copy to src directory. * Add Earthly workflow for building the docker image. * Ignore immediate src, install, and log subdirectories. * Add job for uploading build results archive. * Fix earthly download url. * Add Earthfile artifact for bootstrapping. * Fix spelling of COPY directive. * Set up sources locally. * Fix entrypoint name. * Allow sources to supply output for the next step. * Install all dependencies using apt. They are all available on Ubuntu Jammy. * Use the same directory structure as the existing Docker image. * Save the install directory from a build-testing run. * Add labels from base docker image. * Start to add ikos support. * Revert "Start to add ikos support." This reverts commit 3f2709a8eb0ade84d35e8110b30240521722843a. Ikos support work will continue on the nuclearsandwich/ikos-earthly branch. * Update image tag. * Support VCS_REF arg from Docker image build. * Use the current HEAD as the default VCS ref. * Use earthly to build the base Space ROS image. * Omit GPU arguments and update image / repository name. * Remove unused Dockerfile and corresponding GitHub workflow. * Add push step when the build is on main. * Start updating README.md for Earthly workflow. * Clean and re-fetch sources in automatic build script. --- .github/workflows/earthly-build.yaml | 50 +++++++++ spaceros/.gitignore | 4 + spaceros/Dockerfile | 162 --------------------------- spaceros/Earthfile | 150 +++++++++++++++++++++++++ spaceros/README.md | 31 +++-- spaceros/build.sh | 9 +- spaceros/run.sh | 2 +- 7 files changed, 231 insertions(+), 177 deletions(-) create mode 100644 .github/workflows/earthly-build.yaml create mode 100644 spaceros/.gitignore delete mode 100644 spaceros/Dockerfile create mode 100644 spaceros/Earthfile diff --git a/.github/workflows/earthly-build.yaml b/.github/workflows/earthly-build.yaml new file mode 100644 index 00000000..5666f911 --- /dev/null +++ b/.github/workflows/earthly-build.yaml @@ -0,0 +1,50 @@ +name: Earthly build +on: + push: + branches: ['main'] + pull_request: + schedule: + - cron: '0 11 * * *' +jobs: + build-testing-results: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up earthly + run: | + sudo wget https://github.com/earthly/earthly/releases/latest/download/earthly-linux-amd64 -O /usr/local/bin/earthly + sudo chmod 755 /usr/local/bin/earthly + - name: Build Space ROS + id: build + run: | + cd spaceros + earthly --ci --output +sources + earthly --ci +build-testing + printf "::set-output name=archivename::%s\n" $(basename log/build_results_archives/build_results_*.tar.bz2) + - name: Upload build results archive + uses: actions/upload-artifact@v3 + with: + name: ${{steps.build.outputs.archivename}} + path: spaceros/log/build_results_archives/${{steps.build.outputs.archivename}} + space-ros-image: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up earthly + run: | + sudo wget https://github.com/earthly/earthly/releases/latest/download/earthly-linux-amd64 -O /usr/local/bin/earthly + sudo chmod 755 /usr/local/bin/earthly + - name: Build spaceros image + run: | + cd spaceros + earthly --ci --output +sources + earthly --ci +image + - name: Push spaceros image + env: + DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_RW_TOKEN }} + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: | + echo $DOCKER_HUB_TOKEN | docker login --username osrfbot --password-stdin + earthly --ci --push +image diff --git a/spaceros/.gitignore b/spaceros/.gitignore new file mode 100644 index 00000000..4709e4bb --- /dev/null +++ b/spaceros/.gitignore @@ -0,0 +1,4 @@ +spaceros_ws/ +src/ +install/ +log/ diff --git a/spaceros/Dockerfile b/spaceros/Dockerfile deleted file mode 100644 index 1052288f..00000000 --- a/spaceros/Dockerfile +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2021 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# A Docker configuration script to build the Space ROS image. -# -# The script provides the following build arguments: -# -# VCS_REF - The git revision of the Space ROS source code (no default value). -# VERSION - The version of Space ROS (default: "preview") - -FROM nvidia/cudagl:11.4.1-devel-ubuntu20.04 - -# Define arguments used in the metadata definition -ARG VCS_REF -ARG VERSION="preview" - -# Specify the docker image metadata -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.name="Space ROS" -LABEL org.label-schema.description="Preview version of the Space ROS platform" -LABEL org.label-schema.vendor="Open Robotics" -LABEL org.label-schema.version=${VERSION} -LABEL org.label-schema.url="https://github.com/space-ros" -LABEL org.label-schema.vcs-url="https://github.com/space-ros/docker-images" -LABEL org.label-schema.vcs-ref=${VCS_REF} - -# Disable prompting during package installation -ARG DEBIAN_FRONTEND=noninteractive - -# Define the username and key variables -ENV USERNAME spaceros-user -ENV HOME_DIR=/home/${USERNAME} -ENV SPACEROS_DIR=${HOME_DIR}/spaceros -ARG IKOS_DIR=${HOME_DIR}/ikos -ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" -ENV ROSDISTRO=humble - -# https://github.com/NVIDIA/cuda-repo-management/issues/1 -# Use new GPG keys for nvidia apt repositories that have not yet propagated to base images. -RUN apt-key del 7fa2af80 -RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub - -# Update the Ubuntu software repository -RUN apt-get update - -# Set the locale -RUN apt-get install -y locales -RUN locale-gen en_US en_US.UTF-8 -RUN update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 -ENV LANG=en_US.UTF-8 - -# Add the ROS 2 apt repository -RUN apt-get install -y software-properties-common -RUN add-apt-repository universe -RUN apt-get update && apt-get install -y curl gnupg lsb-release -RUN curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg -RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/ros2.list > /dev/null - -# Install required software development tools and ROS tools (and vim, included for convenience) -RUN apt-get update && apt-get install -y \ - bison \ - build-essential \ - cmake \ - git \ - python3-colcon-common-extensions \ - python3-flake8 \ - python3-pip \ - python3-pytest-cov \ - python3-rosdep \ - python3-setuptools \ - python3-vcstool \ - wget \ - vim - -# Install the required pip packages -RUN python3 -m pip install -U \ - flake8-blind-except \ - flake8-builtins \ - flake8-class-newline \ - flake8-comprehensions \ - flake8-deprecated \ - flake8-docstrings \ - flake8-import-order \ - flake8-quotes \ - pytest-repeat \ - pytest-rerunfailures \ - pytest \ - setuptools - -# Create a spaceros user -RUN useradd -m $USERNAME && \ - echo "$USERNAME:$USERNAME" | chpasswd && \ - usermod --shell /bin/bash $USERNAME && \ - usermod -aG sudo $USERNAME && \ - mkdir -p /etc/sudoers.d && \ - echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/$USERNAME && \ - chmod 0440 /etc/sudoers.d/$USERNAME -USER ${USERNAME} - -## Get the Space ROS source code -RUN mkdir -p ${SPACEROS_DIR}/src -WORKDIR ${SPACEROS_DIR} -ADD --chown=${USERNAME}:${USERNAME} ${REPOS_FILE_URL} ros2.repos -RUN vcs import src < ros2.repos - -# Install system dependencies -RUN sudo rosdep init && \ - rosdep update && \ - rosdep install --from-paths src --ignore-src --rosdistro ${ROSDISTRO} -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" - -# Build the Space ROS source code -RUN colcon build --symlink-install --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - -# Install dependencies needed by the IKOS static analyzer -RUN sudo apt-get install -y \ - clang-9 \ - cmake \ - g++ \ - gcc \ - libboost-dev \ - libboost-filesystem-dev \ - libboost-test-dev \ - libboost-thread-dev \ - libedit-dev \ - libgmp-dev \ - libsqlite3-dev \ - libtbb-dev \ - llvm-9 \ - llvm-9-dev \ - llvm-9-tools \ - python3 \ - python3-pygments \ - zlib1g-dev - -# Get the IKOS source and build it -WORKDIR ${HOME_DIR} -RUN git clone https://github.com/NASA-SW-VnV/ikos.git && cd ikos && git checkout 8b0bab34846b72777cca2a60d473282971806cfc -ENV PYTHONPATH "$PYTHONPATH:${IKOS_DIR}/install/lib/python3.8/site-packages" -WORKDIR ${IKOS_DIR} -RUN mkdir build && cd build && cmake .. -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-9 --no-warn-unused-cli && make && sudo make install -ENV PATH "$PATH:${IKOS_DIR}/install/bin" -ENV IKOS_SCAN_NOTIFIER_FILES "yes" - -# The IKOS installation doesn't install the IKOS egg file, so do it manually (https://github.com/NASA-SW-VnV/ikos/issues/185) -RUN cd ${IKOS_DIR}/install/lib/python3.8/site-packages && sudo python3 /usr/lib/python3/dist-packages/easy_install.py ./ikos-3.0-py3.8.egg - -# Set up the entrypoint -WORKDIR ${SPACEROS_DIR} -COPY ./entrypoint.sh / -ENTRYPOINT ["/entrypoint.sh"] -CMD ["bash"] diff --git a/spaceros/Earthfile b/spaceros/Earthfile new file mode 100644 index 00000000..3548b70f --- /dev/null +++ b/spaceros/Earthfile @@ -0,0 +1,150 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +VERSION 0.6 +FROM ubuntu:jammy + +earthfile: + COPY Earthfile Earthfile + SAVE ARTIFACT Earthfile + +setup: + # Disable prompting during package installation + ARG DEBIAN_FRONTEND=noninteractive + + # The following commands are based on the source install for ROS 2 Rolling Ridley. + # See: https://docs.ros.org/en/ros2_documentation/rolling/Installation/Ubuntu-Development-Setup.html + # The main variation is getting Space ROS sources instead of the Rolling sources. + + # Update the Ubuntu software repository + RUN apt-get update + + # Set the locale + RUN apt-get install -y locales + RUN locale-gen en_US en_US.UTF-8 + RUN update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 + ENV LANG=en_US.UTF-8 + + # Add the ROS 2 apt repository + RUN apt-get install -y software-properties-common + RUN add-apt-repository universe + RUN apt-get update && apt-get install -y curl gnupg lsb-release + RUN curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg + RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/ros2.list > /dev/null + + # Install required software development tools and ROS tools (and vim included for convenience) + RUN apt-get update && apt-get install -y \ + bison \ + build-essential \ + cmake \ + git \ + python3-colcon-common-extensions \ + python3-flake8 \ + python3-flake8-blind-except \ + python3-flake8-builtins \ + python3-flake8-class-newline \ + python3-flake8-comprehensions \ + python3-flake8-deprecated \ + python3-flake8-docstrings \ + python3-flake8-import-order \ + python3-flake8-quotes \ + python3-pip \ + python3-pytest \ + python3-pytest-cov \ + python3-pytest-repeat \ + python3-pytest-rerunfailures \ + python3-rosdep \ + python3-setuptools \ + python3-vcstool \ + wget \ + vim + + # Define the username and key variables + ENV USERNAME spaceros-user + ENV SRC_DIR=/home/${USERNAME}/src + ENV SPACEROS_DIR=${SRC_DIR}/spaceros + ARG IKOS_DIR=${SRC_DIR}/ikos + ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" + ENV ROSDISTRO=humble + + # Create a spaceros user + RUN useradd -m $USERNAME && \ + echo "$USERNAME:$USERNAME" | chpasswd && \ + usermod --shell /bin/bash $USERNAME && \ + usermod -aG sudo $USERNAME && \ + mkdir -p /etc/sudoers.d && \ + echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/$USERNAME && \ + chmod 0440 /etc/sudoers.d/$USERNAME + USER ${USERNAME} + + WORKDIR $SPACEROS_DIR + + +sources: + FROM +setup + COPY github.com/space-ros/space-ros:earthly-wrapper+repos-file/ros2.repos ros2.repos + RUN mkdir src + RUN vcs import src < ros2.repos + SAVE ARTIFACT src AS LOCAL src + +workspace: + FROM +setup + COPY src src/ + +vcs-exact: + FROM +workspace + RUN vcs export --exact src > exact.repos + SAVE ARTIFACT exact.repos AS LOCAL exact.repos + +rosdep: + FROM +workspace + # Install system package dependencies using rosdep + RUN sudo rosdep init && rosdep update + RUN rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" + +build: + FROM +rosdep + RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli + COPY +vcs-exact/exact.repos install/exact.repos + SAVE ARTIFACT install AS LOCAL install + +build-testing: + FROM +rosdep + RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli + RUN colcon test --retest-until-pass 2 --ctest-args -LE "(ikos|xfail)" --pytest-args -m "not xfail" + RUN . install/setup.sh && ros2 run process_sarif make_build_archive + COPY +vcs-exact/exact.repos install/exact.repos + SAVE ARTIFACT log/build_results_archives/build_results*.tar.bz2 AS LOCAL spaceros_ws/log/build_results_archives/ + SAVE ARTIFACT install AS LOCAL install + +image: + FROM +rosdep + + ARG VCS_REF + + # Specify the docker image metadata + LABEL org.label-schema.schema-version="1.0" + LABEL org.label-schema.name="Space ROS" + LABEL org.label-schema.description="Preview version of the Space ROS platform" + LABEL org.label-schema.vendor="Open Robotics" + LABEL org.label-schema.url="https://github.com/space-ros" + LABEL org.label-schema.vcs-url="https://github.com/space-ros/docker-images" + LABEL org.label-schema.vcs-ref=${VCS_REF} + + COPY +build/install ${SPACEROS_DIR}/install + COPY +vcs-exact/exact.repos ${SPACEROS_DIR}/exact.repos + RUN rm -r src + COPY entrypoint.sh /ros_entrypoint.sh + ENTRYPOINT ["/ros_entrypoint.sh"] + CMD ["bash"] + SAVE IMAGE osrf/space-ros:latest diff --git a/spaceros/README.md b/spaceros/README.md index 4b5ce9a7..8ae01855 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -1,11 +1,13 @@ -# Space ROS Docker Image +# Space ROS Docker Image and Earthly configuration -The Dockerfile in this directory builds Space ROS from source code. -To include drivers for NVIDIA GPUs, the Space ROS docker image is based on the NVIDIA CudaGL development image, version 11.4.1, which is, in turn, based on Ubuntu 20.04. +The Earthfile configuration in this directory facilitates builds of Space ROS from source code. +The generated container image is based on Ubuntu 22.04 (Jammy) and can be used with [`rocker`](https://github.com/osrf/rocker) to add X11 and GPU passthrough. ## Building the Docker Image -To build the docker image, run: +The [Earthly](https://earthly.dev/get-earthly) utility is required to build this image. + +To build the image, run: ``` $ ./build.sh @@ -26,11 +28,11 @@ The output will look something like this: ``` $ docker image list REPOSITORY TAG IMAGE ID CREATED SIZE -openrobotics/spaceros latest 629b13cf7b74 1 hour ago 7.8GB -nvidia/cudagl 11.4.1-devel-ubuntu20.04 336416dfcbba 1 day ago 5.35GB +osrf/space-ros latest 109ad8fb7460 4 days ago 2.45GB +ubuntu jammy a8780b506fa4 5 days ago 77.8MB ``` -The new image is named **openrobotics/spaceros:latest**. +The new image is named **osrf/space-ros:latest**. There is a run.sh script provided for convenience that will run the spaceros image in a container. @@ -78,12 +80,21 @@ Commands: Call `ros2 -h` for more detailed usage. ``` -## Running the Space ROS Unit Tests +## Rebuilding Space ROS and running unit tests + +Space ROS sources and intermediate build artifacts are not included in the docker image. -The Space ROS unit tests are not built as part of the Docker image build. To run the unit tests in the container, execute the following command from the top of the Space ROS workspace: +A manifest of the exact sources used to produce the current image is saved as `exact.repos` in the spaceros directory. +To clone all sources from this manifest you can use the command sequence +``` +spaceros-user@d10d85c68f0e:~/spaceros$ mkdir src +spaceros-user@d10d85c68f0e:~/spaceros$ vcs import src < exact.repos +``` +From there you can run a new build and any additional tests. ``` -spaceros-user@d10d85c68f0e:~/spaceros$ colcon test +spaceros-user@d10d85c68f0e:~/spaceros$ colcon build --cmake-args -DCMAKE_BUILD_TYPE=Debug --DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --ctest-args -LE "(ikos|xfail)" --pytest-args -m "not xfail" ``` The tests include running the static analysis tools clang_tidy and cppcheck (which has the MISRA 2012 add-on enabled). diff --git a/spaceros/build.sh b/spaceros/build.sh index e638db2b..199e2873 100755 --- a/spaceros/build.sh +++ b/spaceros/build.sh @@ -4,16 +4,17 @@ ORG=openrobotics IMAGE=spaceros TAG=latest -VCS_REF="" +VCS_REF="$(git rev-parse HEAD)" VERSION=preview echo "" echo "##### Building Space ROS Docker Image #####" echo "" -docker build -t $ORG/$IMAGE:$TAG \ - --build-arg VCS_REF="$VCS_REF" \ - --build-arg VERSION="$VERSION" . +rm -rf src +earthly +sources +earthly +image \ + --VCS_REF="$VCS_REF" echo "" echo "##### Done! #####" diff --git a/spaceros/run.sh b/spaceros/run.sh index 6a868d2a..78a5d9c6 100755 --- a/spaceros/run.sh +++ b/spaceros/run.sh @@ -1,2 +1,2 @@ # Start the container -docker run --rm --gpus all --net=host -e DISPLAY=$DISPLAY --device=/dev/dri:/dev/dri --volume="$HOME/.Xauthority:/home/spaceros-user/.Xauthority:rw" -it openrobotics/spaceros +docker run --rm --net=host -e DISPLAY=$DISPLAY --device=/dev/dri:/dev/dri --volume="$HOME/.Xauthority:/home/spaceros-user/.Xauthority:rw" -it osrf/space-ros From e83ad978fb5c20cfbe3ef9decebb636e0afc8f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Wed, 9 Nov 2022 15:05:58 -0800 Subject: [PATCH 20/54] Warnings in the GitHub actions have suggested updating this. (#37) --- .github/workflows/earthly-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/earthly-build.yaml b/.github/workflows/earthly-build.yaml index 5666f911..1b9ed91c 100644 --- a/.github/workflows/earthly-build.yaml +++ b/.github/workflows/earthly-build.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up earthly run: | sudo wget https://github.com/earthly/earthly/releases/latest/download/earthly-linux-amd64 -O /usr/local/bin/earthly @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up earthly run: | sudo wget https://github.com/earthly/earthly/releases/latest/download/earthly-linux-amd64 -O /usr/local/bin/earthly From 27c4818dd5491b749a205b81c2d8665ed7cbb854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Wed, 9 Nov 2022 15:06:23 -0800 Subject: [PATCH 21/54] Perform image push step in the proper subdirectory. --- .github/workflows/earthly-build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/earthly-build.yaml b/.github/workflows/earthly-build.yaml index 1b9ed91c..bb419224 100644 --- a/.github/workflows/earthly-build.yaml +++ b/.github/workflows/earthly-build.yaml @@ -47,4 +47,5 @@ jobs: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} run: | echo $DOCKER_HUB_TOKEN | docker login --username osrfbot --password-stdin + cd spaceros earthly --ci --push +image From 043d1363dcfb50c69e22a057ef714f4f4e3995b0 Mon Sep 17 00:00:00 2001 From: Dharini Dutia Date: Thu, 17 Nov 2022 10:21:44 -0800 Subject: [PATCH 22/54] Using rocker (#38) * headstart Signed-off-by: Dharini Dutia * updating other demos to use rocker Signed-off-by: Dharini Dutia Signed-off-by: Dharini Dutia --- spaceros/Earthfile | 6 +++--- spaceros/README.md | 2 +- spaceros/build.sh | 4 ---- spaceros/run.sh | 21 ++++++++++++++++++++- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 3548b70f..cb726ce9 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -71,9 +71,9 @@ setup: # Define the username and key variables ENV USERNAME spaceros-user - ENV SRC_DIR=/home/${USERNAME}/src - ENV SPACEROS_DIR=${SRC_DIR}/spaceros - ARG IKOS_DIR=${SRC_DIR}/ikos + ENV HOME_DIR=/home/${USERNAME} + ENV SPACEROS_DIR=${HOME_DIR}/spaceros + ARG IKOS_DIR=${HOME_DIR}/ikos ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" ENV ROSDISTRO=humble diff --git a/spaceros/README.md b/spaceros/README.md index 8ae01855..2934b271 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -93,7 +93,7 @@ spaceros-user@d10d85c68f0e:~/spaceros$ vcs import src < exact.repos From there you can run a new build and any additional tests. ``` -spaceros-user@d10d85c68f0e:~/spaceros$ colcon build --cmake-args -DCMAKE_BUILD_TYPE=Debug --DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli +spaceros-user@d10d85c68f0e:~/spaceros$ colcon build --cmake-args -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --ctest-args -LE "(ikos|xfail)" --pytest-args -m "not xfail" ``` diff --git a/spaceros/build.sh b/spaceros/build.sh index 199e2873..0e46145d 100755 --- a/spaceros/build.sh +++ b/spaceros/build.sh @@ -1,9 +1,5 @@ #!/usr/bin/env bash -ORG=openrobotics -IMAGE=spaceros -TAG=latest - VCS_REF="$(git rev-parse HEAD)" VERSION=preview diff --git a/spaceros/run.sh b/spaceros/run.sh index 78a5d9c6..d4a311ca 100755 --- a/spaceros/run.sh +++ b/spaceros/run.sh @@ -1,2 +1,21 @@ +#!/usr/bin/env bash + +# Runs a docker container with the image created by build.bash +# Requires: +# docker +# an X server +# rocker + + +ROCKER_ARGS="--devices /dev/dri/ --dev-helpers --x11 --user-override-name spaceros-user --network host" + +IMG_NAME=osrf/space-ros + +# Replace `/` with `_` to comply with docker container naming +# And append `_runtime` +CONTAINER_NAME="$(tr '/' '_' <<< "$IMG_NAME")" +ROCKER_ARGS="${ROCKER_ARGS} --name $CONTAINER_NAME" +echo "Using image <$IMG_NAME> to start container <$CONTAINER_NAME>" + # Start the container -docker run --rm --net=host -e DISPLAY=$DISPLAY --device=/dev/dri:/dev/dri --volume="$HOME/.Xauthority:/home/spaceros-user/.Xauthority:rw" -it osrf/space-ros +rocker ${ROCKER_ARGS} $IMG_NAME From b82512c46db872c692e284a1fe35ab3820522bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Mon, 28 Nov 2022 13:21:45 -0800 Subject: [PATCH 23/54] Update image push conditions. (#40) Image pushes weren't being triggered for the scheduled runs. --- .github/workflows/earthly-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/earthly-build.yaml b/.github/workflows/earthly-build.yaml index bb419224..8ff0538a 100644 --- a/.github/workflows/earthly-build.yaml +++ b/.github/workflows/earthly-build.yaml @@ -44,7 +44,7 @@ jobs: - name: Push spaceros image env: DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_RW_TOKEN }} - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + if: ${{ (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/main' }} run: | echo $DOCKER_HUB_TOKEN | docker login --username osrfbot --password-stdin cd spaceros From 65f03bbfb34d5409c9bf10b1be2161aca3c6db3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Wed, 30 Nov 2022 21:19:27 -0800 Subject: [PATCH 24/54] Update GitHub Actions flow (#41) * Fix local artifact location for build result archive. * Restore missing push flag. * Use GITHUB_ENV instead of deprecated set-output. * Error if no artifact is uploaded during build results run. * Save result archive locally for upload. * Configure space-ros image to be pushed. * Try to use environment instead of outputs. --- .github/workflows/earthly-build.yaml | 9 +++++---- spaceros/Earthfile | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/earthly-build.yaml b/.github/workflows/earthly-build.yaml index 8ff0538a..5196fe04 100644 --- a/.github/workflows/earthly-build.yaml +++ b/.github/workflows/earthly-build.yaml @@ -20,13 +20,14 @@ jobs: run: | cd spaceros earthly --ci --output +sources - earthly --ci +build-testing - printf "::set-output name=archivename::%s\n" $(basename log/build_results_archives/build_results_*.tar.bz2) + earthly --ci --output +build-testing + echo "archivename=$(basename log/build_results_archives/build_results_*.tar.bz2)" >> $GITHUB_ENV - name: Upload build results archive uses: actions/upload-artifact@v3 with: - name: ${{steps.build.outputs.archivename}} - path: spaceros/log/build_results_archives/${{steps.build.outputs.archivename}} + name: ${{env.archivename}} + path: spaceros/log/build_results_archives/${{env.archivename}} + if-no-files-found: error space-ros-image: runs-on: ubuntu-latest steps: diff --git a/spaceros/Earthfile b/spaceros/Earthfile index cb726ce9..3acaf9cc 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -124,7 +124,7 @@ build-testing: RUN colcon test --retest-until-pass 2 --ctest-args -LE "(ikos|xfail)" --pytest-args -m "not xfail" RUN . install/setup.sh && ros2 run process_sarif make_build_archive COPY +vcs-exact/exact.repos install/exact.repos - SAVE ARTIFACT log/build_results_archives/build_results*.tar.bz2 AS LOCAL spaceros_ws/log/build_results_archives/ + SAVE ARTIFACT log/build_results_archives/build_results_*.tar.bz2 AS LOCAL log/build_results_archives/ SAVE ARTIFACT install AS LOCAL install image: @@ -147,4 +147,4 @@ image: COPY entrypoint.sh /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] - SAVE IMAGE osrf/space-ros:latest + SAVE IMAGE --push osrf/space-ros:latest From 2a16822fef9ab541c07ae6ee21c3f96f5b7b7894 Mon Sep 17 00:00:00 2001 From: Dharini Dutia Date: Wed, 1 Mar 2023 15:32:18 -0800 Subject: [PATCH 25/54] Updating moveit and canadarm setup (#36) * cmake temp fix and rosinstall_generator) Signed-off-by: Dharini Dutia * removing repos Signed-off-by: Dharini Dutia * moveit tutorials and replaced image for demos Signed-off-by: Dharini Dutia * updates Signed-off-by: Dharini Dutia * shifting to jammy Signed-off-by: Dharini Dutia * update readme Signed-off-by: Dharini Dutia * readme Signed-off-by: Dharini Dutia * cleaning Signed-off-by: Dharini Dutia --------- Signed-off-by: Dharini Dutia --- spaceros/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/spaceros/README.md b/spaceros/README.md index 2934b271..1abf83e7 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -34,6 +34,7 @@ ubuntu jammy a8780b506fa4 5 days ago The new image is named **osrf/space-ros:latest**. +The `rocker` library is required to run the built image, install it by `sudo apt-get install python3-rocker`. There is a run.sh script provided for convenience that will run the spaceros image in a container. ``` From f43bb5634fb86d37a9f37b4ded1b5dc3456eed9c Mon Sep 17 00:00:00 2001 From: Matt Hansen <12618239+mkhansenbot@users.noreply.github.com> Date: Thu, 2 Mar 2023 12:07:19 -0800 Subject: [PATCH 26/54] Start workspace step from end of sources step (#46) Signed-off-by: Matthew Hansen <12618239+mkhansenbot@users.noreply.github.com> --- spaceros/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 3acaf9cc..c0be367a 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -98,7 +98,7 @@ sources: SAVE ARTIFACT src AS LOCAL src workspace: - FROM +setup + FROM +sources COPY src src/ vcs-exact: From 325c109af06faf65c714f7449cc3fa9f43fc0f8e Mon Sep 17 00:00:00 2001 From: Michael Jeronimo Date: Fri, 19 May 2023 08:40:27 -0700 Subject: [PATCH 27/54] Update the OpenGL version to avoid an empty graphics screen. (#50) Fixes #44. Signed-off-by: Michael Jeronimo --- spaceros/Earthfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index c0be367a..442343d8 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -86,9 +86,11 @@ setup: echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/$USERNAME && \ chmod 0440 /etc/sudoers.d/$USERNAME USER ${USERNAME} - WORKDIR $SPACEROS_DIR + # Update the OpenGL version + RUN sudo add-apt-repository ppa:kisak/kisak-mesa + RUN sudo apt update && sudo apt upgrade -y sources: FROM +setup @@ -129,7 +131,6 @@ build-testing: image: FROM +rosdep - ARG VCS_REF # Specify the docker image metadata From 4006ff7f06541cf5ea2ab2d4e168159578463ef0 Mon Sep 17 00:00:00 2001 From: Matt Hansen <12618239+mkhansenbot@users.noreply.github.com> Date: Tue, 15 Aug 2023 00:31:14 +0000 Subject: [PATCH 28/54] Add pipefail to build.sh to ensure script fails if build fails --- spaceros/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spaceros/build.sh b/spaceros/build.sh index 0e46145d..fa877b1d 100755 --- a/spaceros/build.sh +++ b/spaceros/build.sh @@ -3,6 +3,9 @@ VCS_REF="$(git rev-parse HEAD)" VERSION=preview +# Exit script with failure if build fails +set -eo pipefail + echo "" echo "##### Building Space ROS Docker Image #####" echo "" From ed6a638e1a6fac91a949ee57f9b08fa2c8d43297 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sun, 17 Dec 2023 15:00:05 +0100 Subject: [PATCH 29/54] Add refspec #89 --- spaceros/Earthfile | 25 +++++++++++++++++++++---- spaceros/README.md | 6 ++++++ spaceros/build.sh | 1 - 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 442343d8..7d3190df 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -14,6 +14,15 @@ VERSION 0.6 FROM ubuntu:jammy +# current git branch, prefilled by earthly +# https://docs.earthly.dev/docs/earthfile/builtin-args +ARG EARTHLY_GIT_BRANCH + +# can be overriden from cli by passing --GIT_BRANCH=... as an argument +# example -> `earthly +image --SPACEROS_REPO_URL=https://... +ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} +ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" + earthfile: COPY Earthfile Earthfile SAVE ARTIFACT Earthfile @@ -21,7 +30,7 @@ earthfile: setup: # Disable prompting during package installation ARG DEBIAN_FRONTEND=noninteractive - + # The following commands are based on the source install for ROS 2 Rolling Ridley. # See: https://docs.ros.org/en/ros2_documentation/rolling/Installation/Ubuntu-Development-Setup.html # The main variation is getting Space ROS sources instead of the Rolling sources. @@ -74,7 +83,6 @@ setup: ENV HOME_DIR=/home/${USERNAME} ENV SPACEROS_DIR=${HOME_DIR}/spaceros ARG IKOS_DIR=${HOME_DIR}/ikos - ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" ENV ROSDISTRO=humble # Create a spaceros user @@ -94,9 +102,18 @@ setup: sources: FROM +setup - COPY github.com/space-ros/space-ros:earthly-wrapper+repos-file/ros2.repos ros2.repos + RUN mkdir spaceros_repo_dir + + IF[ -z "$(git -C spaceros_repo_dir ls-remote ${SPACEROS_REPO_URL} ${GIT_BRANCH})" ] + RUN echo "branch '${GIT_BRANCH}' does not exist in spaceros repo, cloning main branch" + GIT CLONE ${SPACEROS_REPO_URL} spaceros_repo_dir/ + ELSE + RUN echo "cloning branch '${GIT_BRANCH}' from spaceros repo" + GIT CLONE --branch ${GIT_BRANCH} ${SPACEROS_REPO_URL} spaceros_repo_dir/ + END + RUN mkdir src - RUN vcs import src < ros2.repos + RUN vcs import src < spaceros_repo_dir/ros2.repos SAVE ARTIFACT src AS LOCAL src workspace: diff --git a/spaceros/README.md b/spaceros/README.md index 1abf83e7..75868660 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -15,6 +15,12 @@ $ ./build.sh The build process will take about 20 or 30 minutes, depending on the host computer. +The build process defaults to cloning the `ros2.repos` file from [spaceros](https://github.com/space-ros/space-ros) repository. It looks for a branch with the same name as the current local branch; if it doesn't find one, it falls back to cloning from the main branch. For testing purposes, you can customize both the spaceros repository URL and the branch name by modifying arguments defined in the [Earthfile](./Earthfile). +Example: +```bash +earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.git" +``` + ## Running the Space ROS Docker Image in a Container After building the image, you can see the newly-built image by running: diff --git a/spaceros/build.sh b/spaceros/build.sh index fa877b1d..5a6b2e85 100755 --- a/spaceros/build.sh +++ b/spaceros/build.sh @@ -11,7 +11,6 @@ echo "##### Building Space ROS Docker Image #####" echo "" rm -rf src -earthly +sources earthly +image \ --VCS_REF="$VCS_REF" From 63c5b9bd34aed712592ab2e2d5a4f5affe7cca17 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sun, 17 Dec 2023 15:17:57 +0100 Subject: [PATCH 30/54] simplify comments --- spaceros/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 7d3190df..33c8a850 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -18,10 +18,10 @@ FROM ubuntu:jammy # https://docs.earthly.dev/docs/earthfile/builtin-args ARG EARTHLY_GIT_BRANCH -# can be overriden from cli by passing --GIT_BRANCH=... as an argument +# these can be overriden from cli # example -> `earthly +image --SPACEROS_REPO_URL=https://... -ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" +ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} earthfile: COPY Earthfile Earthfile From e5b4928f2042d5037e3cbd24225e617cd1f5c1b5 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sun, 17 Dec 2023 15:18:49 +0100 Subject: [PATCH 31/54] simplify comments further --- spaceros/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 33c8a850..102610cb 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -18,8 +18,8 @@ FROM ubuntu:jammy # https://docs.earthly.dev/docs/earthfile/builtin-args ARG EARTHLY_GIT_BRANCH -# these can be overriden from cli -# example -> `earthly +image --SPACEROS_REPO_URL=https://... +# these can be overriden from cli. Example: +# `earthly +image --SPACEROS_REPO_URL=https://...` ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} From 1225ce0c1b71deaaa8f4b3a1b7125e7688f90e67 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sun, 17 Dec 2023 15:33:43 +0100 Subject: [PATCH 32/54] revert change to build.sh --- spaceros/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/spaceros/build.sh b/spaceros/build.sh index 5a6b2e85..fa877b1d 100755 --- a/spaceros/build.sh +++ b/spaceros/build.sh @@ -11,6 +11,7 @@ echo "##### Building Space ROS Docker Image #####" echo "" rm -rf src +earthly +sources earthly +image \ --VCS_REF="$VCS_REF" From 00fdbaef571c9daf1c56fb858c51b1bbbc9c54f4 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Mon, 18 Dec 2023 08:12:40 +0100 Subject: [PATCH 33/54] simplify git ls-remote command --- spaceros/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 102610cb..c2a77abd 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -104,7 +104,7 @@ sources: FROM +setup RUN mkdir spaceros_repo_dir - IF[ -z "$(git -C spaceros_repo_dir ls-remote ${SPACEROS_REPO_URL} ${GIT_BRANCH})" ] + IF[ -z "$(git ls-remote ${SPACEROS_REPO_URL} ${GIT_BRANCH})" ] RUN echo "branch '${GIT_BRANCH}' does not exist in spaceros repo, cloning main branch" GIT CLONE ${SPACEROS_REPO_URL} spaceros_repo_dir/ ELSE From a2544fd711b024c227efc53a28351bc537f37a00 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Mon, 18 Dec 2023 16:59:28 +0100 Subject: [PATCH 34/54] temp changes --- spaceros/Earthfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index c2a77abd..d43003d9 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -21,7 +21,7 @@ ARG EARTHLY_GIT_BRANCH # these can be overriden from cli. Example: # `earthly +image --SPACEROS_REPO_URL=https://...` ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" -ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} +ARG SPACEROS_GIT_REF=${EARTHLY_GIT_BRANCH} earthfile: COPY Earthfile Earthfile @@ -104,12 +104,12 @@ sources: FROM +setup RUN mkdir spaceros_repo_dir - IF[ -z "$(git ls-remote ${SPACEROS_REPO_URL} ${GIT_BRANCH})" ] - RUN echo "branch '${GIT_BRANCH}' does not exist in spaceros repo, cloning main branch" + IF[ -z "$(git ls-remote ${SPACEROS_REPO_URL} ${SPACEROS_GIT_REF})" ] + # branch '${SPACEROS_GIT_REF}' does not exist in spaceros repo, cloning main branch GIT CLONE ${SPACEROS_REPO_URL} spaceros_repo_dir/ ELSE - RUN echo "cloning branch '${GIT_BRANCH}' from spaceros repo" - GIT CLONE --branch ${GIT_BRANCH} ${SPACEROS_REPO_URL} spaceros_repo_dir/ + # branch with the same name exists, cloning from it + GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} spaceros_repo_dir/ END RUN mkdir src From 5a96b632432566cde6323e74e9c8879b16853d6b Mon Sep 17 00:00:00 2001 From: xfiderek Date: Tue, 19 Dec 2023 02:53:12 +0100 Subject: [PATCH 35/54] temp --- spaceros/Earthfile | 79 ++++++++++++++++++++++++++++++++++++++-------- spaceros/README.md | 2 +- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index d43003d9..dd834df3 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -18,11 +18,6 @@ FROM ubuntu:jammy # https://docs.earthly.dev/docs/earthfile/builtin-args ARG EARTHLY_GIT_BRANCH -# these can be overriden from cli. Example: -# `earthly +image --SPACEROS_REPO_URL=https://...` -ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" -ARG SPACEROS_GIT_REF=${EARTHLY_GIT_BRANCH} - earthfile: COPY Earthfile Earthfile SAVE ARTIFACT Earthfile @@ -100,20 +95,76 @@ setup: RUN sudo add-apt-repository ppa:kisak/kisak-mesa RUN sudo apt update && sudo apt upgrade -y -sources: - FROM +setup - RUN mkdir spaceros_repo_dir +clone-spaceros-artifacts: + # we must run it in a separate container, so that downstream tasks can be cached if vcs file does not change + FROM ubuntu:jammy + RUN apt-get update && apt-get install -y git wget + + # main purpose of this command is to make sure that the git ls-remote results are not cached + RUN --no-cache echo "Cloning spaceros repo artifacts" + + ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" + # set default spaceros git ref to current branch in local repo + ARG SPACEROS_GIT_REF=${EARTHLY_GIT_BRANCH} # EARTHLY_GIT_BRANCH is populated by earthly + + # if SPACEROS_GIT_REF branch does not exist, artifacts will be cloned from fallback branch + ARG _FALLBACK_BRANCH = "main" + + ARG _GIT_REF_EXISTS = $([ -z $(git ls-remote ${SPACEROS_REPO_URL} ${SPACEROS_GIT_REF}) ] && echo false || echo true) + ARG _MAIN_SPACEROS_REPO_IS_USED = $([ ${SPACEROS_REPO_URL} = "https://github.com/space-ros/space-ros.git" ] && echo true || echo false) + ARG _GIT_REF_OVERRIDEN_BY_USER = $([ ${SPACEROS_GIT_REF} != ${EARTHLY_GIT_BRANCH} ] && echo true || echo false) + + # If main spaceros repo is used then do it using wget to avoid unneccessary file copies + IF [ ${_MAIN_SPACEROS_REPO_IS_USED} ] && [ ${_GIT_REF_EXISTS} ] + RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${SPACEROS_GIT_REF}/ros2.repos + ELSE IF [ $_MAIN_SPACEROS_REPO_IS_USED ] && [ !$_GIT_REF_EXISTS ] && [ !$_GIT_REF_OVERRIDEN_BY_USER ] + RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_FALLBACK_BRANCH}/ros2.repos + ELSE IF [ !$_MAIN_SPACEROS_REPO_IS_USED ] && [ $_GIT_REF_EXISTS ] + GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} + ELSE IF [ !$_MAIN_SPACEROS_REPO_IS_USED ] && [ !$_GIT_REF_EXISTS ] && [ !$_GIT_REF_OVERRIDEN_BY_USER ] + GIT CLONE --branch ${_FALLBACK_BRANCH} ${SPACEROS_REPO_URL} + ELSE + RUN exit 1; + END + + - IF[ -z "$(git ls-remote ${SPACEROS_REPO_URL} ${SPACEROS_GIT_REF})" ] - # branch '${SPACEROS_GIT_REF}' does not exist in spaceros repo, cloning main branch - GIT CLONE ${SPACEROS_REPO_URL} spaceros_repo_dir/ + # ARG _ + + # if branch with the same name does not exist in the spaceros repo, clone from main + # ARG _SPACEROS_GIT_REF_SUBSTITUTED="$( + # if + # then + # echo 'main' + # else + # echo '${SPACEROS_GIT_REF}' + # fi + # ) + # " + + # RUN --no-cache echo $_SPACEROS_GIT_REF_SUBSTITUTED + # $( then echo 'main'; else echo)" + + # we run this command primarily to call `git ls-remote` without caching + + # if branch ${SPACEROS_GIT_REF} does not exist in `spaceros` repo, default to main branch + + # download latest commit. + IF [ ${SPACEROS_REPO_URL} = "https://github.com/space-ros/space-ros.git" ] + + RUN ELSE - # branch with the same name exists, cloning from it - GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} spaceros_repo_dir/ + # otherwise clone whole commit (git clone handles authentication etc) + GIT CLONE --branch ${LATEST_GIT_REF} ${SPACEROS_REPO_URL} . END + SAVE ARTIFACT ros2.repos + +sources: + FROM +setup + COPY +clone-spaceros-artifacts/ros2.repos ros2.repos RUN mkdir src - RUN vcs import src < spaceros_repo_dir/ros2.repos + RUN vcs import src < ros2.repos SAVE ARTIFACT src AS LOCAL src workspace: diff --git a/spaceros/README.md b/spaceros/README.md index 75868660..4cae5ab4 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -18,7 +18,7 @@ The build process will take about 20 or 30 minutes, depending on the host comput The build process defaults to cloning the `ros2.repos` file from [spaceros](https://github.com/space-ros/space-ros) repository. It looks for a branch with the same name as the current local branch; if it doesn't find one, it falls back to cloning from the main branch. For testing purposes, you can customize both the spaceros repository URL and the branch name by modifying arguments defined in the [Earthfile](./Earthfile). Example: ```bash -earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.git" +earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.git" --SPACEROS_GIT_REF="my-branch-name" ``` ## Running the Space ROS Docker Image in a Container From 41bcd89ab35eeb8b3844fff00312632d01b502b2 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Wed, 20 Dec 2023 20:18:15 +0100 Subject: [PATCH 36/54] push working version --- spaceros/Earthfile | 73 +++++++++++----------------------------------- 1 file changed, 17 insertions(+), 56 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index dd834df3..6f553440 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -14,10 +14,6 @@ VERSION 0.6 FROM ubuntu:jammy -# current git branch, prefilled by earthly -# https://docs.earthly.dev/docs/earthfile/builtin-args -ARG EARTHLY_GIT_BRANCH - earthfile: COPY Earthfile Earthfile SAVE ARTIFACT Earthfile @@ -95,73 +91,38 @@ setup: RUN sudo add-apt-repository ppa:kisak/kisak-mesa RUN sudo apt update && sudo apt upgrade -y -clone-spaceros-artifacts: +spaceros-artifacts: # we must run it in a separate container, so that downstream tasks can be cached if vcs file does not change FROM ubuntu:jammy RUN apt-get update && apt-get install -y git wget # main purpose of this command is to make sure that the git ls-remote results are not cached RUN --no-cache echo "Cloning spaceros repo artifacts" - - ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" - # set default spaceros git ref to current branch in local repo - ARG SPACEROS_GIT_REF=${EARTHLY_GIT_BRANCH} # EARTHLY_GIT_BRANCH is populated by earthly - - # if SPACEROS_GIT_REF branch does not exist, artifacts will be cloned from fallback branch - ARG _FALLBACK_BRANCH = "main" - - ARG _GIT_REF_EXISTS = $([ -z $(git ls-remote ${SPACEROS_REPO_URL} ${SPACEROS_GIT_REF}) ] && echo false || echo true) - ARG _MAIN_SPACEROS_REPO_IS_USED = $([ ${SPACEROS_REPO_URL} = "https://github.com/space-ros/space-ros.git" ] && echo true || echo false) - ARG _GIT_REF_OVERRIDEN_BY_USER = $([ ${SPACEROS_GIT_REF} != ${EARTHLY_GIT_BRANCH} ] && echo true || echo false) - - # If main spaceros repo is used then do it using wget to avoid unneccessary file copies - IF [ ${_MAIN_SPACEROS_REPO_IS_USED} ] && [ ${_GIT_REF_EXISTS} ] - RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${SPACEROS_GIT_REF}/ros2.repos - ELSE IF [ $_MAIN_SPACEROS_REPO_IS_USED ] && [ !$_GIT_REF_EXISTS ] && [ !$_GIT_REF_OVERRIDEN_BY_USER ] - RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_FALLBACK_BRANCH}/ros2.repos - ELSE IF [ !$_MAIN_SPACEROS_REPO_IS_USED ] && [ $_GIT_REF_EXISTS ] - GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} - ELSE IF [ !$_MAIN_SPACEROS_REPO_IS_USED ] && [ !$_GIT_REF_EXISTS ] && [ !$_GIT_REF_OVERRIDEN_BY_USER ] - GIT CLONE --branch ${_FALLBACK_BRANCH} ${SPACEROS_REPO_URL} - ELSE - RUN exit 1; - END - - - # ARG _ + # current git branch, prefilled by earthly: https://docs.earthly.dev/docs/earthfile/builtin-args + ARG EARTHLY_GIT_BRANCH - # if branch with the same name does not exist in the spaceros repo, clone from main - # ARG _SPACEROS_GIT_REF_SUBSTITUTED="$( - # if - # then - # echo 'main' - # else - # echo '${SPACEROS_GIT_REF}' - # fi - # ) - # " - - # RUN --no-cache echo $_SPACEROS_GIT_REF_SUBSTITUTED - # $( then echo 'main'; else echo)" - - # we run this command primarily to call `git ls-remote` without caching - - # if branch ${SPACEROS_GIT_REF} does not exist in `spaceros` repo, default to main branch + ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" + # if current local branch does not exist in target repo then use main. note that branch supplied from CLI overrides this behavior. + ARG SPACEROS_GIT_REF="$( [ -n \"$(git ls-remote $SPACEROS_REPO_URL $EARTHLY_GIT_BRANCH)\" ] && echo $EARTHLY_GIT_BRANCH || echo 'main' )" + # get exact commit hash. this makes sure that build will be re-triggered when new commit is pushed + ARG _GIT_COMMIT_HASH = "$(git ls-remote $SPACEROS_REPO_URL $SPACEROS_GIT_REF | cut -f 1)" + + # this means that branch specified by user from CLI does not exist + IF [ -z ${_GIT_COMMIT_HASH} ] + RUN echo "Specified branch ${SPACEROS_GIT_REF} does not exist" && exit 1 + END - # download latest commit. - IF [ ${SPACEROS_REPO_URL} = "https://github.com/space-ros/space-ros.git" ] - - RUN + IF [ $SPACEROS_REPO_URL = "https://github.com/space-ros/space-ros.git" ] + RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_GIT_COMMIT_HASH}/ros2.repos ELSE - # otherwise clone whole commit (git clone handles authentication etc) - GIT CLONE --branch ${LATEST_GIT_REF} ${SPACEROS_REPO_URL} . + GIT CLONE --branch ${_GIT_COMMIT_HASH} ${SPACEROS_REPO_URL} . END SAVE ARTIFACT ros2.repos sources: FROM +setup - COPY +clone-spaceros-artifacts/ros2.repos ros2.repos + COPY +spaceros-artifacts/ros2.repos ros2.repos RUN mkdir src RUN vcs import src < ros2.repos From 21293606b44eebf5e140c711db59c6ed7b60df9c Mon Sep 17 00:00:00 2001 From: xfiderek Date: Wed, 20 Dec 2023 20:20:44 +0100 Subject: [PATCH 37/54] add clarification to wget --- spaceros/Earthfile | 1 + 1 file changed, 1 insertion(+) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 6f553440..192078d8 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -114,6 +114,7 @@ spaceros-artifacts: END IF [ $SPACEROS_REPO_URL = "https://github.com/space-ros/space-ros.git" ] + # run wget because main repo's host is known + it is public RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_GIT_COMMIT_HASH}/ros2.repos ELSE GIT CLONE --branch ${_GIT_COMMIT_HASH} ${SPACEROS_REPO_URL} . From eaffdfdf1e1aa01506936ba37c9def7aac4f4d8d Mon Sep 17 00:00:00 2001 From: Sebastian Castro Date: Thu, 14 Dec 2023 15:37:23 -0500 Subject: [PATCH 38/54] Improve style of shell code blocks in README (#110). This commit styles the READMEs to not include the dollar prompt in code blocks denoting commands to type, to make it easier to copy-and-paste those in a terminal. This also adds bash as the language of the code blocks, so that github styles those correctly. --- spaceros/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/spaceros/README.md b/spaceros/README.md index 4cae5ab4..16420aee 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -9,8 +9,8 @@ The [Earthly](https://earthly.dev/get-earthly) utility is required to build this To build the image, run: -``` -$ ./build.sh +```bash +./build.sh ``` The build process will take about 20 or 30 minutes, depending on the host computer. @@ -25,8 +25,8 @@ earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.g After building the image, you can see the newly-built image by running: -``` -$ docker image list +```bash +docker image list ``` The output will look something like this: @@ -43,8 +43,8 @@ The new image is named **osrf/space-ros:latest**. The `rocker` library is required to run the built image, install it by `sudo apt-get install python3-rocker`. There is a run.sh script provided for convenience that will run the spaceros image in a container. -``` -$ ./run.sh +```bash +./run.sh ``` Upon startup, the container automatically runs the entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: @@ -169,8 +169,8 @@ Sometimes it may be convenient to attach additional terminals to a running Docke With the Space ROS Docker container running, open a second host terminal and then run the following command to determine the container ID: -``` -$ docker container list +```bash +docker container list ``` The output will look something like this: @@ -182,7 +182,7 @@ d10d85c68f0e openrobotics/spaceros "/entrypoint.sh …" 28 minutes ago U The container ID in this case, is *d10d85c68f0e*. So, run the following command in the host terminal: -``` +```bash docker exec -it d10d85c68f0e /bin/bash --init-file "install/setup.bash" ``` From 0dc51c9fbf7aeeb2262de4d2108ce8e27088bc43 Mon Sep 17 00:00:00 2001 From: Sebastian Castro Date: Thu, 11 Jan 2024 04:55:11 -0800 Subject: [PATCH 39/54] Fix style of command name (#114). This commit alters a mention of the `ros2` tool that doesn't use backticks, so it's not correctly formatted. --- spaceros/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spaceros/README.md b/spaceros/README.md index 16420aee..bd953c38 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -53,7 +53,7 @@ Upon startup, the container automatically runs the entrypoint.sh script, which s spaceros-user@d10d85c68f0e:~/spaceros$ ``` -At this point, you can run the 'ros2' command line utility to make sure everything is working OK: +At this point, you can run the `ros2` command line utility to make sure everything is working OK: ``` spaceros-user@d10d85c68f0e:~/spaceros$ ros2 From 0208ff89764d951c37a1cea5d78fd6b525bd256f Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Thu, 11 Jan 2024 04:58:02 -0800 Subject: [PATCH 40/54] Fix style of command name, command options (#114). This commit alters a mention of the `colcon` tool and an option for it that don't use backticks, so it's not correctly formatted. --- spaceros/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/spaceros/README.md b/spaceros/README.md index bd953c38..55066c58 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -106,7 +106,7 @@ spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --ctest-args -LE "(ikos|xfail The tests include running the static analysis tools clang_tidy and cppcheck (which has the MISRA 2012 add-on enabled). -You can use colcon's --packages-select option to run a subset of packages. For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: +You can use colcon's `--packages-select` option to run a subset of packages. For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: ``` spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --event-handlers console_direct+ --packages-select rcpputils @@ -221,10 +221,8 @@ To generate a JUnit XML file for a specific package only, you can add the *--pac spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils ``` -The 'colcon test' command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. After running 'colcon test', you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: +The `colcon test` command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. After running `colcon test`, you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: ``` spaceros-user@d10d85c68f0e:~/spaceros$ more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml - ``` - From 10d5cbfd416928bc57fee51ab9fcf61490a77820 Mon Sep 17 00:00:00 2001 From: Sebastian Castro Date: Thu, 11 Jan 2024 05:09:23 -0800 Subject: [PATCH 41/54] Place sentences in separate lines in READMEs (#113). Having long sentences in READMEs makes them harder to change, and also makes processing the diffs harder (among other disadvantages). This commit implements the one-sentence-per-line principle, by placing different sentences in separate lines in the README. --- spaceros/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/spaceros/README.md b/spaceros/README.md index 55066c58..52bfb857 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -19,7 +19,7 @@ The build process defaults to cloning the `ros2.repos` file from [spaceros](http Example: ```bash earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.git" --SPACEROS_GIT_REF="my-branch-name" -``` +``` ## Running the Space ROS Docker Image in a Container @@ -47,7 +47,8 @@ There is a run.sh script provided for convenience that will run the spaceros ima ./run.sh ``` -Upon startup, the container automatically runs the entrypoint.sh script, which sources the Space ROS environment file (setup.bash). You'll now be running inside the container and should see a prompt similar to this: +Upon startup, the container automatically runs the entrypoint.sh script, which sources the Space ROS environment file (setup.bash). +You'll now be running inside the container and should see a prompt similar to this: ``` spaceros-user@d10d85c68f0e:~/spaceros$ @@ -221,7 +222,8 @@ To generate a JUnit XML file for a specific package only, you can add the *--pac spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils ``` -The `colcon test` command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. After running `colcon test`, you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: +The `colcon test` command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. +After running `colcon test`, you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: ``` spaceros-user@d10d85c68f0e:~/spaceros$ more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml From caf1d41d821729b4f8bc3d0a8c5a2719915f7f25 Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Thu, 11 Jan 2024 05:13:11 -0800 Subject: [PATCH 42/54] Place sentences in separate lines in READMEs (#113). Having long sentences in READMEs makes them harder to change, and also makes processing the diffs harder (among other disadvantages). This commit implements the one-sentence-per-line principle, by placing different sentences in separate lines in the README. --- spaceros/README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/spaceros/README.md b/spaceros/README.md index 52bfb857..ad846317 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -15,7 +15,9 @@ To build the image, run: The build process will take about 20 or 30 minutes, depending on the host computer. -The build process defaults to cloning the `ros2.repos` file from [spaceros](https://github.com/space-ros/space-ros) repository. It looks for a branch with the same name as the current local branch; if it doesn't find one, it falls back to cloning from the main branch. For testing purposes, you can customize both the spaceros repository URL and the branch name by modifying arguments defined in the [Earthfile](./Earthfile). +The build process defaults to cloning the `ros2.repos` file from [spaceros](https://github.com/space-ros/space-ros) repository. +It looks for a branch with the same name as the current local branch; if it doesn't find one, it falls back to cloning from the main branch. +For testing purposes, you can customize both the spaceros repository URL and the branch name by modifying arguments defined in the [Earthfile](./Earthfile). Example: ```bash earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.git" --SPACEROS_GIT_REF="my-branch-name" @@ -107,7 +109,8 @@ spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --ctest-args -LE "(ikos|xfail The tests include running the static analysis tools clang_tidy and cppcheck (which has the MISRA 2012 add-on enabled). -You can use colcon's `--packages-select` option to run a subset of packages. For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: +You can use colcon's `--packages-select` option to run a subset of packages. +For example, to run tests only for the rcpputils package and display the output directly to the console (as well as saving it to a log file), you can run: ``` spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --event-handlers console_direct+ --packages-select rcpputils @@ -115,7 +118,8 @@ spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --event-handlers console_dire ## Viewing Test Output - The output from the tests are stored in XUnit XML files, named *\*.xunit.xml. After running the unit tests, you can scan the build directory for the various *\*.xunit.xml* files. + The output from the tests are stored in XUnit XML files, named *\*.xunit.xml. +After running the unit tests, you can scan the build directory for the various *\*.xunit.xml* files. For example, a clang_tidy.xunit.xml file looks like this: @@ -181,7 +185,8 @@ CONTAINER ID IMAGE COMMAND CREATED d10d85c68f0e openrobotics/spaceros "/entrypoint.sh …" 28 minutes ago Up 28 minutes inspiring_moser ``` -The container ID in this case, is *d10d85c68f0e*. So, run the following command in the host terminal: +The container ID in this case, is *d10d85c68f0e*. +So, run the following command in the host terminal: ```bash docker exec -it d10d85c68f0e /bin/bash --init-file "install/setup.bash" @@ -223,7 +228,8 @@ spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --ins ``` The `colcon test` command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. -After running `colcon test`, you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: +After running `colcon test`, you can view the JUnit XML files. +For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: ``` spaceros-user@d10d85c68f0e:~/spaceros$ more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml From dfe1da70ed236602da20fbc10081f967b1e3cacc Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sat, 13 Jan 2024 13:45:58 +0100 Subject: [PATCH 43/54] Ensure that cobra is executed during tests (#103). Space ROS's main Earthfile's build-testing procedure is invoked in CI to push results of tests and static code analysis. Currently, for all packages cobra-autosar.sarif results are empty. This is because cobra or some dependent executables are not found when colcon test command is invoked. This commit prefixes the call to colcon test in the Earthfile by a call to source that brings all the necessary tools and dependencies into scope. --- spaceros/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 192078d8..7fadc647 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -153,7 +153,7 @@ build: build-testing: FROM +rosdep RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli - RUN colcon test --retest-until-pass 2 --ctest-args -LE "(ikos|xfail)" --pytest-args -m "not xfail" + RUN . install/setup.sh && colcon test --retest-until-pass 2 --ctest-args -LE "(ikos|xfail)" --pytest-args -m "not xfail" RUN . install/setup.sh && ros2 run process_sarif make_build_archive COPY +vcs-exact/exact.repos install/exact.repos SAVE ARTIFACT log/build_results_archives/build_results_*.tar.bz2 AS LOCAL log/build_results_archives/ From fe115ad3c231e089402f0bea31e15d9eecc6f54e Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Thu, 25 Jan 2024 04:59:56 -0800 Subject: [PATCH 44/54] Make IKOS available in the spaceros image (#99). IKOS is one of the tools that Space ROS includes to perform static analysis of ROS 2 applications. Although ament_ikos is provided with the docker image, IKOS itself is not, rendering the former useless and leaving it to users to compile and install IKOS prior to being able to use it. This commit modifies the spaceros Earthfile to install IKOS globally in the system. --- spaceros/Earthfile | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 7fadc647..3345970d 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -73,7 +73,6 @@ setup: ENV USERNAME spaceros-user ENV HOME_DIR=/home/${USERNAME} ENV SPACEROS_DIR=${HOME_DIR}/spaceros - ARG IKOS_DIR=${HOME_DIR}/ikos ENV ROSDISTRO=humble # Create a spaceros user @@ -144,6 +143,28 @@ rosdep: RUN sudo rosdep init && rosdep update RUN rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" + RUN sudo apt-get install --yes \ + gcc g++ cmake libgmp-dev libboost-dev libboost-filesystem-dev \ + libboost-thread-dev libboost-test-dev \ + libsqlite3-dev libtbb-dev libz-dev libedit-dev \ + python3 python3-pip python3-venv \ + llvm-14 llvm-14-dev llvm-14-tools clang-14 + + WORKDIR $SPACEROS_DIR + RUN git clone -b v3.2 --depth 1 https://github.com/NASA-SW-VnV/ikos.git + WORKDIR $SPACEROS_DIR/ikos + RUN mkdir build + WORKDIR $SPACEROS_DIR/ikos/build + RUN cmake \ + -DCMAKE_INSTALL_PREFIX="/opt/ikos" \ + -DCMAKE_BUILD_TYPE="Debug" \ + -DLLVM_CONFIG_EXECUTABLE="/usr/lib/llvm-14/bin/llvm-config" \ + .. + RUN make + RUN sudo make install + WORKDIR $SPACEROS_DIR + RUN sudo rm -rf ikos/ + build: FROM +rosdep RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli From ae6bf4e262992351f710aa98c62e039167f5571b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82a=C5=BCej=20Fiderek?= <57559956+xfiderek@users.noreply.github.com> Date: Sat, 27 Jan 2024 00:24:05 +0100 Subject: [PATCH 45/54] Make excluded pkgs consistent across rosdep and vcs (#88) * Make excluded pkgs consistent across rosdep and vcs. * Add ikos to skip-keys according to discussion https://github.com/space-ros/space-ros/discussions/126#discussioncomment-8247215 Reuse 'excluded-pkgs.txt' for both vcs checkout as well as rosdep. --- spaceros/Earthfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 3345970d..ab71cf54 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -115,14 +115,17 @@ spaceros-artifacts: IF [ $SPACEROS_REPO_URL = "https://github.com/space-ros/space-ros.git" ] # run wget because main repo's host is known + it is public RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_GIT_COMMIT_HASH}/ros2.repos + RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_GIT_COMMIT_HASH}/excluded-pkgs.txt ELSE GIT CLONE --branch ${_GIT_COMMIT_HASH} ${SPACEROS_REPO_URL} . END SAVE ARTIFACT ros2.repos + SAVE ARTIFACT excluded-pkgs.txt sources: FROM +setup COPY +spaceros-artifacts/ros2.repos ros2.repos + COPY +spaceros-artifacts/excluded-pkgs.txt excluded-pkgs.txt RUN mkdir src RUN vcs import src < ros2.repos @@ -141,7 +144,14 @@ rosdep: FROM +workspace # Install system package dependencies using rosdep RUN sudo rosdep init && rosdep update - RUN rosdep install --from-paths src --ignore-src --rosdistro rolling -y --skip-keys "console_bridge fastcdr fastrtps rti-connext-dds-5.3.1 urdfdom_headers rmw_connextdds ros_testing rmw_connextdds rmw_fastrtps_cpp rmw_fastrtps_dynamic_cpp composition demo_nodes_py lifecycle rosidl_typesupport_fastrtps_cpp rosidl_typesupport_fastrtps_c ikos" + RUN rosdep install -y \ + --from-paths src --ignore-src \ + --rosdistro ${ROSDISTRO} \ + # `urdfdom_headers` is cloned from source, however rosdep can't find it. + # It is because package.xml manifest is missing. See: https://github.com/ros/urdfdom_headers + # Additionally, IKOS must be excluded as per: https://github.com/space-ros/docker/issues/99 + --skip-keys "$(tr '\n' ' ' < 'excluded-pkgs.txt') urdfdom_headers ikos" + RUN rm excluded-pkgs.txt RUN sudo apt-get install --yes \ gcc g++ cmake libgmp-dev libboost-dev libboost-filesystem-dev \ From 461060958c9252c6b5b1c103fd1ac5d8c3122292 Mon Sep 17 00:00:00 2001 From: Ivan Perez Date: Wed, 31 Jan 2024 19:37:26 -0800 Subject: [PATCH 46/54] Update GA to push image to Docker Hub only on tags (#93). This commit disables pushes of the spaceros docker image up to Docker Hub on the main branch, and it enables pushes on tags. It also enables manually triggering the build job. This will help ensure that only stable version are pushed to Docker Hub. --- .github/workflows/earthly-build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/earthly-build.yaml b/.github/workflows/earthly-build.yaml index 5196fe04..8b70376e 100644 --- a/.github/workflows/earthly-build.yaml +++ b/.github/workflows/earthly-build.yaml @@ -1,7 +1,9 @@ name: Earthly build on: + workflow_dispatch: push: branches: ['main'] + tags: ['*'] pull_request: schedule: - cron: '0 11 * * *' @@ -45,7 +47,7 @@ jobs: - name: Push spaceros image env: DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_RW_TOKEN }} - if: ${{ (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/main' }} + if: ${{ github.ref_type == 'tag' }} run: | echo $DOCKER_HUB_TOKEN | docker login --username osrfbot --password-stdin cd spaceros From c3133838f06dc367cf1aaf3ac88de50768255602 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Mon, 19 Feb 2024 20:46:36 +0100 Subject: [PATCH 47/54] Ensure that Earthly pulls latest changes from spaceros repos (#134). This commit modifies Earthfile to check whether content of unpinned packages in `ros2.repos` file changed. Prior to this commit, updates to spaceros repos had not been reflected in subsequent builds on the same machine. --- spaceros/Earthfile | 30 ++++++++++++++++++------------ spaceros/README.md | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index ab71cf54..d775062d 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -92,8 +92,7 @@ setup: spaceros-artifacts: # we must run it in a separate container, so that downstream tasks can be cached if vcs file does not change - FROM ubuntu:jammy - RUN apt-get update && apt-get install -y git wget + FROM +setup # main purpose of this command is to make sure that the git ls-remote results are not cached RUN --no-cache echo "Cloning spaceros repo artifacts" @@ -113,22 +112,30 @@ spaceros-artifacts: END IF [ $SPACEROS_REPO_URL = "https://github.com/space-ros/space-ros.git" ] - # run wget because main repo's host is known + it is public + # run wget instead of `GIT CLONE`, because main repo's host is known + it is public RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_GIT_COMMIT_HASH}/ros2.repos RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_GIT_COMMIT_HASH}/excluded-pkgs.txt ELSE - GIT CLONE --branch ${_GIT_COMMIT_HASH} ${SPACEROS_REPO_URL} . + GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} . END - SAVE ARTIFACT ros2.repos + + RUN mkdir src + # we run vcstool inside this task, because some packages in `ros2.repos` are not pinned and otherwise earthly won't pull latest changes + RUN vcs import src < ros2.repos + RUN vcs export --exact src > exact.repos + RUN rm -rf src + + SAVE ARTIFACT ros2.repos # vcs pulled from `spaceros` repo as-is + SAVE ARTIFACT exact.repos # `ros2.repos`, but with pinned versions (e.g. SHAs instead of branches) SAVE ARTIFACT excluded-pkgs.txt sources: FROM +setup COPY +spaceros-artifacts/ros2.repos ros2.repos COPY +spaceros-artifacts/excluded-pkgs.txt excluded-pkgs.txt - + COPY +spaceros-artifacts/exact.repos exact.repos RUN mkdir src - RUN vcs import src < ros2.repos + RUN vcs import src < exact.repos SAVE ARTIFACT src AS LOCAL src workspace: @@ -136,8 +143,7 @@ workspace: COPY src src/ vcs-exact: - FROM +workspace - RUN vcs export --exact src > exact.repos + COPY +spaceros-artifacts/exact.repos exact.repos SAVE ARTIFACT exact.repos AS LOCAL exact.repos rosdep: @@ -178,7 +184,7 @@ rosdep: build: FROM +rosdep RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli - COPY +vcs-exact/exact.repos install/exact.repos + COPY +spaceros-artifacts/exact.repos install/exact.repos SAVE ARTIFACT install AS LOCAL install build-testing: @@ -186,7 +192,7 @@ build-testing: RUN colcon build --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON --no-warn-unused-cli RUN . install/setup.sh && colcon test --retest-until-pass 2 --ctest-args -LE "(ikos|xfail)" --pytest-args -m "not xfail" RUN . install/setup.sh && ros2 run process_sarif make_build_archive - COPY +vcs-exact/exact.repos install/exact.repos + COPY +spaceros-artifacts/exact.repos install/exact.repos SAVE ARTIFACT log/build_results_archives/build_results_*.tar.bz2 AS LOCAL log/build_results_archives/ SAVE ARTIFACT install AS LOCAL install @@ -204,7 +210,7 @@ image: LABEL org.label-schema.vcs-ref=${VCS_REF} COPY +build/install ${SPACEROS_DIR}/install - COPY +vcs-exact/exact.repos ${SPACEROS_DIR}/exact.repos + COPY +spaceros-artifacts/exact.repos ${SPACEROS_DIR}/exact.repos RUN rm -r src COPY entrypoint.sh /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] diff --git a/spaceros/README.md b/spaceros/README.md index ad846317..0aa91dc3 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -234,3 +234,20 @@ For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries ``` spaceros-user@d10d85c68f0e:~/spaceros$ more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml ``` + +## Saving build artifacts locally +`./build.sh` script executes the Earthly `build` task defined in `Earthfile`, which copies specified artifacts from the container to the host. Additional tasks defined in the Earthfile enable the preservation of intermediate artifacts generated during the build process. + +For instance, the `vcs-exact` task can be utilized to copy the exact version of ROS packages pulled during the spaceros build. + +To invoke a task, use the following syntax: +```bash +earthly +TASK_NAME +``` + +Example: +``` +earthly +vcs-exact +``` + +Refer to the [Earthfile](./Earthfile) for tasks that save artifacts locally on the host. Such tasks end with the `SAVE ARTIFACT .. AS LOCAL` syntax. From 8d3724c2b9ef16ecd5443dd7f749dd6fce53b9e7 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Tue, 23 Apr 2024 18:52:00 +0200 Subject: [PATCH 48/54] Add path to IKOS binaries to PATH variable in Earthfile (#138). --- spaceros/Earthfile | 1 + 1 file changed, 1 insertion(+) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index d775062d..fc84e30e 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -178,6 +178,7 @@ rosdep: .. RUN make RUN sudo make install + ENV PATH="/opt/ikos/bin/:$PATH" WORKDIR $SPACEROS_DIR RUN sudo rm -rf ikos/ From 506483960b91d71d445276f4ec8e9eacc5a1af57 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Tue, 23 Apr 2024 18:54:02 +0200 Subject: [PATCH 49/54] Add IKOS_SCAN_NOTIFIER_FILES to entrypoint.sh (#138). Add environment variable to entrypoint of spaceros image. This variable is required for ikos-scan to execute and generate .ikosbin files without having to run the ikos-scan server. --- spaceros/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/spaceros/entrypoint.sh b/spaceros/entrypoint.sh index 4349ddba..93ea40ff 100755 --- a/spaceros/entrypoint.sh +++ b/spaceros/entrypoint.sh @@ -3,4 +3,5 @@ set -e # Setup the Space ROS environment source "${SPACEROS_DIR}/install/setup.bash" +export IKOS_SCAN_NOTIFIER_FILES="" # make ikos create .ikosbin files for compiled packages exec "$@" From 83c6a87baf42985c92ad7cb6d1d41e74ae6e7875 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Tue, 23 Apr 2024 18:55:54 +0200 Subject: [PATCH 50/54] Update instructions on running ikos-scan in README (#138). The current instructions specify how to re-run all tests, which takes substantial time. This change explains how to run ikos-scan specifically, which should be faster than running all tests. --- spaceros/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/spaceros/README.md b/spaceros/README.md index 0aa91dc3..109d1324 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -215,19 +215,19 @@ spaceros-user@d10d85c68f0e:~/spaceros$ CC="ikos-scan-cc" CXX="ikos-scan-c++" LD= ## Generating IKOS Results -To generate JUnit XML files for all of the binaries resulting from the build command in the previous step, you can use **colcon test**, as follows: +To generate JUnit XML/SARIF files for all of the binaries resulting from the build command in the previous step, you can use **colcon test**, as follows: ``` -spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --install-base install_ikos +spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --install-base install_ikos --ctest-args -L "ikos" ``` To generate a JUnit XML file for a specific package only, you can add the *--packages-select* option, as follows: ``` -spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --install-base install_ikos --packages-select rcpputils +spaceros-user@d10d85c68f0e:~/spaceros$ colcon test --build-base build_ikos --install-base install_ikos --ctest-args -L "ikos" --packages-select rcpputils ``` -The `colcon test` command runs various tests, including IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. +The `colcon test` command with the `-L "ikos"` flag runs IKOS report generation, which reads the IKOS database generated in the previous analysis step and generates a JUnit XML report file. After running `colcon test`, you can view the JUnit XML files. For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries you can use the following command: @@ -235,6 +235,11 @@ For example, to view the JUnit XML file for IKOS scan of the rcpputils binaries spaceros-user@d10d85c68f0e:~/spaceros$ more build_ikos/rcpputils/test_results/rcpputils/ikos.xunit.xml ``` +SARIF files are also available in the same path: +``` +spaceros-user@d10d85c68f0e:~/spaceros$ more build_ikos/rcpputils/test_results/rcpputils/ikos.sarif +``` + ## Saving build artifacts locally `./build.sh` script executes the Earthly `build` task defined in `Earthfile`, which copies specified artifacts from the container to the host. Additional tasks defined in the Earthfile enable the preservation of intermediate artifacts generated during the build process. From ebd1d0bfeb15d3706e3ebbe87c8f1a26521993bd Mon Sep 17 00:00:00 2001 From: Matthew Hansen <12618239+mkhansenbot@users.noreply.github.com> Date: Mon, 29 Apr 2024 21:38:34 +0000 Subject: [PATCH 51/54] Remove rocker - fix issue #146 --- spaceros/README.md | 3 +-- spaceros/run.sh | 10 ++-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/spaceros/README.md b/spaceros/README.md index 109d1324..c572e6d0 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -1,7 +1,7 @@ # Space ROS Docker Image and Earthly configuration The Earthfile configuration in this directory facilitates builds of Space ROS from source code. -The generated container image is based on Ubuntu 22.04 (Jammy) and can be used with [`rocker`](https://github.com/osrf/rocker) to add X11 and GPU passthrough. +The generated container image is based on Ubuntu 22.04 (Jammy) ## Building the Docker Image @@ -42,7 +42,6 @@ ubuntu jammy a8780b506fa4 5 days ago The new image is named **osrf/space-ros:latest**. -The `rocker` library is required to run the built image, install it by `sudo apt-get install python3-rocker`. There is a run.sh script provided for convenience that will run the spaceros image in a container. ```bash diff --git a/spaceros/run.sh b/spaceros/run.sh index d4a311ca..7b6845c0 100755 --- a/spaceros/run.sh +++ b/spaceros/run.sh @@ -4,18 +4,12 @@ # Requires: # docker # an X server -# rocker - - -ROCKER_ARGS="--devices /dev/dri/ --dev-helpers --x11 --user-override-name spaceros-user --network host" IMG_NAME=osrf/space-ros - # Replace `/` with `_` to comply with docker container naming # And append `_runtime` CONTAINER_NAME="$(tr '/' '_' <<< "$IMG_NAME")" -ROCKER_ARGS="${ROCKER_ARGS} --name $CONTAINER_NAME" -echo "Using image <$IMG_NAME> to start container <$CONTAINER_NAME>" # Start the container -rocker ${ROCKER_ARGS} $IMG_NAME +docker run --rm -it --name $CONTAINER_NAME --network host \ + -e DISPLAY -e TERM -e QT_X11_NO_MITSHM=1 $IMG_NAME From 0a8dfaee1740d3616aa43b3bd606c23f1b0f3258 Mon Sep 17 00:00:00 2001 From: Matthew Hansen <12618239+mkhansenbot@users.noreply.github.com> Date: Wed, 1 May 2024 15:33:40 +0000 Subject: [PATCH 52/54] Add image tag to Earthfile (issue #148) --- spaceros/Earthfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index fc84e30e..2b991c4e 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -200,6 +200,7 @@ build-testing: image: FROM +rosdep ARG VCS_REF + ARG tag='latest' # Specify the docker image metadata LABEL org.label-schema.schema-version="1.0" @@ -216,4 +217,4 @@ image: COPY entrypoint.sh /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] - SAVE IMAGE --push osrf/space-ros:latest + SAVE IMAGE --push osrf/space-ros:$tag From 72befbedd9612b8b128f1cc78538b43245b685b3 Mon Sep 17 00:00:00 2001 From: Matthew Hansen <12618239+mkhansenbot@users.noreply.github.com> Date: Wed, 1 May 2024 17:12:48 +0000 Subject: [PATCH 53/54] Automatically tag the image when pushing to Dockerhub (issue #148) --- .github/workflows/earthly-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/earthly-build.yaml b/.github/workflows/earthly-build.yaml index 8b70376e..58e93544 100644 --- a/.github/workflows/earthly-build.yaml +++ b/.github/workflows/earthly-build.yaml @@ -51,4 +51,4 @@ jobs: run: | echo $DOCKER_HUB_TOKEN | docker login --username osrfbot --password-stdin cd spaceros - earthly --ci --push +image + earthly --ci --push +image --tag=${{ github.ref_name }} From 326f3f554299689adb93d17472ddaa3d996f6f23 Mon Sep 17 00:00:00 2001 From: Matthew Hansen <12618239+mkhansenbot@users.noreply.github.com> Date: Thu, 2 May 2024 15:47:06 +0000 Subject: [PATCH 54/54] Push image with latest tag and release tag (issue #148) --- spaceros/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 2b991c4e..c2cb6524 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -217,4 +217,4 @@ image: COPY entrypoint.sh /ros_entrypoint.sh ENTRYPOINT ["/ros_entrypoint.sh"] CMD ["bash"] - SAVE IMAGE --push osrf/space-ros:$tag + SAVE IMAGE --push osrf/space-ros:latest osrf/space-ros:$tag