added automated merge based on rostests

.ci/build.sh

@@ -1,24 +1,40 @@
 #!/bin/bash
 
+# This script will build ROS packages from a particular repository into
+# deb packages
+#
+# INPUT:
+# * ./build.sh <package list file name> <{stable/testing/unstable} variant> <repository name>
+# * /tmp/artifacts containts the build artifacts from the previous jobs
+# * /tmp/artifacts/idx.txt contains idx of the previous job
+# * /tmp/artifacts/generated_***_***.yaml rosdep file from the previous jobs
+
+# OUTPUT:
+# * deb packages are put into the "/tmp/artifacts/$IDX" folder where $IDX is the incremented iterator of this build
+# * /tmp/idx.txt with incremented value of $IDX
+# * /tmp/artifacts/generated_***_***.yaml rosdep file with updated definitions from this build
+
 set -e
 
 trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
 trap 'echo "$0: \"${last_command}\" command failed with exit code $?"' ERR
 
+# INPUTS
 LIST=$1
 VARIANT=$2
-PACKAGE_NAME=$3
+REPOSITORY=$3
 WORKSPACE=/tmp/workspace
 ARTIFACTS_FOLDER=/tmp/artifacts
 IDX_FILE=$ARTIFACTS_FOLDER/idx.txt
+ROSDEP_FILE="$ARTIFACTS_FOLDER/generated_${LIST}_${ARCH}.yaml"
 
 YAML_FILE=${LIST}.yaml
 
 # needed for building open_vins
 export ROS_VERSION=1
 
+# determine our architecture
 sudo apt-get -y install dpkg-dev
-
 ARCH=$(dpkg-architecture -qDEB_HOST_ARCH)
 
 # we already have a docker image with ros for the ARM build
@@ -33,8 +49,9 @@ sudo apt-get -y install ros-noetic-catkin python3-catkin-tools
 sudo apt-get -y install fakeroot debhelper
 sudo pip3 install -U bloom
 
-REPOS=$(./.ci/get_repo_source.py $YAML_FILE $VARIANT $ARCH $PACKAGE_NAME)
+REPOS=$(./.ci/get_repo_source.py $YAML_FILE $VARIANT $ARCH $REPOSITORY)
 
+# increment the job's idx
 IDX=`cat "$IDX_FILE"`
 IDX=$(($IDX+1))
 
@@ -86,8 +103,6 @@ echo "$0: catking reported following topological build order:"
 echo "$BUILD_ORDER"
 echo ""
 
-ROSDEP_FILE="$ARTIFACTS_FOLDER/generated_${LIST}_${ARCH}.yaml"
-
 cat $ROSDEP_FILE
 
 if [ -s $ROSDEP_FILE ]; then

.ci/get_repo_source.py

@@ -28,10 +28,14 @@ def main():
 
         if build_for in architecture:
 
+            refs = properties['git_refs']
+
             if variant == "stable":
-                ref = properties['stable_ref'] 
+                ref = refs['stable'] 
+            elif variant == "unstable":
+                ref = refs['unstable'] 
             else:
-                ref = properties['unstable_ref'] 
+                ref = refs['testing'] 
 
             print("{} {} {}".format(repo_name, url, ref))
 

.ci/get_test_matrix.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
+trap 'echo "$0: \"${last_command}\" command failed with exit code $?' ERR
+
+DEBUG=false
+
+LIST=mrs
+ARCH=amd64
+
+YAML_FILE=$LIST.yaml
+
+REPOS=$(./.ci/parse_yaml.py $YAML_FILE $ARCH)
+
+FIRST=true
+
+echo -n "["
+
+echo "$REPOS" | while IFS= read -r REPO; do
+
+  $DEBUG && echo "Cloning $REPO"
+
+  PACKAGE=$(echo "$REPO" | awk '{print $1}')
+  URL=$(echo "$REPO" | awk '{print $2}')
+  TEST=$(echo "$REPO" | awk '{print $6}')
+
+  if [[ "$TEST" != "True" ]]; then
+    continue
+  fi
+
+  if $FIRST; then
+    echo -n "\"$PACKAGE\""
+    FIRST=false
+  else
+    echo -n ", \"$PACKAGE\""
+  fi
+
+done
+
+echo "]"

.ci/merge_push_to_release.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+set -e
+
+trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
+trap 'echo "$0: \"${last_command}\" command failed with exit code $?' ERR
+
+ARCH=amd64
+LOCATION=/tmp/git
+
+REPOS=$(./.ci/parse_yaml.py mrs.yaml $ARCH)
+
+REPOS="$REPOS
+$(./.ci/parse_yaml.py thirdparty.yaml $ARCH)"
+
+REPOS="$REPOS
+$(./.ci/parse_yaml.py nonbloom.yaml $ARCH)"
+
+[ -e $LOCATION ] && rm -rf $LOCATION || echo "$0: nothing to delete"
+mkdir -p $LOCATION
+
+cd $LOCATION
+
+# clone and checkout
+echo "$REPOS" | while IFS= read -r REPO; do
+
+  echo "Cloning $REPO"
+
+  PACKAGE=$(echo "$REPO" | awk '{print $1}')
+  URL=$(echo "$REPO" | awk '{print $2}')
+
+  # strip the https from the url
+  URL=$(echo $URL | sed -r 's|https://(.+)|\1|' | head -n 1)
+
+  RELEASE_BRANCH=$(echo "$REPO" | awk '{print $3}')
+  TESTING_BRANCH=$(echo "$REPO" | awk '{print $4}')
+
+  echo "$0: cloning '$URL --branch $RELEASE_BRANCH' into '$PACKAGE'"
+  [ ! -e $PACKAGE ] && git clone https://$PUSH_TOKEN@$URL --branch $RELEASE_BRANCH $PACKAGE
+
+  echo ""
+
+done
+
+echo "$0: Done cloning"
+echo "$0: Going to merge and dry-run the push"
+
+# merge and try dry-run push
+echo "$REPOS" | while IFS= read -r REPO; do
+
+  PACKAGE=$(echo "$REPO" | awk '{print $1}')
+  TESTING_BRANCH=$(echo "$REPO" | awk '{print $4}')
+
+  echo "$0: going to merge $PACKAGE"
+
+  cd $LOCATION/$PACKAGE
+
+  git merge origin/$TESTING_BRANCH
+
+  git push --dry-run
+
+  echo ""
+
+done
+
+echo "$0: Done merging"
+echo "$0: Going to push"
+
+# merge and try dry-run push
+echo "$REPOS" | while IFS= read -r REPO; do
+
+  PACKAGE=$(echo "$REPO" | awk '{print $1}')
+  TESTING_BRANCH=$(echo "$REPO" | awk '{print $4}')
+
+  echo "$0: going to push $PACKAGE"
+
+  cd $LOCATION/$PACKAGE
+
+  git push
+
+  echo ""
+
+done
+
+echo "$0: Done pushing"

.ci/parse_yaml.py

@@ -31,23 +31,31 @@ def main():
                 stable_ref = "none"
                 testing_ref = "none"
                 unstable_ref = "none"
+                ros_test = 0
+
+                refs = properties['git_refs']
+
+                try:
+                    stable_ref = refs['stable']
+                except:
+                    pass
 
                 try:
-                    stable_ref = properties['stable_ref']
+                    testing_ref = refs['testing']
                 except:
                     pass
 
                 try:
-                    testing_ref = properties['testing_ref']
+                    unstable_ref = refs['unstable']
                 except:
                     pass
 
                 try:
-                    unstable_ref = properties['unstable_ref']
+                    ros_test = bool(properties['ros_test'])
                 except:
                     pass
 
-                print("{} {} {} {} {}".format(package, url, stable_ref, testing_ref, unstable_ref))
+                print("{} {} {} {} {} {}".format(package, url, stable_ref, testing_ref, unstable_ref, ros_test))
 
 if __name__ == '__main__':
     main()

.ci/test.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+set -e
+
+trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
+trap 'echo "$0: \"${last_command}\" command failed with exit code $?"' ERR
+
+LIST=mrs
+VARIANT=testing
+PACKAGE_NAME=$1
+WORKSPACE=/tmp/workspace
+
+YAML_FILE=${LIST}.yaml
+
+# needed for building open_vins
+export ROS_VERSION=1
+
+sudo apt-get -y install dpkg-dev
+
+ARCH=$(dpkg-architecture -qDEB_HOST_ARCH)
+
+# we already have a docker image with ros for the ARM build
+if [[ "$ARCH" != "arm64" ]]; then
+  curl https://ctu-mrs.github.io/ppa-$VARIANT/add_ros_ppa.sh | bash
+fi
+
+curl https://ctu-mrs.github.io/ppa-$VARIANT/add_ppa.sh | bash
+
+sudo apt-get -y -q install ros-noetic-desktop
+sudo apt-get -y -q install ros-noetic-mrs-uav-system
+
+REPOS=$(./.ci/get_repo_source.py $YAML_FILE $VARIANT $ARCH $PACKAGE_NAME)
+
+echo "$0: creating workspace"
+
+mkdir -p $WORKSPACE/src
+cd $WORKSPACE
+source /opt/ros/noetic/setup.bash
+catkin init
+
+catkin config --profile reldeb --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo
+catkin profile set reldeb
+
+cd src
+
+echo "$0: cloning the package"
+
+# clone and checkout
+echo "$REPOS" | while IFS= read -r REPO; do
+
+  PACKAGE=$(echo "$REPO" | awk '{print $1}')
+  URL=$(echo "$REPO" | awk '{print $2}')
+  BRANCH=$(echo "$REPO" | awk '{print $3}')
+
+  echo "$0: cloning '$URL --depth 1 --branch $BRANCH' into '$PACKAGE'"
+  git clone $URL --recurse-submodules --shallow-submodules --depth 1 --branch $BRANCH $PACKAGE
+
+done
+
+cd $WORKSPACE/src
+
+echo "$0: installing rosdep dependencies"
+
+rosdep install --from-path .
+
+echo "$0: building the workspace"
+
+catkin build
+
+echo "$0: testing"
+
+catkin test
+
+echo "$0: tests finished"

.github/workflows/rostest_to_release.yml

@@ -0,0 +1,63 @@
+name: rostest_to_release
+
+on:
+  workflow_dispatch:
+
+  schedule:
+    - cron:  '0 5 * * *' # every day at 7am UTC+2
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+
+  generate-jobs:
+    runs-on: ubuntu-20.04
+    outputs:
+      packages: ${{ steps.generate.outputs.packages }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - id: generate
+        run: |
+          JOB_STRATEGY_MATRIX=$(./.ci/get_test_matrix.sh)
+          echo "packages=$JOB_STRATEGY_MATRIX" >> "$GITHUB_OUTPUT"
+
+  test-job:
+    needs: generate-jobs
+    runs-on: ubuntu-20.04
+    timeout-minutes: 360 # 6 hour timeout
+    strategy:
+      matrix:
+        job: ${{ fromJson(needs.generate-jobs.outputs.packages) }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - name: Checkout CI scripts
+        uses: actions/checkout@v3
+        with:
+          repository: ctu-mrs/ci_scripts
+          ref: master
+          path: .ci_scripts
+          token: ${{ secrets.PUSH_TOKEN }}
+      - id: test
+        run: |
+          .ci/test.sh "${{ matrix.job }}"
+
+  merge_and_push:
+    runs-on: ubuntu-20.04
+    needs: test-job
+    env:
+      PUSH_TOKEN: ${{ secrets.PUSH_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - id: merge
+        run: |
+          .ci/merge_push_to_release.sh