Update compose fixes

.github/workflows/build-test.yaml

@@ -11,10 +11,9 @@ jobs:
   podman-checkout-test:
     runs-on: ubuntu-latest
     env:
-      DOCKER_BUILDKIT: "1"
       AUTHNZ_EMU: "authnz-emu"
-      ANMS_COMPOSE_OPTS: "-f docker-compose.yml -p anms"
-      AGENT_COMPOSE_OPTS: "-f agent-compose.yml -p agents"
+      ANMS_COMPOSE_OPTS: "-f docker-compose.yml"
+      TESTENV_COMPOSE_OPTS: "-f testenv-compose.yml"
       DOCKER_CMD: "podman"
       AUTHNZ_PORT: 8084
       AUTHNZ_HTTPS_PORT: 8443
@@ -37,21 +36,17 @@ jobs:
         run: |
           DOCKER_IMAGE_TAG=$(echo ${{ github.head_ref || github.ref_name }} | sed 's/[^a-zA-Z0-9\-\._]/-/g')
           echo "DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG}" >> $GITHUB_ENV
-      - name: Debug GitHub workspace
-        run: |
-          echo "GITHUB_WORKSPACE=${{ github.workspace }}"
-          ls -al ${{ github.workspace }}
       - name: Build Main
-        run: podman compose build
+        run: podman compose ${ANMS_COMPOSE_OPTS} build
       - name: Build Agents
-        run: podman compose -f agent-compose.yml build
+        run: podman compose ${TESTENV_COMPOSE_OPTS} build
       - name: Build Volume
         run: |
           ./create_volume.sh ./puppet/modules/apl_test/files/anms/tls
       - name: Start
         run: |
-          podman compose up -d --force-recreate
-          podman compose -f agent-compose.yml up -d --force-recreate
+          podman compose ${ANMS_COMPOSE_OPTS} up -d --force-recreate
+          podman compose ${TESTENV_COMPOSE_OPTS} up -d --force-recreate
           sleep 5
       - name: Status
         run: |
@@ -72,16 +67,17 @@ jobs:
       - name: Stop
         if: always()
         run: |
-          for OPTS_NAME in ANMS_COMPOSE_OPTS AGENT_COMPOSE_OPTS; do
-            podman compose ${!OPTS_NAME} rm --stop --force
+          for OPTS_NAME in TESTENV_COMPOSE_OPTS ANMS_COMPOSE_OPTS; do
+            podman compose ${!OPTS_NAME} down --remove-orphans
           done
+
   docker-checkout-test:
     runs-on: ubuntu-latest
     env:
       DOCKER_BUILDKIT: "1"
       AUTHNZ_EMU: "authnz-emu"
-      ANMS_COMPOSE_OPTS: "-f docker-compose.yml -p anms"
-      AGENT_COMPOSE_OPTS: "-f agent-compose.yml -p agents"
+      ANMS_COMPOSE_OPTS: "-f docker-compose.yml"
+      TESTENV_COMPOSE_OPTS: "-f testenv-compose.yml"
       DOCKER_CMD: "docker"
     steps:
       - name: Versions
@@ -102,16 +98,16 @@ jobs:
           echo "GITHUB_WORKSPACE=${{ github.workspace }}"
           ls -al ${{ github.workspace }}
       - name: Build Main
-        run: docker compose build
+        run: docker compose ${ANMS_COMPOSE_OPTS} build
       - name: Build Agents
-        run: docker compose -f agent-compose.yml build
+        run: docker compose ${TESTENV_COMPOSE_OPTS} build
       - name: Build Volume
         run: |
           ./create_volume.sh ./puppet/modules/apl_test/files/anms/tls
       - name: Start
         run: |
-          docker compose up -d --force-recreate
-          docker compose -f agent-compose.yml up -d --force-recreate
+          docker compose ${ANMS_COMPOSE_OPTS} up -d --force-recreate
+          docker compose ${TESTENV_COMPOSE_OPTS} up -d --force-recreate
           sleep 5
       - name: Status
         run: |
@@ -132,6 +128,6 @@ jobs:
       - name: Stop
         if: always()
         run: |
-          for OPTS_NAME in ANMS_COMPOSE_OPTS AGENT_COMPOSE_OPTS; do
-            docker compose ${!OPTS_NAME} rm --stop --force
+          for OPTS_NAME in TESTENV_COMPOSE_OPTS ANMS_COMPOSE_OPTS; do
+            docker compose ${!OPTS_NAME} down --remove-orphans
           done

.gitlab-ci.yml

@@ -49,8 +49,8 @@ variables:
   DOCKER_IMAGE_TAG: $CI_COMMIT_REF_SLUG
   DOCKER_IMAGE_PREFIX: $DOCKER_REGISTRY/$DOCKER_GROUP/
   AUTHNZ_EMU: 1
-  ANMS_COMPOSE_OPTS: -f docker-compose.yml -p anms
-  AGENT_COMPOSE_OPTS: -f agent-compose.yml -p agents
+  ANMS_COMPOSE_OPTS: -f docker-compose.yml
+  TESTENV_COMPOSE_OPTS: -f testenv-compose.yml
 
 stages:
   - build
@@ -88,8 +88,8 @@ stages:
   docker-compose version
 
 .cleanup-docker: &cleanup-docker |
-  for OPTS_NAME in ANMS_COMPOSE_OPTS AGENT_COMPOSE_OPTS; do
-    docker-compose ${!OPTS_NAME} rm --stop --force
+  for OPTS_NAME in TESTENV_COMPOSE_OPTS ANMS_COMPOSE_OPTS; do
+    docker-compose ${!OPTS_NAME} down
   done
   docker network prune -f
   docker volume prune -f
@@ -127,12 +127,12 @@ checkout-test:
     - ./create_volume.sh ./puppet/modules/apl_test/files/anms/tls/
     - echo "running services..."
     - |
-      for OPTS_NAME in ANMS_COMPOSE_OPTS AGENT_COMPOSE_OPTS; do
+      for OPTS_NAME in ANMS_COMPOSE_OPTS TESTENV_COMPOSE_OPTS; do
         docker-compose ${!OPTS_NAME} up --detach --force-recreate
       done
     - sleep 5
     - |
-      for OPTS_NAME in ANMS_COMPOSE_OPTS AGENT_COMPOSE_OPTS; do
+      for OPTS_NAME in ANMS_COMPOSE_OPTS TESTENV_COMPOSE_OPTS; do
         docker-compose ${!OPTS_NAME} ps
       done
     - |

README.md

@@ -93,15 +93,15 @@ Choose the appropriate docker, podman or podman-compose commands in the directio
   - `podman-compose --podman-build-args='--format docker' build`
     - Note: The docker format argument here enables suppoort for HEALTHCHECK. If omitted, the system will run but will be unable to report the health of the system.  This flag does not appear necessary when using the no-dash version of compose.
 - Build Agent images
-  - `docker compose -f agent-compose.yml build`
-  - `podman compose -f agent-compose.yml build`
-  - `podman-compose -f agent-compose.yml --podman-build-args='--format docker' build`
+  - `docker compose -f testenv-compose.yml build`
+  - `podman compose -f testenv-compose.yml build`
+  - `podman-compose -f testenv-compose.yml --podman-build-args='--format docker' build`
 - Start System. Note: You may omit the `-d` argument to keep logs in the foreground.
   - `docker compose up -d`
   - `podman compose up -d`
 - Start additional ION Agent Nodes
-  - `docker compose -f agent-compose.yml up -d`
-  - `podman compose -f agent-compose.yml up -d`
+  - `docker compose -f testenv-compose.yml up -d`
+  - `podman compose -f testenv-compose.yml up -d`
 
 ### Alternative Build.sh setup script (deprecated, docker-only)
 The ANMS repository contains a build script which will build and run multiple Docker containers.
@@ -188,9 +188,9 @@ docker-compose -f docker-compose.yml up -d
 ```
 from within the `anms/` folder.
 
-To restart the agents forcefully, controlled with a different compose file `agent-compose.yml` run:
+To restart the agents forcefully, controlled with a different compose file `testenv-compose.yml` run:
 ```sh
-docker-compose -f agent-compose.yml up -d --force-recreate
+docker-compose -f testenv-compose.yml up -d --force-recreate
 ```
 
 ### Compose Environment and Options

base.Dockerfile

@@ -72,7 +72,7 @@ RUN --mount=type=cache,target=/var/cache/yum \
     pip3 install --upgrade pip pip-tools 
 
 # Submodules with dependencies
-env PY_WHEEL_DIR=/usr/local/lib/wheels
+ENV PY_WHEEL_DIR=/usr/local/lib/wheels
 
 COPY deps/dtnma-ace /usr/src/dtnma-ace
 RUN pip3 wheel /usr/src/dtnma-ace -w ${PY_WHEEL_DIR} --no-deps

build.sh

@@ -102,8 +102,8 @@ then
 fi
 
 echo "Running docker-compose up"
-ANMS_COMPOSE_OPTS="-f docker-compose.yml -p anms"
-AGENT_COMPOSE_OPTS="-f agent-compose.yml -p agents"
+ANMS_COMPOSE_OPTS="-f docker-compose.yml"
+AGENT_COMPOSE_OPTS="-f testenv-compose.yml"
 for OPTS_NAME in ANMS_COMPOSE_OPTS AGENT_COMPOSE_OPTS; do
     docker compose ${!OPTS_NAME} up --detach
 done

checkout-test/Dockerfile

@@ -23,11 +23,13 @@
 # Allow the checkout tests to be run within a docker network
 FROM localhost/anms-base
 
-COPY . /usr/src/checkout-test
 RUN --mount=type=cache,target=/var/cache/yum \
-    dnf -y install python3 python3-pip python3-wheel
+    dnf install -y iputils python3 python3-pip python3-wheel
+
+COPY requirements.txt /usr/src/checkout-test/
 RUN --mount=type=cache,target=/root/.cache/pip \
     pip3 install -r /usr/src/checkout-test/requirements.txt
+COPY . /usr/src/checkout-test
 
 # First argument is the base URL to test
 ENTRYPOINT ["/usr/src/checkout-test/run.sh"]

create_volume.sh

@@ -33,8 +33,8 @@ fi
 
 ${DOCKER_CMD} volume create ${VOLNAME}
 CTRNAME=$(${DOCKER_CMD} run --detach --rm \
-		 -v ${VOLNAME}:${VOLPATH} -it \
-		 docker.io/redhat/ubi9 tail -f /dev/null)
+          -v ${VOLNAME}:${VOLPATH} \
+          docker.io/redhat/ubi9 tail -f /dev/null)
 
 ${DOCKER_CMD} exec ${CTRNAME} rm -rf ${VOLPATH}/*
 for FN in ${SRCPATH}/*

docker-compose.yml

@@ -25,6 +25,8 @@
 # DOCKER_IMAGE_PREFIX to contain a (default empty) image registry and path to all custom images
 # DOCKER_IMAGE_TAG to contain the image tag for custom images
 #
+name: anms
+
 networks:
   default:
     name: ${DOCKER_CTR_PREFIX}anms
@@ -98,11 +100,13 @@ services:
     restart: unless-stopped
     security_opt:
       - "label=type:nginx.process"
-    depends_on:
+    depends_on: # dependencies for name resolution
       - anms-core
       - anms-ui
       - grafana
+      - grafana-image-renderer
       - adminer
+      - ion-manager
 
   opensearch:
     hostname: opensearch
@@ -306,6 +310,8 @@ services:
       builder-acelib:
         # Not expected to run, but needed here to ensure build dependency
         condition: service_started
+      ion-manager:
+        condition: service_started
       postgres:
         condition: service_healthy
       mqtt-broker:

export.sh

@@ -18,7 +18,7 @@ else
     exit 1
 fi
 
-IMAGES=$(${COMPOSE_CMD} -f docker-compose.yml -f agent-compose.yml config | grep --color=auto 'image:' | awk '{print $2}' | sort -u | less)
+IMAGES=$(${COMPOSE_CMD} -f docker-compose.yml -f testenv-compose.yml config | grep --color=auto 'image:' | awk '{print $2}' | sort -u | less)
 echo "${COMPOSE_CMD} reports the following images: ${IMAGES}"
 
 # Determine base command (docker or podman)

puppet/.gitignore

@@ -7,8 +7,7 @@ bolt-debug.log
 .rerun.json
 
 # Files replaced by prep.sh
-modules/anms/files/docker-compose.yml
-modules/anms/files/agent-compose.yml
+modules/anms/files/*-compose.yml
 modules/anms/files/create_volume.sh
 
 # Files replaced by apply_local.sh

puppet/modules/anms/manifests/compose.pp

@@ -52,9 +52,15 @@
       file { "/etc/containers/compose/projects/${title}.env":
         ensure => 'absent',
       }
+      service { "podman-compose@${title}":
+        enable  => false,
+        require => [
+          Systemd::Unit_file['podman-compose@.service'],
+        ],
+      }
     }
     default: {
       fail("Invalid ensure argument: ${ensure}")
     }
   }
-}
+}

puppet/modules/anms/manifests/init.pp

@@ -15,6 +15,7 @@
   Optional[String] $tls_server_key = undef,
   Optional[String] $tls_server_cert = undef,
   Optional[String] $tls_server_ca = undef,
+  Boolean $use_testenv = false,
 ) {
   require Class['anms::hostenv']
 
@@ -107,7 +108,7 @@
       command => "podman login ${ctr_image_prefix} --username \"${ctr_registry_user}\" --password \"${ctr_registry_pass}\"",
       before  => [
         Anms::Compose['anms'],
-        Anms::Compose['agents'],
+        Anms::Compose['testenv'],
       ],
     }
   }
@@ -153,40 +154,35 @@
     ],
   }
 
-  file { '/ammos/anms/agent-compose.yml':
-    ensure => 'file',
-    source => 'puppet:///modules/anms/agent-compose.yml',
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0644',
+  if $use_testenv {
+    file { '/ammos/anms/testenv-compose.yml':
+      ensure => 'file',
+      source => 'puppet:///modules/anms/testenv-compose.yml',
+      owner  => 'root',
+      group  => 'root',
+      mode   => '0644',
+    }
+    anms::compose { 'testenv':
+      ensure       => 'present',
+      directory    => '/ammos/anms',
+      compose_file => 'testenv-compose.yml',
+      require      => [
+        Anms::Compose['anms'], # for the anms network
+      ],
+      subscribe    => [
+        File['/ammos/anms/testenv-compose.yml'],
+        File['/ammos/anms/.env'],
+      ],
+    }
   }
-  anms::compose { 'agents':
-    ensure       => 'present',
-    directory    => '/ammos/anms',
-    compose_file => 'agent-compose.yml',
-    require      => [
-      Anms::Compose['anms'], # for the anms network
-    ],
-    subscribe    => [
-      File['/ammos/anms/agent-compose.yml'],
-      File['/ammos/anms/.env'],
-    ],
+  else {
+    file { '/ammos/anms/testenv-compose.yml':
+      ensure => 'absent',
+    }
+    anms::compose { 'testenv':
+      ensure       => 'absent',
+      directory    => '/ammos/anms',
+      compose_file => 'testenv-compose.yml',
+    }
   }
-
-  # Restart ducts to cache updated DNS resolution
-  $ion_containers = [
-    'ion-manager',
-    'ion-agent2',
-    'ion-agent3',
-  ]
-#  $ion_containers.each |$ctrname| {
-#    exec { "restart-ducts-${ctrname}":
-#      command => "podman exec ${ctrname} ion_restart_ducts",
-#      path    => $facts['path'],
-#      require => [
-#        Anms::Compose['anms'],
-#        Anms::Compose['agents'],
-#      ],
-#    }
-#  }
 }

puppet/prep.sh

@@ -30,7 +30,7 @@ source ${SELFDIR}/getenv.sh
 
 rm -f "${SELFDIR}"/modules/anms/files/*-compose.yml
 cp "${SELFDIR}/../docker-compose.yml" \
-   "${SELFDIR}/../agent-compose.yml" \
+   "${SELFDIR}/../testenv-compose.yml" \
    "${SELFDIR}/../create_volume.sh" \
    "${SELFDIR}/modules/anms/files/"
 

agent-compose.yml → testenv-compose.yml

@@ -25,6 +25,7 @@
 # DOCKER_IMAGE_PREFIX to contain a (default empty) image registry and path to all custom images
 # DOCKER_IMAGE_TAG to contain the image tag for custom images
 #
+name: testenv
 
 networks:
   # This network is created by docker-compose.yml

transcoder/Dockerfile

@@ -21,7 +21,7 @@
 ##
 FROM localhost/dtnma-acelib
 
-ENV APP_WORK_DIR /opt/app
+ENV APP_WORK_DIR=/opt/app
 
 
 # Copy over all required content (source, data, etc.)