OTL-2195: Fluentd disabled by default for salt (#3448)

* OTL-2195: Fluentd disabled by default for salt * include PR in changelog
signalfx · Aug 7, 2023 · 9a427db · 9a427db
1 parent 5938ee7
commit 9a427db
Show file tree

Hide file tree

Showing 8 changed files with 112 additions and 37 deletions.
diff --git a/.github/workflows/salt-test.yml b/.github/workflows/salt-test.yml
@@ -21,7 +21,6 @@ concurrency:
 env:
   PYTHON_VERSION: '3.10'
   REQUIREMENTS_PATH: "internal/buildscripts/packaging/tests/requirements.txt"
-  RESULT_PATH: "~/testresults"
 
 jobs:
   salt-lint-test:
@@ -36,15 +35,37 @@ jobs:
         run: |
           make -C deployments/salt lint
 
+  salt-test-matrix:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Get matrix
+        id: get-matrix
+        run: |
+          # create test matrix for distro
+          dockerfiles=$(find internal/buildscripts/packaging/tests/deployments/salt/images/ -name "Dockerfile.*" | cut -d '.' -f2- | sort -u)
+          if [ -z "$dockerfiles" ]; then
+            echo "Failed to get dockerfiles from internal/buildscripts/packaging/tests/deployments/salt/images!" >&2
+            exit 1
+          fi
+          distro=$(for d in $dockerfiles; do echo -n "\"$d\","; done)
+          arch="\"amd64\", \"arm64\""
+          matrix="{\"DISTRO\": [${distro%,}]}"
+          echo "$matrix" | jq
+          echo "matrix=${matrix}" >> $GITHUB_OUTPUT
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+
   salt-test:
     name: salt-test
     # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
-    runs-on: ubuntu-20.04
-    needs: [salt-lint-test]
+    runs-on: ${{ fromJSON('["ubuntu-20.04", "ubuntu-22.04"]')[matrix.DISTRO == 'amazonlinux-2023'] }}
+    needs: [salt-lint-test, salt-test-matrix]
     strategy:
-      matrix:
-        PACKAGE_TYPE: [ "deb", "rpm" ]
-        TEST_CASE: [ "with_fluentd", "without_fluentd", "with_instrumentation", "service_owner", "custom_env_vars" ]
+      matrix: ${{ fromJSON(needs.salt-test-matrix.outputs.matrix) }}
+      fail-fast: false
     steps:
       - name: Check out the codebase.
         uses: actions/checkout@v3
@@ -62,16 +83,9 @@ jobs:
       - name: Test salt deployment
         timeout-minutes: 45
         run: |
-          mkdir -p ${{ env.RESULT_PATH }}
-          pytest -n2 --verbose -m ${{ matrix.PACKAGE_TYPE }} \
-            -k ${{ matrix.TEST_CASE }} \
-            --junitxml=${{ env.RESULT_PATH }}/results.xml \
-            --html=${{ env.RESULT_PATH }}/results.html \
-            --self-contained-html \
-            internal/buildscripts/packaging/tests/deployments/salt/salt_test.py
-
-      - name: Uploading test result
-        uses: actions/upload-artifact@v3
-        with:
-          name: salt-test-${{ matrix.TEST_CASE}}-${{ matrix.PACKAGE_TYPE }}-result
-          path: ${{ env.RESULT_PATH }}
+          distro="${{ matrix.DISTRO }}"
+          if [[ "$distro" = "amazonlinux-2" ]]; then
+            # workaround for pytest substring matching
+            distro="amazonlinux-2 and not amazonlinux-2023"
+          fi
+          pytest -svx --verbose -k "$distro" internal/buildscripts/packaging/tests/deployments/salt/salt_test.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 ## Unreleased
 
+### 🛑 Breaking changes 🛑
+
+- (Splunk) Fluentd installation ***disabled*** by default for the [`splunk-otel-collector` salt formula](https://github.com/signalfx/splunk-otel-collector/tree/main/deployments/salt) ([#3448](https://github.com/signalfx/splunk-otel-collector/pull/3448))
+  - Specify the `install_fluentd: True` attribute in your pillar to enable installation
+
 ## v0.82.0
 
 ### 🛑 Breaking changes 🛑

diff --git a/deployments/salt/README.md b/deployments/salt/README.md
@@ -7,8 +7,8 @@ Observability Cloud](https://www.splunk.com/en_us/observability.html).
 ## Linux
 Currently, the following Linux distributions and versions are supported:
 
-- Amazon Linux: 2
-- CentOS / Red Hat / Oracle: 7, 8
+- Amazon Linux: 2, 2023 (**Note:** Log collection with Fluentd not currently supported for Amazon Linux 2023.)
+- CentOS / Red Hat / Oracle: 7, 8, 9
 - Debian: 9, 10, 11
 - SUSE: 12, 15 (**Note:** Only for collector versions v0.34.0 or higher. Log collection with fluentd not currently supported.)
 - Ubuntu: 16.04, 18.04, 20.04, 22.04
@@ -32,7 +32,6 @@ splunk-otel-collector:
   splunk_otel_collector_config: '/etc/otel/collector/agent_config.yaml'
   splunk_service_user: splunk-otel-collector
   splunk_service_group: splunk-otel-collector
-
 ```
 
 ## This Salt Formula accepts the following attributes:
@@ -121,7 +120,7 @@ splunk-otel-collector:
   [fluent-plugin-systemd](
   https://github.com/fluent-plugin-systemd/fluent-plugin-systemd) for systemd
   journal log collection, and the required libraries/development tools.
-  (**default:** `True`)
+  (**default:** `False`)
 
 - `td_agent_version`: Version of [td-agent](
   https://td-agent-package-browser.herokuapp.com/) (fluentd package) that will

diff --git a/deployments/salt/splunk-otel-collector/fluentd.sls b/deployments/salt/splunk-otel-collector/fluentd.sls
@@ -68,7 +68,7 @@ Install FluentD Linux capability module dependencies:
     - pkgs:
       - libcap-ng
       - libcap-ng-devel
-{%- if grains['osmajorrelease'] == 8 %}
+{%- if grains['osmajorrelease'] == 8 or grains['osmajorrelease'] == 9 %}
       - pkgconf-pkg-config
 {%- else %}
       - pkgconfig

diff --git a/deployments/salt/splunk-otel-collector/init.sls b/deployments/salt/splunk-otel-collector/init.sls
@@ -15,11 +15,11 @@ Check splunk_realm:
     - failhard: True
 {% endif %}
 
-{% set install_fluentd = salt['pillar.get']('splunk-otel-collector:install_fluentd', True) | to_bool %}
+{% set install_fluentd = salt['pillar.get']('splunk-otel-collector:install_fluentd', False) | to_bool %}
 {% set install_auto_instrumentation = salt['pillar.get']('splunk-otel-collector:install_auto_instrumentation', False) | to_bool %}
 
 include:
-{% if grains['os_family'] == 'Suse' or install_fluentd == False %}
+{% if grains['os_family'] == 'Suse' or install_fluentd == False or (grains['os'] == 'Amazon' and grains['osrelease'] == '2023') %}
   - splunk-otel-collector.install
   - splunk-otel-collector.service_owner
   - splunk-otel-collector.config

diff --git a/...rnal/buildscripts/packaging/tests/deployments/salt/images/rpm/Dockerfile.amazonlinux-2023 b/...rnal/buildscripts/packaging/tests/deployments/salt/images/rpm/Dockerfile.amazonlinux-2023
@@ -0,0 +1,30 @@
+FROM amazonlinux:2023
+
+ENV container docker
+
+RUN yum install -y systemd procps initscripts python3-pip python3-devel gcc
+
+# Only the "amazon/2" repo is currently available, but seems to work for 2023
+RUN rpm --import https://repo.saltproject.io/salt/py3/amazon/2/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub
+RUN curl -fsSL https://repo.saltproject.io/salt/py3/amazon/2/x86_64/latest.repo | tee /etc/yum.repos.d/salt-amzn.repo
+
+RUN yum install -y salt-minion
+
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
+	"systemd-tmpfiles-setup.service" ] || rm -f $i; done); \
+	rm -f /lib/systemd/system/multi-user.target.wants/*;\
+	rm -f /lib/systemd/system/local-fs.target.wants/*; \
+	rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+	rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+	rm -f /lib/systemd/system/basic.target.wants/*;\
+	rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+COPY internal/buildscripts/packaging/tests/deployments/salt/minion /etc/salt/minion
+COPY deployments/salt/splunk-otel-collector /srv/salt/splunk-otel-collector
+COPY deployments/salt/templates /srv/salt/templates
+COPY internal/buildscripts/packaging/tests/deployments/salt/top.sls /srv/pillar/top.sls
+COPY internal/buildscripts/packaging/tests/deployments/salt/top.sls /srv/salt/top.sls
+
+VOLUME [ "/sys/fs/cgroup" ]
+
+CMD ["/usr/sbin/init"]
diff --git a/internal/buildscripts/packaging/tests/deployments/salt/images/rpm/Dockerfile.centos-9 b/internal/buildscripts/packaging/tests/deployments/salt/images/rpm/Dockerfile.centos-9
@@ -0,0 +1,31 @@
+FROM quay.io/centos/centos:stream9
+
+ENV container docker
+
+RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+RUN echo 'fastestmirror=1' >> /etc/yum.conf
+
+RUN dnf install -y systemd procps initscripts python3-pip python3-devel gcc
+
+RUN rpm --import https://repo.saltproject.io/salt/py3/redhat/9/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub
+RUN curl -fsSL https://repo.saltproject.io/salt/py3/redhat/9/x86_64/latest.repo | tee /etc/yum.repos.d/salt.repo
+
+RUN dnf install -y salt-minion
+
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \
+	"systemd-tmpfiles-setup.service" ] || rm -f $i; done); \
+	rm -f /lib/systemd/system/multi-user.target.wants/*;\
+	rm -f /lib/systemd/system/local-fs.target.wants/*; \
+	rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+	rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+	rm -f /lib/systemd/system/basic.target.wants/*;\
+	rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+COPY internal/buildscripts/packaging/tests/deployments/salt/minion /etc/salt/minion
+COPY deployments/salt/splunk-otel-collector /srv/salt/splunk-otel-collector
+COPY deployments/salt/templates /srv/salt/templates
+COPY internal/buildscripts/packaging/tests/deployments/salt/top.sls /srv/pillar/top.sls
+COPY internal/buildscripts/packaging/tests/deployments/salt/top.sls /srv/salt/top.sls
+
+VOLUME [ "/sys/fs/cgroup" ]
+CMD ["/usr/sbin/init"]
diff --git a/internal/buildscripts/packaging/tests/deployments/salt/salt_test.py b/internal/buildscripts/packaging/tests/deployments/salt/salt_test.py
@@ -112,15 +112,15 @@ def test_salt_with_fluentd(distro):
                 run_salt_apply(container, config)
                 verify_env_file(container)
                 assert wait_for(lambda: service_is_running(container))
-                if "opensuse" not in distro:
+                if "opensuse" not in distro and distro != "amazonlinux-2023":
                     assert container.exec_run("systemctl status td-agent").exit_code == 0
                 if collector_version == "latest":
                     verify_package_version(container, "splunk-otel-collector", collector_version, "0.34.0")
                 else:
                     verify_package_version(container, "splunk-otel-collector", collector_version)
         finally:
             run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")
-            if "opensuse" not in distro:
+            if "opensuse" not in distro and distro != "amazonlinux-2023":
                 run_container_cmd(container, "journalctl -u td-agent --no-pager")
                 if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0:
                     run_container_cmd(container, "cat /var/log/td-agent/td-agent.log")
@@ -211,19 +211,14 @@ def test_salt_with_instrumentation(distro):
                 run_salt_apply(container, config)
                 verify_env_file(container)
                 assert wait_for(lambda: service_is_running(container))
-                if "opensuse" not in distro:
-                    assert container.exec_run("systemctl status td-agent").exit_code == 0
+                assert container.exec_run("systemctl status td-agent").exit_code != 0
                 if version == "latest":
                     verify_package_version(container, "splunk-otel-auto-instrumentation", version, "0.48.0")
                 else:
                     verify_package_version(container, "splunk-otel-auto-instrumentation", version)
                 verify_instrumentation_config(container)
         finally:
             run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")
-            if "opensuse" not in distro:
-                run_container_cmd(container, "journalctl -u td-agent --no-pager")
-                if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0:
-                    run_container_cmd(container, "cat /var/log/td-agent/td-agent.log")
 
 
 SERVICE_OWNER_CONFIG = f"""
@@ -234,6 +229,7 @@ def test_salt_with_instrumentation(distro):
   splunk_api_url: '{SPLUNK_API_URL}'
   splunk_service_user: 'test-user'
   splunk_service_group: 'test-user'
+  install_fluentd: True
 """
 
 
@@ -256,13 +252,13 @@ def test_salt_service_owner(distro):
             assert wait_for(lambda: service_is_running(container, service_owner="test-user"))
             _, owner = run_container_cmd(container, f"stat -c '%U:%G' {SPLUNK_ENV_PATH}")
             assert owner.decode("utf-8").strip() == "test-user:test-user"
-            if "opensuse" not in distro:
+            if "opensuse" not in distro and distro != "amazonlinux-2023":
                 assert container.exec_run("systemctl status td-agent").exit_code == 0
                 _, owner = run_container_cmd(container, f"stat -c '%U:%G' {CONFIG_DIR}/fluentd/fluent.conf")
                 assert owner.decode("utf-8").strip() == "td-agent:td-agent"
         finally:
             run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")
-            if "opensuse" not in distro:
+            if "opensuse" not in distro and distro != "amazonlinux-2023":
                 run_container_cmd(container, "journalctl -u td-agent --no-pager")
                 if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0:
                     run_container_cmd(container, "cat /var/log/td-agent/td-agent.log")
@@ -272,7 +268,6 @@ def test_salt_service_owner(distro):
 splunk-otel-collector:
   splunk_access_token: '{SPLUNK_ACCESS_TOKEN}'
   splunk_realm: '{SPLUNK_REALM}'
-  install_fluentd: False
   collector_additional_env_vars:
     MY_CUSTOM_VAR1: value1
     MY_CUSTOM_VAR2: value2
@@ -298,5 +293,6 @@ def test_salt_custom_env_vars(distro):
             run_container_cmd(container, f"grep '^MY_CUSTOM_VAR1=value1$' {SPLUNK_ENV_PATH}")
             run_container_cmd(container, f"grep '^MY_CUSTOM_VAR2=value2$' {SPLUNK_ENV_PATH}")
             assert wait_for(lambda: service_is_running(container))
+            assert container.exec_run("systemctl status td-agent").exit_code != 0
         finally:
             run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager")