CAMEL-25518: camel-hashicorp-vault - Secret refresh detects updates in any engine and before the first check

catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/hashicorp-vault-component.adoc

@@ -651,6 +651,7 @@ The secret names should use the following format:
 - You can use patterns like `database*` to match multiple secrets
 
 Note that `camel.vault.hashicorp.secrets` is not mandatory: if not specified the task responsible for checking updates events will take into account all the properties with a `hashicorp:` prefix.
+Such a secret is tracked in the engine of the property, in the format above (`hashicorp:myengine:mysecret` is tracked as `myengine:mysecret`, `hashicorp:secret:mysecret` as `mysecret`), and its first check compares the version in Vault with the version that was resolved, so an update made before the first check also triggers a reload.
 
 ==== How the Refresh Mechanism Works
 

components/camel-hashicorp-vault/src/main/docs/hashicorp-vault-component.adoc

@@ -651,6 +651,7 @@ The secret names should use the following format:
 - You can use patterns like `database*` to match multiple secrets
 
 Note that `camel.vault.hashicorp.secrets` is not mandatory: if not specified the task responsible for checking updates events will take into account all the properties with a `hashicorp:` prefix.
+Such a secret is tracked in the engine of the property, in the format above (`hashicorp:myengine:mysecret` is tracked as `myengine:mysecret`, `hashicorp:secret:mysecret` as `mysecret`), and its first check compares the version in Vault with the version that was resolved, so an update made before the first check also triggers a reload.
 
 ==== How the Refresh Mechanism Works
 

components/camel-hashicorp-vault/src/main/java/org/apache/camel/component/hashicorp/vault/HashicorpVaultPropertiesFunction.java

@@ -16,9 +16,10 @@
  */
 package org.apache.camel.component.hashicorp.vault;
 
-import java.util.HashSet;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.camel.CamelContext;
 import org.apache.camel.CamelContextAware;
@@ -28,6 +29,8 @@
 import org.apache.camel.util.ObjectHelper;
 import org.apache.camel.util.StringHelper;
 import org.apache.camel.vault.HashicorpVaultConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.vault.authentication.TokenAuthentication;
 import org.springframework.vault.client.VaultEndpoint;
 import org.springframework.vault.core.VaultTemplate;
@@ -72,6 +75,8 @@
 @org.apache.camel.spi.annotations.PropertiesFunction("hashicorp")
 public class HashicorpVaultPropertiesFunction extends ServiceSupport implements PropertiesFunction, CamelContextAware {
 
+    private static final Logger LOG = LoggerFactory.getLogger(HashicorpVaultPropertiesFunction.class);
+
     private static final String CAMEL_HASHICORP_VAULT_TOKEN_ENV = "CAMEL_HASHICORP_VAULT_TOKEN";
     private static final String CAMEL_HASHICORP_VAULT_HOST_ENV = "CAMEL_HASHICORP_VAULT_HOST";
     private static final String CAMEL_HASHICORP_VAULT_PORT_ENV
@@ -84,7 +89,8 @@ public class HashicorpVaultPropertiesFunction extends ServiceSupport implements
             = "CAMEL_HASHICORP_VAULT_NAMESPACE";
     private CamelContext camelContext;
     private VaultTemplate client;
-    private final Set<String> secrets = new HashSet<>();
+    private final Set<String> secrets = ConcurrentHashMap.newKeySet();
+    private final Map<String, Integer> secretVersions = new ConcurrentHashMap<>();
 
     private String engine;
     private String namespace;
@@ -216,8 +222,9 @@ public String apply(String remainder) {
     }
 
     private String getSecretFromSource(String key, String subkey, String defaultValue, String version) {
-        // capture name of secret
-        secrets.add(key);
+        // capture name of secret in the form the refresh task reads the metadata of
+        String secretName = trackedSecretName(engine, key);
+        secrets.add(secretName);
 
         String returnValue = null;
         try {
@@ -235,6 +242,9 @@ private String getSecretFromSource(String key, String subkey, String defaultValu
             VaultResponse rawSecret = client.read(completePath);
             if (ObjectHelper.isNotEmpty(rawSecret)) {
                 returnValue = rawSecret.getData().get("data").toString();
+                if (ObjectHelper.isEmpty(version)) {
+                    captureVersion(secretName, rawSecret);
+                }
             }
             if (ObjectHelper.isNotEmpty(subkey)) {
                 Object field = ((Map) rawSecret.getData().get("data")).get(subkey);
@@ -257,6 +267,28 @@ private String getSecretFromSource(String key, String subkey, String defaultValu
         return returnValue;
     }
 
+    /**
+     * The name of a secret as the refresh task tracks it: engine:secret, or only the secret for the default secret
+     * engine. A lookup without engine is tracked as a secret of the default secret engine, which is also how the
+     * refresh task reads a name without engine.
+     */
+    private static String trackedSecretName(String secretEngine, String key) {
+        return secretEngine == null || "secret".equals(secretEngine) ? key : secretEngine + ":" + key;
+    }
+
+    private void captureVersion(String secretName, VaultResponse rawSecret) {
+        if (rawSecret.getData().get("metadata") instanceof Map<?, ?> metadata && metadata.get("version") != null) {
+            try {
+                // keep the oldest version resolved: a value resolved earlier may still be in use, so the refresh
+                // task must compare with that one (at worst one extra reload)
+                secretVersions.merge(secretName, Integer.valueOf(metadata.get("version").toString()), Math::min);
+            } catch (NumberFormatException e) {
+                LOG.debug("Could not parse secret version for {}: {}", secretName, metadata.get("version"), e);
+                // the refresh task then uses the version it reads first
+            }
+        }
+    }
+
     @Override
     public void setCamelContext(CamelContext camelContext) {
         this.camelContext = camelContext;
@@ -268,9 +300,20 @@ public CamelContext getCamelContext() {
     }
 
     /**
-     * Ids of the secrets in use
+     * Ids of the secrets in use, as engine:secret, or only the secret for the default secret engine
      */
     public Set<String> getSecrets() {
-        return secrets;
+        return Collections.unmodifiableSet(secrets);
+    }
+
+    /**
+     * The oldest version of the secret (named as in {@link #getSecrets()}) that was resolved, or null if not known
+     * (such as when a specific version was requested).
+     *
+     * @param  secretName the name of the secret, as engine:secret, or only the secret for the default secret engine
+     * @return            the oldest version of the secret that was resolved, or null if not known
+     */
+    public Integer getSecretVersion(String secretName) {
+        return secretVersions.get(secretName);
     }
 }

components/camel-hashicorp-vault/src/main/java/org/apache/camel/component/hashicorp/vault/vault/HashicorpVaultReloadTriggerTask.java

@@ -209,6 +209,14 @@ public void run() {
                                 Integer currentVersion = Integer.valueOf(currentVersionObj.toString());
                                 Integer lastKnownVersion = versionsMap.get(secretName);
 
+                                if (lastKnownVersion == null && propertiesFunction != null) {
+                                    // First time seeing this secret: start from the version the properties
+                                    // function resolved, as the secret may have been updated since then
+                                    lastKnownVersion = propertiesFunction.getSecretVersion(secretName);
+                                    if (lastKnownVersion != null) {
+                                        versionsMap.put(secretName, lastKnownVersion);
+                                    }
+                                }
                                 if (lastKnownVersion == null) {
                                     // First time seeing this secret, just record the version
                                     versionsMap.put(secretName, currentVersion);

components/camel-hashicorp-vault/src/test/java/org/apache/camel/component/hashicorp/vault/HashicorpVaultReloadTriggerTaskTest.java

@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.hashicorp.vault;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.camel.CamelContext;
+import org.apache.camel.component.hashicorp.vault.vault.HashicorpVaultReloadTriggerTask;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.apache.camel.vault.HashicorpVaultConfiguration;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Secret refresh against a minimal fake of the HashiCorp Vault KV v2 HTTP API.
+ */
+class HashicorpVaultReloadTriggerTaskTest extends CamelTestSupport {
+
+    private static HttpServer server;
+    // "<engine>/<key>" -> current version
+    private static final Map<String, Integer> VERSIONS = new ConcurrentHashMap<>();
+    private static final List<String> METADATA_READS = new CopyOnWriteArrayList<>();
+
+    private HashicorpVaultReloadTriggerTask task;
+
+    @BeforeAll
+    static void startFakeVault() throws IOException {
+        server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+        server.createContext("/v1/", HashicorpVaultReloadTriggerTaskTest::handle);
+        server.start();
+    }
+
+    @AfterAll
+    static void stopFakeVault() {
+        server.stop(0);
+    }
+
+    @AfterEach
+    void stopTask() {
+        if (task != null) {
+            task.stop();
+        }
+        VERSIONS.clear();
+        METADATA_READS.clear();
+    }
+
+    private static void handle(HttpExchange http) throws IOException {
+        // /v1/<engine>/<data|metadata>/<key>[?version=<n>]
+        String[] parts = http.getRequestURI().getPath().substring("/v1/".length()).split("/", 3);
+        String body = null;
+        if (parts.length == 3) {
+            Integer version = VERSIONS.get(parts[0] + "/" + parts[2]);
+            String query = http.getRequestURI().getQuery();
+            if ("metadata".equals(parts[1])) {
+                METADATA_READS.add(parts[0] + "/metadata/" + parts[2]);
+                if (version != null) {
+                    body = "{\"data\":{\"current_version\":" + version + "}}";
+                }
+            } else if ("data".equals(parts[1]) && version != null) {
+                if (query != null && query.startsWith("version=")) {
+                    // a specific version was requested
+                    version = Integer.valueOf(query.substring("version=".length()));
+                }
+                body = "{\"data\":{\"data\":{\"password\":\"pw-" + version + "\"},\"metadata\":{\"version\":" + version
+                       + "}}}";
+            }
+        }
+        int status = body != null ? 200 : 404;
+        byte[] bytes = (body != null ? body : "{\"errors\":[]}").getBytes(StandardCharsets.UTF_8);
+        http.getResponseHeaders().add("Content-Type", "application/json");
+        http.sendResponseHeaders(status, bytes.length);
+        try (OutputStream os = http.getResponseBody()) {
+            os.write(bytes);
+        }
+    }
+
+    @Override
+    protected CamelContext createCamelContext() throws Exception {
+        CamelContext ctx = super.createCamelContext();
+        HashicorpVaultConfiguration hashicorp = new HashicorpVaultConfiguration();
+        hashicorp.setToken("test-token");
+        hashicorp.setHost("localhost");
+        hashicorp.setPort(String.valueOf(server.getAddress().getPort()));
+        hashicorp.setScheme("http");
+        ctx.getVaultConfiguration().setHashicorpVaultConfiguration(hashicorp);
+        return ctx;
+    }
+
+    private void startTask() {
+        task = new HashicorpVaultReloadTriggerTask();
+        task.setCamelContext(context);
+        task.start();
+    }
+
+    @Test
+    void rotationInNonDefaultEngineIsDetected() {
+        VERSIONS.put("kv/db", 1);
+        assertEquals("pw-1", context.resolvePropertyPlaceholders("{{hashicorp:kv:db#password}}"));
+
+        startTask();
+        task.run();
+        VERSIONS.put("kv/db", 2);
+        task.run();
+
+        assertTrue(METADATA_READS.contains("kv/metadata/db"),
+                "the refresh task must read the metadata of the engine the secret was resolved from, but read "
+                                                              + METADATA_READS);
+        assertFalse(task.getUpdates().isEmpty(),
+                "the new version of the secret kv:db was not detected, updates: " + task.getUpdates());
+    }
+
+    @Test
+    void rotationBeforeFirstCheckIsDetected() {
+        VERSIONS.put("secret/api", 1);
+        assertEquals("pw-1", context.resolvePropertyPlaceholders("{{hashicorp:secret:api#password}}"));
+
+        startTask();
+        // the secret is rotated after it was resolved at startup but before the first check
+        VERSIONS.put("secret/api", 2);
+        task.run();
+
+        assertFalse(task.getUpdates().isEmpty(),
+                "the application resolved version 1 but version 2 was not detected, updates: " + task.getUpdates());
+    }
+
+    @Test
+    void rotationBetweenTwoLookupsBeforeFirstCheckIsDetected() {
+        VERSIONS.put("secret/api", 1);
+        assertEquals("pw-1", context.resolvePropertyPlaceholders("{{hashicorp:secret:api#password}}"));
+        // the secret is rotated, and resolved again (as a dynamic endpoint does) before the first check
+        VERSIONS.put("secret/api", 2);
+        assertEquals("pw-2", context.resolvePropertyPlaceholders("{{hashicorp:secret:api#password}}"));
+
+        HashicorpVaultPropertiesFunction function
+                = (HashicorpVaultPropertiesFunction) context.getPropertiesComponent().getPropertiesFunction("hashicorp");
+        assertEquals(1, function.getSecretVersion("api"), "the oldest resolved version is the one to compare with");
+
+        startTask();
+        task.run();
+
+        assertFalse(task.getUpdates().isEmpty(),
+                "version 1 is still in use by what was resolved first, but version 2 was not detected, updates: "
+                                                 + task.getUpdates());
+    }
+
+    @Test
+    void secretResolvedAtASpecificVersionIsNotComparedWithThatVersion() {
+        VERSIONS.put("secret/api", 2);
+        // the application asks for version 1, so the version it resolved says nothing about updates of the secret
+        assertEquals("pw-1", context.resolvePropertyPlaceholders("{{hashicorp:secret:api#password@1}}"));
+
+        startTask();
+        task.run();
+
+        assertTrue(task.getUpdates().isEmpty(), "the first check compared with the requested version, updates: "
+                                                + task.getUpdates());
+    }
+
+    @Test
+    void secretsAreTrackedWithTheirEngine() {
+        VERSIONS.put("kv/db", 1);
+        VERSIONS.put("secret/api", 1);
+        context.resolvePropertyPlaceholders("{{hashicorp:kv:db#password}}");
+        context.resolvePropertyPlaceholders("{{hashicorp:secret:api#password}}");
+
+        HashicorpVaultPropertiesFunction function
+                = (HashicorpVaultPropertiesFunction) context.getPropertiesComponent().getPropertiesFunction("hashicorp");
+        // the format of camel.vault.hashicorp.secrets: a secret of the default secret engine has no engine prefix
+        assertEquals(Set.of("kv:db", "api"), function.getSecrets());
+    }
+
+    @Test
+    void secretWithoutEngineIsTrackedAsDefaultEngine() {
+        // a fresh function has no engine yet, so a plain hashicorp:api lookup runs with a null engine
+        HashicorpVaultPropertiesFunction function = new HashicorpVaultPropertiesFunction();
+        function.setCamelContext(context);
+        function.start();
+        try {
+            function.apply("api");
+
+            // tracked like a secret of the default secret engine, which is how the refresh task reads a name
+            // without engine, and not as "null:api"
+            assertEquals(Set.of("api"), function.getSecrets());
+        } finally {
+            function.stop();
+        }
+    }
+
+    @Test
+    void unchangedSecretDoesNotTriggerReload() {
+        VERSIONS.put("kv/db", 1);
+        VERSIONS.put("secret/api", 3);
+        context.resolvePropertyPlaceholders("{{hashicorp:kv:db#password}}");
+        context.resolvePropertyPlaceholders("{{hashicorp:secret:api#password}}");
+
+        startTask();
+        task.run();
+        task.run();
+
+        assertTrue(task.getUpdates().isEmpty(), "no secret changed, updates: " + task.getUpdates());
+    }
+}

docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc

@@ -860,6 +860,14 @@ in the Camel CLI) is now bound with the name from the annotation, or else the si
 `@BindToRegistry` class. Before, the bean was bound with the fully qualified class name and the annotation value was
 ignored, so a class in a package, or with a name in the annotation, was not found by the expected name.
 
+=== camel-hashicorp-vault - secret refresh
+
+The secret refresh task now checks a secret used as `hashicorp:myengine:mysecret` in its own engine, and names it
+`myengine:mysecret` in `HashicorpVaultPropertiesFunction.getSecrets()` and in the updates of the task (secrets of the
+`secret` engine keep their names). Before, it checked `mysecret` in the `secret` engine, so an update of a secret in
+any other engine never triggered a reload. The first check now also compares the version in Vault with the version the
+property was resolved with, so an update made between the startup and the first check triggers a reload.
+
 === camel-djl (Breaking change)
 
 DJL library dependency has been upgraded to 0.37+