release: structures 3.8.0

dev-tools/kind/README.md

@@ -254,7 +254,7 @@ Load a locally built Docker image into KinD cluster nodes.
 ./kind-cluster.sh load
 
 # Load specific image
-./kind-cluster.sh load --image mindsignited/structures-server:3.7.0
+./kind-cluster.sh load --image mindsignited/structures-server:3.8.0
 
 # Load into specific cluster
 ./kind-cluster.sh load --name test-cluster

dev-tools/kind/config/structures-server/values.yaml

@@ -18,7 +18,7 @@
 #    ./gradlew :structures-server:bootBuildImage
 #
 # 2. Load into KinD cluster:
-#    kind load docker-image mindsignited/structures-server:3.7.0-SNAPSHOT --name structures-cluster
+#    kind load docker-image mindsignited/structures-server:3.8.0-SNAPSHOT --name structures-cluster
 #
 # OR use the helper script:
 #    ./dev-tools/kind/kind-cluster.sh build --load
@@ -47,7 +47,7 @@ image:
   repository: mindsignited/structures-server
   # Published tag pulled from Docker Hub. Overridden by deploy --tag, or by
   # deploy --build-local which uses the locally built gradle.properties version.
-  tag: 3.7.0
+  tag: 3.8.0
   # Pull from Docker Hub by default; deploy --build-local overrides this to Never
   pullPolicy: IfNotPresent
   sha: ""
@@ -61,7 +61,7 @@ migration:
   image:
     repository: mindsignited/structures-migration
     # Keep in sync with image.tag above; deploy --tag sets both
-    tag: 3.7.0
+    tag: 3.8.0
     # Pull from Docker Hub by default; deploy --build-local overrides this to Never
     pullPolicy: IfNotPresent
     sha: ""

dev-tools/kind/kind-cluster.sh

@@ -405,8 +405,8 @@ Examples:
   # Deploy with inline override
   $(basename "$0") deploy --set replicaCount=3
 
-  # Deploy a specific published tag: the develop snapshot, or a PR image (3.7.0-pr<N>.<sha>)
-  $(basename "$0") deploy --tag 3.7.0-SNAPSHOT
+  # Deploy a specific published tag: the develop snapshot, or a PR image (3.8.0-pr<N>.<sha>)
+  $(basename "$0") deploy --tag 3.8.0-SNAPSHOT
 
   # Build from source and load into the cluster instead of pulling
   $(basename "$0") deploy --build-local
@@ -798,7 +798,7 @@ Examples:
   $(basename "$0") load
 
   # Load specific image
-  $(basename "$0") load --image mindsignited/structures-server:3.7.0
+  $(basename "$0") load --image mindsignited/structures-server:3.8.0
 
   # Load into specific cluster
   $(basename "$0") load --name test-cluster

dev-tools/kind/lib/config.sh

@@ -251,13 +251,13 @@ get_coredns_template_path() {
 #
 # Read version from gradle.properties
 # Returns:
-#   Version string (e.g., "3.7.0-SNAPSHOT")
+#   Version string (e.g., "3.8.0-SNAPSHOT")
 # Example:
 #   version=$(get_structures_version)
 #
 get_structures_version() {
     # Explicit override wins, e.g. deploying the released image from a release
-    # checkout: structuresVersion=3.7.0 ./kind-cluster.sh deploy
+    # checkout: structuresVersion=3.8.0 ./kind-cluster.sh deploy
     if [[ -n "${structuresVersion:-}" ]]; then
         echo "${structuresVersion}"
         return 0
@@ -291,7 +291,7 @@ get_structures_version() {
 #
 # Get image name for structures-server
 # Returns:
-#   Full image name (e.g., "mindsignited/structures-server:3.7.0-SNAPSHOT")
+#   Full image name (e.g., "mindsignited/structures-server:3.8.0-SNAPSHOT")
 # Example:
 #   image=$(get_image_name)
 #
@@ -306,7 +306,7 @@ get_image_name() {
 #
 # Get image name for structures-migration
 # Returns:
-#   Full image name (e.g., "mindsignited/structures-migration:3.7.0-SNAPSHOT")
+#   Full image name (e.g., "mindsignited/structures-migration:3.8.0-SNAPSHOT")
 # Example:
 #   migration_image=$(get_migration_image_name)
 #

dev-tools/kind/lib/images.sh

@@ -81,11 +81,11 @@ build_image() {
 #
 # Check if Docker image exists locally
 # Args:
-#   $1: Image name (e.g., "mindsignited/structures-server:3.7.0-SNAPSHOT")
+#   $1: Image name (e.g., "mindsignited/structures-server:3.8.0-SNAPSHOT")
 # Returns:
 #   0 if exists, 1 otherwise
 # Example:
-#   image_exists "mindsignited/structures-server:3.7.0-SNAPSHOT"
+#   image_exists "mindsignited/structures-server:3.8.0-SNAPSHOT"
 #
 image_exists() {
     local image_name="$1"
@@ -102,7 +102,7 @@ image_exists() {
 # Args:
 #   $1: Image name
 # Example:
-#   get_image_info "mindsignited/structures-server:3.7.0-SNAPSHOT"
+#   get_image_info "mindsignited/structures-server:3.8.0-SNAPSHOT"
 #
 get_image_info() {
     local image_name="$1"
@@ -154,7 +154,7 @@ get_image_info() {
 # Returns:
 #   0 on success, 1 on failure
 # Example:
-#   load_image_into_cluster "structures-cluster" "mindsignited/structures-server:3.7.0-SNAPSHOT"
+#   load_image_into_cluster "structures-cluster" "mindsignited/structures-server:3.8.0-SNAPSHOT"
 #
 load_image_into_cluster() {
     local cluster_name="$1"
@@ -212,7 +212,7 @@ load_image_into_cluster() {
 # Args:
 #   $1: Image name
 # Example:
-#   display_deployment_instructions "mindsignited/structures-server:3.7.0-SNAPSHOT"
+#   display_deployment_instructions "mindsignited/structures-server:3.8.0-SNAPSHOT"
 #
 display_deployment_instructions() {
     local image_name="$1"

docker-compose/compose.yml

@@ -6,7 +6,7 @@ services:
   structures-migration:
     container_name: structures-migration
     pull_policy: always
-    image: mindsignited/structures-migration:${structuresVersion:-3.7.0}
+    image: mindsignited/structures-migration:${structuresVersion:-3.8.0}
     depends_on:
       structures-elasticsearch:
         condition: service_healthy
@@ -17,7 +17,7 @@ services:
   structures-server:
     container_name: structures-server
     pull_policy: always
-    image: mindsignited/structures-server:${structuresVersion:-3.7.0}
+    image: mindsignited/structures-server:${structuresVersion:-3.8.0}
     depends_on:
       structures-elasticsearch:
         condition: service_healthy

docs/kubernetes/KIND_TOOLING.md

@@ -300,7 +300,7 @@ Load a locally built Docker image into KinD cluster nodes.
 ./kind-cluster.sh load
 
 # Load specific image
-./kind-cluster.sh load --image mindsignited/structures-server:3.7.0
+./kind-cluster.sh load --image mindsignited/structures-server:3.8.0
 
 # Load into specific cluster
 ./kind-cluster.sh load --name test-cluster

docs/oidc/OIDC_IMPLEMENTATION.md

@@ -26,19 +26,19 @@ Handles JWKS (JSON Web Key Set) operations:
 - Extracts key information from JWT tokens
 - Provides efficient key lookup by issuer and key ID
 
-### 2. OidcAuthVerifier
+### 2. OidcSecurityService
 Main authentication component:
-- Implements SecurityService interface
-- Validates JWT tokens using JJWT 0.12.x
+- Implements the continuum `SecurityService` interface
+- Validates JWT tokens using JJWT
+- Rejects tokens whose issuer is not the authority of an enabled provider (`Issuer not allowed`)
+- Validates audience and required roles against the matching provider
 - Creates Participant objects from JWT claims
-- Handles issuer and audience validation
-- Supports multiple audience validation
 
-### 3. OidcAuthVerifierProperties
-Configuration properties for OIDC:
+### 3. OidcSecurityServiceProperties
+Configuration properties for OIDC, under `oidc-security-service`:
 - `enabled`: Enable/disable OIDC authentication
-- `allowedIssuers`: List of allowed OIDC issuers
-- `authorizationAudiences`: List of allowed audiences
+- `oidc-providers`: the providers, each with its `authority`, `audience`, roles and frontend settings
+- `jwks-*`: fetch and caching settings, see [Caching Strategy](#caching-strategy)
 
 ### 4. Frontend Integration
 Vue.js frontend components:
@@ -645,15 +645,106 @@ oidc-security-service:
 
 ## Caching Strategy
 
-### JWKS Key Cache
-- **TTL**: 1 hour
-- **Max Size**: 100 keys
-- **Purpose**: Cache individual public keys by issuer and key ID
+Keys are only ever fetched for an issuer that matches the `authority` of an enabled provider. A token from any
+other issuer, or from a disabled provider's issuer, is rejected with `Issuer not allowed` before anything is
+fetched.
+
+Cached documents are refreshed rather than expired. Once one is due, the next lookup starts a refresh in the
+background and is served the cached document meanwhile. A failed refresh keeps the cached document. An IdP
+outage, or a network problem between Structures and the IdP, therefore only fails lookups that need something
+not already cached.
+
+### JWKS Key Set Cache
+
+The key set of each provider is cached by JWKS URL, at most 100 of them. How it behaves in each situation:
+
+| Situation | What happens |
+|---|---|
+| Normal operation | The set is refreshed every `jwks-refresh-interval` (1 hour by default). Refreshes are lazy: the first lookup after the interval starts one and is served the cached set, so no login waits on it. |
+| A refresh fails | The cached set is kept and logged at warn with its age. The next refresh is attempted once `jwks-refresh-interval` passes again, not on every lookup. |
+| A long IdP outage | The last fetched keys keep working for as long as the outage lasts. Set `jwks-max-staleness`, longer than `jwks-refresh-interval`, to cap that. It is counted from the last successful fetch: once the keys are older, the next lookup waits on a fetch, and logins fail until the IdP is reachable. |
+| A token names an unknown key id | The provider may have rotated its keys, so the set is fetched again, at most once per `jwks-refresh-cooldown` (30 seconds by default). Tokens that arrive while that refresh runs, or within the cooldown after it, use its result, so a burst of tokens signed with a new key shares one fetch. |
+| Keys published ahead of use | Picked up by the next routine refresh, so no token waits on a fetch. On a pod with no logins between the key being published and first used, that refresh has not run yet, so the first such token waits on one fetch instead. It is not rejected. |
+| A refresh for an unknown key id fails | The token is rejected, and the fetch error is logged with the rejection. Clients see the gateway's generic authentication error. Tokens with known key ids keep working. |
+
+The cooldown counts only from refreshes that unknown key ids started, so a rotation just after a routine refresh
+is still picked up. A token signed with a rotated key is rejected only if it arrives within the cooldown after
+an unknown key id refresh that ran before the rotation.
+
+#### How this compares to other libraries
+
+Libraries differ mainly in what they do when a refresh fails while keys are cached:
+
+| Library | Routine refresh | Refresh fails while keys are cached | Unknown key id |
+|---|---|---|---|
+| Spring Security `NimbusJwtDecoder` (default) | Cache expires after 5 min | Validation fails | Refetch, not rate limited |
+| Nimbus `JWKSourceBuilder` (default) | 5 min, refreshed 30 s ahead | Fails, unless outage tolerance is enabled (off by default, then 50 min or forever) | Refetch, at most once per 30 s |
+| panva/jose | 10 min max age | Fails | Refetch, 30 s cooldown |
+| auth0 node-jwks-rsa | 10 min max age | Fails, unless a fallback window is set | Refetch, at most 10 per minute |
+| Microsoft.IdentityModel | Every 12 h | Keeps the current keys, no maximum age | Manual refresh at most every 5 min |
+| coreos/go-oidc | Keys do not expire | Keeps the cached keys | Refetch, not rate limited |
+| **Structures** | Every 1 h (`jwks-refresh-interval`) | Keeps the cached keys, optional `jwks-max-staleness` | Refetch, 30 s cooldown (`jwks-refresh-cooldown`) |
+
+Structures takes the availability side, as Microsoft.IdentityModel and go-oidc do: a network problem does not
+reject tokens that are still valid. The cost is that a key the IdP revokes while it cannot be reached is
+trusted until it can be; `jwks-max-staleness` bounds that for deployments that prefer to fail closed.
+
+#### Rotating keys without rejected logins
+
+Publish a new key before signing with it, and every pod picks it up in a routine refresh:
+
+- **Keycloak**: add the new key provider with the key *enabled* but not *active*, so it is published in the
+  JWKS but not used for signing. Wait at least `jwks-refresh-interval`, then make it active, and later disable
+  the old key once tokens signed with it have expired.
+- **Entra ID and Okta** publish upcoming keys ahead of use already.
+
+Without that, a rotation is still picked up through the unknown key id refresh, at the cost of one fetch.
 
 ### Well-known Configuration Cache
-- **TTL**: 24 hours
-- **Max Size**: 10 configurations
-- **Purpose**: Cache OIDC provider discovery documents
+- **Refresh**: every `jwks-refresh-interval`, like key sets; a failed refresh keeps the cached document, and
+  there is no maximum age
+- A lookup that finds both due refreshes both. The two run independently, neither waits for the other, and
+  the lookup is served the cached copies of both. The key set refresh uses the JWKS URL from the cached
+  document
+- A changed `jwks_uri` is used from the first lookup after the discovery refresh completes. That lookup waits
+  on loading the key set from the new URL, and fails if that load fails: keys cached under the old URL are
+  not used as a fallback
+- **Max Size**: 100 configurations
+- **Purpose**: Cache OIDC provider discovery documents, for their `jwks_uri`
+- A document without a `jwks_uri` is treated as a failure and not cached
+- Not used for a provider with `jwks-uri` configured
+
+### Failures and Timeouts
+- Failed fetches, empty key sets and broken discovery documents are never cached. With nothing cached, lookups
+  fail with the last error for `jwks-retry-backoff` (5 seconds by default) before one fetches again, which
+  bounds the requests and log lines an unreachable provider causes
+- Concurrent lookups of the same document share one fetch
+- Fetches do not reuse pooled connections, so a connection silently dropped by a NAT or load balancer cannot
+  stall a fetch until TCP gives up (around 15 minutes on Linux)
+- Every fetch is bounded:
+
+```yaml
+oidc-security-service:
+  jwks-connect-timeout: 5s   # TCP connect to the provider
+  jwks-request-timeout: 10s  # one discovery or JWKS fetch, end to end
+  jwks-refresh-interval: 1h  # how often each key set and discovery document is refreshed
+  jwks-max-staleness:        # unset: keep the last fetched keys through an outage; e.g. 24h to cap it
+  jwks-refresh-cooldown: 30s # minimum time between refreshes caused by unknown key ids
+  jwks-retry-backoff: 5s     # after a failed fetch with nothing cached, wait this long before fetching again
+```
+
+### Fetching keys from a different URL
+When the authority's public URL is not reachable from where Structures runs, set `jwks-uri` on the provider.
+Keys are then fetched from it and the discovery document is not used. The token's issuer must still match
+`authority`. `jwks-uri` is not included in the frontend configuration.
+
+```yaml
+oidc-security-service:
+  oidc-providers:
+    - provider: "keycloak"
+      authority: "https://auth.example.com/realms/structures"
+      jwks-uri: "http://keycloak.auth.svc:8080/realms/structures/protocol/openid-connect/certs"
+```
 
 ## Security Features
 
@@ -858,8 +949,8 @@ console.log('Stored state:', localStorage.getItem('oidc.state.your-client-id'));
 
 ## Performance Considerations
 
-1. **Cache Tuning**: Adjust cache TTL and size based on your OIDC provider
-2. **Network Timeouts**: Configure appropriate timeouts for JWKS fetching
+1. **Refresh Tuning**: `jwks-refresh-interval` and `jwks-refresh-cooldown` set how often Structures fetches from the IdP; the caches hold at most 100 key sets and 100 discovery documents
+2. **Network Timeouts**: `jwks-connect-timeout` and `jwks-request-timeout` bound every JWKS and discovery fetch
 3. **Memory Usage**: Monitor cache memory usage in production
 4. **Concurrent Requests**: The implementation is thread-safe and handles concurrent requests
 

docs/performance/ELASTICSEARCH_WORKLOAD.md

@@ -110,6 +110,8 @@ blunt, put a limiter in front of the writer client:
   it slowly when it recovers (additive increase, multiplicative decrease). Driving it from read
   latency protects reads directly, instead of relying on a number tuned once.
 - If one tenant turns out to use the whole budget, add per-tenant fairness.
+- Bulk items Elasticsearch rejects with a 429 reach clients as a 400 today. See "Give failed bulk
+  items the right status" below.
 
 #### Done: refresh settings for single writes
 
@@ -209,6 +211,58 @@ Follow-ups, not done yet:
     `bulkSave` writes it again every time. If a client mostly sends documents that haven't changed,
     `bulkUpdate` may be cheaper for it. The bulk response marks each skipped document as a `noop`,
     so counting those shows which case a client is in.
+- **Give failed bulk items the right status.** Found in the review of PR #26, not done yet.
+  `doPersistBulkLogic` fails a bulk call with a 409 `VersionConflictException` only when every
+  failed item was a version conflict. Any other failed item makes the whole call an
+  `IllegalArgumentException`, which REST answers with **400 Bad Request**. That includes items
+  Elasticsearch rejected under load (`429 es_rejected_execution_exception`), a `503`, and a call
+  that mixes conflicts with rejections. A client reads the 400 as bad data, so it neither retries
+  the throttled items nor re-reads the conflicted ones. This gets more likely under the write
+  pressure this plan is about. Base the status on the item statuses instead:
+  - **Only conflicts:** 409, as it is now.
+  - **Only client errors** (mapping or parse failures): 400.
+  - **Any 429 or 5xx:** a retryable status, 429 or 503, with the failed item ids listed so the
+    client resends just those.
+
+  This fits with step 2 (chunks report their errors together at the end) and step 3 (the limiter
+  reacts to the same 429s).
+- **Tell bulk callers which items failed.** Not done yet. A failed bulk call rejects with one
+  message that lists each distinct Elasticsearch reason, one per line. Callers can't get a list of
+  the failed items. The best they can do is parse ids out of that text, and that only works for
+  some failures:
+  - **Version conflict reasons** name the stored document id, e.g. `[kinotic-123]: version
+    conflict, ...`. On shared multi-tenant structures that id has the tenant prefixed.
+  - **Many other reasons, such as mapping errors, don't name the document.** Identical reasons are
+    listed once, so 50 items failing the same way show as one line.
+
+  The items that didn't fail are written either way, since Elasticsearch applies each bulk item on
+  its own. Keep listing every reason until this is done, because parsing them is the only option
+  callers have. Elasticsearch returns bulk results in request order, so each failure can be mapped
+  back to its position in the submitted array and to its entity id.
+
+  Over STOMP only the exception's message reaches the JS client. `EventBus` reads just the error
+  header, and Continuum's `ServiceExceptionWrapper` carries only the class, message and stack. So
+  the options are:
+  1. **A result instead of `void`** (preferred): bulk calls return something like
+     `{failures: [{index, id, status, reason}]}`. It works the same over STOMP, REST and Java
+     without changing Continuum. A call that partly failed would no longer reject, so callers that
+     ignore the result would miss failures. Ship it as new methods, or in a major version.
+  2. **A typed exception that carries the failures**: easy for the REST body, but over STOMP it
+     needs Continuum to serialize the extra fields and continuum-client-js to read them.
+  3. **JSON inside the message**: works on every transport today, but callers parse error text.
+     Avoid.
+
+  Do this together with the item above, so each failure also carries a status a client can act on.
+- **Keep Elasticsearch details out of single-write errors.** Found in the second review of PR #26,
+  not done yet. The async client hands back any status it isn't told to expect as the low-level
+  `ResponseException`. It does this for 429, 503 and 409 too (only 400-405 come back as typed
+  errors). That exception's message names the Elasticsearch host and request URI, for example
+  `method [POST], host [http://es-internal:9200], URI [/idx/_update/kinotic-1?...]`, and the caller
+  gets it unchanged. PR #26 fixed this for 409 only: `ElasticVersionConflicts` reads the reason from
+  the response body. A `save` or `update` rejected with a 429 or 503 still answers 500 with the
+  host and URI in its body. Fix it the same way for every `ResponseException`: build the message
+  from the body's `error.reason`, and pick the status from the response, so a 429 or 503 is
+  retryable as described above.
 
 ### Proposed settings