Merge pull request #1352 from 3scale/new-account-proxyconfigs-endpoint

THREESCALE-8508 - /admin/api/account/proxy_configs endpoint for configuration loading

Author: eguzki

CHANGELOG.md

@@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 
 - Opentelemetry support. Opentracing is now deprecated [PR #1379](https://github.com/3scale/APIcast/pull/1379) [THREESCALE-7735](https://issues.redhat.com/browse/THREESCALE-7735)
+- `/admin/api/account/proxy_configs` endpoint for configuration loading [PR #1352](https://github.com/3scale/APIcast/pull/1352) [THREESCALE-8508](https://issues.redhat.com/browse/THREESCALE-8508)
+
+### Removed
+
+- `APICAST_LOAD_SERVICES_WHEN_NEEDED` is dropped and the configuration is fetched "when needed" by default [PR #1352](https://github.com/3scale/APIcast/pull/1352) [THREESCALE-8508](https://issues.redhat.com/browse/THREESCALE-8508)
 
 ## [3.13.2] 2023-02-21
 
@@ -74,6 +79,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Fixed issues with OIDC filters [PR #1304](https://github.com/3scale/APIcast/pull/1304) [PR #1306](https://github.com/3scale/APIcast/pull/1306) [THREESCALE-6042](https://issues.redhat.com/browse/THREESCALE-6042)
 - Fixed issues with OIDC filters [PR #1304](https://github.com/3scale/APIcast/pull/1304) [THREESCALE-6042](https://issues.redhat.com/browse/THREESCALE-6042)
 - Fixed issues with Upstream MTLS certs [PR #1307](https://github.com/3scale/APIcast/pull/1307) [THREESCALE-7508](https://issues.redhat.com/browse/THREESCALE-7508)
+- Fixed warning messages [PR #1318](https://github.com/3scale/APIcast/pull/1318) [THREESCALE-7906](https://issues.redhat.com/browse/THREESCALE-7906)
+- Fixed dirty context [PR #1328](https://github.com/3scale/APIcast/pull/1328) [THREESCALE-8000](https://issues.redhat.com/browse/THREESCALE-8000) [THREESCALE-8007](https://issues.redhat.com/browse/THREESCALE-8007)
+- Fixed jwk alg confusion [PR #1329](https://github.com/3scale/APIcast/pull/1329) [THREESCALE-8249](https://issues.redhat.com/browse/THREESCALE-8249)
+- Fixed issue with resolving target server hostnames to IP when using CONNECT method [PR #1323](https://github.com/3scale/APIcast/pull/1323) [THREESCALE-7967](https://issues.redhat.com/browse/THREESCALE-7967)
+- Fixed issue with resolving target server hostnames to IPs when forwarding requests through http/s proxy [PR #1323](https://github.com/3scale/APIcast/pull/1323) [THREESCALE-7967](https://issues.redhat.com/browse/THREESCALE-7967)
+- Fixed dirty context [PR #1328](https://github.com/3scale/APIcast/pull/1328) [THREESCALE-8000](https://issues.redhat.com/browse/THREESCALE-8000) [THREESCALE-8007](https://issues.redhat.com/browse/THREESCALE-8007) [THREESCALE-8252](https://issues.redhat.com/browse/THREESCALE-8252)
+- Fixed dirty context (part 2 of PR #1328) when tls termination policy is in the policy chain [PR #1333](https://github.com/3scale/APIcast/pull/1333)
+- Fixed NGINX filters policy error [PR #1339](https://github.com/3scale/APIcast/pull/1339) [THREESCALE-7349](https://issues.redhat.com/browse/THREESCALE-7349)
+- Fix to avoid uninitialized variables when request URI is too large [PR #1340](https://github.com/3scale/APIcast/pull/1340) [THREESCALE-7906](https://issues.redhat.com/browse/THREESCALE-7906)
+- Fixed issue where request path is stripped for proxied https requests [PR #1342](https://github.com/3scale/APIcast/pull/1342) [THREESCALE-8426](https://issues.redhat.com/browse/THREESCALE-8426)
+- Bumped liquid-lua to version 0.2.0-2 [PR #1369](https://github.com/3scale/APIcast/pull/1369) - includes: [THREESCALE-8483](https://issues.redhat.com/browse/THREESCALE-8483) and [THREESCALE-8484](https://issues.redhat.com/browse/THREESCALE-8484)
+- New /admin/api/account/proxy_configs endpoint for configuration loading [PR #1352](https://github.com/3scale/APIcast/pull/1352) [THREESCALE-8508](https://issues.redhat.com/browse/THREESCALE-8508)
 
 ### Added
 

Makefile

@@ -244,6 +244,7 @@ clean: ## Remove all docker containers and images
 	make -C $(PROJECT_PATH) -f $(MKFILE_PATH) clean-containers
 	- docker rmi $(DEVEL_IMAGE) $(RUNTIME_IMAGE) apicast-runtime-image:latest apicast-development:latest --force
 	- rm -rf luacov.stats*.out
+	- rm -rf $(PROJECT_PATH)/t/servroot_*
 	make -C $(PROJECT_PATH) -f $(MKFILE_PATH) clean-deps
 
 doc/lua/index.html: $(shell find gateway/src -name '*.lua' 2>/dev/null) | lua_modules $(ROVER)

doc/parameters.md

@@ -62,29 +62,6 @@ Double colon (`:`) separated list of environments (or paths) APIcast should load
 It can be used instead of `-e` or `---environment` parameter on the CLI and for example
 stored in the container image as default environment. Any value passed on the CLI overrides this variable.
 
-### `APICAST_LOAD_SERVICES_WHEN_NEEDED`
-**Values:**
-- `true` or `1` for _true_
-- `false`, `0` or empty for _false_
-
-**Default:** _false_
-
-This option can be used when there are many services configured. However, its
-performance depends on additional factors such as the number of services, the
-latency between APIcast and the 3scale Admin Portal, the Time To Live (TTL) of
-the configuration, etc.
-
-By default, APIcast loads all the services each time it downloads its
-configuration from the Admin Portal. With a large number of services, this could
-become problematic. When this option is enabled, the configurations are loaded
-lazily. APIcast will only load the ones configured for the host specified in the
-host header of the request.
-
-Notes:
-- The caching defined by `APICAST_CONFIGURATION_CACHE` applies.
-- This option will be disabled when `APICAST_CONFIGURATION_LOADER` is `boot`.
-- Not compatible with `APICAST_PATH_ROUTING`.
-
 ### `APICAST_LOG_FILE`
 
 **Default:** _stderr_

gateway/src/apicast/configuration_loader/remote_v2.lua

@@ -100,17 +100,6 @@ local function service_config_endpoint(portal_endpoint, service_id, env, version
   )
 end
 
-local function endpoint_for_services_with_host(portal_endpoint, env, host)
-  local query_args = encode_args({ host = host, version = "latest" })
-
-  return format(
-      "%s/admin/api/account/proxy_configs/%s.json?%s",
-      portal_endpoint,
-      env,
-      query_args
-  )
-end
-
 local function parse_resp_body(self, resp_body)
   local ok, res = pcall(cjson.decode, resp_body)
   if not ok then return nil, res end
@@ -148,72 +137,62 @@ local function parse_resp_body(self, resp_body)
   return cjson.encode(config)
 end
 
-local function load_just_the_services_needed()
-  return resty_env.enabled('APICAST_LOAD_SERVICES_WHEN_NEEDED') and
-         resty_env.value('APICAST_CONFIGURATION_LOADER') == 'lazy'
+local function is_service_filter_by_url_set()
+  if resty_env.value('APICAST_SERVICES_FILTER_BY_URL') then
+    return true
+  else
+    return false
+  end
 end
 
--- When the APICAST_LOAD_SERVICES_WHEN_NEEDED is enabled, but the config loader
--- is boot, APICAST_LOAD_SERVICES_WHEN_NEEDED is going to be ignored. But in
--- that case, env refers to a host and we need to reset it to pick the env
--- again.
-local function reset_env()
-  return resty_env.enabled('APICAST_LOAD_SERVICES_WHEN_NEEDED') and
-         resty_env.value('APICAST_CONFIGURATION_LOADER') == 'boot'
+local function is_service_list_set()
+  if resty_env.value('APICAST_SERVICES_LIST') then
+    return true
+  else
+    return false
+  end
 end
 
-function _M:index(host)
+local function is_service_version_set()
+  local vars = resty_env.list()
+  for n, v in pairs(vars) do
+    if match(n, "APICAST_SERVICE_\\d+_CONFIGURATION_VERSION") and v and v ~= '' then
+        return true
+    end
+  end
+  return false
+end
+
+-- Returns a table that represents paths and query parameters for the current endpoint:
+-- http://${THREESCALE_PORTAL_ENDPOINT}/<env>.json?host=host
+-- http://${THREESCALE_PORTAL_ENDPOINT}/admin/api/account/proxy_configs/<env>.json?host=host&version=version
+local function configuration_endpoint_params(env, host, portal_endpoint_path)
+  return portal_endpoint_path and {path = env, args = {host = host}}
+    or {path = '/admin/api/account/proxy_configs/' .. env, args = {host = host, version = "latest"} }
+end
+
+function _M:index_per_service()
   local http_client = self.http_client
 
   if not http_client then
     return nil, 'not initialized'
   end
 
-  local path = self.path
-
-  if not path then
-    return nil, 'wrong endpoint url'
-  end
-
   local env = resty_env.value('THREESCALE_DEPLOYMENT_ENV')
 
   if not env then
     return nil, 'missing environment'
   end
 
-  local url = resty_url.join(self.endpoint, env .. '.json?' .. encode_args({ host = host }))
-  local res, err = http_client.get(url)
+  local configs = { services = array(), oidc = array() }
+
+  local res, err = self:services()
 
   if not res and err then
-    ngx.log(ngx.DEBUG, 'index get error: ', err, ' url: ', url)
+    ngx.log(ngx.WARN, 'failed to get list of services: ', err, ' url: ', err.url)
     return nil, err
   end
 
-  ngx.log(ngx.DEBUG, 'index get status: ', res.status, ' url: ', url)
-
-  if res.status == 200 then
-    return parse_resp_body(self, res.body)
-  else
-    return nil, 'invalid status'
-  end
-end
-
-function _M:load_configs_for_env_and_host(env, host)
-  local url = endpoint_for_services_with_host(self.endpoint, env, host)
-
-  local response = self.http_client.get(url)
-
-  if response.status == 200 then
-    return parse_resp_body(self, response.body)
-  else
-    ngx.log(ngx.ERR, 'failed to load proxy configs')
-    return false
-  end
-end
-
-function _M:call(environment)
-  local load_just_for_host = load_just_the_services_needed()
-
   local service_regexp_filter  = resty_env.value("APICAST_SERVICES_FILTER_BY_URL")
   if service_regexp_filter then
     local _, err = match("", service_regexp_filter, 'oj')
@@ -223,73 +202,103 @@ function _M:call(environment)
     end
   end
 
-  if self == _M  or not self then
-    local host = environment
-    local m = _M.new()
-    local ret, err = m:index(host)
-
-    if ret then
-      return ret, err
-    end
+  local config
+  for _, object in ipairs(res) do
+    config, err = self:config(object.service, env, 'latest', service_regexp_filter)
 
-    if load_just_for_host then
-      return m:call(host)
+    if config then
+      insert(configs, config)
     else
-      return m:call()
+      ngx.log(ngx.INFO, 'could not get configuration for service ', object.service.id, ': ', err)
     end
   end
 
-  local http_client = self.http_client
+  for i, conf in ipairs(configs) do
+    configs.services[i] = conf.content
 
-  if not http_client then
-    return nil, 'not initialized'
-  end
+    -- Assign false instead of nil to avoid sparse arrays. cjson raises an
+    -- error by default when converting sparse arrays.
+    configs.oidc[i] = conf.oidc or false
 
-  if load_just_for_host then
-    return self:load_configs_for_env_and_host(resty_env.value('THREESCALE_DEPLOYMENT_ENV'), environment)
+    configs[i] = nil
   end
 
-  local env = environment or resty_env.value('THREESCALE_DEPLOYMENT_ENV')
+  return cjson.encode(configs)
+end
+
+function _M:index(host)
+  local http_client = self.http_client
 
-  if reset_env() then
-    env = resty_env.value('THREESCALE_DEPLOYMENT_ENV')
+  if not http_client then
+    return nil, 'not initialized'
   end
 
+  local proxy_config_path = self.path
+  local env = resty_env.value('THREESCALE_DEPLOYMENT_ENV')
+
   if not env then
     return nil, 'missing environment'
   end
 
-  local configs = { services = array(), oidc = array() }
-
-  local res, err = self:services()
+  local endpoint_params = configuration_endpoint_params(env, host, proxy_config_path)
+  local base_url = resty_url.join(self.endpoint, endpoint_params.path .. '.json')
+  local query_args = encode_args(endpoint_params.args) ~= '' and '?'..encode_args(endpoint_params.args)
+  local url = query_args and base_url..query_args or base_url
 
-  if not res and err then
-    ngx.log(ngx.WARN, 'failed to get list of services: ', err, ' url: ', err.url)
+  local res, err = http_client.get(url)
+  if res and res.status == 200 and res.body then
+    ngx.log(ngx.DEBUG, 'index downloaded config from url: ', url)
+    return parse_resp_body(self, res.body)
+  elseif not res and err then
+    ngx.log(ngx.DEBUG, 'index get error: ', err, ' url: ', url)
     return nil, err
   end
 
-  local config
-  for _, object in ipairs(res) do
-    config, err = self:config(object.service, env, 'latest', service_regexp_filter)
+  ngx.log(ngx.DEBUG, 'index get status: ', res.status, ' url: ', url)
 
-    if config then
-      insert(configs, config)
-    else
-      ngx.log(ngx.INFO, 'could not get configuration for service ', object.service.id, ': ', err)
-    end
+  return nil, 'invalid status'
+end
+
+function _M:call(host)
+  if self == _M  or not self then
+    local m = _M.new()
+    return m:call(host)
   end
 
-  for i, conf in ipairs(configs) do
-    configs.services[i] = conf.content
+  local proxy_config_path = self.path
 
-    -- Assign false instead of nil to avoid sparse arrays. cjson raises an
-    -- error by default when converting sparse arrays.
-    configs.oidc[i] = conf.oidc or false
+  -- uses proxy config specific endpoints unless APICAST_SERVICE_%s_CONFIGURATION_VERSION
+  -- When specific version for a specific service is defined,
+  -- loading services one by one is required
+  --
+  -- APICAST_SERVICE_%s_CONFIGURATION_VERSION does not work then THREESCALE_PORTAL_ENDPOINT
+  -- points to master (the API does not allow it), hence error is returned
 
-    configs[i] = nil
+  local use_service_version = is_service_version_set()
+  local use_service_list = is_service_list_set()
+  local use_service_filter_by_url = is_service_filter_by_url_set()
+
+  if use_service_version and proxy_config_path then
+    return nil, 'APICAST_SERVICE_%s_CONFIGURATION_VERSION cannot be used when proxy config path is provided'
   end
 
-  return cjson.encode(configs)
+  if use_service_list and proxy_config_path then
+    return nil, 'APICAST_SERVICES_LIST cannot be used when proxy config path is provided'
+  end
+
+  if use_service_filter_by_url and proxy_config_path then
+    return nil, 'APICAST_SERVICES_FILTER_BY_URL cannot be used when proxy config path is provided'
+  end
+
+  if use_service_version then
+    return self:index_per_service()
+  elseif use_service_list then
+    return self:index_per_service()
+  elseif use_service_filter_by_url then
+    return self:index_per_service()
+  else
+    return self:index(host)
+  end
 end
 
 local services_subset = function()
@@ -401,5 +410,4 @@ function _M:config(service, environment, version, service_regexp_filter)
   end
 end
 
-
 return _M