From eb722798429cbf54fd166c361c329902af03c6f8 Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Tue, 9 May 2023 18:32:36 +0200 Subject: [PATCH 1/7] [DDW-1203] Define a proxy for currency conversions --- flake.nix | 1 + nix/oci-images.nix | 140 ++++++++++++++++++ .../app/config/currencyConfig.coingecko.ts | 8 +- 3 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 nix/oci-images.nix diff --git a/flake.nix b/flake.nix index 46f8bb51cf..7139f89ac6 100644 --- a/flake.nix +++ b/flake.nix @@ -27,6 +27,7 @@ default = package.mainnet; makeSignedInstaller = __mapAttrs (_: a: a.makeSignedInstaller) internal; buildkitePipeline = import ./nix/buildkite-pipeline.nix { inherit inputs targetSystem; }; + ociImages = import ./nix/oci-images.nix { inherit inputs targetSystem; }; }) { x86_64-linux = ./nix/x86_64-linux.nix; x86_64-windows = ./nix/x86_64-windows.nix; diff --git a/nix/oci-images.nix b/nix/oci-images.nix new file mode 100644 index 0000000000..62c990a8f5 --- /dev/null +++ b/nix/oci-images.nix @@ -0,0 +1,140 @@ +{ inputs, targetSystem }: + +let pkgs = inputs.nixpkgs.legacyPackages.${targetSystem}; in + +if targetSystem == "x86_64-linux" then { + + coinGeckoProxy = let + nginx = pkgs.nginxMainline; + nginxCmd = { listenPort }: [ "${nginx}/bin/nginx" "-c" (nginxConf { inherit listenPort; }) ]; + nginxConf = { listenPort }: let + cacheZone = "one"; + nginxConf = '' + daemon off; + user root root; + error_log stderr notice; + events { + worker_connections 1024; + } + http { + access_log /dev/stdout; + include ${nginx}/conf/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + proxy_cache_path /var/cache/nginx/${cacheZone} keys_zone=${cacheZone}:16m inactive=10m max_size=256m; + resolver 8.8.8.8 8.8.4.4; + server { + listen ${toString listenPort}; + server_name localhost; + location / { + return 404; + } + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root ${nginx}/html; + } + location = /api/v3/coins/list { + ${proxyConf} + proxy_cache_valid 200 10m; + proxy_cache_valid any 1m; + set $coingecko https://api.coingecko.com; + proxy_pass $coingecko; + } + location = /api/v3/simple/supported_vs_currencies { + ${proxyConf} + proxy_cache_valid 200 10m; + proxy_cache_valid any 1m; + set $coingecko https://api.coingecko.com; + proxy_pass $coingecko; + } + location = /api/v3/coins/markets { + if ( $args !~ ^ids=cardano&vs_currency=[a-z0-9]+$ ) { return 400; } + ${proxyConf} + proxy_cache_valid 200 400 10m; + proxy_cache_valid any 1m; + set $coingecko https://api.coingecko.com; + proxy_pass $coingecko; + } + } + } + ''; + proxyConf = '' + proxy_cache ${cacheZone}; + proxy_cache_lock on; + proxy_cache_lock_age 10s; + proxy_cache_lock_timeout 10s; + proxy_connect_timeout 60s; + # Don’t let our clients control caching behavior: + proxy_cache_revalidate off; + proxy_cache_use_stale error timeout invalid_header updating + http_500 http_502 http_503 http_504 + http_403 http_404 http_429; + proxy_cache_background_update on; + # Hide some original headers: + proxy_hide_header x-request-id; + proxy_hide_header x-runtime; + proxy_hide_header cf-ray; + proxy_hide_header set-cookie; + proxy_hide_header age; + proxy_hide_header expires; + proxy_hide_header etag; + proxy_hide_header alternate-protocol; + proxy_hide_header cf-cache-status; + # Don’t close upstream connections if our client closes before getting a response: + proxy_ignore_client_abort on; + # Ignore all upstream fields that could interfere with our caching policies: + proxy_ignore_headers X-Accel-Redirect X-Accel-Expires X-Accel-Limit-Rate + X-Accel-Buffering X-Accel-Charset + Expires Cache-Control Set-Cookie Vary; + # Hide errors ≥300: + proxy_intercept_errors on; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_pass_request_headers off; + proxy_ssl_verify on; + proxy_ssl_trusted_certificate /etc/ssl/certs/ca-bundle.crt; + proxy_ssl_server_name on; + add_header X-Cache-Status $upstream_cache_status; + proxy_set_header User-Agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"; + ''; + in pkgs.writeText "nginx-${toString listenPort}.conf" nginxConf; + entrypoint = pkgs.writeShellScript "entrypoint" '' + set -euo pipefail + export PATH="${with pkgs; lib.makeBinPath [ coreutils ]}:$PATH" + + # Let’s pre-populate the cache with the most popular requests: + + ${pkgs.lib.escapeShellArgs (nginxCmd { listenPort = 81; })} & + initial_pid=$! + + for endpoint in \ + '/api/v3/coins/list' \ + '/api/v3/simple/supported_vs_currencies' \ + '/api/v3/coins/markets?ids=cardano&vs_currency=usd' \ + '/api/v3/coins/markets?ids=cardano&vs_currency=jpy' \ + ; do + sleep 2 + echo "Will pre-populate cache with: ‘$endpoint’…" + sleep 3 + ${pkgs.curl}/bin/curl >/dev/null -sS "http://127.0.0.1:81$endpoint" && echo ' ok' || echo ' failed' + done + + kill -s TERM $initial_pid + exec ${pkgs.lib.escapeShellArgs (nginxCmd { listenPort = 80; })} + ''; + in pkgs.dockerTools.buildImage rec { + name = "daedalus-coingecko-proxy"; + tag = inputs.self.rev or "dirty"; + copyToRoot = with pkgs.dockerTools; [ caCertificates fakeNss ]; # XXX: needed for nginx + runAsRoot = '' + #!/bin/sh + set -euo pipefail + mkdir -p /var/log/nginx /var/cache/nginx + exec ${pkgs.lib.escapeShellArgs (nginxCmd { listenPort = 80; })} -t + ''; + config.Expose = "80/tcp"; + config.Entrypoint = [entrypoint]; + }; + +} else {} diff --git a/source/renderer/app/config/currencyConfig.coingecko.ts b/source/renderer/app/config/currencyConfig.coingecko.ts index d40bc663f4..b04a8ede5d 100644 --- a/source/renderer/app/config/currencyConfig.coingecko.ts +++ b/source/renderer/app/config/currencyConfig.coingecko.ts @@ -26,24 +26,30 @@ type CurrencyRateGeckoResponse = Array<{ const id = 'coingecko'; const name = 'CoinGecko'; const website = 'https://www.coingecko.com/en/api'; -const hostname = 'api.coingecko.com'; +const hostname = '127.0.0.1'; const version = 'v3'; const pathBase = `api/${version}`; const requests = { list: [ { hostname, + port: 8095, + protocol: 'http', method: 'GET', path: `/${pathBase}/coins/list`, }, { hostname, + port: 8095, + protocol: 'http', method: 'GET', path: `/${pathBase}/simple/supported_vs_currencies`, }, ], rate: ({ code }: LocalizedCurrency) => ({ hostname, + port: 8095, + protocol: 'http', method: 'GET', path: `/${pathBase}/coins/markets?ids=cardano&vs_currency=${code}`, }), From a08cfa863065928beb50bdc21776e9873c85b230 Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Wed, 10 May 2023 15:04:43 +0200 Subject: [PATCH 2/7] [DDW-1203] Update the PR for Lace, too --- nix/oci-images.nix | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nix/oci-images.nix b/nix/oci-images.nix index 62c990a8f5..08dfdb4b80 100644 --- a/nix/oci-images.nix +++ b/nix/oci-images.nix @@ -56,6 +56,14 @@ if targetSystem == "x86_64-linux" then { set $coingecko https://api.coingecko.com; proxy_pass $coingecko; } + location = /api/v3/simple/price { + if ( $args !~ ^ids=(cardano|bitcoin)&vs_currencies=[a-z0-9,]+(&include_last_updated_at=(true|false))?(&include_24hr_change=(true|false))?$ ) { return 400; } + ${proxyConf} + proxy_cache_valid 200 400 10m; + proxy_cache_valid any 1m; + set $coingecko https://api.coingecko.com; + proxy_pass $coingecko; + } } } ''; @@ -113,6 +121,8 @@ if targetSystem == "x86_64-linux" then { '/api/v3/simple/supported_vs_currencies' \ '/api/v3/coins/markets?ids=cardano&vs_currency=usd' \ '/api/v3/coins/markets?ids=cardano&vs_currency=jpy' \ + '/api/v3/simple/price?ids=cardano&vs_currencies=usd&include_last_updated_at=true' \ + '/api/v3/simple/price?ids=cardano&vs_currencies=usd&include_24hr_change=true' \ ; do sleep 2 echo "Will pre-populate cache with: ‘$endpoint’…" @@ -124,7 +134,7 @@ if targetSystem == "x86_64-linux" then { exec ${pkgs.lib.escapeShellArgs (nginxCmd { listenPort = 80; })} ''; in pkgs.dockerTools.buildImage rec { - name = "daedalus-coingecko-proxy"; + name = "coingecko-proxy"; tag = inputs.self.rev or "dirty"; copyToRoot = with pkgs.dockerTools; [ caCertificates fakeNss ]; # XXX: needed for nginx runAsRoot = '' From 8835ba9a35feef5437f9f52a6758ecd61cff9be2 Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Tue, 16 May 2023 12:05:04 +0200 Subject: [PATCH 3/7] =?UTF-8?q?[DDW-1203]=20Fail=20reqs=20with=20query=20s?= =?UTF-8?q?tring=20where=20it=E2=80=99s=20not=20needed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nix/oci-images.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/oci-images.nix b/nix/oci-images.nix index 08dfdb4b80..968bfb569f 100644 --- a/nix/oci-images.nix +++ b/nix/oci-images.nix @@ -35,6 +35,7 @@ if targetSystem == "x86_64-linux" then { root ${nginx}/html; } location = /api/v3/coins/list { + if ( $args != "" ) { return 400; } ${proxyConf} proxy_cache_valid 200 10m; proxy_cache_valid any 1m; @@ -42,6 +43,7 @@ if targetSystem == "x86_64-linux" then { proxy_pass $coingecko; } location = /api/v3/simple/supported_vs_currencies { + if ( $args != "" ) { return 400; } ${proxyConf} proxy_cache_valid 200 10m; proxy_cache_valid any 1m; From 3c9d0eb3c4acc00ee304193c87868a9ea00bd016 Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Tue, 16 May 2023 12:06:32 +0200 Subject: [PATCH 4/7] [DDW-1203] Remove the proxy definition from `daedalus.git` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s now in --- flake.nix | 1 - nix/oci-images.nix | 152 --------------------------------------------- 2 files changed, 153 deletions(-) delete mode 100644 nix/oci-images.nix diff --git a/flake.nix b/flake.nix index 7139f89ac6..46f8bb51cf 100644 --- a/flake.nix +++ b/flake.nix @@ -27,7 +27,6 @@ default = package.mainnet; makeSignedInstaller = __mapAttrs (_: a: a.makeSignedInstaller) internal; buildkitePipeline = import ./nix/buildkite-pipeline.nix { inherit inputs targetSystem; }; - ociImages = import ./nix/oci-images.nix { inherit inputs targetSystem; }; }) { x86_64-linux = ./nix/x86_64-linux.nix; x86_64-windows = ./nix/x86_64-windows.nix; diff --git a/nix/oci-images.nix b/nix/oci-images.nix deleted file mode 100644 index 968bfb569f..0000000000 --- a/nix/oci-images.nix +++ /dev/null @@ -1,152 +0,0 @@ -{ inputs, targetSystem }: - -let pkgs = inputs.nixpkgs.legacyPackages.${targetSystem}; in - -if targetSystem == "x86_64-linux" then { - - coinGeckoProxy = let - nginx = pkgs.nginxMainline; - nginxCmd = { listenPort }: [ "${nginx}/bin/nginx" "-c" (nginxConf { inherit listenPort; }) ]; - nginxConf = { listenPort }: let - cacheZone = "one"; - nginxConf = '' - daemon off; - user root root; - error_log stderr notice; - events { - worker_connections 1024; - } - http { - access_log /dev/stdout; - include ${nginx}/conf/mime.types; - default_type application/octet-stream; - sendfile on; - keepalive_timeout 65; - proxy_cache_path /var/cache/nginx/${cacheZone} keys_zone=${cacheZone}:16m inactive=10m max_size=256m; - resolver 8.8.8.8 8.8.4.4; - server { - listen ${toString listenPort}; - server_name localhost; - location / { - return 404; - } - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root ${nginx}/html; - } - location = /api/v3/coins/list { - if ( $args != "" ) { return 400; } - ${proxyConf} - proxy_cache_valid 200 10m; - proxy_cache_valid any 1m; - set $coingecko https://api.coingecko.com; - proxy_pass $coingecko; - } - location = /api/v3/simple/supported_vs_currencies { - if ( $args != "" ) { return 400; } - ${proxyConf} - proxy_cache_valid 200 10m; - proxy_cache_valid any 1m; - set $coingecko https://api.coingecko.com; - proxy_pass $coingecko; - } - location = /api/v3/coins/markets { - if ( $args !~ ^ids=cardano&vs_currency=[a-z0-9]+$ ) { return 400; } - ${proxyConf} - proxy_cache_valid 200 400 10m; - proxy_cache_valid any 1m; - set $coingecko https://api.coingecko.com; - proxy_pass $coingecko; - } - location = /api/v3/simple/price { - if ( $args !~ ^ids=(cardano|bitcoin)&vs_currencies=[a-z0-9,]+(&include_last_updated_at=(true|false))?(&include_24hr_change=(true|false))?$ ) { return 400; } - ${proxyConf} - proxy_cache_valid 200 400 10m; - proxy_cache_valid any 1m; - set $coingecko https://api.coingecko.com; - proxy_pass $coingecko; - } - } - } - ''; - proxyConf = '' - proxy_cache ${cacheZone}; - proxy_cache_lock on; - proxy_cache_lock_age 10s; - proxy_cache_lock_timeout 10s; - proxy_connect_timeout 60s; - # Don’t let our clients control caching behavior: - proxy_cache_revalidate off; - proxy_cache_use_stale error timeout invalid_header updating - http_500 http_502 http_503 http_504 - http_403 http_404 http_429; - proxy_cache_background_update on; - # Hide some original headers: - proxy_hide_header x-request-id; - proxy_hide_header x-runtime; - proxy_hide_header cf-ray; - proxy_hide_header set-cookie; - proxy_hide_header age; - proxy_hide_header expires; - proxy_hide_header etag; - proxy_hide_header alternate-protocol; - proxy_hide_header cf-cache-status; - # Don’t close upstream connections if our client closes before getting a response: - proxy_ignore_client_abort on; - # Ignore all upstream fields that could interfere with our caching policies: - proxy_ignore_headers X-Accel-Redirect X-Accel-Expires X-Accel-Limit-Rate - X-Accel-Buffering X-Accel-Charset - Expires Cache-Control Set-Cookie Vary; - # Hide errors ≥300: - proxy_intercept_errors on; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_pass_request_headers off; - proxy_ssl_verify on; - proxy_ssl_trusted_certificate /etc/ssl/certs/ca-bundle.crt; - proxy_ssl_server_name on; - add_header X-Cache-Status $upstream_cache_status; - proxy_set_header User-Agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"; - ''; - in pkgs.writeText "nginx-${toString listenPort}.conf" nginxConf; - entrypoint = pkgs.writeShellScript "entrypoint" '' - set -euo pipefail - export PATH="${with pkgs; lib.makeBinPath [ coreutils ]}:$PATH" - - # Let’s pre-populate the cache with the most popular requests: - - ${pkgs.lib.escapeShellArgs (nginxCmd { listenPort = 81; })} & - initial_pid=$! - - for endpoint in \ - '/api/v3/coins/list' \ - '/api/v3/simple/supported_vs_currencies' \ - '/api/v3/coins/markets?ids=cardano&vs_currency=usd' \ - '/api/v3/coins/markets?ids=cardano&vs_currency=jpy' \ - '/api/v3/simple/price?ids=cardano&vs_currencies=usd&include_last_updated_at=true' \ - '/api/v3/simple/price?ids=cardano&vs_currencies=usd&include_24hr_change=true' \ - ; do - sleep 2 - echo "Will pre-populate cache with: ‘$endpoint’…" - sleep 3 - ${pkgs.curl}/bin/curl >/dev/null -sS "http://127.0.0.1:81$endpoint" && echo ' ok' || echo ' failed' - done - - kill -s TERM $initial_pid - exec ${pkgs.lib.escapeShellArgs (nginxCmd { listenPort = 80; })} - ''; - in pkgs.dockerTools.buildImage rec { - name = "coingecko-proxy"; - tag = inputs.self.rev or "dirty"; - copyToRoot = with pkgs.dockerTools; [ caCertificates fakeNss ]; # XXX: needed for nginx - runAsRoot = '' - #!/bin/sh - set -euo pipefail - mkdir -p /var/log/nginx /var/cache/nginx - exec ${pkgs.lib.escapeShellArgs (nginxCmd { listenPort = 80; })} -t - ''; - config.Expose = "80/tcp"; - config.Entrypoint = [entrypoint]; - }; - -} else {} From 7cfc8e2c05453a4d411a7b4d22a2e35b8fb200cf Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Tue, 16 May 2023 12:09:01 +0200 Subject: [PATCH 5/7] [DDW-1203] Switch to the proxy URL from SRE --- source/renderer/app/config/currencyConfig.coingecko.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/source/renderer/app/config/currencyConfig.coingecko.ts b/source/renderer/app/config/currencyConfig.coingecko.ts index b04a8ede5d..08ac059d08 100644 --- a/source/renderer/app/config/currencyConfig.coingecko.ts +++ b/source/renderer/app/config/currencyConfig.coingecko.ts @@ -26,30 +26,24 @@ type CurrencyRateGeckoResponse = Array<{ const id = 'coingecko'; const name = 'CoinGecko'; const website = 'https://www.coingecko.com/en/api'; -const hostname = '127.0.0.1'; +const hostname = 'coingecko.dev-preprod.eks.lw.iog.io'; const version = 'v3'; const pathBase = `api/${version}`; const requests = { list: [ { hostname, - port: 8095, - protocol: 'http', method: 'GET', path: `/${pathBase}/coins/list`, }, { hostname, - port: 8095, - protocol: 'http', method: 'GET', path: `/${pathBase}/simple/supported_vs_currencies`, }, ], rate: ({ code }: LocalizedCurrency) => ({ hostname, - port: 8095, - protocol: 'http', method: 'GET', path: `/${pathBase}/coins/markets?ids=cardano&vs_currency=${code}`, }), From 72816d7f9f6e7609b0e523b5d63d112ea1623fa1 Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Tue, 16 May 2023 12:13:07 +0200 Subject: [PATCH 6/7] [DDW-1203] Add a changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed55dec87c..4a5be63fc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Chores +- Defined a proxy for currency conversions ([PR 3121](https://github.com/input-output-hk/daedalus/pull/3121)) - Switched Daedalus to be built with Nix flakes ([PR 3008](https://github.com/input-output-hk/daedalus/pull/3008)) - Update `cardano-wallet` to `v2023-04-14` ([PR 3104](https://github.com/input-output-hk/daedalus/pull/3104)) - Prepared an official `aarch64-darwin` build ([PR 3116](https://github.com/input-output-hk/daedalus/pull/3116)) From 8dc18edc355c674dbdcee98380d9d64b9e4cb1e9 Mon Sep 17 00:00:00 2001 From: Michal Rus Date: Wed, 17 May 2023 12:35:23 +0200 Subject: [PATCH 7/7] [DDW-1203] Switch to the final production URL --- source/renderer/app/config/currencyConfig.coingecko.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/renderer/app/config/currencyConfig.coingecko.ts b/source/renderer/app/config/currencyConfig.coingecko.ts index 08ac059d08..daefdf2a83 100644 --- a/source/renderer/app/config/currencyConfig.coingecko.ts +++ b/source/renderer/app/config/currencyConfig.coingecko.ts @@ -26,7 +26,7 @@ type CurrencyRateGeckoResponse = Array<{ const id = 'coingecko'; const name = 'CoinGecko'; const website = 'https://www.coingecko.com/en/api'; -const hostname = 'coingecko.dev-preprod.eks.lw.iog.io'; +const hostname = 'coingecko.live-mainnet.eks.lw.iog.io'; const version = 'v3'; const pathBase = `api/${version}`; const requests = {