Merge branch 'main' into add_onion_tests

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Filecoin Saturn Slack Channel
+    url: https://filecoinproject.slack.com/archives/C03DH0BL02E
+    about: Please ask any questions here
+  - name: Security Issue
+    url: "mailto:saturn@protocol.ai"
+    about: Please report security vulnerabilities here.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,15 @@
+---
+name: Feature request
+about: Use this template to request a feature
+title: Feature request: 
+labels: "Low priority"
+assignees: ""
+---
+
+### What is the feature you would like to see?
+
+### Why do you need this feature?
+
+### How would you like to see this feature implemented?
+
+### Additional context

.github/ISSUE_TEMPLATE/ip-geolocation-override.md → .github/ISSUE_TEMPLATE/fix-incorrect-node-ip-geolocation.md

@@ -1,6 +1,6 @@
 ---
-name: IP geolocation override
-about: Template to request an IP geolocation override
+name: Fix incorrect node IP geolocation
+about: Use this template to fix an incorrect node IP geolocation
 title: Geolocation fix
 labels: ""
 assignees: ""

.github/ISSUE_TEMPLATE/general-issue.md

@@ -0,0 +1,17 @@
+---
+name: General issue
+about: Use this template to report a general issue
+title: General issue
+labels: "Medium priority"
+assignees: ""
+---
+
+### What did you do?
+
+### What happened?
+
+### What would you expect to have happened?
+
+### How can we reproduce this issue?
+
+### Additional context

.github/workflows/gateway-conformance.yml

@@ -0,0 +1,107 @@
+name: Gateway Conformance
+
+on:
+  push:
+    branches:
+      - main
+      - gateway-conformance
+    paths:
+      - ".github/workflows/gateway-conformance.yml"
+      - "container/**"
+      - "Dockerfile"
+  pull_request:
+
+env:
+  L1_ORIGIN: http://127.0.0.1:80
+
+jobs:
+  gateway-conformance:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Start the Kubo gateway
+      - name: Download Kubo gateway
+        uses: ipfs/download-ipfs-distribution-action@v1
+      - name: Start Kubo gateway
+        uses: ipfs/start-ipfs-daemon-action@v1
+
+      # 2. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.2.0
+        with:
+          output: fixtures
+
+      # 3. Populate the Kubo gateway with the gateway-conformance fixtures
+      - name: Import fixtures
+        run: |
+          find fixtures -name '*.car' -exec ipfs dag import --pin-roots=false {} \;
+          sudo rm -rf fixtures
+
+      # 4. Build the L1
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build local Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          tags: saturn-node
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          load: true
+          build-args: |
+            SATURN_NETWORK=local
+            ORCHESTRATOR_URL=http://localhost
+            LOG_INGESTOR_URL=http://localhost
+
+      # 5. Start the L1
+      - name: Start L1
+        run: |
+          export ORCHESTRATOR_REGISTRATION=false
+
+          # Configure lassie to fetch only from the localhost kubo
+          export LASSIE_ALLOW_PROVIDERS=$(ipfs id | jq --raw-output '.Addresses | @csv | sub("\"";"";"g") ')
+          export DOCKER_NETWORK=host
+
+          sh scripts/run.sh &
+
+          num_attempts=0
+          max_attempts=5
+          url=${{ env.L1_ORIGIN }}
+
+          # wait for the container to be up
+          until curl -s "$url" -o /dev/null; do
+              if [ ${num_attempts} -eq ${max_attempts} ];then
+                  echo "Max attempts reached"
+                  exit 1
+              fi
+
+              num_attempts=$(($num_attempts+1))
+              sleep 5;
+          done
+
+      # 6. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        # Wait for release that contains "trustless-block-gateway,trustless-car-gateway" specs
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.2.0
+        with:
+          gateway-url: ${{ env.L1_ORIGIN }}
+          specs: trustless-block-gateway,trustless-car-gateway
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+
+      # 7. Upload the results
+      - name: Upload Markdown summary
+        if: "!cancelled()"
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: "!cancelled()"
+        uses: actions/upload-artifact@v3
+        with:
+          name: gateway-conformance.html
+          path: output.html

.github/workflows/integration-tests.yml

@@ -32,8 +32,8 @@ jobs:
         with:
           context: .
           tags: saturn-node
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=l1
+          cache-to: type=gha,mode=max,scope=l1
           load: true
           build-args: |
             SATURN_NETWORK=local
@@ -66,4 +66,4 @@ jobs:
 
       - name: Print L1 logs
         if: failure()
-        run: cat /home/runner/work/L1-node/L1-node/shared/nginx_log/error.log
+        run: cat /home/runner/work/L1-node/L1-node/shared/nginx_log/error.log || true

.github/workflows/release.yml

@@ -21,7 +21,7 @@ env:
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-latest
+    runs-on: larger
 
     permissions:
       contents: read
@@ -100,8 +100,8 @@ jobs:
             ${{ steps.tags.outputs.mutable_docker_tags }}
           labels: ${{ steps.meta.outputs.labels }}
           platforms: linux/amd64,linux/arm64
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=gha,scope=l1
+          cache-to: type=gha,mode=max,scope=l1
           build-args: |
             NETWORK=${{ env.NETWORK }}
             VERSION=${{ github.run_number }}_${{ steps.git.outputs.sha_short }}

.github/workflows/static-code-analysis.yml

@@ -4,14 +4,15 @@ on:
   push:
     branches: ["main"]
   pull_request:
+    branches: ["main"]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  check:
-    name: "Check linting and formatting"
+  lint-format:
+    name: Check linting and formatting
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -23,3 +24,24 @@ jobs:
       - run: npm ci
       - run: npm run check:eslint
       - run: npm run check:prettier
+
+  codeql-analysis:
+    name: CodeQL Analysis
+    runs-on: ubuntu-latest
+
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: javascript
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

Dockerfile

@@ -9,8 +9,8 @@ ARG NGINX_BRANCH=default
 ARG NGINX_COMMIT=c38588d8376b
 # https://github.com/google/ngx_brotli
 ARG NGX_BROTLI_COMMIT=6e975bcb015f62e1f303054897783355e2a877dc
-ARG NJS_VERSION=0.7.12
-ARG NGX_CAR_RANGE_VERSION="v0.5.0-rc1"
+ARG NJS_VERSION=0.8.0
+ARG NGX_CAR_RANGE_VERSION="v0.6.0"
 
 # Install dependencies
 RUN apt-get update && apt-get install -y \
@@ -121,7 +121,6 @@ COPY --from=build /usr/src/${NGINX_NAME}/objs/ngx_http_brotli_static_module.so /
 COPY --from=build /usr/src/${NGINX_NAME}/objs/ngx_http_js_module.so /usr/lib/nginx/modules/
 COPY --from=build /usr/src/ngx_car_range/target/release/libnginx_car_range.so /usr/lib/nginx/modules/ngx_http_car_range_module.so
 
-# RUN curl -fsSL https://install.speedtest.net/app/cli/install.deb.sh | bash -
 RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
  && curl -fsSL https://packagecloud.io/install/repositories/ookla/speedtest-cli/script.deb.sh | bash - \
  && apt-get install --no-install-recommends -y \
@@ -133,7 +132,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
 
 # Download lassie
 ARG TARGETPLATFORM
-ARG LASSIE_VERSION="v0.13.0"
+ARG LASSIE_VERSION="v0.14.2"
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then ARCHITECTURE=amd64; \
   elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then ARCHITECTURE=arm64; \
   else ARCHITECTURE=386; fi \

README.md

@@ -243,12 +243,14 @@ Having an uptime requirement has the following benefits:
 
 The current uptime requirement is as follows:
 
-- For a node operator to qualify for earnings at the end of the month, their node must be online for at least 7 days during the month. It is important to note that earnings for nodes STILL get calculated before the node satisfies the uptime requirement.
-- If a node is registered after the 21st day of a given month, the uptime requirement will be postponed to the following month. Its earnings will remain pending until it has satisfied the uptime requirement.
-- Nodes that join before the 21st of a given month and do not satisfy the uptime requirement will forfeit their earnings for the given month. Their forfeited earnings will be added to the network reward pool for the following month.
-- The uptime requirement must be satisfied on a monthly basis.
+- For a node operator to qualify for earnings at the end of the month, their node must be online for at least 14 days during the month. It is important to note that earnings for nodes STILL get calculated before the node satisfies the uptime requirement.
+- When a node first registers during a given month, its uptime requirement check will be postponed to the following month to give a fair opportunity for each node to meet the uptime requirement. Its earnings will remain pending until it has satisfied the uptime requirement. If the node satisfied the uptime requirement during the same month it registers, it will be eligible to receive earnings that month.
+- Nodes that do not satisfy the uptime requirement in the months after their registration month will forfeit their earnings for the given month. Their forfeited earnings will be added to the network reward pool for the following month.
+- The uptime requirement must be satisfied on a monthly basis after the registration month is over.
 
-Note: Beginning July 1st, 2023, Saturn’s monthly node uptime requirement will increase from 7 days to 14 days. This means that for a node to qualify for earnings on August 1st, that node must have been online and operational for at least 14 contiguous days in July.
+Note: Beginning July 1st, 2023, Saturn’s monthly node uptime requirement increased from 7 days to 14 days. This means that for a node to qualify for earnings on August 1st, 2023, that node must have been online and operational for at least 14 contiguous days in July.
+
+Read more about Saturn's node uptime requirement in the docs, [here](https://docs.saturn.tech/nodes-uptime-requirement).
 
 ### Obtaining a Filecoin wallet address
 

container/nginx/conf.d/proxy.conf

@@ -1,5 +1,5 @@
 # 2g of memory for keys is about 16 million cached items
-proxy_cache_path /usr/src/app/shared/nginx_cache levels=2:2:2 keys_zone=my_cache:2000m inactive=90d min_free=50G use_temp_path=off manager_files=1000 loader_files=1000;
+proxy_cache_path /usr/src/app/shared/nginx_cache levels=2:2:2 keys_zone=my_cache:2000m inactive=90d min_free=$cache_min_free use_temp_path=off manager_files=1000 loader_files=1000;
 
 upstream node_backend {
     server              127.0.0.1:10361 max_fails=0;

container/nginx/conf.d/shared.conf

@@ -44,6 +44,8 @@ location / {
 
     car_range;
 
+    js_header_filter ipfsResponse.setHeaders;
+
     proxy_pass              http://node_backend;
     proxy_buffering         on;
     proxy_http_version      1.1;
@@ -66,7 +68,7 @@ location / {
     proxy_cache_lock            on;
     proxy_cache_valid           200 301 302 403     365d;
     proxy_cache_valid           any                 1m;
-    proxy_cache_min_uses        1;
+    proxy_cache_min_uses        $cache_min_uses;
     proxy_cache_lock_age        30m;
     proxy_cache_lock_timeout    30m;
     proxy_cache_use_stale       error timeout invalid_header updating http_500 http_503 http_429;
@@ -86,16 +88,17 @@ location / {
     add_header 'Retry-After'                    $retry_after_seconds always;
     add_header 'Access-Control-Allow-Origin'    '*' always;
     add_header 'Access-Control-Allow-Methods'   'GET, POST, OPTIONS' always;
-    add_header 'Access-Control-Allow-Headers'   'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' always;
+    add_header 'Access-Control-Allow-Headers'   'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Traceparent' always;
     add_header 'Access-Control-Expose-Headers'  '*' always;
+    add_header 'Accept-Ranges'                  'none' always;
 
     client_max_body_size 10g;
 
     if ($request_method = 'OPTIONS') {
         add_header 'Timing-Allow-Origin'            '*' always;
         add_header 'Access-Control-Allow-Origin'    '*' always;
         add_header 'Access-Control-Allow-Methods'   'GET, POST, OPTIONS' always;
-        add_header 'Access-Control-Allow-Headers'   'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' always;
+        add_header 'Access-Control-Allow-Headers'   'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Traceparent' always;
         add_header 'Access-Control-Expose-Headers'  '*' always;
         add_header 'Access-Control-Max-Age'         1728000 always;
         add_header 'Content-Type'                   'text/plain; charset=utf-8';

container/nginx/confs/tls_proxy.conf

@@ -58,7 +58,6 @@ server {
 
     include /etc/nginx/conf.d/register.conf;
 
-    # TODO: Temporary block to facilitate bifrost gateway tests.
     location /ipfs/ {
         proxy_http_version      1.1;
         proxy_set_header Host handoff.strn.localhost;