Gh v2 580b czarte and Dan

.githooks/pre-commit

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# File to check (relative to repo root)
+FILENAME="ip_country/src/dbip_country.rs"
+
+if ! git diff --cached --name-only | grep -q "^$FILENAME$"; then
+  exit 0
+fi
+
+# find 'Nonexistents' country in dbip_country.rs
+@cat "$FILENAME" | grep '("NOEX", "Nonexistent")'
+NONEXISTENT=$?
+
+# Get local file size (staged version, not working copy)
+LOCAL_SIZE=$(git ls-files --stage | grep "$FILENAME" | awk '{print $2}' | xargs git cat-file -s)
+
+if [[ $LOCAL_SIZE -gt 1000000 || NONEXISTENT -gt 0 ]]; then
+  if [[ $LOCAL_SIZE -gt 1000000 ]]; then
+    echo "❌ Commit blocked: $FILENAME size it not dev version, we want to keep dev version under 1 MB."
+    echo "   Local file size:  $LOCAL_SIZE bytes"
+  fi
+  if [[ $NONEXISTENT -gt 0 ]]; then
+    echo "❌ Commit blocked: Dev file  must contain '(\"NOEX\", \"Nonexistents\")' country"
+  fi
+  echo ""
+  echo " ⚡  Please sync the file with the remote before committing using:"
+  echo "      git checkout HEAD -- ip_country/src/dbip_country.rs"
+  exit 1
+fi
+
+# All good
+exit 0

.github/workflows/ci-matrix.yml

@@ -20,7 +20,7 @@ jobs:
     name: Build node on ${{ matrix.target.os }}
     runs-on: ${{ matrix.target.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
@@ -36,7 +36,8 @@ jobs:
           toolchain: 1.63.0
           components: rustfmt, clippy
           override: true
-      - uses: actions/cache@v3
+
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/bin/
@@ -45,10 +46,13 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-
 
       - name: Build ${{ matrix.target.os }}
         run: |
+          git fetch
           ./ci/all.sh
           ./ci/multinode_integration_test.sh
           ./ci/collect_results.sh
@@ -66,46 +70,52 @@ jobs:
           df -h /Users/runner/work/Node/Node/node/target/release
   deploy_to_s3:
     needs: build
+    if: success() && (startsWith(github.head_ref, 'GH') || startsWith(github.head_ref, 'v'))
     strategy:
       matrix:
-          os: [linux, macos, windows]
-    runs-on: ubuntu-22.04
+        os: [linux, macos, windows]
+    runs-on: ubuntu-latest
     steps:
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
           with:
             ref: ${{ github.event.pull_request.head.sha }}
+            fetch-depth: 1
 
         - name: Download artifacts
           uses: actions/download-artifact@v4
 
         - name: Display structure of downloaded files
           run: ls -R
 
+        - name: Check artifacts exist
+          run: |
+            if [ ! -d "Node-${{ matrix.os }}/generated/bin/" ]; then
+              echo "Error: Build artifacts not found"
+              exit 1
+            fi
+
+        - name: Configure AWS Credentials
+          uses: aws-actions/configure-aws-credentials@v4
+          with:
+            aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+            aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+            aws-region: us-west-2
+
         - if: startsWith(github.head_ref, 'GH')
           name: Versioned S3 Sync
-          uses: jakejarvis/s3-sync-action@v0.5.1
-          with:
-           args: --acl private --follow-symlinks --delete
-          env:
-            AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-            AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-            AWS_REGION: 'us-west-2'
-            DEST_DIR: 'Node/${{ github.head_ref }}/Node-${{ matrix.os }}'
-            SOURCE_DIR: 'Node-${{ matrix.os }}/generated/bin/'
+          run: |
+            aws s3 sync "Node-${{ matrix.os }}/generated/bin/" "s3://${{ secrets.AWS_S3_BUCKET }}/Node/${{ github.head_ref }}/Node-${{ matrix.os }}" \
+              --delete \
+              --no-progress \
+              --acl private
 
         - if: startsWith(github.head_ref, 'v')
           name: Latest S3 Sync
-          uses: jakejarvis/s3-sync-action@v0.5.1
-          with:
-           args: --acl private --follow-symlinks --delete
-          env:
-            AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-            AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-            AWS_REGION: 'us-west-2'
-            DEST_DIR: 'Node/latest/Node-${{ matrix.os }}'
-            SOURCE_DIR: 'Node-${{ matrix.os }}/generated/bin/'
+          run: |
+            aws s3 sync "Node-${{ matrix.os }}/generated/bin/" "s3://${{ secrets.AWS_S3_BUCKET }}/Node/latest/Node-${{ matrix.os }}" \
+              --delete \
+              --no-progress \
+              --acl private
 
         - name: Invalidate Binaries CloudFront
           uses: chetan/invalidate-cloudfront-action@v2.4

.github/workflows/dbip_download.yml

@@ -0,0 +1,63 @@
+name: Download DBIP data and generate dbip_country.rs
+
+on:
+  schedule:
+    - cron: "0 0 3 * *" # Runs at midnight on the 3rd of every month
+
+env:
+  TEMP_DIR: '/tmp'
+  YEAR_MONTH: '01-9999'
+
+permissions:
+  contents: write
+
+jobs:
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Download DBIP data in MMDB format
+        run: |
+          export YEAR_MONTH=$(date +%Y-%m)
+          echo "TEMP_DIR=$(mktemp -d)" >> $GITHUB_ENV
+          echo "YEAR_MONTH=$YEAR_MONTH" >> $GITHUB_ENV
+          cd ip_country
+          mkdir -p dbip-data
+          curl -L -o dbip-data/dbip-country-lite.mmdb.gz "https://download.db-ip.com/free/dbip-country-lite-$YEAR_MONTH.mmdb.gz"
+          gunzip dbip-data/dbip-country-lite.mmdb.gz
+
+      - name: Generate Rust source file
+        run: |
+          cd ip_country
+          cargo run < "dbip-data/dbip-country-lite.mmdb" > "$TEMP_DIR"/dbip_country.rs
+          ls "$TEMP_DIR"
+
+      - name: Commit and push generated file
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git fetch
+          BRANCH="$(git ls-remote --exit-code --heads origin generated-source 2>/dev/null || true)"
+          if [[ "$BRANCH" == "" ]]
+          then
+               git checkout -B generated-source
+               git rm -rf .
+               mkdir -p ip_country/src
+          else
+               git checkout generated-source
+          fi
+          DIFF="$(diff "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs || true)"
+          if [[ "$DIFF" != "" || "$BRANCH" == "" ]]
+          then
+              mv "${TEMP_DIR}"/dbip_country.rs ip_country/src/dbip_country.rs
+              git add ip_country/src/dbip_country.rs
+              git commit -m "Update generated dbip_country ${YEAR_MONTH} Rust source file" || true
+              git push -u origin HEAD
+          fi

ci/all.sh

@@ -1,12 +1,19 @@
 #!/bin/bash -xev
 # Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved.
 CI_DIR="$( cd "$( dirname "$0" )" && pwd )"
-PARENT_DIR="$1"
+export ARGV_1="$1"
 
 ci/format.sh
 
 export RUST_BACKTRACE=1
-
+#if github checkout fails, you can download csv database from db-ip.com and convert it locally by cargo run ip_country
+if [[ $GITHUB_ACTIONS || "$ARGV_1" == "dbip" ]]; then
+  git checkout origin/generated-source -- ip_country/src/dbip_country.rs
+  ls -la ip_country/src/dbip_country.rs
+fi
+if [[ -z "${GITHUB_ACTIONS}" ]]; then
+  bash ./install-hooks.sh
+fi
 # Remove these two lines to slow down the build
 which sccache || cargo install --version 0.4.1 sccache || echo "Skipping sccache installation"  # Should do significant work only once
 #export CARGO_TARGET_DIR="$CI_DIR/../cargo-cache"
@@ -18,19 +25,19 @@ export RUSTFLAGS="-D warnings -Anon-snake-case"
 echo "*********************************************************************************************************"
 echo "***                                             MASQ_LIB HEAD                                         ***"
 cd "$CI_DIR/../masq_lib"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "***                                             MASQ_LIB TAIL                                         ***"
 echo "*********************************************************************************************************"
 echo "*********************************************************************************************************"
 echo "***                                               NODE HEAD                                           ***"
 cd "$CI_DIR/../node"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "***                                               NODE TAIL                                           ***"
 echo "*********************************************************************************************************"
 echo "*********************************************************************************************************"
 echo "***                                           DNS UTILITY HEAD                                        ***"
 cd "$CI_DIR/../dns_utility"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "***                                           DNS UTILITY TAIL                                        ***"
 echo "*********************************************************************************************************"
 echo "*********************************************************************************************************"
@@ -42,13 +49,12 @@ echo "**************************************************************************
 echo "*********************************************************************************************************"
 echo "***                                             AUTOMAP HEAD                                          ***"
 cd "$CI_DIR/../automap"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "***                                             AUTOMAP TAIL                                          ***"
 echo "*********************************************************************************************************"
 echo "*********************************************************************************************************"
 echo "***                                           IP COUNTRY HEAD                                         ***"
 cd "$CI_DIR/../ip_country"
-ci/all.sh "$PARENT_DIR"
+ci/all.sh
 echo "***                                           IP COUNTRY TAIL                                         ***"
 echo "*********************************************************************************************************"
-