Migrate read queries from REST to GraphQL with simplified preload

.env.example

@@ -1,7 +1,34 @@
-NETBOX_URL=
-NETBOX_TOKEN=
-REPO_URL=https://github.com/netbox-community/devicetype-library.git
+NETBOX_TOKEN=XXXXXXXXX-xxxxxxxxx
+NETBOX_URL=https://netbox.example.org
+
 REPO_BRANCH=master
+REPO_URL=https://github.com/netbox-community/devicetype-library.git
+
+# SSL options — choose one approach if your NetBox uses a self-signed or private CA:
+#
+#   Option A (recommended): point requests at your CA bundle so the certificate
+#   is validated properly.  Set REQUESTS_CA_BUNDLE to the path of your PEM-encoded
+#   CA certificate or bundle file.  This is a standard requests/urllib3 variable
+#   and is also respected by most Python HTTP clients.
+# REQUESTS_CA_BUNDLE=/etc/ssl/certs/my-internal-ca.pem
+#
+#   Option B (insecure – dev/test only): disable certificate verification entirely.
+#   IGNORE_SSL_ERRORS=False is the secure default; set to True only for local/dev use.
+#   WARNING: This bypasses all certificate validation and exposes connections to
+#   man-in-the-middle attacks.  Never enable in production environments.
 IGNORE_SSL_ERRORS=False
-#REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt # you should enable this if you are running on a linux system
+
+# Optional: restrict import to specific device type slugs
 #SLUGS=c9300-48u isr4431 isr4331
+
+# Performance tuning — adjust for your NetBox instance size and network
+# Number of items per GraphQL page (default: 5000).
+# Values up to 25000 have been tested successfully on a default NetBox
+# instance with minimal extra latency.  Most NetBox installs cap
+# MAX_PAGE_SIZE at 1000 by default; the client detects server-side
+# clamping and paginates accordingly.
+#GRAPHQL_PAGE_SIZE=5000
+
+# Number of threads for concurrent component preloading (default: 8).
+# Reduce on smaller instances to lower load; increase for faster imports.
+#PRELOAD_THREADS=8

.envrc → .envrc.example

@@ -1,5 +1,11 @@
-# direnv configuration 
-# Run `direnv allow` to enable
+# direnv configuration — copy to .envrc and run `direnv allow` to enable.
+#
+# Review this file before enabling. It ONLY:
+#   1. Loads secrets from .env (NETBOX_URL, NETBOX_TOKEN, etc.) into the shell
+#   2. Creates a uv virtual environment if one does not exist
+#   3. Syncs Python dependencies via `uv sync`
+#
+# Prerequisites: direnv (https://direnv.net) and uv (https://docs.astral.sh/uv/)
 
 # Source .env file if it exists (contains secrets, not in git)
 if [[ -f .env ]]; then

.github/workflows/test-netbox-main.yaml

@@ -0,0 +1,162 @@
+# SPDX-License-Identifier: MIT
+name: Weekly test against NetBox main
+
+on:
+  schedule:
+    - cron: '0 6 * * 1'  # Every Monday at 06:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  integration-test:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_DB: netbox
+          POSTGRES_USER: netbox
+          POSTGRES_PASSWORD: netbox
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    steps:
+      - name: Checkout importer
+        uses: actions/checkout@v6
+        with:
+          path: importer
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Checkout NetBox main
+        uses: actions/checkout@v6
+        with:
+          repository: netbox-community/netbox
+          path: netbox
+          ref: main
+
+      - name: Install NetBox dependencies
+        run: pip install -r netbox/requirements.txt
+
+      - name: Configure NetBox
+        run: |
+          cat > netbox/netbox/configuration.py << 'EOF'
+          ALLOWED_HOSTS = ['*']
+          DATABASE = {
+              'NAME': 'netbox',
+              'USER': 'netbox',
+              'PASSWORD': 'netbox',
+              'HOST': 'localhost',
+              'PORT': '5432',
+              'CONN_MAX_AGE': 300,
+              'ENGINE': 'django.db.backends.postgresql',
+          }
+          REDIS = {
+              'tasks':   {'HOST': 'localhost', 'PORT': 6379, 'DATABASE': 0, 'SSL': False},
+              'caching': {'HOST': 'localhost', 'PORT': 6379, 'DATABASE': 1, 'SSL': False},
+          }
+          SECRET_KEY = 'test-secret-key-not-for-production-1234567890123456'  # checkov:skip=CKV_SECRET_6
+          API_TOKEN_PEPPERS = {0: 'a' * 64}  # checkov:skip=CKV_SECRET_6
+          DEBUG = True
+          LOGIN_REQUIRED = False
+          EOF
+
+      - name: Run NetBox migrations
+        working-directory: netbox/netbox
+        run: python manage.py migrate --no-input
+
+      - name: Create admin user and API token
+        working-directory: netbox/netbox
+        run: |
+          DJANGO_SUPERUSER_PASSWORD=admin python manage.py createsuperuser \
+            --username=admin --email=admin@example.com --no-input
+          TOKEN=$(python manage.py shell -c "
+          from django.apps import apps
+          Token = apps.get_model('users', 'Token')
+          from django.contrib.auth import get_user_model
+          user = get_user_model().objects.get(username='admin')
+          t = Token.objects.create(user=user)
+          print(t.key)
+          ")
+          echo "NETBOX_TOKEN=$TOKEN" >> "$GITHUB_ENV"
+          echo "Created API token"
+
+      - name: Start NetBox dev server
+        working-directory: netbox/netbox
+        run: |
+          rm -f /tmp/netbox.log && touch /tmp/netbox.log
+          python manage.py runserver 0.0.0.0:8000 >> /tmp/netbox.log 2>&1 &
+          echo $! > /tmp/netbox.pid
+          # Wait up to 60 s for NetBox to respond
+          for i in $(seq 1 30); do
+            if curl -sf http://localhost:8000/api/ > /dev/null 2>&1; then
+              echo "NetBox is ready (attempt $i)"
+              break
+            fi
+            echo "Waiting for NetBox... ($i/30)"
+            sleep 2
+          done
+          curl -sf http://localhost:8000/api/ > /dev/null || { echo "NetBox did not start"; exit 1; }
+
+      - name: Install importer dependencies
+        working-directory: importer
+        run: uv sync
+
+      - name: Set up test-fixtures git repo
+        run: |
+          mkdir -p /tmp/test-fixtures/device-types/TestVendor
+          mkdir -p /tmp/test-fixtures/module-types/TestVendor
+          mkdir -p /tmp/test-fixtures/elevation-images/TestVendor
+          cp importer/tests/fixtures/device-types/TestVendor/*.yaml \
+             /tmp/test-fixtures/device-types/TestVendor/
+          cp importer/tests/fixtures/module-types/TestVendor/*.yaml \
+             /tmp/test-fixtures/module-types/TestVendor/
+          cp importer/tests/fixtures/elevation-images/TestVendor/*.png \
+             /tmp/test-fixtures/elevation-images/TestVendor/
+          cd /tmp/test-fixtures
+          git init -b main
+          git config user.email "ci@test.local"
+          git config user.name "CI"
+          git add .
+          git commit -m "test fixtures"
+
+      - name: Run integration tests
+        working-directory: importer
+        env:
+          NETBOX_URL: http://localhost:8000
+          NETBOX_TOKEN: ${{ env.NETBOX_TOKEN }}
+          REPO_URL: file:///tmp/test-fixtures
+          REPO_BRANCH: main
+        run: uv run python tests/integration/test_import.py
+
+      - name: Print NetBox version on failure
+        if: failure()
+        run: |
+          curl -s http://localhost:8000/api/status/ | python3 -m json.tool || true
+          echo "--- NetBox server log ---"
+          cat /tmp/netbox.log 2>/dev/null | tail -50 || true

.gitignore

@@ -104,7 +104,7 @@ celerybeat.pid
 # Environments
 .env*
 !.env.example
-!.envrc
+!.envrc.example
 .venv
 env/
 venv/

README.md

@@ -1,9 +1,28 @@
 # NetBox Device Type Import
 
+[![Tests](https://github.com/marcinpsk/Device-Type-Library-Import/actions/workflows/tests.yml/badge.svg)](https://github.com/marcinpsk/Device-Type-Library-Import/actions/workflows/tests.yml)
+[![NetBox main](https://github.com/marcinpsk/Device-Type-Library-Import/actions/workflows/test-netbox-main.yaml/badge.svg)](https://github.com/marcinpsk/Device-Type-Library-Import/actions/workflows/test-netbox-main.yaml)
+[![NetBox](https://img.shields.io/badge/NetBox-3.2%2B_through_4.5%2B-blue)](https://netbox.dev)
+[![Python](https://img.shields.io/badge/python-3.12%2B-blue)](https://www.python.org)
+
 This library is intended to be your friend and help you import all the device-types defined within
 the [NetBox Device Type Library Repository](https://github.com/netbox-community/devicetype-library).
 
-> Tested working with 2.9.4, 2.10.4
+> **Tested working with NetBox 3.2+ through 4.5+** (weekly CI run against NetBox `main`)
+
+---
+
+> ⚠️ **direnv users** — This repo ships a `.envrc.example` file.  If you use
+> [direnv](https://direnv.net/), **review the file before enabling it**:
+>
+> ```shell
+> cp .envrc.example .envrc
+> cat .envrc          # confirm it only loads .env vars and syncs uv
+> direnv allow
+> ```
+>
+> The file exclusively loads variables from `.env` into your shell and runs
+> `uv sync` to keep dependencies up to date.  Your `.envrc` is git-ignored.
 
 ## Description
 
@@ -13,42 +32,31 @@ into the NetBox UI.
 
 ## Getting Started
 
-1. This script uses `uv` for dependency management.
+1. Install dependencies with `uv`:
 
    ```shell
    uv sync
    ```
 
-1. There are two variables that are required when using this script to import device types
-   into your NetBox installation. (1) Your NetBox instance URL and (2) a token with
-   **write rights**.
+1. Copy `.env.example` to `.env` and fill in your NetBox URL and API token
+   (the token needs **write rights**):
 
-Copy the existing `.env.example` to your own `.env` file, and fill in the variables.
+   ```shell
+   cp .env.example .env
+   vim .env
+   ```
 
-```shell
-cp .env.example .env
-vim .env
-```
+1. Run the script:
 
-Finally, we are able to execute the script and import some device templates!
+   ```shell
+   uv run nb-dt-import.py
+   ```
 
 ## Usage
 
-To use the script, simply execute the script as follows. `uv` will handle the virtual
-environment.
-
-```shell
-uv run nb-dt-import.py
-```
-
-This will clone the latest master branch from the `netbox-community/devicetype-library`
-from GitHub and install it into the `repo` subdirectory. If this directory already exists,
-it will perform a `git pull` to update the repository instead.
-
-Next, it will loop over every manufacturer and every device of every manufacturer and begin
-checking if your NetBox install already has them, and if not, creates them. It will skip
-preexisting manufacturers, devices, interfaces, etc. so as to not end up with duplicate
-entries in your NetBox instance.
+Running the script clones (or updates) the `netbox-community/devicetype-library` repository
+into the `repo` subdirectory, then loops over every manufacturer and device, creating anything
+that is missing from NetBox while skipping entries that already exist.
 
 ### Arguments
 
@@ -58,13 +66,13 @@ import devices.
 To import only device by APC, for example:
 
 ```shell
-./nb-dt-import.py --vendors apc
+uv run nb-dt-import.py --vendors apc
 ```
 
 `--vendors` can also accept a comma-separated list of vendors if you want to import multiple.
 
 ```shell
-./nb-dt-import.py --vendors apc,juniper
+uv run nb-dt-import.py --vendors apc,juniper
 ```
 
 #### Update Mode
@@ -73,7 +81,7 @@ By default, the script only creates new device types and skips existing ones. To
 existing device types:
 
 ```shell
-./nb-dt-import.py --update
+uv run nb-dt-import.py --update
 ```
 
 This will:
@@ -91,7 +99,7 @@ If you've changed a device type definition (for example, converting interfaces t
 to support SFP modules), you can remove obsolete components with:
 
 ```shell
-./nb-dt-import.py --update --remove-components
+uv run nb-dt-import.py --update --remove-components
 ```
 
 This will delete any components (interfaces, ports, bays, etc.) that exist in NetBox but are
@@ -109,41 +117,6 @@ no longer present in the YAML definition.
 - Review the change detection report before enabling component removal
 - Test on a staging NetBox instance first if possible
 
-## Docker build
-
-It's possible to use this project as a docker container.
-
-To build:
-
-```shell
-docker build -t netbox-devicetype-import-library .
-```
-
-Alternatively you can pull a pre-built image from GitHub Container Registry (ghcr.io):
-
-```shell
-docker pull ghcr.io/minitriga/netbox-device-type-library-import
-```
-
-The container supports the following env var as configuration:
-
-- `REPO_URL`, the repo to look for device types
-  (defaults to `https://github.com/netbox-community/devicetype-library.git`)
-- `REPO_BRANCH`, the branch to check out if appropriate, defaults to master.
-- `NETBOX_URL`, used to access netbox
-- `NETBOX_TOKEN`, token for accessing netbox
-- `VENDORS`, a comma-separated list of vendors to import (defaults to None)
-- `REQUESTS_CA_BUNDLE`, path to a CA_BUNDLE for validation if you are using
-  self-signed certificates (file must be included in the container)
-
-To run:
-
-```shell
-docker run -e "NETBOX_URL=http://netbox:8080/" \
-  -e "NETBOX_TOKEN=98765434567890" \
-  ghcr.io/minitriga/netbox-device-type-library-import
-```
-
 ## Contributing
 
 We're happy about any pull requests!