Add citizenlab url management

ooniapi/common/src/common/config.py

@@ -63,3 +63,10 @@ class Settings(BaseSettings):
             "type": "cloudfront",
         },
     ]
+
+    # citizenlab github configuration
+    working_dir: str = ""
+    github_user: str = ""
+    github_token: str = ""
+    origin_repo: str = ""
+    push_repo: str = ""

ooniapi/common/src/common/errors.py

@@ -0,0 +1,129 @@
+"""
+API errors with support for localization
+
+"""
+
+from typing import Dict, Optional
+from pydantic import Field
+from fastapi import HTTPException
+
+
+class BaseOONIException(HTTPException):
+    """Custom exception class for OONI-related errors."""
+    status_code: int = 400
+    err_str: str = "err_generic_ooni_exception"
+    err_args: Optional[Dict[str, str]] = None
+    description: str = "Generic OONI error"
+
+    def __init__(
+        self,
+        description: Optional[str] = None,
+        err_args: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Initialize the BaseOONIException.
+
+        Args:
+            description (Optional[str]): A description of the error.
+            err_args (Optional[Dict[str, str]]): Additional arguments related to the error.
+        """
+        if description != None:
+            self.description = description
+        if err_args != None:
+            self.err_args = err_args
+
+        super().__init__(status_code=self.status_code,
+            detail={"description": self.description,
+                    "err_args": self.err_args,
+                    "err_str": self.err_str})
+
+
+class BadURL(BaseOONIException):
+    """Exception raised for invalid URLs."""
+    err_str = "err_bad_url"
+    description = "Invalid URL"
+
+
+class BadCategoryCode(BaseOONIException):
+    """Exception raised for invalid category codes."""
+    err_str = "err_bad_category_code"
+    description = "Invalid category code"
+
+
+class BadCategoryDescription(BaseOONIException):
+    """Exception raised for invalid category descriptions."""
+    err_str = "err_bad_category_description"
+    description = "Invalid category description"
+
+
+class BadDate(BaseOONIException):
+    """Exception raised for invalid date formats."""
+    err_str = "err_bad_date"
+    description = "Invalid date"
+
+
+class CountryNotSupported(BaseOONIException):
+    """Exception raised when a country is not supported."""
+    err_str = "err_country_not_supported"
+    description = "Country Not Supported"
+
+
+class InvalidCountryCode(BaseOONIException):
+    """Exception raised for invalid country codes."""
+    err_str = "err_invalid_country_code"
+    description = "Country code is invalid"
+
+
+class EmptyTranslation(BaseOONIException):
+    """Exception raised for empty translation fields."""
+    err_str = "err_empty_translation_field"
+    description = "Empty translation field"
+
+
+class DuplicateURLError(BaseOONIException):
+    """Exception raised for duplicate URLs."""
+    err_str = "err_duplicate_url"
+    description = "Duplicate URL"
+
+
+class DuplicateRuleError(BaseOONIException):
+    """Exception raised for duplicate rules."""
+    err_str = "err_duplicate_rule"
+    description = "Duplicate rule"
+
+
+class RuleNotFound(BaseOONIException):
+    """Exception raised when a rule is not found."""
+    code = 404
+    err_str = "err_rule_not_found"
+    description = "Rule not found error"
+
+
+class CannotClosePR(BaseOONIException):
+    """Exception raised when unable to close a pull request (PR)."""
+    err_str = "err_cannot_close_pr"
+    description = "Unable to close PR. Please reload data."
+
+
+class CannotUpdateList(BaseOONIException):
+    """Exception raised when unable to update due to changes in the URL list."""
+    err_str = "err_cannot_update_list"
+    description = "Unable to update. The URL list has changed in the meantime."
+
+
+class NoProposedChanges(BaseOONIException):
+    """Exception raised when there are no proposed changes."""
+    err_str = "err_no_proposed_changes"
+    description = "No changes are being proposed"
+
+
+class OwnershipPermissionError(BaseOONIException):
+    """Exception raised for ownership permission errors."""
+    err_str = "err_ownership"
+    description = "Attempted to create, update, or delete an item belonging to another user."
+
+
+class InvalidRequest(BaseOONIException):
+    """Exception raised for invalid parameters in a request."""
+    err_str = "err_request_params"
+    description = "Invalid parameters in the request"

ooniapi/services/citizenlab/.dockerignore

@@ -0,0 +1,10 @@
+.DS_Store
+*.log
+*.pyc
+*.swp
+*.env
+.coverage
+coverage.xml
+dist/
+.venv/
+__pycache__/

ooniapi/services/citizenlab/.gitignore

@@ -0,0 +1,3 @@
+/dist
+/coverage_html
+*.coverage*

ooniapi/services/citizenlab/Dockerfile

@@ -0,0 +1,33 @@
+# Python builder
+FROM python:3.11-bookworm as builder
+ARG BUILD_LABEL=docker
+
+WORKDIR /build
+
+RUN python -m pip install hatch
+
+COPY . /build
+
+# When you build stuff on macOS you end up with ._ files
+# https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them
+RUN find /build -type f -name '._*' -delete
+
+RUN echo "$BUILD_LABEL" > /build/src/citizenlab/BUILD_LABEL
+
+RUN hatch build
+
+### Actual image running on the host
+FROM python:3.11-bookworm as runner
+
+WORKDIR /app
+
+COPY --from=builder /build/README.md /app/
+COPY --from=builder /build/dist/*.whl /app/
+RUN pip install /app/*whl && rm /app/*whl
+
+COPY --from=builder /build/src/citizenlab/common/alembic/ /app/alembic/
+COPY --from=builder /build/src/citizenlab/common/alembic.ini /app/
+RUN rm -rf /app/alembic/__pycache__
+
+CMD ["uvicorn", "citizenlab.main:app", "--host", "0.0.0.0", "--port", "80"]
+EXPOSE 80

ooniapi/services/citizenlab/LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ooniapi/services/citizenlab/Makefile

@@ -0,0 +1,64 @@
+SERVICE_NAME ?= citizenlab
+
+ECS_CONTAINER_NAME ?= ooniapi-service-$(SERVICE_NAME)
+IMAGE_NAME ?= ooni/api-$(SERVICE_NAME)
+DATE := $(shell python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d'))")
+GIT_FULL_SHA ?= $(shell git rev-parse HEAD)
+SHORT_SHA := $(shell echo ${GIT_FULL_SHA} | cut -c1-8)
+PKG_VERSION := $(shell hatch version)
+
+BUILD_LABEL := $(DATE)-$(SHORT_SHA)
+VERSION_LABEL = v$(PKG_VERSION)
+ENV_LABEL ?= latest
+
+print-labels:
+	echo "ECS_CONTAINER_NAME=${ECS_CONTAINER_NAME}"
+	echo "PKG_VERSION=${PKG_VERSION}"
+	echo "BUILD_LABEL=${BUILD_LABEL}"
+	echo "VERSION_LABEL=${VERSION_LABEL}"
+	echo "ENV_LABEL=${ENV_LABEL}"
+
+init:
+	hatch env create
+
+docker-build:
+	# We need to use tar -czh to resolve the common dir symlink
+	tar -czh . | docker build \
+		--build-arg BUILD_LABEL=${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${BUILD_LABEL} \
+		-t ${IMAGE_NAME}:${VERSION_LABEL} \
+		-t ${IMAGE_NAME}:${ENV_LABEL} \
+		-
+	echo "built image: ${IMAGE_NAME}:${BUILD_LABEL} (${IMAGE_NAME}:${VERSION_LABEL} ${IMAGE_NAME}:${ENV_LABEL})"
+
+docker-push:
+	# We need to use tar -czh to resolve the common dir symlink
+	docker push ${IMAGE_NAME}:${BUILD_LABEL}
+	docker push ${IMAGE_NAME}:${VERSION_LABEL}
+	docker push ${IMAGE_NAME}:${ENV_LABEL}
+
+docker-smoketest:
+	./scripts/docker-smoketest.sh ${IMAGE_NAME}:${BUILD_LABEL}
+
+imagedefinitions.json:
+	echo '[{"name":"${ECS_CONTAINER_NAME}","imageUri":"${IMAGE_NAME}:${BUILD_LABEL}"}]' > imagedefinitions.json
+
+test:
+	hatch run test
+
+test-cov:
+	hatch run test-cov
+
+build:
+	hatch build
+
+clean:
+	hatch clean
+	rm -f imagedefinitions.json
+	rm -rf build dist *eggs *.egg-info
+	rm -rf .venv
+
+run:
+	hatch run uvicorn $(SERVICE_NAME).main:app
+
+.PHONY: init test build clean docker print-labels

ooniapi/services/citizenlab/README.md

@@ -0,0 +1 @@
+

ooniapi/services/citizenlab/buildspec.yml

@@ -0,0 +1,29 @@
+version: 0.2
+env:
+  variables:
+    OONI_CODE_PATH: ooniapi/services/citizenlab
+    DOCKERHUB_SECRET_ID: oonidevops/dockerhub/access_token
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+
+  pre_build:
+    commands:
+      - echo "Logging in to dockerhub"
+      - DOCKER_SECRET=$(aws secretsmanager get-secret-value --secret-id $DOCKERHUB_SECRET_ID --query SecretString --output text)
+      - echo $DOCKER_SECRET | docker login --username ooni --password-stdin
+
+  build:
+    commands:
+      - export GIT_FULL_SHA=${CODEBUILD_RESOLVED_SOURCE_VERSION}
+      - cd $OONI_CODE_PATH
+      - make docker-build
+      - make docker-smoketest
+      - make docker-push
+      - make imagedefinitions.json
+      - cat imagedefinitions.json | tee ${CODEBUILD_SRC_DIR}/imagedefinitions.json
+
+artifacts:
+  files: imagedefinitions.json