From d1f5cfeeda284f5957098a124d1dbbc04d1f8c33 Mon Sep 17 00:00:00 2001 From: emileten Date: Thu, 12 Oct 2023 17:42:48 +0900 Subject: [PATCH 01/18] import eoapi-cdk constructs and reuse them --- infrastructure/aws/cdk/app.py | 370 ++++++++---------------- infrastructure/aws/cdk/config.py | 5 +- infrastructure/aws/requirements-cdk.txt | 8 +- 3 files changed, 133 insertions(+), 250 deletions(-) diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index b327440..45a435f 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -1,19 +1,14 @@ """ CDK Stack definition code for EOAPI """ -import json import os from typing import Any -from aws_cdk import App, CfnOutput, CustomResource, Duration, RemovalPolicy, Stack, Tags -from aws_cdk import aws_apigatewayv2_alpha as apigw +from aws_cdk import App, CfnOutput, Duration, RemovalPolicy, Stack, Tags from aws_cdk import aws_ec2 as ec2 -from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda from aws_cdk import aws_logs as logs from aws_cdk import aws_rds as rds -from aws_cdk import aws_secretsmanager as secretsmanager -from aws_cdk.aws_apigatewayv2_integrations_alpha import HttpLambdaIntegration from config import ( eoAPISettings, eoDBSettings, @@ -22,103 +17,16 @@ eoVectorSettings, ) from constructs import Construct +from eoapi_cdk import ( + PgStacApiLambda, + PgStacDatabase, + TiPgApiLambda, + TitilerPgstacApiLambda, +) eoapi_settings = eoAPISettings() -class BootstrappedDb(Construct): - """ - Given an RDS database, connect to DB and create a database, user, and - password - """ - - def __init__( - self, - scope: Construct, - id: str, - db: rds.DatabaseInstance, - new_dbname: str, - new_username: str, - secrets_prefix: str, - pgstac_version: str, - enable_context: bool = False, - enable_mosaic_index: bool = False, - context_dir: str = "../../", - ) -> None: - """Update RDS database.""" - super().__init__(scope, id) - - # TODO: Utilize a singleton function. - handler = aws_lambda.Function( - self, - "DatabaseBootstrapper", - handler="handler.handler", - runtime=aws_lambda.Runtime.PYTHON_3_10, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.db", - build_args={"PYTHON_VERSION": "3.10", "PGSTAC_VERSION": pgstac_version}, - platform="linux/amd64", - ), - timeout=Duration.minutes(5), - vpc=db.vpc, - allow_public_subnet=True, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - - self.secret = secretsmanager.Secret( - self, - id, - secret_name=os.path.join( - secrets_prefix, id.replace(" ", "_"), self.node.addr - ), - generate_secret_string=secretsmanager.SecretStringGenerator( - secret_string_template=json.dumps( - { - "dbname": new_dbname, - "engine": "postgres", - "port": 5432, - "host": db.instance_endpoint.hostname, - "username": new_username, - }, - ), - generate_string_key="password", - exclude_punctuation=True, - ), - description=f"Deployed by {Stack.of(self).stack_name}", - ) - - self.resource = CustomResource( - scope=scope, - id="BootstrappedDbResource", - service_token=handler.function_arn, - properties={ - # By setting pgstac_version in the properties assures - # that Create/Update events will be passed to the service token - "pgstac_version": pgstac_version, - "context": enable_context, - "mosaic_index": enable_mosaic_index, - "conn_secret_arn": db.secret.secret_arn, - "new_user_secret_arn": self.secret.secret_arn, - }, - # We do not need to run the custom resource on STAC Delete - # Custom Resource are not physical resources so it's OK to `Retain` it - removal_policy=RemovalPolicy.RETAIN, - ) - - # Allow lambda to... - # read new user secret - self.secret.grant_read(handler) - # read database secret - db.secret.grant_read(handler) - # connect to database - db.connections.allow_from(handler, port_range=ec2.Port.tcp(5432)) - - def is_required_by(self, construct: Construct): - """Register required services.""" - return construct.node.add_dependency(self.resource) - - class eoAPIconstruct(Stack): """Earth Observation API CDK application""" @@ -181,62 +89,76 @@ def __init__( # noqa: C901 vpc.add_gateway_endpoint(key, service=service) eodb_settings = eoDBSettings() - db = rds.DatabaseInstance( + + pgstac_db = PgStacDatabase( self, - f"{id}-postgres-db", + "pgstac-db", vpc=vpc, - engine=rds.DatabaseInstanceEngine.POSTGRES, + engine=rds.DatabaseInstanceEngine.postgres( + version=rds.PostgresEngineVersion.VER_14 + ), + vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), + allocated_storage=eodb_settings.allocated_storage, instance_type=ec2.InstanceType.of( ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize(eodb_settings.instance_size), ), database_name="postgres", - # should set the subnet to `PRIVATE_ISOLATED` but then we need either a bastion host to connect to the db - # or an API to ingest/delete data in the DB - vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), backup_retention=Duration.days(7), deletion_protection=eoapi_settings.stage.lower() == "production", removal_policy=RemovalPolicy.SNAPSHOT if eoapi_settings.stage.lower() == "production" else RemovalPolicy.DESTROY, - ) - - setup_db = BootstrappedDb( - self, - "STAC DB for eoapi", - db=db, - new_dbname=eodb_settings.dbname, - new_username=eodb_settings.user, + custom_resource_properties={ + "pgstac_version": eodb_settings.pgstac_version, + "context": eodb_settings.context, + "mosaic_index": eodb_settings.mosaic_index, + }, + bootstrapper_lambda_function_options={ + "handler": "handler.handler", + "runtime": aws_lambda.Runtime.PYTHON_3_10, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.db", + build_args={ + "PYTHON_VERSION": "3.10", + "PGSTAC_VERSION": eodb_settings.pgstac_version, + }, + platform="linux/amd64", + ), + "timeout": Duration.minutes(5), + "allow_public_subnet": True, + "log_retention": logs.RetentionDays.ONE_WEEK, + }, + pgstac_db_name=eodb_settings.dbname, + pgstac_username=eodb_settings.user, secrets_prefix=os.path.join(stage, name), - pgstac_version=eodb_settings.pgstac_version, - enable_context=eodb_settings.context, - enable_mosaic_index=eodb_settings.mosaic_index, - context_dir=context_dir, ) CfnOutput( self, f"{id}-database-secret-arn", - value=db.secret.secret_arn, + value=pgstac_db.pgstac_secret.secret_arn, description="Arn of the SecretsManager instance holding the connection info for Postgres DB", ) # eoapi.raster if "raster" in eoapi_settings.functions: + db_secrets = { - "POSTGRES_HOST": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -245,76 +167,56 @@ def __init__( # noqa: C901 env = eoraster_settings.env or {} if "DB_MAX_CONN_SIZE" not in env: env["DB_MAX_CONN_SIZE"] = "1" + env.update(db_secrets) - eoraster_function = aws_lambda.Function( + eoraster = TitilerPgstacApiLambda( self, f"{id}-raster-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.raster", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, vpc=vpc, - vpc_subnets=ec2.SubnetSelection( + subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), - allow_public_subnet=True, - handler="handler.handler", - memory_size=eoraster_settings.memory, - timeout=Duration.seconds(eoraster_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eoraster_function.add_environment(key=k, value=str(v)) - - eoraster_function.add_to_role_policy( - iam.PolicyStatement( - actions=["s3:GetObject"], - resources=[ - f"arn:aws:s3:::{bucket}/{eoraster_settings.key}" - for bucket in eoraster_settings.buckets - ], - ) - ) - - db.connections.allow_from(eoraster_function, port_range=ec2.Port.tcp(5432)) - - raster_api = apigw.HttpApi( - self, - f"{id}-raster-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-raster-integration", - eoraster_function, - ), + api_env=env, + lambda_function_options={ + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.raster", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "allow_public_subnet": True, + "handler": "handler.handler", + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "memory_size": eoraster_settings.memory, + "timeout": Duration.seconds(eoraster_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, + buckets=eoraster_settings.buckets, ) - CfnOutput(self, "eoAPI-raster", value=raster_api.url.strip("/")) - - setup_db.is_required_by(eoraster_function) # eoapi.stac if "stac" in eoapi_settings.functions: db_secrets = { - "POSTGRES_HOST_READER": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST_READER": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_HOST_WRITER": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST_WRITER": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -325,65 +227,54 @@ def __init__( # noqa: C901 env["DB_MAX_CONN_SIZE"] = "1" if "DB_MIN_CONN_SIZE" not in env: env["DB_MIN_CONN_SIZE"] = "1" - - eostac_function = aws_lambda.Function( - self, - f"{id}-stac-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.stac", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), - vpc=vpc, - handler="handler.handler", - memory_size=eostac_settings.memory, - timeout=Duration.seconds(eostac_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eostac_function.add_environment(key=k, value=str(v)) - + env.update(db_secrets) # If raster is deployed we had the TITILER_ENDPOINT env to add the Proxy extension if "raster" in eoapi_settings.functions: - eostac_function.add_environment( - key="TITILER_ENDPOINT", value=raster_api.url.strip("/") - ) + env["TITILER_ENDPOINT"] = eoraster.url.strip("/") - db.connections.allow_from(eostac_function, port_range=ec2.Port.tcp(5432)) - - stac_api = apigw.HttpApi( + PgStacApiLambda( self, - f"{id}-stac-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-stac-integration", - eostac_function, + id=f"{id}-stac-lambda", + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, + vpc=vpc, + subnet_selection=ec2.SubnetSelection( + subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), + api_env=env, + lambda_function_options={ + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.stac", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "handler": "handler.handler", + "memory_size": eostac_settings.memory, + "timeout": Duration.seconds(eostac_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, ) - CfnOutput(self, "eoAPI-stac", value=stac_api.url.strip("/")) - - setup_db.is_required_by(eostac_function) # eoapi.vector if "vector" in eoapi_settings.functions: db_secrets = { - "POSTGRES_HOST": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -396,41 +287,34 @@ def __init__( # noqa: C901 if "DB_MIN_CONN_SIZE" not in env: env["DB_MIN_CONN_SIZE"] = "1" - eovector_function = aws_lambda.Function( + env.update(db_secrets) + + TiPgApiLambda( self, f"{id}-vector-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.vector", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), vpc=vpc, - handler="handler.handler", - memory_size=eovector_settings.memory, - timeout=Duration.seconds(eovector_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eovector_function.add_environment(key=k, value=str(v)) - - db.connections.allow_from(eovector_function, port_range=ec2.Port.tcp(5432)) - - vector_api = apigw.HttpApi( - self, - f"{id}-vector-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-vector-integration", - eovector_function, + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, + subnet_selection=ec2.SubnetSelection( + subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), + api_env=env, + lambda_function_options={ + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.vector", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "handler": "handler.handler", + "memory_size": eovector_settings.memory, + "timeout": Duration.seconds(eovector_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, ) - CfnOutput(self, "eoAPI-vector", value=vector_api.url.strip("/")) - - setup_db.is_required_by(eovector_function) app = App() diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index 0450bf9..80d8bba 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -42,7 +42,7 @@ class eoDBSettings(BaseSettings): instance_size: str = "SMALL" context: bool = True mosaic_index: bool = True - + allocated_storage: int = 20 model_config = { "env_prefix": "CDK_EOAPI_DB_", "env_file": ".env", @@ -90,9 +90,6 @@ class eoRasterSettings(BaseSettings): # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html buckets: List = ["*"] - # S3 key pattern to limit the access to specific items (e.g: "my_data/*.tif") - key: str = "*" - timeout: int = 10 memory: int = 3008 diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt index c134c87..404e9f1 100644 --- a/infrastructure/aws/requirements-cdk.txt +++ b/infrastructure/aws/requirements-cdk.txt @@ -1,9 +1,11 @@ # aws cdk -aws-cdk-lib==2.94.0 -aws_cdk-aws_apigatewayv2_alpha==2.94.0a0 -aws_cdk-aws_apigatewayv2_integrations_alpha==2.94.0a0 +aws-cdk-lib==2.99.1 +aws_cdk-aws_apigatewayv2_alpha==2.99.1a0 +aws_cdk-aws_apigatewayv2_integrations_alpha==2.99.1a0 constructs>=10.0.0 # pydantic settings pydantic~=2.0 pydantic-settings~=2.0 + +../eoapi-cdk/dist/python/eoapi_cdk-5.4.0-py3-none-any.whl \ No newline at end of file From 8e40f0c49e9835f720502e07dcf3773376800f26 Mon Sep 17 00:00:00 2001 From: emileten Date: Tue, 31 Oct 2023 16:45:23 +0900 Subject: [PATCH 02/18] add ingestor, browser, bump eoapi-cdk --- infrastructure/aws/cdk/app.py | 139 ++++++++++++++++++------ infrastructure/aws/cdk/config.py | 4 +- infrastructure/aws/requirements-cdk.txt | 5 +- 3 files changed, 113 insertions(+), 35 deletions(-) diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index 45a435f..b253d84 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -4,11 +4,14 @@ import os from typing import Any +import boto3 from aws_cdk import App, CfnOutput, Duration, RemovalPolicy, Stack, Tags from aws_cdk import aws_ec2 as ec2 +from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda from aws_cdk import aws_logs as logs from aws_cdk import aws_rds as rds +from aws_cdk import aws_s3 as s3 from config import ( eoAPISettings, eoDBSettings, @@ -20,6 +23,8 @@ from eoapi_cdk import ( PgStacApiLambda, PgStacDatabase, + StacBrowser, + StacIngestor, TiPgApiLambda, TitilerPgstacApiLambda, ) @@ -42,8 +47,6 @@ def __init__( # noqa: C901 """Define stack.""" super().__init__(scope, id, **kwargs) - # vpc = ec2.Vpc(self, f"{id}-vpc", nat_gateways=0) - vpc = ec2.Vpc( self, f"{id}-vpc", @@ -52,23 +55,8 @@ def __init__( # noqa: C901 name="ingress", cidr_mask=24, subnet_type=ec2.SubnetType.PUBLIC, - ), - ec2.SubnetConfiguration( - name="application", - cidr_mask=24, - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS, - ), - ec2.SubnetConfiguration( - name="rds", - cidr_mask=28, - subnet_type=ec2.SubnetType.PRIVATE_ISOLATED, - ), + ) ], - nat_gateways=1, - ) - print( - """The eoAPI stack use AWS NatGateway for the Raster service so it can reach the internet. -This might incurs some cost (https://docs.aws.amazon.com/vpc/latest/userguide/vpc-nat-gateway.html).""" ) interface_endpoints = [ @@ -135,6 +123,9 @@ def __init__( # noqa: C901 secrets_prefix=os.path.join(stage, name), ) + # allow connections from anywhere to the DB + pgstac_db.db.connections.allow_default_port_from_any_ipv4() + CfnOutput( self, f"{id}-database-secret-arn", @@ -174,10 +165,6 @@ def __init__( # noqa: C901 f"{id}-raster-lambda", db=pgstac_db.db, db_secret=pgstac_db.pgstac_secret, - vpc=vpc, - subnet_selection=ec2.SubnetSelection( - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS - ), api_env=env, lambda_function_options={ "code": aws_lambda.Code.from_docker_build( @@ -232,15 +219,11 @@ def __init__( # noqa: C901 if "raster" in eoapi_settings.functions: env["TITILER_ENDPOINT"] = eoraster.url.strip("/") - PgStacApiLambda( + eostac = PgStacApiLambda( self, id=f"{id}-stac-lambda", db=pgstac_db.db, db_secret=pgstac_db.pgstac_secret, - vpc=vpc, - subnet_selection=ec2.SubnetSelection( - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS - ), api_env=env, lambda_function_options={ "runtime": aws_lambda.Runtime.PYTHON_3_11, @@ -259,6 +242,38 @@ def __init__( # noqa: C901 }, ) + if eostac_settings.stac_browser_github_tag is not None: + assert ( + eostac_settings.stac_api_custom_domain_name is not None + ), "stac_api_custom_domain_name must be set if stac_browser_github_tag is not None. The browser deployment needs a resolved STAC API url at deployment time and so needs to rely on a predefined custom domain name." + stac_browser_bucket = s3.Bucket( + self, + "stac-browser-bucket", + bucket_name=f"{id.lower()}-stac-browser", + removal_policy=RemovalPolicy.DESTROY, + auto_delete_objects=True, + website_index_document="index.html", + public_read_access=True, + block_public_access=s3.BlockPublicAccess( + block_public_acls=False, + block_public_policy=False, + ignore_public_acls=False, + restrict_public_buckets=False, + ), + object_ownership=s3.ObjectOwnership.OBJECT_WRITER, + ) + + # need to build this manually, the attribute eostac.url is not resolved yet. + + StacBrowser( + self, + "stac-browser", + github_repo_tag=eostac_settings.stac_browser_github_tag, + stac_catalog_url=eostac_settings.stac_api_custom_domain_name, + website_index_document="index.html", + bucket_arn=stac_browser_bucket.bucket_arn, + ) + # eoapi.vector if "vector" in eoapi_settings.functions: db_secrets = { @@ -292,12 +307,8 @@ def __init__( # noqa: C901 TiPgApiLambda( self, f"{id}-vector-lambda", - vpc=vpc, db=pgstac_db.db, db_secret=pgstac_db.pgstac_secret, - subnet_selection=ec2.SubnetSelection( - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS - ), api_env=env, lambda_function_options={ "runtime": aws_lambda.Runtime.PYTHON_3_11, @@ -316,6 +327,72 @@ def __init__( # noqa: C901 }, ) + if "ingestor" in eoapi_settings.functions: + + data_access_role = self._create_data_access_role() + + stac_ingestor = StacIngestor( + self, + "stac-ingestor", + stac_url=eostac.url, + stage=eoapi_settings.stage, + data_access_role=data_access_role, + stac_db_secret=pgstac_db.pgstac_secret, + stac_db_security_group=pgstac_db.db.connections.security_groups[0], + api_env={"REQUESTER_PAYS": "True"}, + ) + + data_access_role = self._grant_assume_role_with_principal_pattern( + data_access_role, stac_ingestor.handler_role.role_name + ) + + def _create_data_access_role(self) -> iam.Role: + + """ + Creates an IAM role with full S3 read access. + """ + + data_access_role = iam.Role( + self, + "data-access-role", + assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), + ) + + data_access_role.add_managed_policy( + iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3FullAccess") + ) + + return data_access_role + + def _grant_assume_role_with_principal_pattern( + self, + role_to_assume: iam.Role, + principal_pattern: str, + account_id: str = boto3.client("sts").get_caller_identity().get("Account"), + ) -> iam.Role: + """ + Grants assume role permissions to the role of the given + account with the given name pattern. Default account + is the current account. + """ + + role_to_assume.assume_role_policy.add_statements( + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + principals=[iam.AnyPrincipal()], + actions=["sts:AssumeRole"], + conditions={ + "StringLike": { + "aws:PrincipalArn": [ + f"arn:aws:iam::{account_id}:role/{principal_pattern}" + ] + } + }, + ) + ) + + return role_to_assume + app = App() diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index 80d8bba..c177bd8 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -12,6 +12,7 @@ class functionName(str, Enum): stac = "stac" raster = "raster" vector = "vector" + ingestor = "ingestor" class eoAPISettings(BaseSettings): @@ -56,7 +57,8 @@ class eoSTACSettings(BaseSettings): timeout: int = 10 memory: int = 256 - + stac_browser_github_tag: None | str = "v3.1.0" # if not none, will try to deploy this version of radiant earth stac browser + stac_api_custom_domain_name: None | str = "https://stac.eoapi.dev" model_config = { "env_prefix": "CDK_EOAPI_STAC_", "env_file": ".env", diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt index 404e9f1..1a49cd7 100644 --- a/infrastructure/aws/requirements-cdk.txt +++ b/infrastructure/aws/requirements-cdk.txt @@ -3,9 +3,8 @@ aws-cdk-lib==2.99.1 aws_cdk-aws_apigatewayv2_alpha==2.99.1a0 aws_cdk-aws_apigatewayv2_integrations_alpha==2.99.1a0 constructs>=10.0.0 - +boto3==1.28.71 # pydantic settings pydantic~=2.0 pydantic-settings~=2.0 - -../eoapi-cdk/dist/python/eoapi_cdk-5.4.0-py3-none-any.whl \ No newline at end of file +eoapi-cdk==6.0.0 \ No newline at end of file From 0f138f5fc7e4285b77641b1e854e909ee6e001b9 Mon Sep 17 00:00:00 2001 From: emileten Date: Wed, 1 Nov 2023 12:55:34 +0900 Subject: [PATCH 03/18] remove default API url --- infrastructure/aws/cdk/app.py | 6 +++--- infrastructure/aws/cdk/config.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index b253d84..7a1bfc9 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -242,10 +242,10 @@ def __init__( # noqa: C901 }, ) - if eostac_settings.stac_browser_github_tag is not None: + if eostac_settings.stac_api_custom_domain_name is not None: assert ( - eostac_settings.stac_api_custom_domain_name is not None - ), "stac_api_custom_domain_name must be set if stac_browser_github_tag is not None. The browser deployment needs a resolved STAC API url at deployment time and so needs to rely on a predefined custom domain name." + eostac_settings.stac_browser_github_tag is not None + ), "stac_browser_github_tag must be set if stac_api_custom_domain_name is not None." stac_browser_bucket = s3.Bucket( self, "stac-browser-bucket", diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index c177bd8..28cbba4 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -57,8 +57,10 @@ class eoSTACSettings(BaseSettings): timeout: int = 10 memory: int = 256 - stac_browser_github_tag: None | str = "v3.1.0" # if not none, will try to deploy this version of radiant earth stac browser - stac_api_custom_domain_name: None | str = "https://stac.eoapi.dev" + stac_browser_github_tag: None | str = "v3.1.0" + stac_api_custom_domain_name: None | str = ( + None # if not none, will try to deploy a browser with the above tag + ) model_config = { "env_prefix": "CDK_EOAPI_STAC_", "env_file": ".env", From f5cd23de8b87da1fad9f317e1ec708a45ff5859f Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 3 Nov 2023 14:47:17 +0100 Subject: [PATCH 04/18] update raster and vector services --- .github/workflows/tests/test_raster.py | 72 ++++++--- docker-compose.yml | 4 +- runtime/eoapi/raster/eoapi/raster/app.py | 152 ++++++++++++------ .../raster/templates/mosaic-builder.html | 2 +- runtime/eoapi/raster/pyproject.toml | 2 +- runtime/eoapi/vector/pyproject.toml | 2 +- 6 files changed, 164 insertions(+), 70 deletions(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 17556ca..4e6dfb9 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -16,21 +16,23 @@ def test_raster_api(): def test_mosaic_api(): """test mosaic.""" query = {"collections": ["noaa-emergency-response"], "filter-lang": "cql-json"} - resp = httpx.post(f"{raster_endpoint}/mosaic/register", json=query) + resp = httpx.post(f"{raster_endpoint}/searches/register", json=query) assert resp.headers["content-type"] == "application/json" assert resp.status_code == 200 - assert resp.json()["searchid"] + assert resp.json()["id"] assert resp.json()["links"] - searchid = resp.json()["searchid"] + searchid = resp.json()["id"] - resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/-85.6358,36.1624/assets") + resp = httpx.get(f"{raster_endpoint}/searches/{searchid}/-85.6358,36.1624/assets") assert resp.status_code == 200 assert len(resp.json()) == 1 assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"] assert resp.json()[0]["id"] == "20200307aC0853900w361030" - resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/tiles/15/8589/12849/assets") + resp = httpx.get( + f"{raster_endpoint}/searches/{searchid}/tiles/15/8589/12849/assets" + ) assert resp.status_code == 200 assert len(resp.json()) == 1 assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"] @@ -38,7 +40,37 @@ def test_mosaic_api(): z, x, y = 15, 8589, 12849 resp = httpx.get( - f"{raster_endpoint}/mosaic/{searchid}/tiles/{z}/{x}/{y}", + f"{raster_endpoint}/searches/{searchid}/tiles/{z}/{x}/{y}", + params={"assets": "cog"}, + headers={"Accept-Encoding": "br, gzip"}, + timeout=10.0, + ) + assert resp.status_code == 200 + assert resp.headers["content-type"] == "image/jpeg" + assert "content-encoding" not in resp.headers + + +def test_mosaic_collection_api(): + """test mosaic collection.""" + resp = httpx.get( + f"{raster_endpoint}/collections/noaa-emergency-response/-85.6358,36.1624/assets" + ) + assert resp.status_code == 200 + assert len(resp.json()) == 1 + assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"] + assert resp.json()[0]["id"] == "20200307aC0853900w361030" + + resp = httpx.get( + f"{raster_endpoint}/collections/noaa-emergency-response/tiles/15/8589/12849/assets" + ) + assert resp.status_code == 200 + assert len(resp.json()) == 1 + assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"] + assert resp.json()[0]["id"] == "20200307aC0853900w361030" + + z, x, y = 15, 8589, 12849 + resp = httpx.get( + f"{raster_endpoint}/collections/noaa-emergency-response/tiles/{z}/{x}/{y}", params={"assets": "cog"}, headers={"Accept-Encoding": "br, gzip"}, timeout=10.0, @@ -102,11 +134,11 @@ def test_mosaic_search(): }, ] for search in searches: - resp = httpx.post(f"{raster_endpoint}/mosaic/register", json=search) + resp = httpx.post(f"{raster_endpoint}/searches/register", json=search) assert resp.status_code == 200 - assert resp.json()["searchid"] + assert resp.json()["id"] - resp = httpx.get(f"{raster_endpoint}/mosaic/list") + resp = httpx.get(f"{raster_endpoint}/searches/list") assert resp.headers["content-type"] == "application/json" assert resp.status_code == 200 assert ( @@ -122,9 +154,11 @@ def test_mosaic_search(): assert len(links) == 2 assert links[0]["rel"] == "self" assert links[1]["rel"] == "next" - assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=10&offset=10" + assert links[1]["href"] == f"{raster_endpoint}/searches/list?limit=10&offset=10" - resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"limit": 1, "offset": 1}) + resp = httpx.get( + f"{raster_endpoint}/searches/list", params={"limit": 1, "offset": 1} + ) assert resp.status_code == 200 assert resp.json()["context"]["matched"] > 10 assert resp.json()["context"]["limit"] == 1 @@ -133,33 +167,33 @@ def test_mosaic_search(): links = resp.json()["links"] assert len(links) == 3 assert links[0]["rel"] == "self" - assert links[0]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=1" + assert links[0]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=1" assert links[1]["rel"] == "next" - assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=2" + assert links[1]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=2" assert links[2]["rel"] == "prev" - assert links[2]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=0" + assert links[2]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=0" # Filter on mosaic metadata - resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"owner": "vincent"}) + resp = httpx.get(f"{raster_endpoint}/searches/list", params={"owner": "vincent"}) assert resp.status_code == 200 assert resp.json()["context"]["matched"] == 7 assert resp.json()["context"]["limit"] == 10 assert resp.json()["context"]["returned"] == 7 # sortBy - resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "lastused"}) + resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "lastused"}) assert resp.status_code == 200 - resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "usecount"}) + resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "usecount"}) assert resp.status_code == 200 - resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "-owner"}) + resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "-owner"}) assert resp.status_code == 200 assert ( "owner" not in resp.json()["searches"][0]["search"]["metadata"] ) # some mosaic don't have owners - resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "owner"}) + resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "owner"}) assert resp.status_code == 200 assert "owner" in resp.json()["searches"][0]["search"]["metadata"] diff --git a/docker-compose.yml b/docker-compose.yml index b6f54c4..a29863d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: # At the time of writing, rasterio and psycopg wheels are not available for arm64 arch # so we force the image to be built with linux/amd64 platform: linux/amd64 - image: ghcr.io/stac-utils/titiler-pgstac:0.8.0 + image: ghcr.io/stac-utils/titiler-pgstac:1.0.0a3 ports: - "${MY_DOCKER_IP:-127.0.0.1}:8082:8082" environment: @@ -89,7 +89,7 @@ services: - ./dockerfiles/scripts:/tmp/scripts tipg: - image: ghcr.io/developmentseed/tipg:0.4.4 + image: ghcr.io/developmentseed/tipg:0.5.0 ports: - "${MY_DOCKER_IP:-127.0.0.1}:8083:8083" environment: diff --git a/runtime/eoapi/raster/eoapi/raster/app.py b/runtime/eoapi/raster/eoapi/raster/app.py index 0ae1fb1..12c37e6 100644 --- a/runtime/eoapi/raster/eoapi/raster/app.py +++ b/runtime/eoapi/raster/eoapi/raster/app.py @@ -21,8 +21,13 @@ from titiler.core.middleware import CacheControlMiddleware from titiler.mosaic.errors import MOSAIC_STATUS_CODES from titiler.pgstac.db import close_db_connection, connect_to_db -from titiler.pgstac.dependencies import ItemPathParams -from titiler.pgstac.factory import MosaicTilerFactory +from titiler.pgstac.dependencies import CollectionIdParams, ItemIdParams, SearchIdParams +from titiler.pgstac.extensions import searchInfoExtension +from titiler.pgstac.factory import ( + MosaicTilerFactory, + add_search_list_route, + add_search_register_route, +) from titiler.pgstac.reader import PgSTACReader try: @@ -86,56 +91,100 @@ async def lifespan(app: FastAPI): ) ############################################################################### -# MOSAIC Endpoints -mosaic = MosaicTilerFactory( - router_prefix="/mosaic", - # add /statistics [POST] +# `Secret` endpoint for mosaic builder. Do not need to be public (in the OpenAPI docs) +@app.get("/collections", include_in_schema=False) +async def list_collection(request: Request): + """list collections.""" + with request.app.state.dbpool.connection() as conn: + with conn.cursor(row_factory=dict_row) as cursor: + cursor.execute("SELECT * FROM pgstac.all_collections();") + r = cursor.fetchone() + return r.get("all_collections", []) + + +############################################################################### +# STAC Search Endpoints +searches = MosaicTilerFactory( + path_dependency=SearchIdParams, + router_prefix="/searches/{search_id}", add_statistics=True, - # add /map viewer add_viewer=True, - # add /mosaic/list endpoint - add_mosaic_list=True, - # add `/bbox` and `/feature [POST]` endpoint - add_part=False, + add_part=True, + extensions=[ + searchInfoExtension(), + ], +) +app.include_router( + searches.router, tags=["STAC Search"], prefix="/searches/{search_id}" +) + +add_search_register_route( + app, + prefix="/searches", + tile_dependencies=[ + searches.layer_dependency, + searches.dataset_dependency, + searches.pixel_selection_dependency, + searches.tile_dependency, + searches.process_dependency, + searches.rescale_dependency, + searches.colormap_dependency, + searches.render_dependency, + searches.pgstac_dependency, + searches.reader_dependency, + searches.backend_dependency, + ], + tags=["STAC Search"], ) +add_search_list_route(app, prefix="/searches", tags=["STAC Search"]) -@mosaic.router.get("/builder", response_class=HTMLResponse) -async def mosaic_builder(request: Request): +@app.get("/searches/builder", response_class=HTMLResponse, tags=["STAC Search"]) +async def virtual_mosaic_builder(request: Request): """Mosaic Builder Viewer.""" + base_url = str(request.base_url) return templates.TemplateResponse( name="mosaic-builder.html", context={ "request": request, - "register_endpoint": mosaic.url_for(request, "register_search"), - "collections_endpoint": str(request.url_for("list_collection")), + "register_endpoint": str( + app.url_path_for("register_search").make_absolute_url(base_url=base_url) + ), + "collections_endpoint": str( + app.url_path_for("list_collection").make_absolute_url(base_url=base_url) + ), }, media_type="text/html", ) -# `Secret` endpoint for mosaic builder. Do not need to be public (in the OpenAPI docs) -@app.get("/collections", include_in_schema=False) -async def list_collection(request: Request): - """list collections.""" - with request.app.state.dbpool.connection() as conn: - with conn.cursor(row_factory=dict_row) as cursor: - cursor.execute("SELECT * FROM pgstac.all_collections();") - r = cursor.fetchone() - return r.get("all_collections", []) - +############################################################################### +# STAC COLLECTION Endpoints +collection = MosaicTilerFactory( + path_dependency=CollectionIdParams, + router_prefix="/collections/{collection_id}", + add_statistics=True, + add_viewer=True, + add_part=True, +) +app.include_router( + collection.router, tags=["STAC Collection"], prefix="/collections/{collection_id}" +) -app.include_router(mosaic.router, tags=["Mosaic"], prefix="/mosaic") ############################################################################### # STAC Item Endpoints stac = MultiBaseTilerFactory( reader=PgSTACReader, - path_dependency=ItemPathParams, + path_dependency=ItemIdParams, router_prefix="/collections/{collection_id}/items/{item_id}", - # add /map viewer add_viewer=True, ) +app.include_router( + stac.router, + tags=["STAC Item"], + prefix="/collections/{collection_id}/items/{item_id}", +) @stac.router.get("/viewer", response_class=HTMLResponse) @@ -155,9 +204,12 @@ def viewer(request: Request, item: pystac.Item = Depends(stac.path_dependency)): app.include_router( - stac.router, tags=["Item"], prefix="/collections/{collection_id}/items/{item_id}" + stac.router, + tags=["STAC Item"], + prefix="/collections/{collection_id}/items/{item_id}", ) + ############################################################################### # Tiling Schemes Endpoints tms = TMSFactory() @@ -216,44 +268,52 @@ def landing(request: Request): "rel": "service-doc", }, { - "title": "STAC Item Asset's Info (template URL)", - "href": stac.url_for(request, "info"), + "title": "PgSTAC Virtual Mosaic list (JSON)", + "href": str(app.url_path_for("list_searches")), "type": "application/json", "rel": "data", }, { - "title": "STAC Item Viewer (template URL)", - "href": stac.url_for(request, "viewer"), + "title": "PgSTAC Virtual Mosaic builder", + "href": str(app.url_path_for("virtual_mosaic_builder")), "type": "text/html", "rel": "data", }, { - "title": "STAC Mosaic List (JSON)", - "href": mosaic.url_for(request, "list_mosaic"), - "type": "application/json", + "title": "PgSTAC Virtual Mosaic viewer (template URL)", + "href": str(app.url_path_for("map_viewer", search_id="{search_id}")), + "type": "text/html", "rel": "data", }, { - "title": "STAC Mosaic Builder", - "href": mosaic.url_for(request, "mosaic_builder"), + "title": "PgSTAC Collection viewer (template URL)", + "href": str( + app.url_path_for("map_viewer", collection_id="{collection_id}") + ), "type": "text/html", "rel": "data", }, { - "title": "STAC Mosaic Metadata (template URL)", - "href": mosaic.url_for(request, "info_search", searchid="{searchid}"), - "type": "application/json", + "title": "PgSTAC Item viewer (template URL)", + "href": str( + app.url_path_for( + "map_viewer", + collection_id="{collection_id}", + item_id="{item_id}", + ) + ), + "type": "text/html", "rel": "data", }, { - "title": "STAC Mosaic viewer (template URL)", - "href": mosaic.url_for(request, "map_viewer", searchid="{searchid}"), + "title": "TiTiler-PgSTAC Documentation (external link)", + "href": "https://stac-utils.github.io/titiler-pgstac/", "type": "text/html", - "rel": "data", + "rel": "doc", }, { - "title": "TiTiler-pgSTAC Documentation (external link)", - "href": "https://stac-utils.github.io/titiler-pgstac/", + "title": "TiTiler-PgSTAC source code (external link)", + "href": "https://github.com/stac-utils/titiler-pgstac", "type": "text/html", "rel": "doc", }, diff --git a/runtime/eoapi/raster/eoapi/raster/templates/mosaic-builder.html b/runtime/eoapi/raster/eoapi/raster/templates/mosaic-builder.html index 7b3710f..9ec1bfb 100644 --- a/runtime/eoapi/raster/eoapi/raster/templates/mosaic-builder.html +++ b/runtime/eoapi/raster/eoapi/raster/templates/mosaic-builder.html @@ -596,7 +596,7 @@ var info_link = data.links.filter((link) => link.rel == 'metadata'); var tilejson_link = data.links.filter((link) => link.rel == 'tilejson'); var map_link = data.links.filter((link) => link.rel == 'map'); - var mosaic_info_str = `

MosaicID: ${data.searchid}

Links:

` + var mosaic_info_str = `

MosaicID: ${data.id}

Links:

` if (info_link) { mosaic_info_str += `

- Mosaic Info Link

` } diff --git a/runtime/eoapi/raster/pyproject.toml b/runtime/eoapi/raster/pyproject.toml index 723a8ca..9ed9869 100644 --- a/runtime/eoapi/raster/pyproject.toml +++ b/runtime/eoapi/raster/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.pgstac==0.8.0", + "titiler.pgstac==1.0.0a3", "starlette-cramjam>=0.3,<0.4", "importlib_resources>=1.1.0;python_version<'3.9'", ] diff --git a/runtime/eoapi/vector/pyproject.toml b/runtime/eoapi/vector/pyproject.toml index 956c83a..0265067 100644 --- a/runtime/eoapi/vector/pyproject.toml +++ b/runtime/eoapi/vector/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "tipg==0.4.4", + "tipg==0.5.0", ] [project.optional-dependencies] From 7828dcc7b54ee21c56e01b7e1fd53ad8c3dc3e38 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 9 Nov 2023 21:24:41 +0100 Subject: [PATCH 05/18] add /cog/ endpoints in raster service --- .github/workflows/tests/test_raster.py | 12 ++++++++++++ runtime/eoapi/raster/eoapi/raster/app.py | 19 ++++++++++++++++++- runtime/eoapi/raster/pyproject.toml | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 4e6dfb9..0bd21cd 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -235,3 +235,15 @@ def test_collections(): collections = resp.json() assert len(collections) == 1 assert collections[0]["id"] == "noaa-emergency-response" + + +def test_cog_endpoints(): + """test /cog endpoints""" + resp = httpx.get( + f"{raster_endpoint}/cog/info", + params={ + "url": "https://noaa-eri-pds.s3.us-east-1.amazonaws.com/2020_Nashville_Tornado/20200307a_RGB/20200307aC0854500w361030n.tif", + }, + ) + assert resp.status_code == 200 + assert resp.headers["content-type"] == "application/json" diff --git a/runtime/eoapi/raster/eoapi/raster/app.py b/runtime/eoapi/raster/eoapi/raster/app.py index 12c37e6..fbbb85d 100644 --- a/runtime/eoapi/raster/eoapi/raster/app.py +++ b/runtime/eoapi/raster/eoapi/raster/app.py @@ -17,8 +17,14 @@ from starlette.templating import Jinja2Templates from starlette_cramjam.middleware import CompressionMiddleware from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers -from titiler.core.factory import AlgorithmFactory, MultiBaseTilerFactory, TMSFactory +from titiler.core.factory import ( + AlgorithmFactory, + MultiBaseTilerFactory, + TilerFactory, + TMSFactory, +) from titiler.core.middleware import CacheControlMiddleware +from titiler.extensions import cogViewerExtension from titiler.mosaic.errors import MOSAIC_STATUS_CODES from titiler.pgstac.db import close_db_connection, connect_to_db from titiler.pgstac.dependencies import CollectionIdParams, ItemIdParams, SearchIdParams @@ -210,6 +216,17 @@ def viewer(request: Request, item: pystac.Item = Depends(stac.path_dependency)): ) +############################################################################### +# COG Endpoints +cog = TilerFactory( + router_prefix="/cog", + extensions=[ + cogViewerExtension(), + ], +) + +app.include_router(cog.router, prefix="/cog", tags=["Cloud Optimized GeoTIFF"]) + ############################################################################### # Tiling Schemes Endpoints tms = TMSFactory() diff --git a/runtime/eoapi/raster/pyproject.toml b/runtime/eoapi/raster/pyproject.toml index 9ed9869..8bdd8c4 100644 --- a/runtime/eoapi/raster/pyproject.toml +++ b/runtime/eoapi/raster/pyproject.toml @@ -20,6 +20,7 @@ classifiers = [ dynamic = ["version"] dependencies = [ "titiler.pgstac==1.0.0a3", + "titiler.extensions", "starlette-cramjam>=0.3,<0.4", "importlib_resources>=1.1.0;python_version<'3.9'", ] From 5eaf5057d655b94163f649968e2cbdec0ecea3a9 Mon Sep 17 00:00:00 2001 From: emileten Date: Sat, 11 Nov 2023 19:07:16 +0900 Subject: [PATCH 06/18] add stac browser to docker deployment, change tiler in aws deployment to our tiler --- docker-compose.custom.yml | 9 +++++ docker-compose.yml | 10 +++++ dockerfiles/Dockerfile.browser | 34 ++++++++++++++++ dockerfiles/browser_config.js | 39 +++++++++++++++++++ infrastructure/aws/cdk/app.py | 1 + infrastructure/aws/cdk/stac_browser_config.js | 39 +++++++++++++++++++ infrastructure/aws/requirements-cdk.txt | 2 +- 7 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 dockerfiles/Dockerfile.browser create mode 100644 dockerfiles/browser_config.js create mode 100644 infrastructure/aws/cdk/stac_browser_config.js diff --git a/docker-compose.custom.yml b/docker-compose.custom.yml index 8e257f1..e1a0b4c 100644 --- a/docker-compose.custom.yml +++ b/docker-compose.custom.yml @@ -259,6 +259,15 @@ services: volumes: - ./.pgdata:/var/lib/postgresql/data + stac-browser: + build: + context: dockerfiles + dockerfile: Dockerfile.browser + ports: + - "${MY_DOCKER_IP:-127.0.0.1}:8084:8080" + depends_on: + - stac + networks: default: name: eoapi-network diff --git a/docker-compose.yml b/docker-compose.yml index a29863d..f7764dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,6 +132,16 @@ services: volumes: - ./.pgdata:/var/lib/postgresql/data + # change to official image when available https://github.com/radiantearth/stac-browser/pull/386 + stac-browser: + build: + context: dockerfiles + dockerfile: Dockerfile.browser + ports: + - "${MY_DOCKER_IP:-127.0.0.1}:8084:8080" + depends_on: + - stac-fastapi + networks: default: name: eoapi-network diff --git a/dockerfiles/Dockerfile.browser b/dockerfiles/Dockerfile.browser new file mode 100644 index 0000000..f564524 --- /dev/null +++ b/dockerfiles/Dockerfile.browser @@ -0,0 +1,34 @@ +# Copyright Radiant Earth Foundation + +FROM node:lts-alpine3.18 AS build-step +ARG DYNAMIC_CONFIG=true + +WORKDIR /app + +RUN apk add --no-cache git +RUN git clone https://github.com/radiantearth/stac-browser.git . +# remove the default config.js +RUN rm config.js +RUN npm install +# replace the default config.js with our config file +COPY ./browser_config.js ./config.js +RUN \[ "${DYNAMIC_CONFIG}" == "true" \] && sed -i 's//