Skip to content

Commit

Permalink
1974 add default storage used by all dsr policies by default (#2438)
Browse files Browse the repository at this point in the history
Co-authored-by: Paul Sanders <pau@ethyca.com>
  • Loading branch information
adamsachs and Paul Sanders authored Feb 8, 2023
1 parent 1451026 commit b069fe5
Show file tree
Hide file tree
Showing 34 changed files with 2,271 additions and 220 deletions.
26 changes: 26 additions & 0 deletions .fides/db_dataset.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,29 @@ dataset:
data_categories:
- system.operations
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: applicationconfig
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
fields:
- name: api_set
data_categories:
- system.operations
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: updated_at
data_categories:
- system.operations
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: created_at
data_categories:
- system.operations
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: id
data_categories:
- system.operations
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: single_row
data_categories:
- system.operations
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: auditlog
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
fields:
Expand Down Expand Up @@ -1432,6 +1455,9 @@ dataset:
- name: id
data_categories: [system.operations]
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: is_default
data_categories: [system.operations]
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
- name: key
data_categories: [system.operations]
data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
Expand Down
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ The types of changes are:

* Issue addressing missing field in dataset migration [#2510](https://github.com/ethyca/fides/pull/2510)

### Added
* Add default storage configuration functionality and associated APIs [#2438](https://github.com/ethyca/fides/pull/2438)

## [2.6.1](https://github.com/ethyca/fides/compare/2.6.0...2.6.1)

### Fixed
Expand Down
6 changes: 1 addition & 5 deletions scripts/load_examples.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,9 @@
)

# Create an S3 storage config to store DSR results
storage_key = constants.DEFAULT_STORAGE_KEY
if get_secret("AWS_SECRETS")["access_key_id"]:
print("AWS secrets provided, attempting to configure S3 storage...")
create_s3_storage(auth_header=auth_header, key=constants.S3_STORAGE_KEY)
storage_key = constants.S3_STORAGE_KEY
create_s3_storage(auth_header=auth_header)

# Edit the default DSR policies to use for testing privacy requests
# NOTE: We use the default policies to test the default privacy center
Expand All @@ -50,14 +48,12 @@
auth_header=auth_header,
policy_key=constants.DEFAULT_ACCESS_POLICY,
rule_key=constants.DEFAULT_ACCESS_POLICY_RULE,
storage_key=storage_key,
action_type="access",
)
create_rule(
auth_header=auth_header,
policy_key=constants.DEFAULT_ERASURE_POLICY,
rule_key=constants.DEFAULT_ERASURE_POLICY_RULE,
storage_key=storage_key,
action_type="erasure",
)

Expand Down
2 changes: 0 additions & 2 deletions scripts/setup/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
DEFAULT_ACCESS_POLICY_RULE,
DEFAULT_ERASURE_POLICY,
DEFAULT_ERASURE_POLICY_RULE,
DEFAULT_STORAGE_KEY,
)
from fides.api.ops.api.v1 import urn_registry as urls

Expand All @@ -23,7 +22,6 @@
FIDES_URL = "http://fides:8080"
BASE_URL = FIDES_URL + urls.V1_URL_PREFIX

LOCAL_STORAGE_KEY = DEFAULT_STORAGE_KEY
S3_STORAGE_KEY = "s3_storage"
S3_STORAGE_BUCKET = "fides-test-privacy-requests"

Expand Down
2 changes: 0 additions & 2 deletions scripts/setup/dsr_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,10 @@ def create_rule(
auth_header: Dict[str, str],
policy_key: str = constants.DEFAULT_ACCESS_POLICY,
rule_key: str = constants.DEFAULT_ACCESS_POLICY_RULE,
storage_key: str = constants.DEFAULT_STORAGE_KEY,
action_type: str = "access",
):
rules = [
{
"storage_destination_key": storage_key,
"name": f"Example {action_type} Rule",
"action_type": action_type,
"key": rule_key,
Expand Down
50 changes: 28 additions & 22 deletions scripts/setup/s3_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,41 +10,34 @@

def create_s3_storage(
auth_header: Dict[str, str],
key: str = constants.S3_STORAGE_KEY,
bucket: str = constants.S3_STORAGE_BUCKET,
):
logger.info(f"Configuring S3 storage with key={key} and bucket={bucket}")
url = f"{constants.BASE_URL}{urls.STORAGE_CONFIG}"
response = requests.patch(
logger.info(f"Configuring S3 storage with bucket={bucket}")
url = f"{constants.BASE_URL}{urls.STORAGE_DEFAULT}"
response = requests.put(
url,
headers=auth_header,
json=[
{
"name": key,
"key": key,
"type": "s3",
"format": "json",
"details": {
"auth_method": "secret_keys",
"bucket": bucket,
"naming": "request_id",
},
json={
"type": "s3",
"format": "json",
"details": {
"auth_method": "secret_keys",
"bucket": bucket,
"naming": "request_id",
},
],
},
)

logger.info(f"Configuring S3 storage secrets for key={key}")
logger.info(f"Configuring S3 storage secrets for default s3 storage")
if not response.ok:
raise RuntimeError(
f"fides storage creation failed! response.status_code={response.status_code}, response.json()={response.json()}"
)
else:
storage = (response.json())["succeeded"]
if len(storage) > 0:
logger.info(f"Created fides storage with key={key} via {url}")

logger.info(f"Created default s3 storage via {url}")

# Update the storage config with the appropriate secrets
storage_secrets_path = urls.STORAGE_SECRETS.format(config_key=key)
storage_secrets_path = urls.STORAGE_DEFAULT_SECRETS.format(storage_type="s3")
url = f"{constants.BASE_URL}{storage_secrets_path}"
response = requests.put(
url,
Expand All @@ -58,3 +51,16 @@ def create_s3_storage(
raise RuntimeError(
f"fides storage secrets failed! response.status_code={response.status_code}, response.json()={response.json()}"
)

url = f"{constants.BASE_URL}{urls.CONFIG}"
response = requests.patch(
url,
headers=auth_header,
json={"storage": {"active_default_storage_type": "s3"}},
)
if not response.ok:
raise RuntimeError(
f"Failed to make s3 storage the active default storage type!"
)

logger.info(f"Successfully made s3 storage the active default storage type")
37 changes: 3 additions & 34 deletions src/fides/api/ctl/database/seed.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,6 @@
PRIVACY_REQUEST_TRANSFER,
)
from fides.api.ops.models.policy import ActionType, DrpAction, Policy, Rule, RuleTarget
from fides.api.ops.models.storage import StorageConfig
from fides.api.ops.schemas.storage.storage import (
FileNaming,
ResponseFormat,
StorageDetails,
StorageType,
)
from fides.core.config import get_config
from fides.lib.db.base_class import FidesBase
from fides.lib.exceptions import KeyOrNameAlreadyExists
Expand All @@ -36,7 +29,6 @@

CONFIG = get_config()
FIDESOPS_AUTOGENERATED_CLIENT_KEY = "default_oauth_client"
DEFAULT_STORAGE_KEY = "default_local_storage"
DEFAULT_ACCESS_POLICY = "default_access_policy"
DEFAULT_ACCESS_POLICY_RULE = "default_access_policy_rule"
DEFAULT_ERASURE_POLICY = "default_erasure_policy"
Expand Down Expand Up @@ -179,28 +171,6 @@ def load_default_access_policy(
f"Skipping {DEFAULT_ACCESS_POLICY} creation as it already exists in the database"
)

local_storage_config = StorageConfig.get_by(
db_session, field="key", value=DEFAULT_STORAGE_KEY
)
if not local_storage_config:
log.info(f"Creating: {DEFAULT_STORAGE_KEY}...")
local_storage_config = StorageConfig.create(
db=db_session,
data={
"name": "Default Local Storage Config",
"key": DEFAULT_STORAGE_KEY,
"type": StorageType.local,
"details": {
StorageDetails.NAMING.value: FileNaming.request_id.value,
},
"format": ResponseFormat.json,
},
)
else:
log.info(
f"Skipping {DEFAULT_STORAGE_KEY} creation as it already exists in the database"
)

access_rule: Optional[FidesBase] = Rule.get_by(
db_session, field="key", value=DEFAULT_ACCESS_POLICY_RULE
)
Expand All @@ -213,7 +183,6 @@ def load_default_access_policy(
"name": "Default Access Rule",
"key": DEFAULT_ACCESS_POLICY_RULE,
"policy_id": access_policy.id,
"storage_destination_id": local_storage_config.id,
"client_id": client_id,
},
)
Expand All @@ -231,7 +200,7 @@ def load_default_access_policy(
db=db_session,
data=data,
)
except KeyOrNameAlreadyExists:
except KeyOrNameAlreadyExists: # pragma: no cover
# This rule target already exists against the Policy
pass
else:
Expand Down Expand Up @@ -296,7 +265,7 @@ def load_default_erasure_policy(
db=db_session,
data=data,
)
except KeyOrNameAlreadyExists:
except KeyOrNameAlreadyExists: # pragma: no cover
# This rule target already exists against the Policy
pass
else:
Expand Down Expand Up @@ -415,7 +384,7 @@ async def load_default_taxonomy(async_session: AsyncSession) -> None:
await create_resource(
sql_model_map[resource_type], resource, async_session
)
except QueryError:
except QueryError: # pragma: no cover
pass # The create_resource function will log the error
else:
log.info(f"INSERTED {len(resources)} {resource_type} resource(s)")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""add application config table
Revision ID: 5d62bab40b71
Revises: 7fd4601ef88b
Create Date: 2023-01-27 22:56:33.367379
"""
import sqlalchemy as sa
import sqlalchemy_utils
from alembic import op

# revision identifiers, used by Alembic.
revision = "5d62bab40b71"
down_revision = "7fd4601ef88b"
branch_labels = None
depends_on = None


def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"applicationconfig",
sa.Column("id", sa.String(length=255), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=True,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=True,
),
sa.Column(
"api_set",
sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(),
nullable=True,
),
sa.Column("single_row", sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.CheckConstraint("single_row", name="single_row_check"),
)
op.create_index(
op.f("ix_applicationconfig_id"), "applicationconfig", ["id"], unique=False
)
# ### end Alembic commands ###


def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_applicationconfig_id"), table_name="applicationconfig")
op.drop_table("applicationconfig")
# ### end Alembic commands ###
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
"""add is_default column to storage config
Revision ID: 7fd4601ef88b
Revises: d6c6c6555c86
Create Date: 2023-01-26 14:13:22.538719
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy import text
from sqlalchemy.sql.elements import TextClause

# revision identifiers, used by Alembic.
revision = "7fd4601ef88b"
down_revision = "392992c7733a"
branch_labels = None
depends_on = None


def upgrade():
# schema migration - add a nullable storageconfig.is_default field. We will shortly make it non-nullable
op.add_column("storageconfig", sa.Column("is_default", sa.Boolean(), nullable=True))
op.create_index(
op.f("ix_storageconfig_is_default"),
"storageconfig",
["is_default"],
unique=False,
)
op.create_index(
"only_one_default_per_type",
"storageconfig",
["type", "is_default"],
unique=True,
postgresql_where=sa.text("is_default"),
)

# Data migration - automatically populate storageconfig.is_default column
bind = op.get_bind()
existing_storageconfigs: TextClause = bind.execute(
text("SELECT id FROM storageconfig;")
)
for row in existing_storageconfigs:
update_storage_config_query: TextClause = text(
"UPDATE storageconfig SET is_default = false WHERE id= :storageconfig_id"
)
bind.execute(update_storage_config_query, {"storageconfig_id": row["id"]})

# now follow up to make it not nullable
op.alter_column(
"storageconfig", "is_default", existing_type=sa.Boolean(), nullable=False
)

# ### end Alembic commands ###


def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(
"only_one_default_per_type",
table_name="storageconfig",
postgresql_where=sa.text("is_default"),
)
op.drop_index(op.f("ix_storageconfig_is_default"), table_name="storageconfig")
op.drop_column("storageconfig", "is_default")
# ### end Alembic commands ###
Loading

0 comments on commit b069fe5

Please sign in to comment.