Skip to content

Commit a2a82b0

Browse files
gtoonstragtoonstra
authored
Merge pull request #17 from datafold/gerard-eng-1027-deploy-google-with-changes-in-helm-charts
fix: Several azure fixes
2 parents 883e33a + 9ec0a82 commit a2a82b0

File tree

24 files changed

+565
-70
lines changed

24 files changed

+565
-70
lines changed

‎README.md

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -110,20 +110,6 @@ k9s
110110

111111
### Initializing the application
112112

113-
The deployment is created and the initjob should have created the databases and done the
114-
initialization of the site settings.
115-
116-
If that didn't complete successfully, try to restart the job.
117-
118-
Once the deployment is complete and the initjob succeeded, we can set the install to that for false in config.yaml:
119-
120-
```
121-
initjob:
122-
install: false
123-
```
124-
125-
Alternatively, here are the manual steps to achieve the same:
126-
127113
Establish a shell into the `<deployment>-dfshell` container.
128114
It is likely that the scheduler and server containers are crashing in a loop.
129115

‎examples/deployment/application/config.yaml

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,63 @@
11
global:
2+
cloudProvider: "azure"
23
datadog:
3-
env: prod
4+
env: test
45
install: true
56
operator:
67
allowRollback: true
7-
releaseChannel: stable
8+
releaseChannel: foo
89
backupCronSchedule: 0 1 * * *
910
maintenanceWindow: ""
1011
postgres:
1112
install: false
1213
nginx:
1314
ingress:
1415
deploy: true
15-
initjob:
16+
datadog:
17+
configuration:
18+
apm: false
19+
monitorPostgres: true
20+
npm:
21+
enabled: true
22+
dnsstats: true
23+
config:
24+
portalCertData: '-----BEGIN CERTIFICATE-----\nMIIF0DCCBLigAwIBAgIQAVQqsCg8OcOVGz9sD29ewTANBgkqhkiG9w0BAQsFADA8\nMQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRwwGgYDVQQDExNBbWF6b24g\nUlNBIDIwNDggTTAzMB4XDTI0MTIxNjAwMDAwMFoXDTI2MDExNDIzNTk1OVowHTEb\nMBkGA1UEAxMScG9ydGFsLmRhdGFmb2xkLmlvMIIBIjANBgkqhkiG9w0BAQEFAAOC\nAQ8AMIIBCgKCAQEA49uvagYKuBZsXEmq+aPpBwWyw/j8Ke/oO/b1RPqtWjNuZfpd\n4MrjpT9oi1Aq9J29HCN2i7PSZHznaovCm362fEDWqLdfPoVMxicF1Q5qRDe7vvWo\nNZcA0PJckD7RRWB9yInGMxH8IoxbkJ+7qUdOGIreIs6u5c3EVa04wLTjrdiJSAQv\n5UG/qMRkw4YVcx82E6rDwiaft+iONLQwDhs2oPGF/HtteJ57e6Krpy2AymRuq1Oh\nJFPe5Ng2UIEN2ny7kb4uB4SC/Ia9EKvZllXJFKeJwRPWIKwofRCCbzUNHCyWka/J\nhr8Y/W8Q2RzE/spORH97X2uZ4D+3dMg/tH/r9wIDAQABo4IC6zCCAucwHwYDVR0j\nBBgwFoAUVdkYX9IczAHhWLS+q9lVQgHXLgIwHQYDVR0OBBYEFN8sZRehNoz13gJY\nP/kvtJobCGttMB0GA1UdEQQWMBSCEnBvcnRhbC5kYXRhZm9sZC5pbzATBgNVHSAE\nDDAKMAgGBmeBDAECATAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUH\nAwEGCCsGAQUFBwMCMDsGA1UdHwQ0MDIwMKAuoCyGKmh0dHA6Ly9jcmwucjJtMDMu\nYW1hem9udHJ1c3QuY29tL3IybTAzLmNybDB1BggrBgEFBQcBAQRpMGcwLQYIKwYB\nBQUHMAGGIWh0dHA6Ly9vY3NwLnIybTAzLmFtYXpvbnRydXN0LmNvbTA2BggrBgEF\nBQcwAoYqaHR0cDovL2NydC5yMm0wMy5hbWF6b250cnVzdC5jb20vcjJtMDMuY2Vy\nMAwGA1UdEwEB/wQCMAAwggF+BgorBgEEAdZ5AgQCBIIBbgSCAWoBaAB1AA5XlLzz\nrqk+MxssmQez95Dfm8I9cTIl3SGpJaxhxU4hAAABk8+aBkEAAAQDAEYwRAIgJ6Wz\nfFRMIaaKc9GLH1v/BP3JNosLoJtUj5qu7zOLThwCIEwR38uEvcFFix91UnTLs4U1\nUhYLn4SVligtuT9TDsCHAHcAZBHEbKQS7KeJHKICLgC8q08oB9QeNSer6v7VA8l9\nzfAAAAGTz5oGcQAABAMASDBGAiEAs/RTBqGcneECjfiUp4pjEixPo1QnmIMMK86c\n1PwD308CIQDXwFzCQEn+iTpJpSNOP1cXt88oxpuHSXaBIy3YVHXtDAB2AEmcm2ne\nHXzs/DbezYdkprhbrwqHgBnRVVL76esp3fjDAAABk8+aBogAAAQDAEcwRQIgb7vI\nAmTCYkx1kmafnndESNO+YkvPNKzrSpxXhJebVL4CIQCgNffwKFEODQSerrN0h7pf\nwNdL4U66YT4FBCu9OCT43DANBgkqhkiG9w0BAQsFAAOCAQEAHpWxAKJRIoNodVeb\ndhJ3pX3cESS4AjjJaKUR94pJPi0KRoJAFsveR+0fOicCyBsn30hfNl+07P7VEUU0\nP4KZquITE0WmTGoQvFPcrTrqMV5MX8uh0oS6UaBFlZmSBWvO4pX3+D5xJI3RWxrl\nICM49BwHFSWIy51zeiFtiohQ8t8IcQhPCScM+UYqoIOp7DxcLetNY6DDi6nU4UO6\nfy3DakMrCY3XFYuI2LFDGphoyif1HHzmtVysR+E6YBRadoeLikRgg3VNGM2il0GO\n0WO36x7DxrfnPeSRmIprJXXQD8f3ePLMcUzrdyVDdTgyDqKQ8F2UK70v61EdyVHP\nOqkgMw==\n-----END CERTIFICATE-----\n'
25+
worker:
26+
resources:
27+
limits:
28+
memory: 8000Mi
29+
requests:
30+
memory: 8000Mi
31+
worker-catalog:
32+
resources:
33+
limits:
34+
memory: 19000Mi
35+
requests:
36+
memory: 19000Mi
37+
worker-monitor:
1638
install: true
39+
replicaCount: 1
40+
worker:
41+
queues: "alerts"
42+
count: 10
43+
memory: "2000000"
44+
resources:
45+
limits:
46+
memory: 6Gi
47+
requests:
48+
memory: 6Gi
49+
worker-portal:
50+
install: true
51+
storage-worker:
52+
install: true
53+
replicaCount: 1
54+
worker:
55+
tasks_ack_late: "true"
56+
terminationGracePeriodSeconds: "18000"
57+
resources:
58+
limits:
59+
memory: 5Gi
60+
requests:
61+
memory: 5Gi
62+
storage:
63+
dataSize: 100Gi

‎examples/deployment/application/versions.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ terraform {
1212
}
1313

1414
locals {
15-
operator_version = "1.1.4"
16-
helm_version = "0.6.40"
15+
operator_version = "1.2.8"
16+
helm_version = "0.6.83"
1717
crd_version = "0.1.1"
1818
}

‎examples/deployment/infra/config.tf

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@ resource "local_file" "infra_config" {
44
"${path.module}/../templates/infra_settings.tpl",
55
{
66
aws_target_group_arn = "",
7-
clickhouse_access_key = "",
8-
clickhouse_secret_key = "",
9-
clickhouse_backup_sa = "",
7+
gcp_backup_account = "",
108
clickhouse_data_size = local.clickhouse_data_size,
11-
clickhouse_data_volume_id = "",
9+
clickhouse_data_volume_id = module.azure[0].clickhouse_data_volume_id,
1210
clickhouse_gcs_bucket = "",
1311
clickhouse_logs_size = local.clickhouse_logs_size,
14-
clickhouse_log_volume_id = "",
12+
clickhouse_log_volume_id = module.azure[0].clickhouse_logs_volume_id,
1513
clickhouse_s3_bucket = "",
1614
clickhouse_s3_region = "",
15+
clickhouse_s3_backup_role = "",
16+
clickhouse_azblob_client_id = try(module.azure[0].service_account_configs[local.clickhouse_backup_sa].azure_identity.client_id, "")
1717
clickhouse_azblob_account_name = module.azure[0].azure_blob_account_name,
18-
clickhouse_azblob_account_key = module.azure[0].azure_blob_account_key,
1918
clickhouse_azblob_container = module.azure[0].azure_blob_container,
2019
cloud_provider = module.azure[0].cloud_provider,
2120
cluster_name = module.azure[0].cluster_name,
@@ -29,9 +28,35 @@ resource "local_file" "infra_config" {
2928
postgres_server = module.azure[0].postgres_host,
3029
postgres_user = module.azure[0].postgres_username,
3130
redis_data_size = local.redis_data_size,
32-
redis_data_volume_id = "",
31+
redis_data_volume_id = module.azure[0].redis_data_volume_id,
3332
server_name = module.azure[0].domain_name,
3433
vpc_cidr = module.azure[0].vpc_cidr,
34+
35+
# service accounts vars
36+
dfshell_role_arn = try(module.azure[0].dfshell_role_arn, "")
37+
dfshell_service_account_name = try(module.azure[0].dfshell_service_account_name, "datafold-dfshell")
38+
worker_portal_role_arn = try(module.azure[0].worker_portal_role_arn, "")
39+
worker_portal_service_account_name = try(module.azure[0].worker_portal_service_account_name, "datafold-worker-portal")
40+
operator_role_arn = try(module.azure[0].operator_role_arn, "")
41+
operator_service_account_name = try(module.azure[0].operator_service_account_name, "datafold-operator")
42+
server_role_arn = try(module.azure[0].server_role_arn, "")
43+
server_service_account_name = try(module.azure[0].server_service_account_name, "datafold-server")
44+
scheduler_role_arn = try(module.azure[0].scheduler_role_arn, "")
45+
scheduler_service_account_name = try(module.azure[0].scheduler_service_account_name, "datafold-scheduler")
46+
worker_role_arn = try(module.azure[0].worker_role_arn, "")
47+
worker_service_account_name = try(module.azure[0].worker_service_account_name, "datafold-worker")
48+
worker_catalog_role_arn = try(module.azure[0].worker_catalog_role_arn, "")
49+
worker_catalog_service_account_name = try(module.azure[0].worker_catalog_service_account_name, "datafold-worker-catalog")
50+
worker_interactive_role_arn = try(module.azure[0].worker_interactive_role_arn, "")
51+
worker_interactive_service_account_name = try(module.azure[0].worker_interactive_service_account_name, "datafold-worker-interactive")
52+
worker_singletons_role_arn = try(module.azure[0].worker_singletons_role_arn, "")
53+
worker_singletons_service_account_name = try(module.azure[0].worker_singletons_service_account_name, "datafold-worker-singletons")
54+
worker_lineage_role_arn = try(module.azure[0].worker_lineage_role_arn, "")
55+
worker_lineage_service_account_name = try(module.azure[0].worker_lineage_service_account_name, "datafold-worker-lineage")
56+
worker_monitor_role_arn = try(module.azure[0].worker_monitor_role_arn, "")
57+
worker_monitor_service_account_name = try(module.azure[0].worker_monitor_service_account_name, "datafold-worker-monitor")
58+
storage_worker_role_arn = try(module.azure[0].storage_worker_role_arn, "")
59+
storage_worker_service_account_name = try(module.azure[0].storage_worker_service_account_name, "datafold-storage-worker")
3560
}
3661
)
3762

‎examples/deployment/infra/main.tf

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,12 @@
77
# ┣━┫┏━┛┃ ┃┣┳┛┣╸
88
# ╹ ╹┗━╸┗━┛╹┗╸┗━╸
99

10+
locals {
11+
storage_account_name = replace("${local.deployment_name}-storage", "-", "")
12+
}
13+
1014
module "azure" {
11-
source = "datafold/datafold/aws"
15+
source = "datafold/datafold/azure"
1216
version = "1.0.0"
1317

1418
providers = {
@@ -34,4 +38,30 @@ module "azure" {
3438

3539
# Nodes
3640
node_pool_vm_size = "Standard_E8s_v3"
41+
42+
service_accounts = {
43+
"${local.clickhouse_backup_sa}" = {
44+
namespace = local.deployment_name
45+
create_azure_identity = true
46+
identity_name = local.clickhouse_backup_sa
47+
role_assignments = [
48+
{
49+
role = "Storage Blob Data Contributor"
50+
scope = "/subscriptions/${local.azure_subscription_id}/resourceGroups/${local.resource_group_name}/providers/Microsoft.Storage/storageAccounts/${local.storage_account_name}"
51+
}
52+
]
53+
},
54+
}
55+
56+
# Certificate note
57+
# Example:
58+
# [profile acme]
59+
# role_arn = arn:aws:iam::1234567890:role/ACMERoute53CertificateChallenger
60+
# source_profile = default
61+
# region = us-west-2
62+
63+
acme_provider = "route53"
64+
acme_config = {
65+
AWS_PROFILE = "acme"
66+
}
3767
}

‎examples/deployment/infra/versions.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ terraform {
22
required_providers {
33
azurerm = {
44
source = "hashicorp/azurerm"
5-
version = "~>3.17"
5+
version = "~>4.35.0"
66
}
77

88
tls = {

‎examples/deployment/locals.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ locals {
1313
redis_data_size = "10"
1414
postgres_port = "5432"
1515
ssl_cert_name = "ssl"
16+
clickhouse_backup_sa = "datafold-clickhouse"
1617

1718
# Common tags to be assigned to all resources
1819
common_tags = {

‎examples/deployment/templates/cloud-init.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ssh_authorized_keys: [ssh-rsa AAAA.... youruser@email.com, ssh-rsa AAAA... otheruser@email.com]

‎examples/deployment/templates/infra_settings.tpl

Lines changed: 73 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,16 @@ clickhouse:
33
gcs_bucket: ${clickhouse_gcs_bucket}
44
s3_bucket: ${clickhouse_s3_bucket}
55
s3_region: ${clickhouse_s3_region}
6+
s3_backup_role: ${clickhouse_s3_backup_role}
7+
gcp_backup_account: ${gcp_backup_account}
8+
azblob_backup_client_id: ${clickhouse_azblob_client_id}
69
azblob_account_name: ${clickhouse_azblob_account_name}
710
azblob_container: ${clickhouse_azblob_container}
811
storage:
912
dataSize: ${clickhouse_data_size}
1013
dataVolumeId: ${clickhouse_data_volume_id}
1114
logSize: ${clickhouse_logs_size}
1215
logVolumeId: ${clickhouse_log_volume_id}
13-
secrets:
14-
access_key: ${clickhouse_access_key}
15-
secret_key: ${clickhouse_secret_key}
16-
clickhouse_backup_sa: ${clickhouse_backup_sa}
17-
azblob_account_key: ${clickhouse_azblob_account_key}
1816

1917
redis:
2018
storage:
@@ -47,3 +45,73 @@ secrets:
4745
password: ${postgres_password}
4846
port: ${postgres_port}
4947
user: ${postgres_user}
48+
49+
dfshell:
50+
serviceAccount:
51+
name: ${dfshell_service_account_name}
52+
roleArn: ${dfshell_role_arn}
53+
54+
worker-portal:
55+
serviceAccount:
56+
name: ${worker_portal_service_account_name}
57+
roleArn: ${worker_portal_role_arn}
58+
59+
operator:
60+
serviceAccount:
61+
name: ${operator_service_account_name}
62+
roleArn: ${operator_role_arn}
63+
64+
server:
65+
serviceAccount:
66+
name: ${server_service_account_name}
67+
roleArn: ${server_role_arn}
68+
69+
scheduler:
70+
serviceAccount:
71+
name: ${scheduler_service_account_name}
72+
roleArn: ${scheduler_role_arn}
73+
74+
worker:
75+
serviceAccount:
76+
name: ${worker_service_account_name}
77+
roleArn: ${worker_role_arn}
78+
79+
worker2:
80+
serviceAccount:
81+
name: ${worker_service_account_name}
82+
roleArn: ${worker_role_arn}
83+
84+
worker3:
85+
serviceAccount:
86+
name: ${worker_service_account_name}
87+
roleArn: ${worker_role_arn}
88+
89+
worker-catalog:
90+
serviceAccount:
91+
name: ${worker_catalog_service_account_name}
92+
roleArn: ${worker_catalog_role_arn}
93+
94+
worker-interactive:
95+
serviceAccount:
96+
name: ${worker_interactive_service_account_name}
97+
roleArn: ${worker_interactive_role_arn}
98+
99+
worker-singletons:
100+
serviceAccount:
101+
name: ${worker_singletons_service_account_name}
102+
roleArn: ${worker_singletons_role_arn}
103+
104+
worker-lineage:
105+
serviceAccount:
106+
name: ${worker_lineage_service_account_name}
107+
roleArn: ${worker_lineage_role_arn}
108+
109+
worker-monitor:
110+
serviceAccount:
111+
name: ${worker_monitor_service_account_name}
112+
roleArn: ${worker_monitor_role_arn}
113+
114+
storage-worker:
115+
serviceAccount:
116+
name: ${storage_worker_service_account_name}
117+
roleArn: ${storage_worker_role_arn}

0 commit comments

Comments
 (0)