-
Notifications
You must be signed in to change notification settings - Fork 1
/
opensearch-to-clickhouse.tf
204 lines (176 loc) · 7.74 KB
/
opensearch-to-clickhouse.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
# Infrastructure for Managed Service for OpenSearch cluster, Managed Service for ClickHouse cluster, and Data Transfer.
# RU: https://cloud.yandex.ru/ru/docs/data-transfer/tutorials/opensearch-to-clickhouse
# EN: https://cloud.yandex.com/en/docs/data-transfer/tutorials/opensearch-to-clickhouse
# Specify the following settings:
locals {
# Settings for the Managed Service for OpenSearch cluster:
mos_version = "" # Desired version of the Opensearch. For available versions, see the documentation main page: https://yandex.cloud/en/docs/managed-opensearch/.
source_admin_password = "" # Password of admin in Managed Service for OpenSearch
# Settings for the Managed Service for ClickHouse cluster:
mch_db_name = "" # Name of the Managed Service for ClickHouse database
mch_username = "" # Name of the Managed Service for ClickHouse user
mch_user_password = "" # Password of the Managed Service for ClickHouse user
# Specify these settings ONLY AFTER the clusters are created. Then run "terraform apply" command again.
# You should set up the source endpoint using GUI to obtain its ID
source_endpoint_id = "" # Source endpoint ID
transfer_enabled = 0 # Set to 1 to enable creation of target endpoint and transfer
# Setting for the YC CLI that allows running CLI command to activate the transfer
profile_name = "" # Name of the YC CLI profile
# The following settings are predefined. Change them only if necessary.
opensearch_port = 9200 # Managed Service for OpenSearch port for Internet connection
mch_https_port = 8443 # Managed Service for ClickHouse HTTPS port
mch_client_port = 9440 # Managed Service for ClickHouse client port
network_name = "mynet" # Name of the network for Managed Service for OpenSearch cluster and Managed Service for ClickHouse cluster
subnet_name = "mysubnet" # Name of the subnet for Managed Service for OpenSearch cluster and Managed Service for ClickHouse cluster
sg_name = "mos-mch-sg" # Name of the security group for Managed Service for OpenSearch cluster and Managed Service for ClickHouse cluster
mos_cluster_name = "mos-cluster-3845-2" # Name of the Managed Service for OpenSearch cluster
node_group_name = "mos-group" # Node group name in the Managed Service for OpenSearch cluster
dashboards_name = "dashboards" # Name of the dashboards node group in the Managed Service for OpenSearch cluster
mch_cluster_name = "mch-cluster" # Name of the Managed Service for ClickHouse cluster
target_endpoint_name = "mch-target" # Name of the target endpoint
transfer_name = "mos-to-mch-transfer" # Name of the Data Transfer
}
resource "yandex_vpc_network" "mynet" {
description = "Network for Managed Service for OpenSearch cluster and Managed Service for ClickHouse cluster"
name = local.network_name
}
resource "yandex_vpc_subnet" "mysubnet" {
description = "Subnet for for Managed Service for OpenSearch cluster and Managed Service for ClickHouse cluster"
name = local.subnet_name
zone = "ru-central1-a"
network_id = yandex_vpc_network.mynet.id
v4_cidr_blocks = ["10.1.0.0/16"]
}
resource "yandex_vpc_security_group" "mos-mch-sg" {
description = "Security group for Managed Service for OpenSearch cluster and Managed Service for ClickHouse cluster"
name = local.sg_name
network_id = yandex_vpc_network.mynet.id
ingress {
description = "Allow connections to the Managed Service for OpenSearch cluster from the Internet"
protocol = "TCP"
port = local.opensearch_port
v4_cidr_blocks = ["0.0.0.0/0"]
}
ingress {
description = "Allow connections to the Managed Service for ClickHouse via HTTPS"
port = local.mch_https_port
protocol = "TCP"
v4_cidr_blocks = ["0.0.0.0/0"]
}
ingress {
description = "Allow connections to the Managed Service for ClickHouse via Clickhouse-client"
port = local.mch_client_port
protocol = "TCP"
v4_cidr_blocks = ["0.0.0.0/0"]
}
egress {
description = "The rule allows all outgoing traffic"
protocol = "ANY"
v4_cidr_blocks = ["0.0.0.0/0"]
from_port = 0
to_port = 65535
}
}
resource "yandex_mdb_opensearch_cluster" "my-os-cluster" {
description = "Managed Service for OpenSearch cluster"
name = local.mos_cluster_name
environment = "PRODUCTION"
network_id = yandex_vpc_network.mynet.id
security_group_ids = [yandex_vpc_security_group.mos-mch-sg.id]
config {
version = local.mos_version
admin_password = local.source_admin_password
opensearch {
node_groups {
name = local.node_group_name
assign_public_ip = true
hosts_count = 1
zone_ids = ["ru-central1-a"]
subnet_ids = [yandex_vpc_subnet.mysubnet.id]
roles = ["DATA", "MANAGER"]
resources {
resource_preset_id = "s2.micro" # 2 vCPU, 8 GB RAM
disk_size = 10737418240 # Bytes
disk_type_id = "network-ssd"
}
}
}
dashboards {
node_groups {
name = local.dashboards_name
assign_public_ip = true
hosts_count = 1
zone_ids = ["ru-central1-a"]
subnet_ids = [yandex_vpc_subnet.mysubnet.id]
resources {
resource_preset_id = "s2.micro" # 2 vCPU, 8 GB RAM
disk_size = 10737418240 # Bytes
disk_type_id = "network-ssd"
}
}
}
}
maintenance_window {
type = "ANYTIME"
}
}
resource "yandex_mdb_clickhouse_cluster" "mych" {
description = "Managed Service for ClickHouse cluster"
name = local.mch_cluster_name
environment = "PRESTABLE"
network_id = yandex_vpc_network.mynet.id
security_group_ids = [yandex_vpc_security_group.mos-mch-sg.id]
clickhouse {
resources {
resource_preset_id = "s2.micro"
disk_type_id = "network-ssd"
disk_size = 32
}
}
host {
type = "CLICKHOUSE"
zone = "ru-central1-a"
subnet_id = yandex_vpc_subnet.mysubnet.id
assign_public_ip = true
}
database {
name = local.mch_db_name
}
user {
name = local.mch_username
password = local.mch_user_password
permission {
database_name = local.mch_db_name
}
}
}
resource "yandex_datatransfer_endpoint" "managed-clickhouse-target" {
description = "Target endpoint for the Managed Service for ClickHouse cluster"
count = local.transfer_enabled
name = local.target_endpoint_name
settings {
clickhouse_target {
connection {
connection_options {
mdb_cluster_id = yandex_mdb_clickhouse_cluster.mych.id
database = local.mch_db_name
user = local.mch_username
password {
raw = local.mch_user_password
}
}
}
}
}
}
resource "yandex_datatransfer_transfer" "mos-to-mch-transfer" {
description = "Transfer from the Managed Service for OpenSearch cluster to the Managed Service for ClickHouse cluster"
count = local.transfer_enabled
name = local.transfer_name
source_id = local.source_endpoint_id
target_id = yandex_datatransfer_endpoint.managed-clickhouse-target[count.index].id
type = "SNAPSHOT_ONLY" # Copy all data from the source server
provisioner "local-exec" {
command = "yc --profile ${local.profile_name} datatransfer transfer activate ${yandex_datatransfer_transfer.mos-to-mch-transfer[count.index].id}"
}
}