diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml index e5bf55461..694235f1e 100644 --- a/.github/workflows/testflows-sink-connector-lightweight.yml +++ b/.github/workflows/testflows-sink-connector-lightweight.yml @@ -93,7 +93,7 @@ jobs: - name: Upload artifacts to Altinity Test Reports S3 bucket if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} - working-directory: sink-connector/tests/integration/logs + working-directory: sink-connector-lightweight/tests/integration/logs env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -109,3 +109,69 @@ jobs: sink-connector-lightweight/tests/integration/env/auto/configs/*.yml if-no-files-found: error retention-days: 60 + testflows-lightweight-replicated: + runs-on: [self-hosted, on-demand, type-cpx51, image-x86-app-docker-ce] + + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v3 + if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} + with: + name: clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz + + - name: Load Docker image + if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} + run: | + docker load < clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz + docker image ls + + - name: Runner ssh command + working-directory: sink-connector/tests/integration + run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" + + - name: Install all dependencies + working-directory: sink-connector-lightweight/tests/integration + run: pip3 install -r requirements.txt + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d_%H%M%S')" >> $GITHUB_OUTPUT + + - name: Add ~./local/bin to the PATH + if: always() + working-directory: sink-connector-lightweight/tests/integration + run: echo ~/.local/bin >> $GITHUB_PATH + + - name: Run testflows tests + working-directory: sink-connector-lightweight/tests/integration + run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end -o classic --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log + + - name: Create tfs results report + if: always() + working-directory: sink-connector-lightweight/tests/integration/logs + run: cat raw.log | tfs report results | tfs document convert > report.html + + - name: Create tfs coverage report + if: always() + working-directory: sink-connector-lightweight/tests/integration/logs + run: cat raw.log | tfs report coverage ../requirements/requirements.py | tfs document convert > coverage.html + + - name: Upload artifacts to Altinity Test Reports S3 bucket + if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} + working-directory: sink-connector-lightweight/tests/integration/logs + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: 'eu-west-2' + run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink_lw/ --recursive --exclude "*" --include "*.log" --include "*.html" + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: testflows-sink-connector-lightweight-replicated-artefacts + path: | + sink-connector-lightweight/tests/integration/logs/*.log + sink-connector-lightweight/tests/integration/env/auto_replicated/configs/*.yml + if-no-files-found: error + retention-days: 60 \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/common.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/common.xml new file mode 100644 index 000000000..0ba01589b --- /dev/null +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + :: + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ports.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ports.xml new file mode 100644 index 000000000..01ce4adf5 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,7 @@ + + + 8443 + 9440 + 9005 + 9004 + \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/remote.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/remote.xml index 926aaf09f..cc0dacc14 100644 --- a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/remote.xml +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/remote.xml @@ -12,6 +12,10 @@ false + + clickhouse + 9000 + clickhouse1 9000 @@ -27,6 +31,12 @@ + + + clickhouse + 9000 + + clickhouse1 diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ssl.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 000000000..1083c8d74 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,18 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + /etc/clickhouse-server/ssl/dhparam.pem + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/storage.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/storage.xml new file mode 100644 index 000000000..5c23dfdfd --- /dev/null +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,99 @@ + + + + + + s3 + http://minio:9001/root/data/ + minio + minio123 + + + 1024 + + + /mnt/new_disk/ + 10485760 + + + /var/lib/clickhouse/disk1/ + + + /var/lib/clickhouse/disk2/ + + + /jbod1/ + + + /jbod2/ + 10485760 + + + + + +
+ s3_disk +
+ + + + + + default + + + + + +
+ default + 1073741824 +
+ + new_disk + + + + + + + disk1 + + + + + + + disk2 + + + + + + + disk1 + + + s3_disk + + + 0.2 + + + + + jbod1 + + + jbod2 + + + s3_disk + + + + + + + diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/dhparam.pem b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 000000000..2e6cee079 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.crt b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.crt new file mode 100644 index 000000000..7ade2d962 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.key b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.key new file mode 100644 index 000000000..f0fb61ac4 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse1/config.d/macros.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse1/config.d/macros.xml index 46879388c..6422ccb1c 100644 --- a/sink-connector-lightweight/tests/integration/configs/clickhouse1/config.d/macros.xml +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse1/config.d/macros.xml @@ -1,6 +1,7 @@ + replicated_cluster clickhouse1 01 diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse2/config.d/macros.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse2/config.d/macros.xml index 658914602..6f745f230 100644 --- a/sink-connector-lightweight/tests/integration/configs/clickhouse2/config.d/macros.xml +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse2/config.d/macros.xml @@ -1,8 +1,9 @@ + replicated_cluster clickhouse2 - 02 + 01 diff --git a/sink-connector-lightweight/tests/integration/configs/clickhouse3/config.d/macros.xml b/sink-connector-lightweight/tests/integration/configs/clickhouse3/config.d/macros.xml index 684de2307..46a2f1f31 100644 --- a/sink-connector-lightweight/tests/integration/configs/clickhouse3/config.d/macros.xml +++ b/sink-connector-lightweight/tests/integration/configs/clickhouse3/config.d/macros.xml @@ -1,8 +1,8 @@ + replicated_cluster clickhouse3 - 02 - + 01 diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/clickhouse-service.yml b/sink-connector-lightweight/tests/integration/env/auto_replicated/clickhouse-service.yml new file mode 100644 index 000000000..13eb4daf8 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/clickhouse-service.yml @@ -0,0 +1,31 @@ +version: '2.3' + +services: + clickhouse: +# image: clickhouse/clickhouse-integration-test:28741 + image: altinityinfra/clickhouse-regression-multiarch:1.0 + init: true + expose: + - "9000" + - "8123" + - "2181" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/logs.xml:/etc/clickhouse-server/config.d/logs.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/remote.xml:/etc/clickhouse-server/config.d/remote.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "tail -f /dev/null" + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable + diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/clickhouse-sink-connector-lt-service.yml b/sink-connector-lightweight/tests/integration/env/auto_replicated/clickhouse-sink-connector-lt-service.yml new file mode 100644 index 000000000..dd3182645 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/clickhouse-sink-connector-lt-service.yml @@ -0,0 +1,17 @@ +version: "2.3" + +services: + clickhouse-sink-connector-lt: + hostname: clickhouse-sink-connector-lt + image: ${SINK_CONNECTOR_IMAGE} + restart: "no" + expose: + - "8083" + - "1976" + - "5005" + extra_hosts: + - "host.docker.internal:host-gateway" + entrypoint: bash -c "tail -f /dev/null" + volumes: + - ./configs:/configs + - ../../logs:/logs \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/configs/config.yml b/sink-connector-lightweight/tests/integration/env/auto_replicated/configs/config.yml new file mode 100644 index 000000000..c98a77ad1 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/configs/config.yml @@ -0,0 +1,49 @@ +name: my_connector +database.hostname: mysql-master +database.port: '3306' +database.user: root +database.password: root +database.server.name: ER54 +database.include.list: test +clickhouse.server.url: clickhouse +clickhouse.server.user: root +clickhouse.server.password: root +clickhouse.server.port: '8123' +clickhouse.server.database: test +database.allowPublicKeyRetrieval: 'true' +snapshot.mode: initial +offset.flush.interval.ms: '5000' +connector.class: io.debezium.connector.mysql.MySqlConnector +offset.storage: io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore +offset.storage.jdbc.offset.table.name: altinity_sink_connector.replica_source_info +offset.storage.jdbc.url: jdbc:clickhouse://clickhouse:8123/altinity_sink_connector +offset.storage.jdbc.user: root +offset.storage.jdbc.password: root +offset.storage.jdbc.offset.table.ddl: |- + CREATE TABLE if not exists %s + ( + `id` String, + `offset_key` String, + `offset_val` String, + `record_insert_ts` DateTime, + `record_insert_seq` UInt64, + `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9)) + ) + ENGINE = ReplacingMergeTree(_version) + ORDER BY id + SETTINGS index_granularity = 8198 +offset.storage.jdbc.offset.table.delete: delete from %s where 1=1 +schema.history.internal: io.debezium.storage.jdbc.history.JdbcSchemaHistory +schema.history.internal.jdbc.url: jdbc:clickhouse://clickhouse:8123/altinity_sink_connector +schema.history.internal.jdbc.user: root +schema.history.internal.jdbc.password: root +schema.history.internal.jdbc.schema.history.table.ddl: |- + CREATE TABLE if not exists %s + (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id +schema.history.internal.jdbc.schema.history.table.name: altinity_sink_connector.replicate_schema_history +replacingmergetree.delete.column: _sign +enable.snapshot.ddl: 'true' +database.connectionTimeZone: UTC +database.serverTimezone: UTC +clickhouse.datetime.timezone: UTC +auto.create.tables: 'true' diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/docker-compose.yml b/sink-connector-lightweight/tests/integration/env/auto_replicated/docker-compose.yml new file mode 100644 index 000000000..080c314ec --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/docker-compose.yml @@ -0,0 +1,110 @@ +version: "2.3" + + +services: + mysql-master: + image: docker.io/bitnami/mysql:8.0 + restart: "no" + expose: + - "3306" + environment: + - MYSQL_ROOT_PASSWORD=root + - MYSQL_DATABASE=test + - MYSQL_REPLICATION_MODE=master + - MYSQL_REPLICATION_USER=repl_user + - ALLOW_EMPTY_PASSWORD=yes + volumes: + - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/share_folder:/tmp/share_folder" + healthcheck: + test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ] + interval: 15s + timeout: 5s + retries: 6 + + + clickhouse-sink-connector-lt: + extends: + file: clickhouse-sink-connector-lt-service.yml + service: clickhouse-sink-connector-lt + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + + clickhouse: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse/etc/:/etc/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse0/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances_auto/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml" + depends_on: + zookeeper: + condition: service_healthy + + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependencies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse: + condition: service_healthy + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/docker.env b/sink-connector-lightweight/tests/integration/env/auto_replicated/docker.env new file mode 100644 index 000000000..a217ebef5 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/docker.env @@ -0,0 +1,19 @@ +export database.hostname="mysql-master" +export database.port="3306" +export database.user="root" +export database.password="root" +export database.include.list=test +#export table.include.list=sbtest1 +export database.allowPublicKeyRetrieval"="true" +export snapshot.mode="initial" +export connector.class="io.debezium.connector.mysql.MySqlConnector" +export clickhouse.server.database="test" +export replacingmergetree.delete.column="_sign" +export metrics.port=8083 +export clickhouse.server.url="clickhouse" +export clickhouse.server.user="root" +export clickhouse.server.password="root" +export clickhouse.server.port="8123" +export thread.pool.size="10" +export provide.transaction.metadata="false" +export disable.ddl="false" \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/mysqld.cnf b/sink-connector-lightweight/tests/integration/env/auto_replicated/mysqld.cnf new file mode 100644 index 000000000..2facad3a1 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/mysqld.cnf @@ -0,0 +1,6 @@ +[mysqld] +max_connections=100000 + + +gtid-mode = on +enforce-gtid-consistency = true \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/zookeeper-service.yml b/sink-connector-lightweight/tests/integration/env/auto_replicated/zookeeper-service.yml new file mode 100755 index 000000000..f27405b97 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.6.2 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable diff --git a/sink-connector-lightweight/tests/integration/helpers/create_config.py b/sink-connector-lightweight/tests/integration/helpers/create_config.py index 0237d60b5..0f7ef03f4 100644 --- a/sink-connector-lightweight/tests/integration/helpers/create_config.py +++ b/sink-connector-lightweight/tests/integration/helpers/create_config.py @@ -57,6 +57,20 @@ def create_default_sink_config(self, path="env/auto/configs/config.yml"): config.save(filename=path) +@TestStep(Given) +def create_default_sink_config_replicated( + self, path="env/auto_replicated/configs/replicated_config.yml" +): + """Create the default sink connector configuration.""" + config = self.context.config + + with By(f"creating the default sink connector configuration file"): + config.update( + {"auto.create.tables.replicated": "true", "auto.create.tables": "true"} + ) + config.save(filename=path) + + @TestStep(Given) def update_sink_config(self, new_data: dict, path="env/auto/configs/config.yml"): """Update the sink connector configuration.""" @@ -75,3 +89,27 @@ def remove_configuration(self, key, path): with By(f"removing the sink connector configuration key {key}"): config.remove(key) config.save(filename=path) + + +@TestStep(Given) +def include_all_databases_with_rrmt(self, node=None, config_file=None): + """Create and use ClickHouse Sink Connector configuration which allows the following: + - Monitors all databases in source and replicates them in destination. + - Tables created automatically are with ReplicatedReplacingMergeTree engine. + """ + config = self.context.config + + if node is None: + node = self.context.sink_node + + with By("removing the ClickHouse Sink Connector configuration"): + config.remove("database.include.list") + config.update( + {"auto.create.tables.replicated": "true", "auto.create.tables": "true"} + ) + config.save(filename=config_file) + + with And( + "restarting the ClickHouse Sink Connector and using the new configuration file" + ): + node.restart_sink_connector(config_file=config_file) diff --git a/sink-connector-lightweight/tests/integration/regression.py b/sink-connector-lightweight/tests/integration/regression.py index 7c4d3ee1e..24a448be6 100755 --- a/sink-connector-lightweight/tests/integration/regression.py +++ b/sink-connector-lightweight/tests/integration/regression.py @@ -42,13 +42,18 @@ def regression( self.context.stress = stress - with Pool(2) as pool: + with Pool(1) as pool: try: Feature( test=load("regression_auto", "regression"), parallel=True, executor=pool, )(**args) + Feature( + test=load("regression_auto_replicated", "regression"), + parallel=True, + executor=pool, + )(**args) finally: join() diff --git a/sink-connector-lightweight/tests/integration/regression_auto.py b/sink-connector-lightweight/tests/integration/regression_auto.py index b30b1970c..767d1c69f 100755 --- a/sink-connector-lightweight/tests/integration/regression_auto.py +++ b/sink-connector-lightweight/tests/integration/regression_auto.py @@ -170,6 +170,7 @@ def regression( "zookeeper": ("zookeeper",), } + self.context.nodes = nodes self.context.clickhouse_version = clickhouse_version self.context.config = SinkConfig() create_default_sink_config() @@ -197,6 +198,7 @@ def regression( self.context.env = env self.context.clickhouse_table_engines = ["ReplacingMergeTree"] + self.context.clickhouse_table_engine = "ReplacingMergeTree" self.context.database = "test" diff --git a/sink-connector-lightweight/tests/integration/regression_auto_replicated.py b/sink-connector-lightweight/tests/integration/regression_auto_replicated.py new file mode 100755 index 000000000..4a3cd5661 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/regression_auto_replicated.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 + +import os +import sys +import time + +from testflows.core import * + +from integration.tests.steps.sink_configurations import ( + config_with_replicated_table, +) + +append_path(sys.path, "..") + +from integration.helpers.argparser import argparser +from integration.helpers.common import check_clickhouse_version +from integration.helpers.common import create_cluster +from integration.helpers.create_config import * +from integration.requirements.requirements import * +from integration.tests.steps.clickhouse import * + +ffails = { + "primary keys/no primary key": ( + Skip, + "https://github.com/Altinity/clickhouse-sink-connector/issues/39", + ), + "delete/no primary key innodb": (Skip, "doesn't work in raw"), + "delete/no primary key": (Skip, "doesn't work in raw"), + "update/no primary key innodb": (Skip, "makes delete"), + "update/no primary key": (Skip, "makes delete"), + "/mysql to clickhouse replication/auto replicated table creation/truncate/no primary key innodb/{'ReplacingMergeTree'}/*": ( + Skip, + "doesn't work", + ), + "/mysql to clickhouse replication/auto replicated table creation/truncate/no primary key/{'ReplacingMergeTree'}/*": ( + Skip, + "doesn't work", + ), + "/mysql to clickhouse replication/auto replicated table creation/truncate/no primary key": ( + Skip, + "doesn't work", + ), + "partition limits": (Skip, "doesn't ready"), + "delete/many partition many parts/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "delete/one million datapoints/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "delete/many partition one part/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "delete/one partition one part/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "delete/one partition mixed parts/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "delete/many partition mixed parts/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "delete/parallel/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "update/many partition many parts": ( + Skip, + "doesn't work without primary key and doesn't insert duplicates of primary key", + ), + "update/one million datapoints": ( + Skip, + "doesn't work without primary key and doesn't insert duplicates of primary key", + ), + "update/many partition one part": ( + Skip, + "doesn't work without primary key and doesn't insert duplicates of primary key", + ), + "insert/many partition many parts/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "insert/one million datapoints/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "insert/many partition one part/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "insert/one partition one part/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "insert/one partition mixed parts/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "insert/many partition mixed parts/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "insert/parallel/*_no_primary_key": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "/mysql to clickhouse replication/auto replicated table creation/insert/*": ( + Skip, + "doesn't work without primary key as only last row of insert is replicated", + ), + "/mysql to clickhouse replication/auto replicated table creation/partitions/*": ( + Skip, + "https://github.com/Altinity/clickhouse-sink-connector/issues/461", + ), + "/mysql to clickhouse replication/auto replicated table creation/truncate/no primary key innodb/*": ( + Skip, + "Sometimes when inserting two values, only one values is replicated. Seems to be a config issue.", + ), + "/mysql to clickhouse replication/auto replicated table creation/truncate/no primary key/*": ( + Skip, + "Sometimes when inserting two values, only one values is replicated. Seems to be a config issue.", + ), + "/mysql to clickhouse replication/auto replicated table creation/schema only/*": ( + Skip, + "Seems to be broken in CI/CD. need oto fix.", + ), + "/mysql to clickhouse replication/auto replicated table creation/cli/*": ( + Skip, + "Seems to be broken in CI/CD. need oto fix.", + ), +} + +xflags = {} + + +@TestModule +@ArgumentParser(argparser) +@FFails(ffails) +@XFlags(xflags) +@Name("auto replicated table creation") +@Requirements( + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MySQLStorageEngines_ReplicatedReplacingMergeTree( + "1.0" + ), +) +@Specifications(SRS030_MySQL_to_ClickHouse_Replication) +def regression( + self, + local, + clickhouse_binary_path, + clickhouse_version, + env="env/auto_replicated", + stress=None, + thread_fuzzer=None, + collect_service_logs=None, +): + """ClickHouse regression for MySQL to ClickHouse replication with auto replicated table creation.""" + nodes = { + "clickhouse-sink-connector-lt": ("clickhouse-sink-connector-lt",), + "mysql-master": ("mysql-master",), + "clickhouse": ("clickhouse", "clickhouse1", "clickhouse2", "clickhouse3"), + "bash-tools": ("bash-tools",), + "zookeeper": ("zookeeper",), + } + + self.context.nodes = nodes + self.context.clickhouse_version = clickhouse_version + self.context.config = SinkConfig() + create_default_sink_config_replicated() + + if stress is not None: + self.context.stress = stress + + if collect_service_logs is not None: + self.context.collect_service_logs = collect_service_logs + + with Given("docker-compose cluster"): + cluster = create_cluster( + local=local, + clickhouse_binary_path=clickhouse_binary_path, + thread_fuzzer=thread_fuzzer, + collect_service_logs=collect_service_logs, + stress=stress, + nodes=nodes, + docker_compose_project_dir=os.path.join(current_dir(), env), + caller_dir=os.path.join(current_dir()), + ) + + self.context.cluster = cluster + + self.context.env = env + + self.context.clickhouse_table_engine = "ReplicatedReplacingMergeTree" + self.context.clickhouse_table_engines = ["ReplicatedReplacingMergeTree"] + + self.context.database = "test" + + if check_clickhouse_version("<21.4")(self): + skip(reason="only supported on ClickHouse version >= 21.4") + + self.context.node = cluster.node("clickhouse") + + with And("I create test database in ClickHouse"): + create_clickhouse_database(name="test") + + with And("I start sink-connector-lightweight"): + self.context.sink_node = cluster.node("clickhouse-sink-connector-lt") + + self.context.sink_node.start_sink_connector( + config_file="env/auto_replicated/configs/replicated_config.yml" + ) + + with Pool(1) as executor: + Feature( + run=load("tests.sanity", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.autocreate", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.insert", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.alter", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.compound_alters", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.parallel_alters", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.truncate", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.deduplication", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.types", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.columns_inconsistency", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.snowflake_id", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.databases", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.table_names", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.is_deleted", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.calculated_columns", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.datatypes", "module"), + parallel=True, + executor=executor, + ) + Feature( + run=load("tests.retry_on_fail", "module"), + parallel=True, + executor=executor, + ) + + join() + + Feature( + run=load("tests.schema_only", "module"), + ) + Feature( + run=load("tests.sink_cli_commands", "module"), + ) + Feature( + run=load("tests.multiple_databases", "module"), + ) + + +if __name__ == "__main__": + regression() diff --git a/sink-connector-lightweight/tests/integration/requirements.txt b/sink-connector-lightweight/tests/integration/requirements.txt index 869791f58..28ce2b366 100644 --- a/sink-connector-lightweight/tests/integration/requirements.txt +++ b/sink-connector-lightweight/tests/integration/requirements.txt @@ -1,4 +1,5 @@ docker-compose==1.29.2 testflows==2.1.5 awscli==1.27.36 -docker==6.1.3 \ No newline at end of file +docker==6.1.3 +requests==2.31.0 diff --git a/sink-connector-lightweight/tests/integration/requirements/requirements.md b/sink-connector-lightweight/tests/integration/requirements/requirements.md index 9b9994cdf..983162238 100644 --- a/sink-connector-lightweight/tests/integration/requirements/requirements.md +++ b/sink-connector-lightweight/tests/integration/requirements/requirements.md @@ -221,40 +221,69 @@ * 31.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.Corrupted](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsdiskcorrupted) * 32 [Prometheus](#prometheus) * 32.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus](#rqsrs-030clickhousemysqltoclickhousereplicationprometheus) -* 33 [Multiple Databases](#multiple-databases) - * 33.1 [Test Schema - Multiple Databases ](#test-schema---multiple-databases-) - * 33.2 [Databases on Source and Destination](#databases-on-source-and-destination) - * 33.2.1 [Multiple Databases on Source and Destination](#multiple-databases-on-source-and-destination) - * 33.2.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabases) - * 33.2.2 [Multiple Databases on Source and One Database on Destination](#multiple-databases-on-source-and-one-database-on-destination) - * 33.2.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceMultipleDestinationOne](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourcemultipledestinationone) - * 33.2.3 [One Database on Source and Multiple Databases on Destination](#one-database-on-source-and-multiple-databases-on-destination) - * 33.2.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationMultiple](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourceonedestinationmultiple) - * 33.2.4 [One Database on Source and One Database on Destination](#one-database-on-source-and-one-database-on-destination) - * 33.2.4.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationOne](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourceonedestinationone) - * 33.3 [Table Structure on Source and Destination Databases](#table-structure-on-source-and-destination-databases) - * 33.3.1 [Two Tables with the Same Name and Different Structure on Different Databases](#two-tables-with-the-same-name-and-different-structure-on-different-databases) - * 33.3.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameDifferentStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablessamenamedifferentstructure) - * 33.3.2 [Two Tables with the Same Name and the Same Structure on Different Databases](#two-tables-with-the-same-name-and-the-same-structure-on-different-databases) - * 33.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameSameStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablessamenamesamestructure) - * 33.3.3 [Two Tables with the Different Name and the Same Structure on Different Databases](#two-tables-with-the-different-name-and-the-same-structure-on-different-databases) - * 33.3.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.DifferentNameSameStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablesdifferentnamesamestructure) - * 33.4 [Configuration Values](#configuration-values) - * 33.4.1 [Include Specific List of Databases To Replicate](#include-specific-list-of-databases-to-replicate) - * 33.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.IncludeList](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesincludelist) - * 33.4.2 [Replicate All Databases](#replicate-all-databases) - * 33.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.ReplicateAll](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesreplicateall) - * 33.5 [Table Operations](#table-operations) - * 33.5.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations) - * 33.5.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename) - * 33.5.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name) - * 33.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename) - * 33.6 [Error Handling](#error-handling) - * 33.6.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination) - * 33.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist) - * 33.7 [Concurrent Actions](#concurrent-actions) - * 33.7.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently) - * 33.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions) +* 33 [ReplicatedReplacingMergeTree](#replicatedreplacingmergetree) + * 33.1 [Test Schema For ReplicatedReplacingMergeTree](#test-schema-for-replicatedreplacingmergetree) + * 33.2 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetree) + * 33.3 [Types of Clusters That Can Be Used for ReplicatedReplacingMergeTree](#types-of-clusters-that-can-be-used-for-replicatedreplacingmergetree) + * 33.3.1 [Multiple Shards and Replicas](#multiple-shards-and-replicas) + * 33.3.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.MultipleShardsAndReplicas](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreeclustertypesmultipleshardsandreplicas) + * 33.3.2 [One Shard and One Replica](#one-shard-and-one-replica) + * 33.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.OneShardOneReplica](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreeclustertypesoneshardonereplica) + * 33.3.3 [Secure Cluster with One Shard and One Replica](#secure-cluster-with-one-shard-and-one-replica) + * 33.3.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterOneShardOneReplica](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreeclustertypessecureclusteroneshardonereplica) + * 33.3.4 [Secure Cluster with Multiple Shards and Replicas](#secure-cluster-with-multiple-shards-and-replicas) + * 33.3.4.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterMultipleShardsAndReplicas](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreeclustertypessecureclustermultipleshardsandreplicas) + * 33.4 [Possible Events](#possible-events) + * 33.4.1 [Node Related Events](#node-related-events) + * 33.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.Killed](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsnoderelatedeventskilled) + * 33.4.1.2 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.AllKilled](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsnoderelatedeventsallkilled) + * 33.4.1.3 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.ChangeLeader](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsnoderelatedeventschangeleader) + * 33.4.2 [Replica Related Events](#replica-related-events) + * 33.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.BehindLeader](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsreplicarelatedeventsbehindleader) + * 33.4.2.2 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.DataInconsistency](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsreplicarelatedeventsdatainconsistency) + * 33.4.2.3 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.NewReplica](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsreplicarelatedeventsnewreplica) + * 33.4.2.4 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.RemovedReplica](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsreplicarelatedeventsremovedreplica) + * 33.4.3 [Connection Related Events](#connection-related-events) + * 33.4.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ConnectionRelatedEvents.Interrupted](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsconnectionrelatedeventsinterrupted) + * 33.4.4 [Disk Related Events](#disk-related-events) + * 33.4.4.1 [Out of Space](#out-of-space) + * 33.4.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.OutOfSpace](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsdiskoutofspace) + * 33.4.4.2 [Corrupted Disk](#corrupted-disk) + * 33.4.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.Corrupted](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsdiskcorrupted) +* 34 [Multiple Databases](#multiple-databases) + * 34.1 [Test Schema - Multiple Databases ](#test-schema---multiple-databases-) + * 34.2 [Databases on Source and Destination](#databases-on-source-and-destination) + * 34.2.1 [Multiple Databases on Source and Destination](#multiple-databases-on-source-and-destination) + * 34.2.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabases) + * 34.2.2 [Multiple Databases on Source and One Database on Destination](#multiple-databases-on-source-and-one-database-on-destination) + * 34.2.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceMultipleDestinationOne](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourcemultipledestinationone) + * 34.2.3 [One Database on Source and Multiple Databases on Destination](#one-database-on-source-and-multiple-databases-on-destination) + * 34.2.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationMultiple](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourceonedestinationmultiple) + * 34.2.4 [One Database on Source and One Database on Destination](#one-database-on-source-and-one-database-on-destination) + * 34.2.4.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationOne](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourceonedestinationone) + * 34.3 [Table Structure on Source and Destination Databases](#table-structure-on-source-and-destination-databases) + * 34.3.1 [Two Tables with the Same Name and Different Structure on Different Databases](#two-tables-with-the-same-name-and-different-structure-on-different-databases) + * 34.3.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameDifferentStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablessamenamedifferentstructure) + * 34.3.2 [Two Tables with the Same Name and the Same Structure on Different Databases](#two-tables-with-the-same-name-and-the-same-structure-on-different-databases) + * 34.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameSameStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablessamenamesamestructure) + * 34.3.3 [Two Tables with the Different Name and the Same Structure on Different Databases](#two-tables-with-the-different-name-and-the-same-structure-on-different-databases) + * 34.3.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.DifferentNameSameStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablesdifferentnamesamestructure) + * 34.4 [Configuration Values](#configuration-values) + * 34.4.1 [Include Specific List of Databases To Replicate](#include-specific-list-of-databases-to-replicate) + * 34.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.IncludeList](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesincludelist) + * 34.4.2 [Replicate All Databases](#replicate-all-databases) + * 34.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.ReplicateAll](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesreplicateall) + * 34.5 [Table Operations](#table-operations) + * 34.5.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations) + * 34.5.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename) + * 34.5.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name) + * 34.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename) + * 34.6 [Error Handling](#error-handling) + * 34.6.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination) + * 34.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist) + * 34.7 [Concurrent Actions](#concurrent-actions) + * 34.7.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently) + * 34.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions) ## Introduction @@ -1609,6 +1638,144 @@ version: 1.0 [Altinity Sink Connector] SHALL support expose data transfer representation to [Prometheus] service. +## ReplicatedReplacingMergeTree + +### Test Schema For ReplicatedReplacingMergeTree + +```yaml +ReplicatedReplacingMergeTree: + Clusters: + - Cluster with multiple shards and replicas + - Cluster with one shard and one replica + - Secure cluster with one shard and one replica + - Secure cluster with multiple shards and replicas + Possible Events: + Node Related Events: + - Some of the nodes where replicas are running are killed + - All of the nodes where replicas are running are killed + - Change of the leader node during the replication process + Replica Related Events: + - One or more replicas are behind the leader replica + - Data inconsistency between replicas + - New replica added durin replication process + - Replica removed during replication process + Connection Related Event: + - Connection between replicas is interrupted + Disk: + OutOfSpace: + - Out of disk space on the disk used by one of replicas in source database cluster + Corruptions: + - Corruption on a disk used by one of replicas in source database cluster +``` + +### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree +version: 1.0 + +[Altinity Sink Connector] SHALL support table replication from source database to the destination database and store the table as `ReplicatedReplacingMergeTree` [ClickHouse] table engine. + + +In order for [ALtinity Sink Connector] to replicate a source table as `ReplicatedReplacingMergeTree` in [ClickHouse] the configuration file should contain the following setting: + +```yaml +auto.create.tables.replicated: "true" +``` +### Types of Clusters That Can Be Used for ReplicatedReplacingMergeTree + +#### Multiple Shards and Replicas + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.MultipleShardsAndReplicas +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a cluster with multiple shards and replicas. + +#### One Shard and One Replica + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.OneShardOneReplica +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a cluster with one shard and one replica. + +#### Secure Cluster with One Shard and One Replica + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterOneShardOneReplica +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a secure cluster with one shard and one replica. + +#### Secure Cluster with Multiple Shards and Replicas + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterMultipleShardsAndReplicas +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a secure cluster with multiple shards and replicas. + +### Possible Events + +#### Node Related Events + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.Killed +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when some of the nodes where replicas are running are killed. + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.AllKilled +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when all the nodes where replicas are running are killed. + + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.ChangeLeader +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when the leader node is changed during the replication process. + +#### Replica Related Events + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.BehindLeader +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when one or more replicas are behind the leader replica. +Replication process from destination to source database SHALL not be interrupted in this case. + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.DataInconsistency +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when there is data inconsistency between replicas. + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.NewReplica +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when a new replica is added during the replication process. + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.RemovedReplica +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when a replica is removed during the replication process. + +#### Connection Related Events + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ConnectionRelatedEvents.Interrupted +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when the connection between replicas is interrupted. + +#### Disk Related Events + +##### Out of Space + +###### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.OutOfSpace +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when one of the replicas in the source database cluster is out of disk space. + +##### Corrupted Disk + +###### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.Corrupted +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when one of the replicas in the source database cluster has a corrupted disk. + + ## Multiple Databases ### Test Schema - Multiple Databases diff --git a/sink-connector-lightweight/tests/integration/requirements/requirements.py b/sink-connector-lightweight/tests/integration/requirements/requirements.py index 20c7e5c7a..4c38c388a 100644 --- a/sink-connector-lightweight/tests/integration/requirements/requirements.py +++ b/sink-connector-lightweight/tests/integration/requirements/requirements.py @@ -2026,6 +2026,255 @@ num="32.1", ) +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support table replication from source database to the destination database and store the table as `ReplicatedReplacingMergeTree` [ClickHouse] table engine.\n" + "\n" + "\n" + "In order for [ALtinity Sink Connector] to replicate a source table as `ReplicatedReplacingMergeTree` in [ClickHouse] the configuration file should contain the following setting:\n" + "\n" + "```yaml\n" + 'auto.create.tables.replicated: "true"\n' + "```\n" + ), + link=None, + level=2, + num="33.2", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_ClusterTypes_MultipleShardsAndReplicas = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.MultipleShardsAndReplicas", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a cluster with multiple shards and replicas.\n" + "\n" + ), + link=None, + level=4, + num="33.3.1.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_ClusterTypes_OneShardOneReplica = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.OneShardOneReplica", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a cluster with one shard and one replica.\n" + "\n" + ), + link=None, + level=4, + num="33.3.2.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_ClusterTypes_SecureClusterOneShardOneReplica = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterOneShardOneReplica", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a secure cluster with one shard and one replica.\n" + "\n" + ), + link=None, + level=4, + num="33.3.3.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_ClusterTypes_SecureClusterMultipleShardsAndReplicas = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterMultipleShardsAndReplicas", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a secure cluster with multiple shards and replicas.\n" + "\n" + ), + link=None, + level=4, + num="33.3.4.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_NodeRelatedEvents_Killed = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.Killed", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when some of the nodes where replicas are running are killed.\n" + "\n" + ), + link=None, + level=4, + num="33.4.1.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_NodeRelatedEvents_AllKilled = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.AllKilled", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when all the nodes where replicas are running are killed.\n" + "\n" + "\n" + ), + link=None, + level=4, + num="33.4.1.2", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_NodeRelatedEvents_ChangeLeader = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.ChangeLeader", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when the leader node is changed during the replication process.\n" + "\n" + ), + link=None, + level=4, + num="33.4.1.3", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ReplicaRelatedEvents_BehindLeader = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.BehindLeader", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when one or more replicas are behind the leader replica. \n" + "Replication process from destination to source database SHALL not be interrupted in this case.\n" + "\n" + ), + link=None, + level=4, + num="33.4.2.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ReplicaRelatedEvents_DataInconsistency = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.DataInconsistency", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when there is data inconsistency between replicas.\n" + "\n" + ), + link=None, + level=4, + num="33.4.2.2", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ReplicaRelatedEvents_NewReplica = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.NewReplica", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when a new replica is added during the replication process.\n" + "\n" + ), + link=None, + level=4, + num="33.4.2.3", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ReplicaRelatedEvents_RemovedReplica = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.RemovedReplica", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when a replica is removed during the replication process.\n" + "\n" + ), + link=None, + level=4, + num="33.4.2.4", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ConnectionRelatedEvents_Interrupted = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ConnectionRelatedEvents.Interrupted", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when the connection between replicas is interrupted.\n" + "\n" + ), + link=None, + level=4, + num="33.4.3.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_Disk_OutOfSpace = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.OutOfSpace", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when one of the replicas in the source database cluster is out of disk space.\n" + "\n" + ), + link=None, + level=5, + num="33.4.4.1.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_Disk_Corrupted = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.Corrupted", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from source database to the destination database when one of the replicas in the source database cluster has a corrupted disk.\n" + "\n" + "\n" + ), + link=None, + level=5, + num="33.4.4.2.1", +) + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases = Requirement( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases", version="1.0", @@ -2047,7 +2296,7 @@ ), link=None, level=4, - num="33.2.1.1", + num="34.2.1.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_SourceMultipleDestinationOne = Requirement( @@ -2070,7 +2319,7 @@ ), link=None, level=4, - num="33.2.2.1", + num="34.2.2.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_SourceOneDestinationMultiple = Requirement( @@ -2093,7 +2342,7 @@ ), link=None, level=4, - num="33.2.3.1", + num="34.2.3.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_SourceOneDestinationOne = Requirement( @@ -2114,7 +2363,7 @@ ), link=None, level=4, - num="33.2.4.1", + num="34.2.4.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_Tables_SameNameDifferentStructure = Requirement( @@ -2130,7 +2379,7 @@ ), link=None, level=4, - num="33.3.1.1", + num="34.3.1.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_Tables_SameNameSameStructure = Requirement( @@ -2146,7 +2395,7 @@ ), link=None, level=4, - num="33.3.2.1", + num="34.3.2.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_Tables_DifferentNameSameStructure = Requirement( @@ -2162,7 +2411,7 @@ ), link=None, level=4, - num="33.3.3.1", + num="34.3.3.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_IncludeList = Requirement( @@ -2185,7 +2434,7 @@ ), link=None, level=4, - num="33.4.1.1", + num="34.4.1.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_ReplicateAll = Requirement( @@ -2201,7 +2450,7 @@ ), link=None, level=4, - num="33.4.2.1", + num="34.4.2.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_TableOperations_SpecifyDatabaseName = Requirement( @@ -2223,7 +2472,7 @@ ), link=None, level=4, - num="33.5.1.1", + num="34.5.1.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_TableOperations_NoSpecifyDatabaseName = Requirement( @@ -2245,7 +2494,7 @@ ), link=None, level=4, - num="33.5.2.1", + num="34.5.2.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ErrorHandling_DatabaseNotExist = Requirement( @@ -2261,7 +2510,7 @@ ), link=None, level=4, - num="33.6.1.1", + num="34.6.1.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConcurrentActions = Requirement( @@ -2287,7 +2536,7 @@ ), link=None, level=4, - num="33.7.1.1", + num="34.7.1.1", ) SRS030_MySQL_to_ClickHouse_Replication = Specification( @@ -2971,139 +3220,240 @@ level=2, num="32.1", ), - Heading(name="Multiple Databases", level=1, num="33"), - Heading(name="Test Schema - Multiple Databases ", level=2, num="33.1"), - Heading(name="Databases on Source and Destination", level=2, num="33.2"), + Heading(name="ReplicatedReplacingMergeTree", level=1, num="33"), + Heading( + name="Test Schema For ReplicatedReplacingMergeTree", level=2, num="33.1" + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree", + level=2, + num="33.2", + ), + Heading( + name="Types of Clusters That Can Be Used for ReplicatedReplacingMergeTree", + level=2, + num="33.3", + ), + Heading(name="Multiple Shards and Replicas", level=3, num="33.3.1"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.MultipleShardsAndReplicas", + level=4, + num="33.3.1.1", + ), + Heading(name="One Shard and One Replica", level=3, num="33.3.2"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.OneShardOneReplica", + level=4, + num="33.3.2.1", + ), Heading( - name="Multiple Databases on Source and Destination", level=3, num="33.2.1" + name="Secure Cluster with One Shard and One Replica", level=3, num="33.3.3" + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterOneShardOneReplica", + level=4, + num="33.3.3.1", + ), + Heading( + name="Secure Cluster with Multiple Shards and Replicas", + level=3, + num="33.3.4", + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterMultipleShardsAndReplicas", + level=4, + num="33.3.4.1", + ), + Heading(name="Possible Events", level=2, num="33.4"), + Heading(name="Node Related Events", level=3, num="33.4.1"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.Killed", + level=4, + num="33.4.1.1", + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.AllKilled", + level=4, + num="33.4.1.2", + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.ChangeLeader", + level=4, + num="33.4.1.3", + ), + Heading(name="Replica Related Events", level=3, num="33.4.2"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.BehindLeader", + level=4, + num="33.4.2.1", + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.DataInconsistency", + level=4, + num="33.4.2.2", + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.NewReplica", + level=4, + num="33.4.2.3", + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.RemovedReplica", + level=4, + num="33.4.2.4", + ), + Heading(name="Connection Related Events", level=3, num="33.4.3"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ConnectionRelatedEvents.Interrupted", + level=4, + num="33.4.3.1", + ), + Heading(name="Disk Related Events", level=3, num="33.4.4"), + Heading(name="Out of Space", level=4, num="33.4.4.1"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.OutOfSpace", + level=5, + num="33.4.4.1.1", + ), + Heading(name="Corrupted Disk", level=4, num="33.4.4.2"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.Corrupted", + level=5, + num="33.4.4.2.1", + ), + Heading(name="Multiple Databases", level=1, num="34"), + Heading(name="Test Schema - Multiple Databases ", level=2, num="34.1"), + Heading(name="Databases on Source and Destination", level=2, num="34.2"), + Heading( + name="Multiple Databases on Source and Destination", level=3, num="34.2.1" ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases", level=4, - num="33.2.1.1", + num="34.2.1.1", ), Heading( name="Multiple Databases on Source and One Database on Destination", level=3, - num="33.2.2", + num="34.2.2", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceMultipleDestinationOne", level=4, - num="33.2.2.1", + num="34.2.2.1", ), Heading( name="One Database on Source and Multiple Databases on Destination", level=3, - num="33.2.3", + num="34.2.3", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationMultiple", level=4, - num="33.2.3.1", + num="34.2.3.1", ), Heading( name="One Database on Source and One Database on Destination", level=3, - num="33.2.4", + num="34.2.4", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationOne", level=4, - num="33.2.4.1", + num="34.2.4.1", ), Heading( name="Table Structure on Source and Destination Databases", level=2, - num="33.3", + num="34.3", ), Heading( name="Two Tables with the Same Name and Different Structure on Different Databases", level=3, - num="33.3.1", + num="34.3.1", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameDifferentStructure", level=4, - num="33.3.1.1", + num="34.3.1.1", ), Heading( name="Two Tables with the Same Name and the Same Structure on Different Databases", level=3, - num="33.3.2", + num="34.3.2", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameSameStructure", level=4, - num="33.3.2.1", + num="34.3.2.1", ), Heading( name="Two Tables with the Different Name and the Same Structure on Different Databases", level=3, - num="33.3.3", + num="34.3.3", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.DifferentNameSameStructure", level=4, - num="33.3.3.1", + num="34.3.3.1", ), - Heading(name="Configuration Values", level=2, num="33.4"), + Heading(name="Configuration Values", level=2, num="34.4"), Heading( name="Include Specific List of Databases To Replicate", level=3, - num="33.4.1", + num="34.4.1", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.IncludeList", level=4, - num="33.4.1.1", + num="34.4.1.1", ), - Heading(name="Replicate All Databases", level=3, num="33.4.2"), + Heading(name="Replicate All Databases", level=3, num="34.4.2"), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.ReplicateAll", level=4, - num="33.4.2.1", + num="34.4.2.1", ), - Heading(name="Table Operations", level=2, num="33.5"), + Heading(name="Table Operations", level=2, num="34.5"), Heading( - name="Specify Database Name in Table Operations", level=3, num="33.5.1" + name="Specify Database Name in Table Operations", level=3, num="34.5.1" ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName", level=4, - num="33.5.1.1", + num="34.5.1.1", ), Heading( name="Table Operations Without Specifying Database Name", level=3, - num="33.5.2", + num="34.5.2", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName", level=4, - num="33.5.2.1", + num="34.5.2.1", ), - Heading(name="Error Handling", level=2, num="33.6"), + Heading(name="Error Handling", level=2, num="34.6"), Heading( name="When Replicated Database Does Not Exist on the Destination", level=3, - num="33.6.1", + num="34.6.1", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist", level=4, - num="33.6.1.1", + num="34.6.1.1", ), - Heading(name="Concurrent Actions", level=2, num="33.7"), + Heading(name="Concurrent Actions", level=2, num="34.7"), Heading( name="Perform Table Operations on Each Database Concurrently", level=3, - num="33.7.1", + num="34.7.1", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions", level=4, - num="33.7.1.1", + num="34.7.1.1", ), ), requirements=( @@ -3214,6 +3564,21 @@ RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Disk_OutOfSpace, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Disk_Corrupted, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Prometheus, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_ClusterTypes_MultipleShardsAndReplicas, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_ClusterTypes_OneShardOneReplica, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_ClusterTypes_SecureClusterOneShardOneReplica, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_ClusterTypes_SecureClusterMultipleShardsAndReplicas, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_NodeRelatedEvents_Killed, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_NodeRelatedEvents_AllKilled, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_NodeRelatedEvents_ChangeLeader, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ReplicaRelatedEvents_BehindLeader, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ReplicaRelatedEvents_DataInconsistency, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ReplicaRelatedEvents_NewReplica, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ReplicaRelatedEvents_RemovedReplica, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_ConnectionRelatedEvents_Interrupted, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_Disk_OutOfSpace, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ReplicatedReplacingMergeTree_PossibleEvents_Disk_Corrupted, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_SourceMultipleDestinationOne, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_SourceOneDestinationMultiple, @@ -3452,40 +3817,69 @@ * 31.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.Corrupted](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsdiskcorrupted) * 32 [Prometheus](#prometheus) * 32.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus](#rqsrs-030clickhousemysqltoclickhousereplicationprometheus) -* 33 [Multiple Databases](#multiple-databases) - * 33.1 [Test Schema - Multiple Databases ](#test-schema---multiple-databases-) - * 33.2 [Databases on Source and Destination](#databases-on-source-and-destination) - * 33.2.1 [Multiple Databases on Source and Destination](#multiple-databases-on-source-and-destination) - * 33.2.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabases) - * 33.2.2 [Multiple Databases on Source and One Database on Destination](#multiple-databases-on-source-and-one-database-on-destination) - * 33.2.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceMultipleDestinationOne](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourcemultipledestinationone) - * 33.2.3 [One Database on Source and Multiple Databases on Destination](#one-database-on-source-and-multiple-databases-on-destination) - * 33.2.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationMultiple](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourceonedestinationmultiple) - * 33.2.4 [One Database on Source and One Database on Destination](#one-database-on-source-and-one-database-on-destination) - * 33.2.4.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationOne](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourceonedestinationone) - * 33.3 [Table Structure on Source and Destination Databases](#table-structure-on-source-and-destination-databases) - * 33.3.1 [Two Tables with the Same Name and Different Structure on Different Databases](#two-tables-with-the-same-name-and-different-structure-on-different-databases) - * 33.3.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameDifferentStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablessamenamedifferentstructure) - * 33.3.2 [Two Tables with the Same Name and the Same Structure on Different Databases](#two-tables-with-the-same-name-and-the-same-structure-on-different-databases) - * 33.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameSameStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablessamenamesamestructure) - * 33.3.3 [Two Tables with the Different Name and the Same Structure on Different Databases](#two-tables-with-the-different-name-and-the-same-structure-on-different-databases) - * 33.3.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.DifferentNameSameStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablesdifferentnamesamestructure) - * 33.4 [Configuration Values](#configuration-values) - * 33.4.1 [Include Specific List of Databases To Replicate](#include-specific-list-of-databases-to-replicate) - * 33.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.IncludeList](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesincludelist) - * 33.4.2 [Replicate All Databases](#replicate-all-databases) - * 33.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.ReplicateAll](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesreplicateall) - * 33.5 [Table Operations](#table-operations) - * 33.5.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations) - * 33.5.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename) - * 33.5.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name) - * 33.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename) - * 33.6 [Error Handling](#error-handling) - * 33.6.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination) - * 33.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist) - * 33.7 [Concurrent Actions](#concurrent-actions) - * 33.7.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently) - * 33.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions) +* 33 [ReplicatedReplacingMergeTree](#replicatedreplacingmergetree) + * 33.1 [Test Schema For ReplicatedReplacingMergeTree](#test-schema-for-replicatedreplacingmergetree) + * 33.2 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetree) + * 33.3 [Types of Clusters That Can Be Used for ReplicatedReplacingMergeTree](#types-of-clusters-that-can-be-used-for-replicatedreplacingmergetree) + * 33.3.1 [Multiple Shards and Replicas](#multiple-shards-and-replicas) + * 33.3.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.MultipleShardsAndReplicas](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreeclustertypesmultipleshardsandreplicas) + * 33.3.2 [One Shard and One Replica](#one-shard-and-one-replica) + * 33.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.OneShardOneReplica](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreeclustertypesoneshardonereplica) + * 33.3.3 [Secure Cluster with One Shard and One Replica](#secure-cluster-with-one-shard-and-one-replica) + * 33.3.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterOneShardOneReplica](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreeclustertypessecureclusteroneshardonereplica) + * 33.3.4 [Secure Cluster with Multiple Shards and Replicas](#secure-cluster-with-multiple-shards-and-replicas) + * 33.3.4.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterMultipleShardsAndReplicas](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreeclustertypessecureclustermultipleshardsandreplicas) + * 33.4 [Possible Events](#possible-events) + * 33.4.1 [Node Related Events](#node-related-events) + * 33.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.Killed](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsnoderelatedeventskilled) + * 33.4.1.2 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.AllKilled](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsnoderelatedeventsallkilled) + * 33.4.1.3 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.ChangeLeader](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsnoderelatedeventschangeleader) + * 33.4.2 [Replica Related Events](#replica-related-events) + * 33.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.BehindLeader](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsreplicarelatedeventsbehindleader) + * 33.4.2.2 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.DataInconsistency](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsreplicarelatedeventsdatainconsistency) + * 33.4.2.3 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.NewReplica](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsreplicarelatedeventsnewreplica) + * 33.4.2.4 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.RemovedReplica](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsreplicarelatedeventsremovedreplica) + * 33.4.3 [Connection Related Events](#connection-related-events) + * 33.4.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ConnectionRelatedEvents.Interrupted](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsconnectionrelatedeventsinterrupted) + * 33.4.4 [Disk Related Events](#disk-related-events) + * 33.4.4.1 [Out of Space](#out-of-space) + * 33.4.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.OutOfSpace](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsdiskoutofspace) + * 33.4.4.2 [Corrupted Disk](#corrupted-disk) + * 33.4.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.Corrupted](#rqsrs-030clickhousemysqltoclickhousereplicationreplicatedreplacingmergetreepossibleeventsdiskcorrupted) +* 34 [Multiple Databases](#multiple-databases) + * 34.1 [Test Schema - Multiple Databases ](#test-schema---multiple-databases-) + * 34.2 [Databases on Source and Destination](#databases-on-source-and-destination) + * 34.2.1 [Multiple Databases on Source and Destination](#multiple-databases-on-source-and-destination) + * 34.2.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabases) + * 34.2.2 [Multiple Databases on Source and One Database on Destination](#multiple-databases-on-source-and-one-database-on-destination) + * 34.2.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceMultipleDestinationOne](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourcemultipledestinationone) + * 34.2.3 [One Database on Source and Multiple Databases on Destination](#one-database-on-source-and-multiple-databases-on-destination) + * 34.2.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationMultiple](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourceonedestinationmultiple) + * 34.2.4 [One Database on Source and One Database on Destination](#one-database-on-source-and-one-database-on-destination) + * 34.2.4.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.SourceOneDestinationOne](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasessourceonedestinationone) + * 34.3 [Table Structure on Source and Destination Databases](#table-structure-on-source-and-destination-databases) + * 34.3.1 [Two Tables with the Same Name and Different Structure on Different Databases](#two-tables-with-the-same-name-and-different-structure-on-different-databases) + * 34.3.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameDifferentStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablessamenamedifferentstructure) + * 34.3.2 [Two Tables with the Same Name and the Same Structure on Different Databases](#two-tables-with-the-same-name-and-the-same-structure-on-different-databases) + * 34.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.SameNameSameStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablessamenamesamestructure) + * 34.3.3 [Two Tables with the Different Name and the Same Structure on Different Databases](#two-tables-with-the-different-name-and-the-same-structure-on-different-databases) + * 34.3.3.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.Tables.DifferentNameSameStructure](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestablesdifferentnamesamestructure) + * 34.4 [Configuration Values](#configuration-values) + * 34.4.1 [Include Specific List of Databases To Replicate](#include-specific-list-of-databases-to-replicate) + * 34.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.IncludeList](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesincludelist) + * 34.4.2 [Replicate All Databases](#replicate-all-databases) + * 34.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.ReplicateAll](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesreplicateall) + * 34.5 [Table Operations](#table-operations) + * 34.5.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations) + * 34.5.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename) + * 34.5.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name) + * 34.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename) + * 34.6 [Error Handling](#error-handling) + * 34.6.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination) + * 34.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist) + * 34.7 [Concurrent Actions](#concurrent-actions) + * 34.7.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently) + * 34.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions) ## Introduction @@ -4840,6 +5234,144 @@ [Altinity Sink Connector] SHALL support expose data transfer representation to [Prometheus] service. +## ReplicatedReplacingMergeTree + +### Test Schema For ReplicatedReplacingMergeTree + +```yaml +ReplicatedReplacingMergeTree: + Clusters: + - Cluster with multiple shards and replicas + - Cluster with one shard and one replica + - Secure cluster with one shard and one replica + - Secure cluster with multiple shards and replicas + Possible Events: + Node Related Events: + - Some of the nodes where replicas are running are killed + - All of the nodes where replicas are running are killed + - Change of the leader node during the replication process + Replica Related Events: + - One or more replicas are behind the leader replica + - Data inconsistency between replicas + - New replica added durin replication process + - Replica removed during replication process + Connection Related Event: + - Connection between replicas is interrupted + Disk: + OutOfSpace: + - Out of disk space on the disk used by one of replicas in source database cluster + Corruptions: + - Corruption on a disk used by one of replicas in source database cluster +``` + +### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree +version: 1.0 + +[Altinity Sink Connector] SHALL support table replication from source database to the destination database and store the table as `ReplicatedReplacingMergeTree` [ClickHouse] table engine. + + +In order for [ALtinity Sink Connector] to replicate a source table as `ReplicatedReplacingMergeTree` in [ClickHouse] the configuration file should contain the following setting: + +```yaml +auto.create.tables.replicated: "true" +``` +### Types of Clusters That Can Be Used for ReplicatedReplacingMergeTree + +#### Multiple Shards and Replicas + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.MultipleShardsAndReplicas +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a cluster with multiple shards and replicas. + +#### One Shard and One Replica + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.OneShardOneReplica +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a cluster with one shard and one replica. + +#### Secure Cluster with One Shard and One Replica + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterOneShardOneReplica +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a secure cluster with one shard and one replica. + +#### Secure Cluster with Multiple Shards and Replicas + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.ClusterTypes.SecureClusterMultipleShardsAndReplicas +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database that is stored on a secure cluster with multiple shards and replicas. + +### Possible Events + +#### Node Related Events + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.Killed +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when some of the nodes where replicas are running are killed. + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.AllKilled +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when all the nodes where replicas are running are killed. + + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.NodeRelatedEvents.ChangeLeader +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when the leader node is changed during the replication process. + +#### Replica Related Events + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.BehindLeader +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when one or more replicas are behind the leader replica. +Replication process from destination to source database SHALL not be interrupted in this case. + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.DataInconsistency +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when there is data inconsistency between replicas. + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.NewReplica +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when a new replica is added during the replication process. + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ReplicaRelatedEvents.RemovedReplica +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when a replica is removed during the replication process. + +#### Connection Related Events + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.ConnectionRelatedEvents.Interrupted +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when the connection between replicas is interrupted. + +#### Disk Related Events + +##### Out of Space + +###### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.OutOfSpace +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when one of the replicas in the source database cluster is out of disk space. + +##### Corrupted Disk + +###### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ReplicatedReplacingMergeTree.PossibleEvents.Disk.Corrupted +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from source database to the destination database when one of the replicas in the source database cluster has a corrupted disk. + + ## Multiple Databases ### Test Schema - Multiple Databases diff --git a/sink-connector-lightweight/tests/integration/tests/autocreate.py b/sink-connector-lightweight/tests/integration/tests/autocreate.py index 2ddcb91b5..bd9bce5a7 100644 --- a/sink-connector-lightweight/tests/integration/tests/autocreate.py +++ b/sink-connector-lightweight/tests/integration/tests/autocreate.py @@ -19,11 +19,9 @@ def create_all_data_types( f"I create MySQL to CH replicated table with all supported NOT NULL data types", description=table_name, ): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When(f"I check MySQL table {table_name} was created"): diff --git a/sink-connector-lightweight/tests/integration/tests/calculated_columns.py b/sink-connector-lightweight/tests/integration/tests/calculated_columns.py index c4c93b8f5..4dd72c916 100644 --- a/sink-connector-lightweight/tests/integration/tests/calculated_columns.py +++ b/sink-connector-lightweight/tests/integration/tests/calculated_columns.py @@ -33,11 +33,10 @@ def string_concatenation(self): with Given( f"I create a {table_name} table with calculated column with string concatenation" ): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"first_name VARCHAR(50) NOT NULL,last_name VARCHAR(50) NOT NULL,fullname varchar(101) " + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"first_name VARCHAR(50) NOT NULL,last_name VARCHAR(50) NOT NULL,fullname varchar(101) " f"GENERATED ALWAYS AS (CONCAT(first_name,' ',last_name)),email VARCHAR(100) NOT NULL", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], ) with And(f"inserting data into the {table_name} table"): @@ -60,10 +59,9 @@ def basic_arithmetic_operations(self): b = 4 with Given(f"I create a {table_name} table with calculated column"): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"a INT, b INT, sum_col INT AS (a + b), diff_col INT AS (a - b), prod_col INT AS (a * b), div_col DOUBLE AS (a / b)", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"a INT, b INT, sum_col INT AS (a + b), diff_col INT AS (a - b), prod_col INT AS (a * b), div_col DOUBLE AS (a / b)", ) with And(f"inserting data into the {table_name} table"): @@ -87,10 +85,9 @@ def complex_expressions(self): bonus_rate = "520.65" with Given(f"I create a {table_name} table with calculated column"): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"base_salary DECIMAL(10,2), bonus_rate DECIMAL(5,2), total_compensation DECIMAL(12,2) AS (base_salary + (base_salary * bonus_rate / 100))", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"base_salary DECIMAL(10,2), bonus_rate DECIMAL(5,2), total_compensation DECIMAL(12,2) AS (base_salary + (base_salary * bonus_rate / 100))", ) with And(f"inserting data into the {table_name} table"): @@ -116,10 +113,9 @@ def nested(self): b = "2" with Given(f"I create a {table_name} table with calculated column"): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"a INT, b INT, c INT AS (a + b), d INT AS (c * 2)", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"a INT, b INT, c INT AS (a + b), d INT AS (c * 2)", ) with And(f"inserting data into the {table_name} table"): diff --git a/sink-connector-lightweight/tests/integration/tests/columns_inconsistency.py b/sink-connector-lightweight/tests/integration/tests/columns_inconsistency.py index 0e3e8de09..577a18b11 100644 --- a/sink-connector-lightweight/tests/integration/tests/columns_inconsistency.py +++ b/sink-connector-lightweight/tests/integration/tests/columns_inconsistency.py @@ -14,11 +14,9 @@ def mysql_to_clickhouse_insert( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_table_engine=clickhouse_table_engine, - clickhouse_columns=clickhouse_columns, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When("I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/databases.py b/sink-connector-lightweight/tests/integration/tests/databases.py index b3627a118..efafd3b50 100644 --- a/sink-connector-lightweight/tests/integration/tests/databases.py +++ b/sink-connector-lightweight/tests/integration/tests/databases.py @@ -21,12 +21,9 @@ def databases_tables( table_name = f"databases_{getuid()}" with Given(f"I create MySQL table {table_name})"): - create_mysql_to_clickhouse_replicated_table( - version_column=version_column, - name=table_name, - clickhouse_columns=clickhouse_columns, - mysql_columns=mysql_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with And( diff --git a/sink-connector-lightweight/tests/integration/tests/datatypes.py b/sink-connector-lightweight/tests/integration/tests/datatypes.py index 60bf39a5a..0e15866d0 100644 --- a/sink-connector-lightweight/tests/integration/tests/datatypes.py +++ b/sink-connector-lightweight/tests/integration/tests/datatypes.py @@ -35,10 +35,9 @@ def create_table_with_datetime_column(self, table_name, data, precision): clickhouse_node = self.context.clickhouse_node with By(f"creating a {table_name} table with datetime column"): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"date DATETIME({precision})", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"date DATETIME({precision})", ) with And(f"inserting data to MySQL {table_name} table"): diff --git a/sink-connector-lightweight/tests/integration/tests/deduplication.py b/sink-connector-lightweight/tests/integration/tests/deduplication.py index 1755f2e15..378f305d8 100644 --- a/sink-connector-lightweight/tests/integration/tests/deduplication.py +++ b/sink-connector-lightweight/tests/integration/tests/deduplication.py @@ -15,11 +15,9 @@ def deduplication( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns="age INT", - clickhouse_columns="age Int32", - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns="age INT", ) if inserts: diff --git a/sink-connector-lightweight/tests/integration/tests/delete.py b/sink-connector-lightweight/tests/integration/tests/delete.py index 3c830c301..7a80a9d14 100644 --- a/sink-connector-lightweight/tests/integration/tests/delete.py +++ b/sink-connector-lightweight/tests/integration/tests/delete.py @@ -21,13 +21,10 @@ def simple_delete( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, primary_key=primary_key, - engine=engine, ) with When(f"I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/insert.py b/sink-connector-lightweight/tests/integration/tests/insert.py index d9adb215e..e8e51f4e1 100644 --- a/sink-connector-lightweight/tests/integration/tests/insert.py +++ b/sink-connector-lightweight/tests/integration/tests/insert.py @@ -18,11 +18,9 @@ def simple_insert( with Given( f"I create MySQL to ClickHouse replicated table", description=table_name ): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_table_engine=clickhouse_table_engine, - clickhouse_columns=clickhouse_columns, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When("I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/is_deleted.py b/sink-connector-lightweight/tests/integration/tests/is_deleted.py index 7a072231a..61a951806 100644 --- a/sink-connector-lightweight/tests/integration/tests/is_deleted.py +++ b/sink-connector-lightweight/tests/integration/tests/is_deleted.py @@ -18,10 +18,9 @@ def create_table_with_is_deleted( with By( f"creating a {table_name} table with is_deleted column and {datatype} datatype" ): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"col1 varchar(255), col2 int, {column} {datatype}", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"col1 varchar(255), col2 int, {column} {datatype}", ) with And(f"inserting data into the {table_name} table"): diff --git a/sink-connector-lightweight/tests/integration/tests/manual_section.py b/sink-connector-lightweight/tests/integration/tests/manual_section.py index 0fdd3d5a2..a3af8ea9a 100644 --- a/sink-connector-lightweight/tests/integration/tests/manual_section.py +++ b/sink-connector-lightweight/tests/integration/tests/manual_section.py @@ -18,11 +18,9 @@ def mysql_to_clickhouse_connection( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When(f"I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/multiple_databases.py b/sink-connector-lightweight/tests/integration/tests/multiple_databases.py index ddcd94431..fe7402d5d 100644 --- a/sink-connector-lightweight/tests/integration/tests/multiple_databases.py +++ b/sink-connector-lightweight/tests/integration/tests/multiple_databases.py @@ -1,4 +1,5 @@ from integration.helpers.common import change_sink_configuration +from integration.helpers.create_config import include_all_databases_with_rrmt from integration.tests.steps.clickhouse import ( check_if_table_was_created, create_clickhouse_database, @@ -39,6 +40,19 @@ def replicate_all_databases(self): ) +@TestStep(Given) +def replicate_all_databases_rrmt(self): + """Set ClickHouse Sink Connector configuration to replicate all databases when tables have ReplicatedReplacingMergeTree engine.""" + with By( + "creating a ClickHouse Sink Connector configuration without database.include.list values specified" + ): + include_all_databases_with_rrmt( + config_file=os.path.join( + self.context.config_file, "multiple_databases_rrmt.yml" + ) + ) + + @TestStep(Given) def create_source_database(self, database_name): """Create a MySQL database.""" @@ -98,11 +112,10 @@ def create_table_and_insert_values( table_values = create_sample_values() with By("creating a sample table in MySQL"): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"col1 varchar(255), col2 int", + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"col1 varchar(255), col2 int", database_name=database_name, - clickhouse_table_engine=self.context.clickhouse_table_engines[0], ) with And("inserting data into the table"): @@ -636,6 +649,8 @@ def module( Check that auto table replication works when there are multiple databases in MySQL. """ + engine = self.context.clickhouse_table_engine + self.context.clickhouse_node = self.context.cluster.node(clickhouse_node) self.context.mysql_node = self.context.cluster.node(mysql_node) @@ -655,12 +670,18 @@ def module( ): create_databases(databases=self.context.list_of_databases) - self.context.config_file = os.path.join("env", "auto", "configs") + if engine == "ReplacingMergeTree": + self.context.config_file = os.path.join("env", "auto", "configs") + elif engine == "ReplicatedReplacingMergeTree": + self.context.config_file = os.path.join("env", "auto_replicated", "configs") with And( "I create a new ClickHouse Sink Connector configuration to monitor all of the databases" ): - replicate_all_databases() + if engine == "ReplacingMergeTree": + replicate_all_databases() + elif engine == "ReplicatedReplacingMergeTree": + replicate_all_databases_rrmt() with Pool(parallel_cases) as executor: Feature(run=inserts, parallel=True, executor=executor) diff --git a/sink-connector-lightweight/tests/integration/tests/multiple_tables.py b/sink-connector-lightweight/tests/integration/tests/multiple_tables.py index 905724c47..0dce0e3ec 100644 --- a/sink-connector-lightweight/tests/integration/tests/multiple_tables.py +++ b/sink-connector-lightweight/tests/integration/tests/multiple_tables.py @@ -21,11 +21,9 @@ def multiple_table_auto_creation( for i in range(number_of_tables): table_name = f"users{i}" with Given(f"I create MySQL table {table_name}"): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_table_engine=clickhouse_table_engine, - clickhouse_columns=clickhouse_columns, + create_mysql_table( + table_name=table_name, + olumns=mysql_columns, ) with When(f"I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/parallel.py b/sink-connector-lightweight/tests/integration/tests/parallel.py index ba8eb5a16..dde796065 100644 --- a/sink-connector-lightweight/tests/integration/tests/parallel.py +++ b/sink-connector-lightweight/tests/integration/tests/parallel.py @@ -11,11 +11,9 @@ def insert_update_delete(self): with Given("I create MySQL to ClickHouse replicated table"): for clickhouse_table_engine in self.context.clickhouse_table_engines: - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns="x INT", - clickhouse_columns="x Int32", - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns="x INT", ) with When( diff --git a/sink-connector-lightweight/tests/integration/tests/partition_limits.py b/sink-connector-lightweight/tests/integration/tests/partition_limits.py index 6f124c355..ac13c2be1 100644 --- a/sink-connector-lightweight/tests/integration/tests/partition_limits.py +++ b/sink-connector-lightweight/tests/integration/tests/partition_limits.py @@ -22,11 +22,9 @@ def partition_limits( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL table {table_name}"): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_table_engine=clickhouse_table_engine, - clickhouse_columns=clickhouse_columns, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, partition_by="id", ) diff --git a/sink-connector-lightweight/tests/integration/tests/partitions.py b/sink-connector-lightweight/tests/integration/tests/partitions.py index 1a745acd6..ead069737 100644 --- a/sink-connector-lightweight/tests/integration/tests/partitions.py +++ b/sink-connector-lightweight/tests/integration/tests/partitions.py @@ -12,11 +12,10 @@ def create_table_partitioned_by_range(self): for clickhouse_table_engine in self.context.clickhouse_table_engines: with Given(f"I create the table {table_name} partitioned by range"): - create_mysql_to_clickhouse_replicated_table( - name=f"{table_name}", - mysql_columns="order_id INT AUTO_INCREMENT, order_date DATE, total_amount DECIMAL(10, 2)", + create_mysql_table( + table_name=f"{table_name}", + columns="order_id INT AUTO_INCREMENT, order_date DATE, total_amount DECIMAL(10, 2)", primary_key="order_id, order_date", - clickhouse_table_engine=clickhouse_table_engine, partition_by_mysql="RANGE (YEAR(order_date)) (PARTITION p1 VALUES LESS THAN (2020))", ) diff --git a/sink-connector-lightweight/tests/integration/tests/primary_keys.py b/sink-connector-lightweight/tests/integration/tests/primary_keys.py index be9a59c81..fcc29c1c3 100644 --- a/sink-connector-lightweight/tests/integration/tests/primary_keys.py +++ b/sink-connector-lightweight/tests/integration/tests/primary_keys.py @@ -24,13 +24,10 @@ def check_different_primary_keys( f"I create MySQL to CH replicated table with some primary key", description=table_name, ): - create_mysql_to_clickhouse_replicated_table( + create_mysql_table( name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + columns=mysql_columns, primary_key=primary_key, - engine=engine, ) with When(f"I insert data in MySQL table {table_name}"): diff --git a/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py b/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py index 3792a9961..cec1a6e5c 100644 --- a/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py +++ b/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py @@ -15,10 +15,9 @@ def retry_on_fail(self): clickhouse_node.stop_clickhouse(safe=False) with When("I creat a table in MySQL"): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"retry VARCHAR(16)", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"retry VARCHAR(16)", ) with And("I insert data into the MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/sanity.py b/sink-connector-lightweight/tests/integration/tests/sanity.py index 7b8e3b44f..eb3278098 100644 --- a/sink-connector-lightweight/tests/integration/tests/sanity.py +++ b/sink-connector-lightweight/tests/integration/tests/sanity.py @@ -18,11 +18,9 @@ def mysql_to_clickhouse_connection( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When(f"I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/schema_changes.py b/sink-connector-lightweight/tests/integration/tests/schema_changes.py index 45454192e..e066355c8 100644 --- a/sink-connector-lightweight/tests/integration/tests/schema_changes.py +++ b/sink-connector-lightweight/tests/integration/tests/schema_changes.py @@ -26,11 +26,9 @@ def check_datatype_replication( ) with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When(f"I insert data in MySQL table {table_name}"): diff --git a/sink-connector-lightweight/tests/integration/tests/schema_only.py b/sink-connector-lightweight/tests/integration/tests/schema_only.py index 074eb4a5c..39e1f2052 100644 --- a/sink-connector-lightweight/tests/integration/tests/schema_only.py +++ b/sink-connector-lightweight/tests/integration/tests/schema_only.py @@ -1,10 +1,6 @@ -from integration.requirements.requirements import ( - RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_TableSchemaCreation, -) -from integration.helpers.create_config import * -from integration.tests.steps.service_settings import * from integration.tests.steps.mysql import * -from integration.helpers.common import change_sink_configuration +from integration.tests.steps.service_settings import * +from integration.tests.steps.sink_configurations import config_with_schema_only @TestStep(Given) @@ -14,10 +10,9 @@ def create_table_structure(self, table_name): clickhouse_node = self.context.clickhouse_node with By(f"creating a {table_name} table"): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"col1 varchar(255), col2 int", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"col1 varchar(255), col2 int", ) with And(f"inserting data into the {table_name} table"): @@ -69,9 +64,7 @@ def module( with Given( "I create a new ClickHouse Sink Connector configuration with schema-only mode" ): - change_sink_configuration( - values={"snapshot.mode": "schema_only"}, config_file=config_file - ) + config_with_schema_only() for scenario in loads(current_module(), Scenario): Scenario(run=scenario) diff --git a/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py b/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py index 4f425b480..2ceb8f98b 100644 --- a/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py +++ b/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py @@ -46,10 +46,9 @@ def create_and_validate_table(self, table_name): with By( "creating a table in MySQL and checking that it was also created in ClickHouse" ): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns=f"col1 varchar(255), col2 int", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns=f"col1 varchar(255), col2 int", ) check_if_table_was_created(table_name=table_name) diff --git a/sink-connector-lightweight/tests/integration/tests/snowflake_id.py b/sink-connector-lightweight/tests/integration/tests/snowflake_id.py index 44f19ee21..dea95fc8b 100644 --- a/sink-connector-lightweight/tests/integration/tests/snowflake_id.py +++ b/sink-connector-lightweight/tests/integration/tests/snowflake_id.py @@ -19,11 +19,9 @@ def mysql_to_clickhouse_snowflake( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When(f"I insert data in MySQL table"): @@ -96,11 +94,9 @@ def mysql_to_clickhouse_snowflake_with_mysql_restart( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When(f"I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/steps/clickhouse.py b/sink-connector-lightweight/tests/integration/tests/steps/clickhouse.py index f19f8a3a8..410087b14 100644 --- a/sink-connector-lightweight/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector-lightweight/tests/integration/tests/steps/clickhouse.py @@ -11,7 +11,7 @@ def drop_database(self, database_name=None, node=None): node = self.context.cluster.node("clickhouse") with By("executing drop database query"): node.query( - f"DROP DATABASE IF EXISTS {database_name} ON CLUSTER sharded_replicated_cluster;" + f"DROP DATABASE IF EXISTS {database_name} ON CLUSTER replicated_cluster;" ) @@ -66,7 +66,7 @@ def create_clickhouse_database(self, name=None, node=None): drop_database(database_name=name) node.query( - f"CREATE DATABASE IF NOT EXISTS {name} ON CLUSTER sharded_replicated_cluster" + f"CREATE DATABASE IF NOT EXISTS {name} ON CLUSTER replicated_cluster" ) yield finally: @@ -143,9 +143,17 @@ def check_if_table_was_created( if node is None: node = self.context.cluster.node("clickhouse") - retry(node.query, timeout=timeout, delay=3)( - f"EXISTS {database_name}.{table_name}", message=f"{message}" - ) + if self.context.clickhouse_table_engine == "ReplicatedReplacingMergeTree": + for node in self.context.cluster.nodes["clickhouse"]: + retry(self.context.cluster.node(node).query, timeout=timeout, delay=3)( + f"EXISTS {database_name}.{table_name}", message=f"{message}" + ) + elif self.context.clickhouse_table_engine == "ReplacingMergeTree": + retry(node.query, timeout=timeout, delay=3)( + f"EXISTS {database_name}.{table_name}", message=f"{message}" + ) + else: + raise Exception("Unknown ClickHouse table engine") @TestStep(Then) @@ -160,12 +168,27 @@ def validate_data_in_clickhouse_table( if node is None: node = self.context.cluster.node("clickhouse") - for retry in retries(timeout=40): - with retry: - data = node.query( - f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" - ) - assert data.output.strip().replace('"', "") == expected_output, error() + if self.context.clickhouse_table_engine == "ReplicatedReplacingMergeTree": + for node in self.context.cluster.nodes["clickhouse"]: + for retry in retries(timeout=40): + with retry: + data = self.context.cluster.node(node).query( + f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + ) + + assert ( + data.output.strip().replace('"', "") == expected_output + ), error() + elif self.context.clickhouse_table_engine == "ReplacingMergeTree": + for retry in retries(timeout=40): + with retry: + data = node.query( + f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + ) + assert data.output.strip().replace('"', "") == expected_output, error() + + else: + raise Exception("Unknown ClickHouse table engine") @TestStep(Then) diff --git a/sink-connector-lightweight/tests/integration/tests/steps/mysql.py b/sink-connector-lightweight/tests/integration/tests/steps/mysql.py index 6892ffc76..dab4cc4e9 100644 --- a/sink-connector-lightweight/tests/integration/tests/steps/mysql.py +++ b/sink-connector-lightweight/tests/integration/tests/steps/mysql.py @@ -97,35 +97,62 @@ def create_mysql_database(self, node=None, database_name=None): with Finally(f"I delete MySQL database {database_name}"): node.query(f"DROP DATABASE IF EXISTS {database_name};") - @TestStep(Given) -def create_mysql_table(self, name=None, statement=None, node=None, database_name=None): - """Creation of default MySQL table for tests.""" +def create_mysql_table( + self, + table_name, + columns, + mysql_node=None, + clickhouse_node=None, + database_name=None, + primary_key="id", + engine=True, + partition_by_mysql=False, +): + """Create MySQL table that will be auto created in ClickHouse.""" if database_name is None: database_name = "test" - if node is None: - node = self.context.cluster.node("mysql-master") - if name is None: - name = "users" - if statement is None: - statement = ( - f"CREATE TABLE IF NOT EXISTS {database_name}.{name}" - f" (id INT AUTO_INCREMENT," - f" age INT, PRIMARY KEY (id)) ORDER BY tuple() ENGINE = InnoDB;" - ) + + if mysql_node is None: + mysql_node = self.context.cluster.node("mysql-master") + + if clickhouse_node is None: + clickhouse_node = self.context.cluster.node("clickhouse") try: - with Given(f"I create MySQL table {name}"): - node.query(statement) + key = "" + if primary_key is not None: + key = f"{primary_key} INT NOT NULL," + + with Given(f"I create MySQL table", description=name): + query = f"CREATE TABLE IF NOT EXISTS {database_name}.{table_name} ({key}{columns}" + + if primary_key is not None: + query += f", PRIMARY KEY ({primary_key}))" + else: + query += ")" + + if engine: + query += f" ENGINE = InnoDB" + + if partition_by_mysql: + query += f", PARTITION BY {partition_by_mysql}" + + query += ";" + + mysql_node.query(query) + yield finally: - with Finally("I clean up by deleting table in MySQL"): - node.query(f"DROP TABLE IF EXISTS {name};") - self.context.cluster.node("clickhouse").query( - f"DROP TABLE IF EXISTS test.{name} ON CLUSTER sharded_replicated_cluster;" + with Finally( + "I clean up by deleting MySQL to ClickHouse replicated table", + description={name}, + ): + mysql_node.query(f"DROP TABLE IF EXISTS {database_name}.{table_name};") + clickhouse_node.query( + f"DROP TABLE IF EXISTS {database_name}.{table_name} ON CLUSTER replicated_cluster;" ) - time.sleep(5) @TestStep @@ -156,59 +183,6 @@ def create_mysql_to_clickhouse_replicated_table( clickhouse_node = self.context.cluster.node("clickhouse") try: - if self.context.env.endswith("auto"): - if clickhouse_table_engine == "ReplacingMergeTree": - pass - else: - raise NotImplementedError( - f"table '{clickhouse_table_engine}' not supported" - ) - - elif self.context.env.endswith("manual"): - if clickhouse_table_engine == "ReplicatedReplacingMergeTree": - with Given( - f"I create ReplicatedReplacingMergeTree as a replication table", - description=name, - ): - clickhouse_node.query( - f"CREATE TABLE IF NOT EXISTS test.{name} ON CLUSTER sharded_replicated_cluster" - f"(id Int32,{clickhouse_columns}, {sign_column} " - f"Int8, {version_column} UInt64) " - f"ENGINE = ReplicatedReplacingMergeTree(" - "'/clickhouse/tables/{shard}" - f"/{name}'," - " '{replica}'," - f" {version_column}) " - f"{f'PRIMARY KEY ({primary_key}) ORDER BY ({primary_key})' if primary_key is not None else 'ORDER BY tuple()'}" - f"{f'PARTITION BY ({partition_by})' if partition_by is not None else ''}" - f" SETTINGS " - f"index_granularity = 8192;", - ) - elif clickhouse_table_engine == "ReplacingMergeTree": - with Given( - f"I create ClickHouse table as replication table to MySQL test.{name}" - ): - clickhouse_node.query( - f"CREATE TABLE IF NOT EXISTS test.{name} " - f"(id Int32,{clickhouse_columns}, {sign_column} " - f"Int8, {version_column} UInt64) " - f"ENGINE = ReplacingMergeTree({version_column}) " - f"{f'PRIMARY KEY ({primary_key}) ORDER BY ({primary_key})' if primary_key is not None else 'ORDER BY tuple()'}" - f"{f'PARTITION BY ({partition_by})' if partition_by is not None else ''}" - f" SETTINGS " - f"index_granularity = 8192;", - ) - - else: - raise NotImplementedError( - f"table '{clickhouse_table_engine}' not supported" - ) - - else: - raise NotImplementedError( - f"table creation method '{self.context.env}' not supported" - ) - with Given(f"I create MySQL table", description=name): query = f"CREATE TABLE IF NOT EXISTS {database_name}.{name} (id INT NOT NULL,{mysql_columns}" @@ -234,7 +208,7 @@ def create_mysql_to_clickhouse_replicated_table( ): mysql_node.query(f"DROP TABLE IF EXISTS {database_name}.{name};") clickhouse_node.query( - f"DROP TABLE IF EXISTS {database_name}.{name} ON CLUSTER sharded_replicated_cluster;" + f"DROP TABLE IF EXISTS {database_name}.{name} ON CLUSTER replicated_cluster;" ) time.sleep(5) diff --git a/sink-connector-lightweight/tests/integration/tests/steps/sink_configurations.py b/sink-connector-lightweight/tests/integration/tests/steps/sink_configurations.py new file mode 100644 index 000000000..2936e8f6a --- /dev/null +++ b/sink-connector-lightweight/tests/integration/tests/steps/sink_configurations.py @@ -0,0 +1,42 @@ +import os + +from integration.helpers.create_config import * +from integration.helpers.common import change_sink_configuration + +default_config_path = os.path.join("env", "auto", "configs") + + +@TestStep(Given) +def config_with_replicated_table( + self, config_file=default_config_path + "replicated_replacing_merge_tree.yml" +): + """Create the Sink Connector configuration with the ReplicatedReplacingMergeTree table.""" + change_sink_configuration( + values={"auto.create.tables.replicated": "true", "auto.create.tables": "true"}, + config_file=config_file, + ) + + +@TestStep(Given) +def config_with_replicated_table_and_disabled_auto_create( + self, + config_file=default_config_path + + "replicated_replacing_merge_tree_no_auto_create.yml", +): + """Create the Sink Connector configuration with the ReplicatedReplacingMergeTree table.""" + change_sink_configuration( + values={ + "auto.create.tables.replicated": "true", + "auto.create.tables": "false", + "enable.snapshot.ddl": "false", + }, + config_file=config_file, + ) + + +@TestStep(Given) +def config_with_schema_only(self, config_file=default_config_path + "schema_only.yml"): + """Create the Sink Connector configuration with the schema only.""" + change_sink_configuration( + values={"snapshot.mode": "schema_only"}, config_file=config_file + ) diff --git a/sink-connector-lightweight/tests/integration/tests/table_names.py b/sink-connector-lightweight/tests/integration/tests/table_names.py index e958d54e5..f74323602 100644 --- a/sink-connector-lightweight/tests/integration/tests/table_names.py +++ b/sink-connector-lightweight/tests/integration/tests/table_names.py @@ -51,11 +51,9 @@ def check_table_names(self, table_name): clickhouse_node = self.context.clickhouse_node with Given(f"I create the {table_name} table"): - create_mysql_to_clickhouse_replicated_table( - name=f"\`{table_name}\`", - mysql_columns="x INT", - clickhouse_columns="x Int32", - clickhouse_table_engine=self.context.clickhouse_table_engines[0], + create_mysql_table( + table_name=f"\`{table_name}\`", + columns="x INT", ) with And("I insert data into the table"): diff --git a/sink-connector-lightweight/tests/integration/tests/truncate.py b/sink-connector-lightweight/tests/integration/tests/truncate.py index 0f85f31f1..9d96057f8 100644 --- a/sink-connector-lightweight/tests/integration/tests/truncate.py +++ b/sink-connector-lightweight/tests/integration/tests/truncate.py @@ -20,13 +20,10 @@ def truncate( mysql = self.context.cluster.node("mysql-master") with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, primary_key=primary_key, - engine=engine, ) with When(f"I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/types.py b/sink-connector-lightweight/tests/integration/tests/types.py index 05b78d36a..60b233ed7 100644 --- a/sink-connector-lightweight/tests/integration/tests/types.py +++ b/sink-connector-lightweight/tests/integration/tests/types.py @@ -26,11 +26,9 @@ def check_datatype_replication( ) with Given(f"I create MySQL to CH replicated table", description=table_name): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with When(f"I insert data in MySQL table {table_name}"): diff --git a/sink-connector-lightweight/tests/integration/tests/update.py b/sink-connector-lightweight/tests/integration/tests/update.py index 129b9d714..4871d72d5 100644 --- a/sink-connector-lightweight/tests/integration/tests/update.py +++ b/sink-connector-lightweight/tests/integration/tests/update.py @@ -23,13 +23,10 @@ def simple_update( with Given( f"I create MySQL to ClickHouse replicated table", description=table_name ): - create_mysql_to_clickhouse_replicated_table( - name=table_name, - mysql_columns=mysql_columns, - clickhouse_columns=clickhouse_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, primary_key=primary_key, - engine=engine, ) with When(f"I insert data in MySQL table"): diff --git a/sink-connector-lightweight/tests/integration/tests/virtual_columns.py b/sink-connector-lightweight/tests/integration/tests/virtual_columns.py index 7882526f7..ecaec77f4 100644 --- a/sink-connector-lightweight/tests/integration/tests/virtual_columns.py +++ b/sink-connector-lightweight/tests/integration/tests/virtual_columns.py @@ -25,12 +25,9 @@ def virtual_column_names( table_name = f"virtual_columns_{getuid()}" with Given(f"I create MySQL table {table_name})"): - create_mysql_to_clickhouse_replicated_table( - version_column=version_column, - name=table_name, - clickhouse_columns=clickhouse_columns, - mysql_columns=mysql_columns, - clickhouse_table_engine=clickhouse_table_engine, + create_mysql_table( + table_name=table_name, + columns=mysql_columns, ) with Then(f"I make check that ClickHouse table virtual column names are correct"): diff --git a/sink-connector/tests/integration/requirements.txt b/sink-connector/tests/integration/requirements.txt index c190f0357..a89621f2e 100644 --- a/sink-connector/tests/integration/requirements.txt +++ b/sink-connector/tests/integration/requirements.txt @@ -2,3 +2,4 @@ docker-compose==1.29.2 testflows==2.1.5 docker==6.1.3 awscli==1.27.36 +requests==2.31.0