Merge pull request #1 from barloc/initial

Initial: add main functionality

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 110
+ignore = E501
+exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.eggs,*.egg,node_modules,*pa/strategy/report*
+max-complexity = 19

.gitignore

@@ -0,0 +1,2 @@
+venv
+__pycache__

Dockerfile

@@ -0,0 +1,14 @@
+FROM alpine:3.18
+
+ENV PYTHONUNBUFFERED 1
+
+RUN apk update && apk add postgresql gzip bash && apk add python3 py3-pip && pip3 install --no-cache --upgrade pip setuptools wheel
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+WORKDIR /app
+COPY utils/ /app/utils/
+COPY chmfc.py /app/
+
+CMD ["python", "/app/chmfc.py"]

Makefile

@@ -0,0 +1,28 @@
+NAME=clickhouse-migrations-for-cluster
+DEV_COMPOSE_FLAGS=-f test-dockerfiles/docker-compose.yml -f test-dockerfiles/docker-compose.dev.yml -p dev
+
+.PHONY: env_up env_test env_down
+env_up: env_down
+	docker compose $(COMPOSE_FLAGS) up -d
+env_down:
+	docker compose $(COMPOSE_FLAGS) down -v
+
+.PHONY: dev_env_up dev_env_deploy dev_env_test dev_env_down
+dev_env_up: COMPOSE_FLAGS=${DEV_COMPOSE_FLAGS}
+dev_env_up: env_up
+dev_env_down: COMPOSE_FLAGS=${DEV_COMPOSE_FLAGS}
+dev_env_down: env_down
+
+.PHONY: linter
+linter:
+	docker-compose -f test-dockerfiles/docker-compose.ci.yml rm -f
+	docker compose -f test-dockerfiles/docker-compose.linter.yml down
+	docker compose -f test-dockerfiles/docker-compose.linter.yml up --build --force-recreate
+	docker compose -f test-dockerfiles/docker-compose.linter.yml down
+
+.PHONY: ci
+ci:
+	docker-compose -f test-dockerfiles/docker-compose.ci.yml rm -f
+	docker compose -f test-dockerfiles/docker-compose.ci.yml down
+	docker compose -f test-dockerfiles/docker-compose.ci.yml up --build chmfc
+	docker compose -f test-dockerfiles/docker-compose.ci.yml down

README.md

@@ -0,0 +1,83 @@
+# clickhouse migrations for cluster mode
+
+Service applies migrations from the directory to the clickhouse cluster. Main goal of the service - dev or test environments, which are identical with production environment.
+
+## Important
+
+Clickhouse doesn't have transactions for replicated tables, so we can not provide apllying migration in the single transaction. It is always applying migration queries step by step and after it writing success message to the trunsactions table in the cluster.
+
+## Usage
+
+Add your migrations directory to the docker image and then execute it.
+
+`docker run -v ... -e ... barloc/clickhouse-migrations-for-cluster:0.0.1`
+
+### Params
+
+Only via envs.
+
+* CHMFC_CH_HOST - hostname of the cluster (default: localhost)
+* CHMFC_CH_PORT - port (default: 9000)
+* CHMFC_CH_LOGIN - login (default: test)
+* CHMFC_CH_PASSWORD - password (default: test)
+* CHMFC_CH_DATABASE - database for connection (default: test)
+* CHMFC_MIG_PATH - path to the directory with migrations (default: test-migrations)
+* CHMFC_MIG_TABLE - name of the table with applying migrations (default: test._migrations)
+
+### Migrations name (file)
+
+* Every file with migration must have prefix with number (0001, 01, 010...), it is version of migrations. Migrations applies in the order of this prefix (version).
+* Every file contains one or many query with `;` between.
+
+## Local development
+
+You need: docker, make, python >3.11.
+
+### Install environment:
+
+Install `venv`:
+
+```
+python3 -m venv ./venv
+```
+
+Activate:
+
+```
+source ./venv/bin/activate
+```
+
+Install dependencies:
+```
+pip install -r requirements.txt
+```
+
+### Linter
+
+```
+make linter
+```
+
+### Local env
+
+Up clickhouse cluster for work:
+
+```
+make dev_env_up
+```
+
+Down cluster:
+
+```
+make dev_env_down
+```
+
+### Integration test
+
+```
+make ci
+```
+
+### Clickhouse version
+
+Tested with yandex/clickhouse-server:21.8.5.7. If you want up version then rewrite it in the test-dockerfiles/docker-compose.*.yml.

chmfc.py

@@ -0,0 +1,88 @@
+import os
+import logging
+
+from clickhouse_driver import Client
+
+from utils.data import Migrations, Migration
+from utils.errors import CHMFCBaseError, CHMFCBadQueryError
+from utils.queries import ClickhouseQueries
+
+# set looger
+logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
+logger = logging.getLogger("chmfc")
+
+# get env vars
+clickhouse_host = os.getenv('CHMFC_CH_HOST', 'localhost')
+clickhouse_port = os.getenv('CHMFC_CH_PORT', '9000')
+clickhouse_login = os.getenv('CHMFC_CH_LOGIN', 'test')
+clickhouse_password = os.getenv('CHMFC_CH_PASSWORD', 'test')
+clickhouse_database = os.getenv('CHMFC_CH_DATABASE', 'test')
+migrations_path = os.getenv('CHMFC_MIG_PATH', 'test-migrations')
+migrations_table = os.getenv('CHMFC_MIG_TABLE', 'default._migrations')
+
+rule_queries = ClickhouseQueries(migrations_table=migrations_table)
+
+
+def check_or_create_migrations_table(client: Client):
+    # check migrations table and create it if needs
+    exists = True
+    try:
+        ch_client.execute(rule_queries.exists_migrations_table())
+    except Exception as e:
+        if f"Table {migrations_table} doesn't exist" in str(e):
+            create_shards, create_distributed = rule_queries.create_migrations_table()
+            ch_client.execute(create_shards)
+            ch_client.execute(create_distributed)
+
+            exists = False
+            logger.info(f"Table {migrations_table} has been created.")
+        else:
+            raise CHMFCBaseError(e)
+
+    if exists:
+        logger.info(f"Table {migrations_table} exists.")
+
+
+def handle_migration(client: Client, version: int, v: Migration):
+    writed_migration = client.execute(rule_queries.get_migration_by_version(version))
+
+    # if exists
+    if writed_migration:
+        for row in writed_migration:
+            if row[1] != v.checksum:
+                logger.info(f'ALERT - {version} - {v.filename} - checksums are not equal')
+        logger.info(f'PASS - {version} - {v.filename} - {v.checksum}')
+    else:
+        logger.debug(f'Migration {version} in clickhouse? {writed_migration if writed_migration else "No."}')
+
+        # if migrations doesn't contain any query
+        if not v.queries:
+            logger.info(f'EMPTY - {version} - {v.filename}')
+            client.execute(rule_queries.write_migration_to_the_table(version, v.filename, v.checksum))
+
+        # else apply all queries in the migration
+        for query in v.queries:
+            try:
+                client.execute(query)
+            except Exception as e:
+                exception_by_lines = str(e).split('\n')
+                reason_ = ''
+                for line in exception_by_lines:
+                    if line.startswith('DB::Exception: '):
+                        reason_ = line.replace('DB::Exception: ', '').replace('Stack trace:', '').strip()
+                        break
+                raise CHMFCBadQueryError(query, reason_)
+        # and write migration version to the migrations table
+        client.execute(rule_queries.write_migration_to_the_table(version, v.filename, v.checksum))
+        logger.info(f'OK - {version} - {v.filename} - {v.checksum}')
+
+
+if __name__ == "__main__":
+    with Client.from_url(
+        f'tcp://{clickhouse_login}:{clickhouse_password}@{clickhouse_host}:{clickhouse_port}/{clickhouse_database}'
+    ) as ch_client:
+
+        check_or_create_migrations_table(ch_client)
+
+        for version, v in sorted(Migrations().get_from_path(migrations_path).items()):
+            handle_migration(ch_client, version, v)

requirements.txt

@@ -0,0 +1 @@
+clickhouse-driver==0.2.6

test-dockerfiles/Dockerfile.linter

@@ -0,0 +1,7 @@
+FROM alpine/flake8:6.0.0
+
+RUN pip install flake8-bugbear pep8-naming flake8-builtins
+
+COPY utils /apps/
+COPY chmfc.py /apps/
+COPY .flake8 /apps/

test-dockerfiles/clickhouse/macros.xml

@@ -0,0 +1,9 @@
+<yandex>
+    <macros>
+        <installation>test</installation>
+        <all-sharded-shard>0</all-sharded-shard>
+        <cluster>test_cluster</cluster>
+        <shard>0</shard>
+        <replica from_env="REPLICA_NAME" />
+    </macros>
+</yandex>

test-dockerfiles/clickhouse/remote_servers.xml

@@ -0,0 +1,17 @@
+<yandex>
+    <remote_servers>
+        <test_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster>
+    </remote_servers>
+</yandex>

test-dockerfiles/clickhouse/zookeeper.xml

@@ -0,0 +1,11 @@
+<yandex>
+    <default_replica_path>/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}</default_replica_path>
+    <default_replica_name>{replica}</default_replica_name>
+
+    <zookeeper>
+        <node>
+            <host>zookeeper</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+</yandex>

test-dockerfiles/docker-compose.ci.yml

@@ -0,0 +1,74 @@
+version: '2.4'
+
+services:
+
+  chmfc:
+    build:
+      context: ../
+      dockerfile: Dockerfile
+    container_name: chmfc
+    environment:
+      CHMFC_CH_HOST: clickhouse
+      CHMFC_CH_LOGIN: test
+      CHMFC_CH_PASSWORD: test
+      CHMFC_CH_DATABASE: default
+      CHMFC_MIG_PATH: /migrations
+      CHMFC_MIG_TABLE: default.migrations
+    volumes:
+      - ../test-migrations:/migrations
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+      clickhouse1:
+        condition: service_healthy
+
+  clickhouse:
+    image: yandex/clickhouse-server:21.8.5.7
+    container_name: test_clickhouse
+    volumes:
+      - ./clickhouse/macros.xml:/etc/clickhouse-server/conf.d/macros.xml
+      - ./clickhouse/zookeeper.xml:/etc/clickhouse-server/conf.d/zookeeper.xml
+      - ./clickhouse/remote_servers.xml:/etc/clickhouse-server/config.d/remote_servers.xml
+    environment:
+      CLICKHOUSE_DB: test
+      CLICKHOUSE_USER: test
+      CLICKHOUSE_PASSWORD: test
+      REPLICA_NAME: clickhouse
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+    depends_on:
+      zookeeper:
+        condition: service_started
+
+  clickhouse1:
+    image: yandex/clickhouse-server:21.8.5.7
+    container_name: test_clickhouse1
+    volumes:
+      - ./clickhouse/macros.xml:/etc/clickhouse-server/conf.d/macros.xml
+      - ./clickhouse/zookeeper.xml:/etc/clickhouse-server/conf.d/zookeeper.xml
+      - ./clickhouse/remote_servers.xml:/etc/clickhouse-server/config.d/remote_servers.xml
+    environment:
+      CLICKHOUSE_DB: test
+      CLICKHOUSE_USER: test
+      CLICKHOUSE_PASSWORD: test
+      REPLICA_NAME: clickhouse1
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+    depends_on:
+      zookeeper:
+        condition: service_started
+
+  zookeeper:
+    image: zookeeper:3.5
+    container_name: zookeeper
+    hostname: zookeeper
+
+networks:
+  default:
+    name: test

test-dockerfiles/docker-compose.dev.yml

@@ -0,0 +1,12 @@
+version: '2.4'
+
+services:
+
+  clickhouse:
+    ports:
+      - 8123:8123
+      - 9000:9000
+
+networks:
+  default:
+    name: test

test-dockerfiles/docker-compose.linter.yml

@@ -0,0 +1,10 @@
+version: '2.4'
+
+services:
+
+  chmfc-linter:
+    build:
+      context: ../
+      dockerfile: test-dockerfiles/Dockerfile.linter
+    container_name: chmfc_linter
+    command: /apps