🧪 Add chaos testing example

ydb-platform · Oct 28, 2024 · 45308fb · 45308fb
1 parent 5a5caa4
commit 45308fb
Show file tree

Hide file tree

Showing 3 changed files with 297 additions and 11 deletions.
diff --git a/.github/workflows/ydb-slo-action-test-init.yml b/.github/workflows/ydb-slo-action-test-init.yml
@@ -18,18 +18,33 @@ jobs:
             - name: Checkout code
               uses: actions/checkout@v4
 
-            - name: Inititialize YDB SLO Test
+            - name: Initialize YDB SLO Test
               uses: ydb-platform/ydb-slo-action/init@main
               with:
                   github_pull_request_number: ${{ github.event.inputs.github_pull_request_number }}
                   sdk_name: ${{ matrix.sdk }}
 
+            - name: Download chaos testing tool
+              uses: robinraju/release-downloader@v1
+              with:
+                  repository: 'alexei-led/pumba'
+                  tag: '0.10.1'
+                  fileName: 'pumba_linux_amd64'
+                  out-file-path: './pumba'
+
+            - run: |
+                ifconfig
+                ipconfig getiflist
+                ipconfig getsummary en0
+                ipconfig getsummary bridge0
+
             - name: Run chaos testing
               run: |
                   echo 'Performing chaos testing...'
-                  sleep 10
+                  chmod +x ./pumba
+                  ./pumba --random --interval 30s restart re2:^ydb-dynamic &
 
             - name: Run user workload
               run: |
                   echo 'Running user workload...';
-                  sleep 10
+                  sleep 600
diff --git a/init/main.js b/init/main.js
diff --git a/init/src/configs.test.ts b/init/src/configs.test.ts
@@ -0,0 +1,271 @@
+import {
+	HOST,
+	PROMETHEUS_PORT,
+	PROMETHEUS_PUSHGATEWAY_PORT,
+	YDB_ENDPOINT,
+	YDB_GRPC_PORT,
+	YDB_IC_PORT,
+	YDB_MON_PORT,
+	YDB_TENANT,
+} from './constants'
+
+export const prometheusConfig = /** YAML */ `
+global:
+  scrape_interval: 1s
+  evaluation_interval: 1s
+
+scrape_configs:
+  - job_name: 'pushgateway'
+    static_configs:
+      - targets: ['${HOST}:${PROMETHEUS_PUSHGATEWAY_PORT}']
+`
+
+export const ydbConfig = /** YAML */ `
+actor_system_config:
+  cpu_count: 1
+  node_type: STORAGE
+  use_auto_config: true
+blob_storage_config:
+  service_set:
+    groups:
+      - erasure_species: none
+        rings:
+          - fail_domains:
+              - vdisk_locations:
+                  - node_id: 1
+                    path: SectorMap:1:64
+                    pdisk_category: SSD
+channel_profile_config:
+  profile:
+    - channel:
+        - erasure_species: none
+          pdisk_category: 0
+          storage_pool_kind: ssd
+        - erasure_species: none
+          pdisk_category: 0
+          storage_pool_kind: ssd
+        - erasure_species: none
+          pdisk_category: 0
+          storage_pool_kind: ssd
+      profile_id: 0
+domains_config:
+  domain:
+    - name: Root
+      storage_pool_types:
+        - kind: ssd
+          pool_config:
+            box_id: 1
+            erasure_species: none
+            kind: ssd
+            pdisk_filter:
+              - property:
+                  - type: SSD
+            vdisk_kind: Default
+  state_storage:
+    - ring:
+        node:
+          - 1
+        nto_select: 1
+      ssid: 1
+grpc_config:
+  port: ${YDB_GRPC_PORT}
+host_configs:
+  - drive:
+      - path: SectorMap:1:64
+        type: SSD
+    host_config_id: 1
+hosts:
+  - host: ${HOST}
+    host_config_id: 1
+    node_id: 1
+    port: ${YDB_IC_PORT}
+    walle_location:
+      body: 1
+      data_center: az-1
+      rack: "0"
+static_erasure: none
+`
+
+// Generate YDB Static Node
+let generateStaticNode = () =>
+	/** YAML */ `
+  static-0:
+    <<: *ydb-common
+    container_name: ydb-static-0
+    command:
+      - /opt/ydb/bin/ydbd
+      - server
+      - --grpc-port
+      - "${YDB_GRPC_PORT}"
+      - --mon-port
+      - "${YDB_MON_PORT}"
+      - --ic-port
+      - "${YDB_IC_PORT}"
+      - --yaml-config
+      - /opt/ydb/cfg/config.yaml
+      - --node
+      - static
+      - --label
+      - deployment=docker
+    ports:
+      - ${YDB_GRPC_PORT}:${YDB_GRPC_PORT}
+      - ${YDB_MON_PORT}:${YDB_MON_PORT}
+      - ${YDB_IC_PORT}:${YDB_IC_PORT}
+    healthcheck:
+      test: bash -c "exec 6<> /dev/tcp/${HOST}/${YDB_GRPC_PORT}"
+      interval: 10s
+      timeout: 1s
+      retries: 3
+      start_period: 30s
+    deploy:
+      <<: *ydb-deploy
+
+  static-init:
+    <<: *ydb-common
+    restart: on-failure
+    container_name: ydb-static-init
+    command:
+      - /opt/ydb/bin/ydbd
+      - -s
+      - ${YDB_ENDPOINT}
+      - admin
+      - blobstorage
+      - config
+      - init
+      - --yaml-file
+      - /opt/ydb/cfg/config.yaml
+    depends_on:
+      static-0:
+        condition: service_healthy
+
+  tenant-init:
+    <<: *ydb-common
+    restart: on-failure
+    container_name: ydb-tenant-init
+    command:
+      - /opt/ydb/bin/ydbd
+      - -s
+      - ${YDB_ENDPOINT}
+      - admin
+      - database
+      - ${YDB_TENANT}
+      - create
+      - ssd:1
+    depends_on:
+      static-init:
+        condition: service_completed_successfully
+`.slice(1)
+
+// Generate YDB Dynamic Node
+let generateDynamicNode = (idx: number) =>
+	/** YAML */ `
+  dynamic-${idx}:
+    <<: *ydb-common
+    container_name: ydb-dynamic-${idx}
+    command:
+      - /opt/ydb/bin/ydbd
+      - server
+      - --grpc-port
+      - "${YDB_GRPC_PORT + idx}"
+      - --mon-port
+      - "${YDB_MON_PORT + idx}"
+      - --ic-port
+      - "${YDB_IC_PORT + idx}"
+      - --yaml-config
+      - /opt/ydb/cfg/config.yaml
+      - --tenant
+      - ${YDB_TENANT}
+      - --node-broker
+      - ${YDB_ENDPOINT}
+      - --label
+      - deployment=docker
+    ports:
+      - ${YDB_GRPC_PORT + idx}:${YDB_GRPC_PORT + idx}
+      - ${YDB_MON_PORT + idx}:${YDB_MON_PORT + idx}
+      - ${YDB_IC_PORT + idx}:${YDB_IC_PORT + idx}
+    healthcheck:
+      test: bash -c "exec 6<> /dev/tcp/${HOST}/${YDB_GRPC_PORT + idx}"
+      interval: 10s
+      timeout: 1s
+      retries: 3
+      start_period: 30s
+    depends_on:
+      static-0:
+        condition: service_healthy
+      static-init:
+        condition: service_completed_successfully
+      tenant-init:
+        condition: service_completed_successfully
+    deploy:
+      <<: *ydb-deploy
+`.slice(1)
+
+// Generate Monitoring
+let generateMonitoring = () =>
+	/** YAML */ `
+  prometheus:
+    image: prom/prometheus
+    restart: unless-stopped
+    <<: *runtime
+    ports:
+      - "${PROMETHEUS_PORT}:${PROMETHEUS_PORT}"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    deploy: &monitoring-deploy
+      resources:
+        limits:
+          cpus: '0.1'
+          memory: 1000M
+        reservations:
+          cpus: '0.001'
+          memory: 50M
+
+  prometheus-pushgateway:
+    image: prom/pushgateway
+    restart: unless-stopped
+    <<: *runtime
+    ports:
+      - "${PROMETHEUS_PUSHGATEWAY_PORT}:${PROMETHEUS_PUSHGATEWAY_PORT}"
+    network_mode: host
+    deploy:
+      <<: *monitoring-deploy
+`.slice(1)
+
+// Generate Compose
+export let generateComposeFile = (
+	ydbDatabaseNodeCount: number
+) => /** YAML */ `# Code generated by Github Action; DO NOT EDIT.
+
+x-runtime: &runtime
+  platform: linux/amd64
+  privileged: true
+  network_mode: host
+
+x-node: &ydb-common
+  image: cr.yandex/crptqonuodf51kdj7a7d/ydb:24.2.7
+  restart: always
+  hostname: ${HOST}
+  <<: *runtime
+  volumes:
+    - ./ydb.yaml:/opt/ydb/cfg/config.yaml
+
+x-deploy: &ydb-deploy
+  restart_policy:
+    condition: any
+  resources:
+    limits:
+      cpus: '1'
+      memory: 1000M
+    reservations:
+      cpus: '0.1'
+      memory: 250M
+
+name: ydb
+
+services:
+${generateStaticNode()}
+${Array.from({ length: ydbDatabaseNodeCount }, (_, i) => i + 1)
+	.map(generateDynamicNode)
+	.join('\n')}
+${generateMonitoring()}
+`