From 6a0910274d7ce1e65415064639551d23d56a0b02 Mon Sep 17 00:00:00 2001 From: Jose Luis Franco Arza Date: Wed, 24 Apr 2024 23:56:58 +0200 Subject: [PATCH 01/10] Add backup and restore test with multi-tenant class --- .github/workflows/tests.yaml | 20 ++ .../Dockerfile | 14 ++ .../backup_and_restore_multi_tenant.py | 227 ++++++++++++++++++ .../requirements.txt | 4 + backup_and_restore_multi_node_multi_tenant.sh | 54 +++++ 5 files changed, 319 insertions(+) create mode 100644 apps/backup_and_restore_multi_tenant/Dockerfile create mode 100644 apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py create mode 100644 apps/backup_and_restore_multi_tenant/requirements.txt create mode 100755 backup_and_restore_multi_node_multi_tenant.sh diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index cffc15a8..7c347a29 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -507,6 +507,26 @@ jobs: password: ${{secrets.DOCKER_PASSWORD}} - name: Run chaos test run: ./backup_and_restore_multi_node_crud.sh + backup-restore-crud-multi-node-50k-tenants: + name: Backup & Restore multi node with several thounsands of tenants + runs-on: ubuntu-latest-4-cores + timeout-minutes: 60 + env: + PERSISTENCE_LSM_ACCESS_STRATEGY: ${{inputs.lsm_access_strategy}} + steps: + - uses: actions/checkout@v3 + # - name: Polar Signals Continuous Profiling + # uses: polarsignals/gh-actions-ps-profiling@v0.0.1 + # with: + # polarsignals_cloud_token: ${{ secrets.POLARSIGNALS_TOKEN }} + # labels: 'job=${{ github.job }};gh_run_id=${{ github.run_id }}' + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{secrets.DOCKER_USERNAME}} + password: ${{secrets.DOCKER_PASSWORD}} + - name: Run chaos test multi-tenant backup & restore + run: ./backup_and_restore_multi_node_multi_tenant.sh backup-restore-version-compat: name: Backup & Restore version compatibility runs-on: ubuntu-latest-8-cores diff --git a/apps/backup_and_restore_multi_tenant/Dockerfile b/apps/backup_and_restore_multi_tenant/Dockerfile new file mode 100644 index 00000000..80d4c54b --- /dev/null +++ b/apps/backup_and_restore_multi_tenant/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.10-slim-bullseye + +WORKDIR /workdir + +ARG backend +ENV BACKUP_BACKEND_PROVIDER=${backend} + +ARG expected_shard_count +ENV EXPECTED_SHARD_COUNT=${expected_shard_count} + +COPY requirements.txt . +RUN pip3 install -r requirements.txt + +COPY backup_and_restore_multi_tenant.py ./ diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py new file mode 100644 index 00000000..97531139 --- /dev/null +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -0,0 +1,227 @@ +import datetime +import numpy as np +import os +import random +import requests +import sys +import time +import uuid +import weaviate +from weaviate import Tenant + +from loguru import logger +from typing import Optional + + +def assert_expected_shard_count(client: weaviate.Client): + expected = os.environ.get("EXPECTED_SHARD_COUNT") + if expected is None or expected == "": + expected = 1 + else: + expected = int(expected) + + schema = client.schema.get()["classes"] + class_shard_counts = [ + {"class": cls["class"], "actualCount": cls["shardingConfig"]["actualCount"]} + for cls in schema + ] + + logger.info(f"Expected shard count per class: {expected}") + logger.info(f"Actual shard counts: {class_shard_counts}") + + assert len(class_shard_counts) > 0 and len(class_shard_counts) == len(schema) + assert all(list(map(lambda cls: cls["actualCount"] == expected, class_shard_counts))) + + +def other_classes(all_classes, self): + return [c for c in all_classes if c != self] + + +def reset_schema(client: weaviate.Client, class_names): + client.schema.delete_all() + for class_name in class_names: + class_obj = { + "vectorizer": "none", + "vectorIndexConfig": { + "efConstruction": 128, + "maxConnections": 16, + "ef": 256, + "cleanupIntervalSeconds": 10, + }, + "class": class_name, + "invertedIndexConfig": { + "indexTimestamps": False, + }, + "multiTenancyConfig": {"enabled": True}, + "replicationConfig": { + "factor": 2, + }, + "properties": [ + { + "dataType": ["boolean"], + "name": "should_be_deleted", + }, + { + "dataType": ["boolean"], + "name": "is_divisible_by_four", + }, + { + "dataType": ["int"], + "name": "index_id", + }, + { + "dataType": ["string"], + "name": "stage", + }, + ], + } + + client.schema.create_class(class_obj) + + for class_name in class_names: + for other in other_classes(class_names, class_name): + add_prop = { + "dataType": [ + other, + ], + "name": f"to_{other}", + } + + client.schema.property.create(class_name, add_prop) + + #assert_expected_shard_count(client) + + +def handle_errors(results: Optional[dict]) -> None: + """ + Handle error message from batch requests logs the message as an info message. + Parameters + ---------- + results : Optional[dict] + The returned results for Batch creation. + """ + + if results is not None: + for result in results: + if ( + "result" in result + and "errors" in result["result"] + and "error" in result["result"]["errors"] + ): + for message in result["result"]["errors"]["error"]: + logger.error(message["message"]) + + +def create_tenants(client: weaviate.Client, class_name, tenants, stage="stage_1"): + client.schema.add_class_tenants(class_name=class_name, tenants=[Tenant(name=f"tenant_{i}") for i in range(tenants)]) + +def delete_records(client: weaviate.Client, class_name): + client.batch.delete_objects( + class_name=class_name, + where={"operator": "Equal", "path": ["should_be_deleted"], "valueBoolean": True}, + output="minimal", + dry_run=False, + ) + + +def fatal(msg): + logger.error(msg) + sys.exit(1) + + +def success(msg): + logger.success(msg) + +def backend_provider(): + backend = os.environ.get("BACKUP_BACKEND_PROVIDER") + if backend is None or backend == "": + return "filesystem" + return backend + + +def temp_backup_url_create(): + return f"http://localhost:8080/v1/backups/{backend_provider()}" + + +def temp_backup_url_create_status(backup_name): + return f"http://localhost:8080/v1/backups/{backend_provider()}/{backup_name}" + + +def temp_backup_url_restore(backup_name): + return f"http://localhost:8080/v1/backups/{backend_provider()}/{backup_name}/restore" + + +def temp_backup_url_restore_status(backup_name): + return f"http://localhost:8080/v1/backups/{backend_provider()}/{backup_name}/restore" + + +def create_backup(client: weaviate.Client, name): + create_body = {"id": name} + res = requests.post(temp_backup_url_create(), json=create_body) + if res.status_code > 399: + fatal(f"Backup Create returned status code {res.status_code} with body: {res.json()}") + + while True: + time.sleep(3) + res = requests.get(temp_backup_url_create_status(name)) + res_json = res.json() + if res_json["status"] == "SUCCESS": + success(f"Backup creation successful") + break + if res_json["status"] == "FAILED": + fatal(f"Backup failed with res: {res_json}") + + +def restore_backup(client: weaviate.Client, name): + restore_body = {"id": name} + res = requests.post(temp_backup_url_restore(name), json=restore_body) + if res.status_code > 399: + fatal(f"Backup Restore returned status code {res.status_code} with body: {res.json()}") + + while True: + time.sleep(3) + res = requests.get(temp_backup_url_restore_status(name)) + res_json = res.json() + if res_json["status"] == "SUCCESS": + success(f"Restore succeeded") + break + if res_json["status"] == "FAILED": + fatal(f"Restore failed with res: {res_json}") + + +client = weaviate.Client("http://localhost:8080") + +backup_name = f"{int(datetime.datetime.now().timestamp())}_stage_1" + +class_names = ["Class_A", "Class_B"] +tenants = 12_000 + +logger.info(f"Step 0, reset everything, import schema") +reset_schema(client, class_names) + +logger.info(f"Step 1, create {tenants} tenants per class") +for class_name in class_names: + create_tenants( + client, + class_name, + tenants=tenants, + stage="stage_1", + ) + + +logger.info("Step 2, create backup of current instance including all classes") +create_backup(client, backup_name) + +logger.info("Step 3, delete all classes") +client.schema.delete_all() + +logger.info("Step 4, restore backup mark") +restore_backup(client, backup_name) + +logger.info("Step 10, run test and make sure results are same as on original instance at stage 1") +for class_name in class_names: + + actual_tenants = client.schema.get_class_tenants(class_name) + logger.info(f"{class_name}: {len(actual_tenants)} tenants") + assert len(actual_tenants) == tenants, f"Expected {tenants} tenants, but got {len(actual_tenants)}" + diff --git a/apps/backup_and_restore_multi_tenant/requirements.txt b/apps/backup_and_restore_multi_tenant/requirements.txt new file mode 100644 index 00000000..057d5838 --- /dev/null +++ b/apps/backup_and_restore_multi_tenant/requirements.txt @@ -0,0 +1,4 @@ +weaviate-client>=3.9.0 +numpy==1.22.2 +loguru==0.5.3 + diff --git a/backup_and_restore_multi_node_multi_tenant.sh b/backup_and_restore_multi_node_multi_tenant.sh new file mode 100755 index 00000000..3530a4d0 --- /dev/null +++ b/backup_and_restore_multi_node_multi_tenant.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -e + +function wait_weaviate_cluster() { + echo "Wait for Weaviate to be ready" + local node1_ready=false + local node2_ready=false + for _ in {1..120}; do + if curl -sf -o /dev/null localhost:8080/v1/.well-known/ready; then + echo "Weaviate node1 is ready" + node1_ready=true + fi + + if curl -sf -o /dev/null localhost:8081; then + echo "Weaviate node2 is ready" + node2_ready=true + fi + + if $node1_ready && $node2_ready; then + return 0 + fi + + echo "Weaviate cluster is not ready, trying again in 1s" + sleep 1 + done + if ! $node1_ready; then + echo "ERROR: Weaviate node1 is not ready after 120s" + fi + if ! $node2_ready; then + echo "ERROR: Weaviate node2 is not ready after 120s" + fi + exit 1 +} + +echo "Building all required containers" +( cd apps/backup_and_restore_multi_tenant/ && docker build -t backup_and_restore_multi_tenant \ + --build-arg backend="s3" --build-arg expected_shard_count=150000 . ) + +export WEAVIATE_NODE_1_VERSION=$WEAVIATE_VERSION +export WEAVIATE_NODE_2_VERSION=$WEAVIATE_VERSION + +echo "Starting Weaviate..." +docker-compose -f apps/weaviate/docker-compose-backup.yml up -d weaviate-node-1 weaviate-node-2 backup-s3 + +wait_weaviate_cluster + +echo "Creating S3 bucket..." +docker-compose -f apps/weaviate/docker-compose-backup.yml up create-s3-bucket + +echo "Run multi-node backup and restore with Multi-tenant class" +docker run --network host -t backup_and_restore_multi_tenant python3 backup_and_restore_multi_tenant.py + +echo "Passed!" From 35136a5ab6dce1ce9b2189025d0b6e9e08515a3a Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Thu, 2 May 2024 12:37:12 +0100 Subject: [PATCH 02/10] Make port configurable and add hot/cold tenants, decrease num tenants to reduce flakiness --- .../backup_and_restore_multi_tenant.py | 34 ++++++++++++------- backup_and_restore_multi_node_multi_tenant.sh | 3 ++ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index 97531139..722195b0 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -7,10 +7,12 @@ import time import uuid import weaviate -from weaviate import Tenant +from weaviate import Tenant, schema from loguru import logger -from typing import Optional +from typing import Optional, List + +WEAVIATE_PORT = 8080 def assert_expected_shard_count(client: weaviate.Client): @@ -112,8 +114,8 @@ def handle_errors(results: Optional[dict]) -> None: logger.error(message["message"]) -def create_tenants(client: weaviate.Client, class_name, tenants, stage="stage_1"): - client.schema.add_class_tenants(class_name=class_name, tenants=[Tenant(name=f"tenant_{i}") for i in range(tenants)]) +def create_tenants(client: weaviate.Client, class_name: str, tenants: List[Tenant], stage="stage_1"): + client.schema.add_class_tenants(class_name=class_name, tenants=tenants) def delete_records(client: weaviate.Client, class_name): client.batch.delete_objects( @@ -140,19 +142,19 @@ def backend_provider(): def temp_backup_url_create(): - return f"http://localhost:8080/v1/backups/{backend_provider()}" + return f"http://localhost:{WEAVIATE_PORT}/v1/backups/{backend_provider()}" def temp_backup_url_create_status(backup_name): - return f"http://localhost:8080/v1/backups/{backend_provider()}/{backup_name}" + return f"http://localhost:{WEAVIATE_PORT}/v1/backups/{backend_provider()}/{backup_name}" def temp_backup_url_restore(backup_name): - return f"http://localhost:8080/v1/backups/{backend_provider()}/{backup_name}/restore" + return f"http://localhost:{WEAVIATE_PORT}/v1/backups/{backend_provider()}/{backup_name}/restore" def temp_backup_url_restore_status(backup_name): - return f"http://localhost:8080/v1/backups/{backend_provider()}/{backup_name}/restore" + return f"http://localhost:{WEAVIATE_PORT}/v1/backups/{backend_provider()}/{backup_name}/restore" def create_backup(client: weaviate.Client, name): @@ -189,17 +191,25 @@ def restore_backup(client: weaviate.Client, name): fatal(f"Restore failed with res: {res_json}") -client = weaviate.Client("http://localhost:8080") +client = weaviate.Client(f"http://localhost:{WEAVIATE_PORT}") backup_name = f"{int(datetime.datetime.now().timestamp())}_stage_1" class_names = ["Class_A", "Class_B"] -tenants = 12_000 +num_tenants_hot = 1_000 +num_tenants_cold = num_tenants_hot * 2 logger.info(f"Step 0, reset everything, import schema") reset_schema(client, class_names) -logger.info(f"Step 1, create {tenants} tenants per class") +logger.info(f"Step 1, create {num_tenants_hot} hot tenants and {num_tenants_cold} cold tenants per class") +tenants = [ + Tenant(name=f"{i}_tenant", activity_status=schema.TenantActivityStatus.HOT) + for i in range(num_tenants_hot) +] + [ + Tenant(name=f"{i + num_tenants_hot}_tenant", activity_status=schema.TenantActivityStatus.COLD) + for i in range(num_tenants_cold) +] for class_name in class_names: create_tenants( client, @@ -223,5 +233,5 @@ def restore_backup(client: weaviate.Client, name): actual_tenants = client.schema.get_class_tenants(class_name) logger.info(f"{class_name}: {len(actual_tenants)} tenants") - assert len(actual_tenants) == tenants, f"Expected {tenants} tenants, but got {len(actual_tenants)}" + assert len(actual_tenants) == len(tenants), f"Expected {tenants} tenants, but got {len(actual_tenants)}" diff --git a/backup_and_restore_multi_node_multi_tenant.sh b/backup_and_restore_multi_node_multi_tenant.sh index 3530a4d0..9a601db4 100755 --- a/backup_and_restore_multi_node_multi_tenant.sh +++ b/backup_and_restore_multi_node_multi_tenant.sh @@ -37,6 +37,9 @@ echo "Building all required containers" ( cd apps/backup_and_restore_multi_tenant/ && docker build -t backup_and_restore_multi_tenant \ --build-arg backend="s3" --build-arg expected_shard_count=150000 . ) +# verify WEAVIATE_VERSION is set +[ -z "$WEAVIATE_VERSION" ] && echo "ERROR: Need to set the WEAVIATE_VERSION env var to the docker tag for the image (e.g.: X.X.X-raft-XXXXXXX)" && exit 1 + export WEAVIATE_NODE_1_VERSION=$WEAVIATE_VERSION export WEAVIATE_NODE_2_VERSION=$WEAVIATE_VERSION From 194bdf386bc4a3e0fecefa98b9e25f530fa6ecb3 Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Fri, 3 May 2024 11:57:48 +0100 Subject: [PATCH 03/10] try on github actions with 60k tenants --- .../backup_and_restore_multi_tenant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index 722195b0..75105abc 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -196,7 +196,7 @@ def restore_backup(client: weaviate.Client, name): backup_name = f"{int(datetime.datetime.now().timestamp())}_stage_1" class_names = ["Class_A", "Class_B"] -num_tenants_hot = 1_000 +num_tenants_hot = 20_000 num_tenants_cold = num_tenants_hot * 2 logger.info(f"Step 0, reset everything, import schema") From 3438a4c041138b2b22de635b906ece3c25bd4a91 Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Fri, 3 May 2024 12:06:42 +0100 Subject: [PATCH 04/10] add comment to trigger new run --- .../backup_and_restore_multi_tenant.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index 75105abc..bdeee8c3 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -196,6 +196,7 @@ def restore_backup(client: weaviate.Client, name): backup_name = f"{int(datetime.datetime.now().timestamp())}_stage_1" class_names = ["Class_A", "Class_B"] +# 60k total tenants num_tenants_hot = 20_000 num_tenants_cold = num_tenants_hot * 2 From 425f54385e554234083c49097ad3f43a5ebd3af2 Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Fri, 3 May 2024 12:36:15 +0100 Subject: [PATCH 05/10] have some retries if getting the backup status fails --- .../backup_and_restore_multi_tenant.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index bdeee8c3..0f94b09e 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -157,15 +157,24 @@ def temp_backup_url_restore_status(backup_name): return f"http://localhost:{WEAVIATE_PORT}/v1/backups/{backend_provider()}/{backup_name}/restore" -def create_backup(client: weaviate.Client, name): +def create_backup(client: weaviate.Client, name, max_num_connection_errors: int = 120): create_body = {"id": name} res = requests.post(temp_backup_url_create(), json=create_body) if res.status_code > 399: fatal(f"Backup Create returned status code {res.status_code} with body: {res.json()}") + num_connection_errors = 0 while True: - time.sleep(3) - res = requests.get(temp_backup_url_create_status(name)) + time.sleep(10) + try: + res = requests.get(temp_backup_url_create_status(name)) + except requests.exceptions.ConnectionError as e: + # sometimes the server becomes unresponsive during the backup but recovers later + num_connection_errors += 1 + if num_connection_errors >= max_num_connection_errors: + raise e + logger.info(f"Failed to get backup status, will keep trying: {e}. {max_num_connection_errors - num_connection_errors} tries left.") + continue res_json = res.json() if res_json["status"] == "SUCCESS": success(f"Backup creation successful") From dc263ff879da69147f4a34b754f88e8e588779ab Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Fri, 3 May 2024 13:00:01 +0100 Subject: [PATCH 06/10] keep trying on restore as well --- .../backup_and_restore_multi_tenant.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index 0f94b09e..994a96b5 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -169,7 +169,7 @@ def create_backup(client: weaviate.Client, name, max_num_connection_errors: int try: res = requests.get(temp_backup_url_create_status(name)) except requests.exceptions.ConnectionError as e: - # sometimes the server becomes unresponsive during the backup but recovers later + # sometimes the server becomes unresponsive during the backup but comes back online later num_connection_errors += 1 if num_connection_errors >= max_num_connection_errors: raise e @@ -183,15 +183,25 @@ def create_backup(client: weaviate.Client, name, max_num_connection_errors: int fatal(f"Backup failed with res: {res_json}") -def restore_backup(client: weaviate.Client, name): +def restore_backup(client: weaviate.Client, name, max_num_connection_errors: int = 120): restore_body = {"id": name} res = requests.post(temp_backup_url_restore(name), json=restore_body) if res.status_code > 399: fatal(f"Backup Restore returned status code {res.status_code} with body: {res.json()}") + num_connection_errors = 0 while True: - time.sleep(3) - res = requests.get(temp_backup_url_restore_status(name)) + time.sleep(10) + try: + res = requests.get(temp_backup_url_restore_status(name)) + except requests.exceptions.ConnectionError as e: + # sometimes the server becomes unresponsive during the recoverery but comes back online later + num_connection_errors += 1 + if num_connection_errors >= max_num_connection_errors: + raise e + logger.info(f"Failed to get restore status, will keep trying: {e}. {max_num_connection_errors - num_connection_errors} tries left.") + continue + res_json = res.json() if res_json["status"] == "SUCCESS": success(f"Restore succeeded") @@ -206,7 +216,7 @@ def restore_backup(client: weaviate.Client, name): class_names = ["Class_A", "Class_B"] # 60k total tenants -num_tenants_hot = 20_000 +num_tenants_hot = 2_000 num_tenants_cold = num_tenants_hot * 2 logger.info(f"Step 0, reset everything, import schema") From aa5838dd9e0daf06bc4a7b6c46958efd1f779cb2 Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Fri, 3 May 2024 13:17:03 +0100 Subject: [PATCH 07/10] try with 3k --- .../backup_and_restore_multi_tenant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index 994a96b5..007203fd 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -216,7 +216,7 @@ def restore_backup(client: weaviate.Client, name, max_num_connection_errors: int class_names = ["Class_A", "Class_B"] # 60k total tenants -num_tenants_hot = 2_000 +num_tenants_hot = 3_000 num_tenants_cold = num_tenants_hot * 2 logger.info(f"Step 0, reset everything, import schema") From 0523c4aba2cc6739a6869ec14bce48b5f726d577 Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Fri, 3 May 2024 13:43:20 +0100 Subject: [PATCH 08/10] try with 5k tenants --- .../backup_and_restore_multi_tenant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index 007203fd..8e2d1a74 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -216,7 +216,7 @@ def restore_backup(client: weaviate.Client, name, max_num_connection_errors: int class_names = ["Class_A", "Class_B"] # 60k total tenants -num_tenants_hot = 3_000 +num_tenants_hot = 5_000 num_tenants_cold = num_tenants_hot * 2 logger.info(f"Step 0, reset everything, import schema") From 54fe55e4ee46915557baa43a4945cf2196280e3c Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Fri, 3 May 2024 14:42:19 +0100 Subject: [PATCH 09/10] try with 30k tenants --- .../backup_and_restore_multi_tenant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index 8e2d1a74..86c3a6e3 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -216,7 +216,7 @@ def restore_backup(client: weaviate.Client, name, max_num_connection_errors: int class_names = ["Class_A", "Class_B"] # 60k total tenants -num_tenants_hot = 5_000 +num_tenants_hot = 10_000 num_tenants_cold = num_tenants_hot * 2 logger.info(f"Step 0, reset everything, import schema") From 6c0f35186005085dbe59e9077c584e34d1898408 Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Fri, 3 May 2024 15:34:21 +0100 Subject: [PATCH 10/10] try with 45k tenants --- .../backup_and_restore_multi_tenant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py index 86c3a6e3..c86d73cd 100644 --- a/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py +++ b/apps/backup_and_restore_multi_tenant/backup_and_restore_multi_tenant.py @@ -216,7 +216,7 @@ def restore_backup(client: weaviate.Client, name, max_num_connection_errors: int class_names = ["Class_A", "Class_B"] # 60k total tenants -num_tenants_hot = 10_000 +num_tenants_hot = 15_000 num_tenants_cold = num_tenants_hot * 2 logger.info(f"Step 0, reset everything, import schema")