Skip to content

Commit

Permalink
First second go
Browse files Browse the repository at this point in the history
  • Loading branch information
i-am-tom committed Jan 15, 2024
1 parent 82454f0 commit cb62c40
Show file tree
Hide file tree
Showing 32 changed files with 201 additions and 201 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/benchmarks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ name: Postgres NDC component benchmarks

permissions:
contents: write
deployments: write
ndc_metadatas: write

jobs:
benchmark:
Expand Down Expand Up @@ -34,7 +34,7 @@ jobs:
cd benchmarks/component
docker compose up --detach --wait postgres grafana
- name: Generate the deployment configuration 🚧
- name: Generate the ndc_metadata configuration 🚧
run: |
set -e -u -o pipefail
cd benchmarks/component
Expand All @@ -43,7 +43,7 @@ jobs:
CONFIGURATION_SERVER_PORT="$(docker compose port agent-configuration 9100 | sed 's/.\+://')"
CONFIGURATION_SERVER="localhost:${CONFIGURATION_SERVER_PORT}"
../../scripts/new-configuration.sh "$CONFIGURATION_SERVER" 'postgresql://postgres:password@postgres' \
| tee ./generated/deployment.json
| tee ./generated/ndc-metadata.json
docker compose down agent-configuration
- name: Run benchmarks 🏃
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/cargo-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,18 +122,18 @@ jobs:
version: "25.x"
repo-token: ${{ secrets.GITHUB_TOKEN }}

- name: setup deployment
- name: setup ndc_metadata
env:
AURORA_CONNECTION_STRING: ${{ secrets.AURORA_CONNECTION_STRING }}
run: |
# take connection string from env, create deployment file with it
cat static/aurora/v1-chinook-deployment-template.json \
# take connection string from env, create ndc_metadata file with it
cat static/aurora/v1-chinook-ndc-metadata-template.json \
| jq '.connectionUri={"uri":{"value":(env | .AURORA_CONNECTION_STRING)}}' \
> static/aurora/v1-chinook-deployment.json
> static/aurora/v1-chinook-ndc-metadata.json
cat static/aurora/v2-chinook-deployment-template.json \
cat static/aurora/v2-chinook-ndc-metadata-template.json \
| jq '.connectionUri={"uri":{"value":(env | .AURORA_CONNECTION_STRING)}}' \
> static/aurora/v2-chinook-deployment.json
> static/aurora/v2-chinook-ndc-metadata.json
- name: install tools
run: |
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/component/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Running `run.sh` with a benchmark name as an argument will:

1. build the PostgreSQL data connector Docker image,
2. start the database with Chinook data,
3. start the agent using an associated deployment, and
3. start the agent using an associated ndc_metadata, and
4. run a benchmark using k6.

Running without arguments will list available benchmarks.
Expand Down
6 changes: 3 additions & 3 deletions benchmarks/component/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,13 @@ services:
ports:
- "8100:8100"
environment:
CONFIGURATION_FILE: "/deployment.json"
CONFIGURATION_FILE: "/ndc-metadata.json"
# we don't care about traces right now, and the benchmarks flood the batch buffer
OTEL_TRACES_SAMPLER: "always_off"
volumes:
- type: bind
source: ./generated/deployment.json
target: /deployment.json
source: ./generated/ndc-metadata.json
target: /ndc-metadata.json
read_only: true
healthcheck:
test:
Expand Down
6 changes: 3 additions & 3 deletions benchmarks/component/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ info 'Starting the dependencies'
docker compose up --wait postgres grafana
POSTGRESQL_SOCKET="$(docker compose port postgres 5432)"

info 'Generating the deployment configuration'
info 'Generating the ndc_metadata configuration'
mkdir -p generated
cargo run -p ndc-postgres --quiet --release -- configuration serve &
AGENT_PID=$!
Expand All @@ -37,7 +37,7 @@ if ! kill -0 "$AGENT_PID"; then
exit 1
fi
../../scripts/new-configuration.sh localhost:9100 "postgresql://postgres:password@${POSTGRESQL_SOCKET}" \
> ./generated/deployment.json
> ./generated/ndc-metadata.json
kill "$AGENT_PID" && wait "$AGENT_PID" || :
rm -f ./agent.pid

Expand All @@ -50,7 +50,7 @@ fi
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT='http://localhost:4317' \
OTEL_SERVICE_NAME='ndc-postgres' \
cargo run -p ndc-postgres --quiet --release -- \
serve --configuration=./generated/deployment.json \
serve --configuration=./generated/ndc-metadata.json \
>& agent.log &
AGENT_PID=$!
echo "$AGENT_PID" > ./agent.pid
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ pub async fn validate_raw_configuration(
Ok(config)
}

/// Construct the deployment configuration by introspecting the database.
/// Construct the ndc_metadata configuration by introspecting the database.
pub async fn configure(
args: RawConfiguration,
) -> Result<RawConfiguration, connector::UpdateConfigurationError> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ pub async fn validate_raw_configuration(
Ok(config)
}

/// Construct the deployment configuration by introspecting the database.
/// Construct the ndc_metadata configuration by introspecting the database.
pub async fn configure(
args: RawConfiguration,
) -> Result<RawConfiguration, connector::UpdateConfigurationError> {
Expand Down
6 changes: 3 additions & 3 deletions crates/tests/databases-tests/src/aurora/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
use std::env;

pub const CHINOOK_DEPLOYMENT_PATH: &str = "static/aurora/v2-chinook-deployment.json";
pub const CHINOOK_NDC_METADATA_PATH: &str = "static/aurora/v2-chinook-ndc-metadata.json";

/// We get our connection string from an env var so that it can be stored in secrets in CI
pub fn get_connection_string() -> String {
env::var("AURORA_CONNECTION_STRING").unwrap()
}

/// Creates a router with a fresh state from the test deployment.
/// Creates a router with a fresh state from the test ndc_metadata.
pub async fn create_router() -> axum::Router {
tests_common::router::create_router(CHINOOK_DEPLOYMENT_PATH).await
tests_common::router::create_router(CHINOOK_NDC_METADATA_PATH).await
}
6 changes: 3 additions & 3 deletions crates/tests/databases-tests/src/citus/common.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
//! Common functions used across test cases.
pub const CHINOOK_DEPLOYMENT_PATH: &str = "static/citus/v2-chinook-deployment.json";
pub const CHINOOK_NDC_METADATA_PATH: &str = "static/citus/v2-chinook-ndc-metadata.json";

pub const CONNECTION_STRING: &str =
"postgresql://postgres:password@localhost:64004?sslmode=disable";

/// Creates a router with a fresh state from the test deployment.
/// Creates a router with a fresh state from the test ndc_metadata.
pub async fn create_router() -> axum::Router {
tests_common::router::create_router(CHINOOK_DEPLOYMENT_PATH).await
tests_common::router::create_router(CHINOOK_NDC_METADATA_PATH).await
}
4 changes: 2 additions & 2 deletions crates/tests/databases-tests/src/citus/configuration_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ mod configuration_tests {
async fn test_configure_is_idempotent() {
common_tests::configuration_v2_tests::configure_is_idempotent(
common::CONNECTION_STRING,
common::CHINOOK_DEPLOYMENT_PATH,
common::CHINOOK_NDC_METADATA_PATH,
)
.await
}

#[test]
fn configuration_conforms_to_the_schema() {
common_tests::configuration_v2_tests::configuration_conforms_to_the_schema(
common::CHINOOK_DEPLOYMENT_PATH,
common::CHINOOK_NDC_METADATA_PATH,
)
}
}
6 changes: 3 additions & 3 deletions crates/tests/databases-tests/src/cockroach/common.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
//! Common functions used across test cases.
pub const CHINOOK_DEPLOYMENT_PATH: &str = "static/cockroach/v2-chinook-deployment.json";
pub const CHINOOK_NDC_METADATA_PATH: &str = "static/cockroach/v2-chinook-ndc-metadata.json";

pub const CONNECTION_STRING: &str = "postgresql://postgres:password@localhost:64003/defaultdb";

/// Creates a router with a fresh state from the test deployment.
/// Creates a router with a fresh state from the test ndc_metadata.
pub async fn create_router() -> axum::Router {
tests_common::router::create_router(CHINOOK_DEPLOYMENT_PATH).await
tests_common::router::create_router(CHINOOK_NDC_METADATA_PATH).await
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ mod configuration_tests {
async fn test_configure_is_idempotent() {
common_tests::configuration_v2_tests::configure_is_idempotent(
common::CONNECTION_STRING,
common::CHINOOK_DEPLOYMENT_PATH,
common::CHINOOK_NDC_METADATA_PATH,
)
.await
}

#[test]
fn configuration_conforms_to_the_schema() {
common_tests::configuration_v2_tests::configuration_conforms_to_the_schema(
common::CHINOOK_DEPLOYMENT_PATH,
common::CHINOOK_NDC_METADATA_PATH,
)
}
}
14 changes: 7 additions & 7 deletions crates/tests/databases-tests/src/deployment_snapshot_tests.rs
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
#[cfg(test)]
mod deployment_snapshots {
mod ndc_metadata_snapshots {
use ndc_postgres::configuration;
use std::fs;
use std::path::PathBuf;

// each time we run `just generate-chinook-configuration` we save the old
// Postgres deployment in `static/deployment-snapshots`. This test parses each snapshot to
// Postgres ndc_metadata in `static/ndc-metadata-snapshots`. This test parses each snapshot to
// ensure we are still able to understand old versions
#[test_each::path(glob = "static/deployment-snapshots/*.json", name(segments = 2))]
fn test_snapshot(deployment_path: PathBuf) {
let file = fs::File::open(deployment_path).expect("fs::File::open");
#[test_each::path(glob = "static/ndc-metadata-snapshots/*.json", name(segments = 2))]
fn test_snapshot(ndc_metadata_path: PathBuf) {
let file = fs::File::open(ndc_metadata_path).expect("fs::File::open");

let deployment_json_value: serde_json::Value =
let ndc_metadata_json_value: serde_json::Value =
serde_json::from_reader(file).expect("serde_json::from_reader");

let _decoded_configuration: configuration::RawConfiguration =
serde_json::from_value(deployment_json_value.clone())
serde_json::from_value(ndc_metadata_json_value.clone())
.expect("Unable to deserialize as RawConfiguration");
}
}
2 changes: 1 addition & 1 deletion crates/tests/databases-tests/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ pub mod cockroach;
pub mod postgres;

pub mod capabilities_tests;
pub mod deployment_snapshot_tests;
pub mod ndc_metadata_snapshot_tests;
8 changes: 4 additions & 4 deletions crates/tests/databases-tests/src/postgres/common.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
//! Common functions used across test cases.
pub const CHINOOK_DEPLOYMENT_PATH_V2: &str = "static/postgres/v2-chinook-deployment.json";
pub const CHINOOK_DEPLOYMENT_PATH_V1: &str = "static/postgres/v1-chinook-deployment.json";
pub const CHINOOK_NDC_METADATA_PATH_V2: &str = "static/postgres/v2-chinook-ndc-metadata.json";
pub const CHINOOK_NDC_METADATA_PATH_V1: &str = "static/postgres/v1-chinook-ndc-metadata.json";

pub const CONNECTION_STRING: &str = "postgresql://postgres:password@localhost:64002";

/// Creates a router with a fresh state from the test deployment.
/// Creates a router with a fresh state from the test ndc_metadata.
pub async fn create_router() -> axum::Router {
tests_common::router::create_router(CHINOOK_DEPLOYMENT_PATH_V2).await
tests_common::router::create_router(CHINOOK_NDC_METADATA_PATH_V2).await
}
10 changes: 5 additions & 5 deletions crates/tests/databases-tests/src/postgres/configuration_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
//! The github CI setup runs these tests subject to the filtering logic in
//! '.github/test-configuration.json'. Naming a test with the prefix 'postgres_current_only` will
//! ensure they only run on the latest version of postgres being tested. This is necessary because
//! they rely on supporting data (the chinook deployment configuration) which we maintain only for
//! they rely on supporting data (the chinook ndc_metadata configuration) which we maintain only for
//! the latest version.
#[cfg(test)]
Expand All @@ -25,15 +25,15 @@ mod configuration_tests {
async fn postgres_current_only_configure_v2_is_idempotent() {
common_tests::configuration_v2_tests::configure_is_idempotent(
common::CONNECTION_STRING,
common::CHINOOK_DEPLOYMENT_PATH_V2,
common::CHINOOK_NDC_METADATA_PATH_V2,
)
.await
}

#[test]
fn configuration_v2_conforms_to_the_schema() {
common_tests::configuration_v2_tests::configuration_conforms_to_the_schema(
common::CHINOOK_DEPLOYMENT_PATH_V2,
common::CHINOOK_NDC_METADATA_PATH_V2,
)
}

Expand All @@ -60,15 +60,15 @@ mod configuration_tests {
async fn postgres_current_only_configure_v1_is_idempotent() {
common_tests::configuration_v1_tests::configure_is_idempotent(
common::CONNECTION_STRING,
common::CHINOOK_DEPLOYMENT_PATH_V1,
common::CHINOOK_NDC_METADATA_PATH_V1,
)
.await
}

#[test]
fn configuration_v1_conforms_to_the_schema() {
common_tests::configuration_v1_tests::configuration_conforms_to_the_schema(
common::CHINOOK_DEPLOYMENT_PATH_V1,
common::CHINOOK_NDC_METADATA_PATH_V1,
)
}

Expand Down
Loading

0 comments on commit cb62c40

Please sign in to comment.