From 0702818e2efbc33ac013830c3cfb3e622873a136 Mon Sep 17 00:00:00 2001 From: Natoandro Date: Sat, 28 Oct 2023 17:24:38 +0300 Subject: [PATCH] feat(prisma/migrations): Default value on new column (#465) - [x] Display a more concise error message for new column that failed the NON NULL constraint during the migration. - [x] Enable user to set default value on new NON NULL column. --------- Signed-off-by: Natoandro Signed-off-by: Teo Stocco Co-authored-by: Teo Stocco --- libs/common/src/typegraph/runtimes/prisma.rs | 1 + typegate/deno.json | 5 +- typegate/deno.lock | 87 +++++++++ .../runtimes/prisma/hooks/generate_schema.ts | 4 + .../runtimes/prisma/hooks/run_migrations.ts | 26 ++- typegate/src/typegraph/types.ts | 1 + typegate/tests/e2e/cli/.gitignore | 1 + .../deploy_test.ts} | 167 +++++++++++++++++- typegate/tests/e2e/cli/select.sh | 9 + typegate/tests/e2e/cli/templates/migration.py | 20 +++ .../tests/e2e/typegraph/typegraph_test.ts | 7 +- typegate/tests/metatype.yml | 8 + .../tests/runtimes/temporal/temporal_test.ts | 6 +- typegate/tests/utils/meta.ts | 15 +- typegate/tests/utils/migrations.ts | 5 +- typegate/tests/utils/shell.ts | 50 ++++-- typegate/tests/utils/test.ts | 2 +- typegate/tests/utils/test_module.ts | 25 +++ typegraph/core/src/errors.rs | 2 +- typegraph/core/src/runtimes/mod.rs | 2 +- typegraph/core/src/runtimes/prisma/context.rs | 1 + typegraph/core/src/runtimes/prisma/model.rs | 12 ++ typegraph/core/src/validation/types.rs | 10 +- website/static/specs/0.0.3.json | 3 +- 24 files changed, 427 insertions(+), 42 deletions(-) create mode 100644 typegate/tests/e2e/cli/.gitignore rename typegate/tests/e2e/{cli_deploy_test.ts => cli/deploy_test.ts} (51%) create mode 100755 typegate/tests/e2e/cli/select.sh create mode 100644 typegate/tests/e2e/cli/templates/migration.py create mode 100644 typegate/tests/utils/test_module.ts diff --git a/libs/common/src/typegraph/runtimes/prisma.rs b/libs/common/src/typegraph/runtimes/prisma.rs index 1e2edc3a78..1eb1e144c8 100644 --- a/libs/common/src/typegraph/runtimes/prisma.rs +++ b/libs/common/src/typegraph/runtimes/prisma.rs @@ -90,6 +90,7 @@ pub struct ScalarProperty { pub injection: Option, pub unique: bool, pub auto: bool, + pub default_value: Option, } #[cfg_attr(feature = "codegen", derive(JsonSchema))] diff --git a/typegate/deno.json b/typegate/deno.json index b2e5d1c235..1eb9c53f80 100644 --- a/typegate/deno.json +++ b/typegate/deno.json @@ -8,7 +8,7 @@ "comment1": "echo cwd is by default the directory of deno.json", "comment2": "echo cannot restrict ffi to a lib https://github.com/denoland/deno/issues/15511", "run": "cd .. && deno run --config=typegate/deno.json --unstable --allow-run=hostname --allow-sys --allow-env --allow-hrtime --allow-write=tmp --allow-ffi --allow-read=. --allow-net typegate/src/main.ts", - "test": "cd .. && deno test --trace-ops --config=typegate/deno.json --unstable --allow-run=cargo,hostname,target/debug/meta,git,python3,rm,mkdir --allow-sys --allow-env --allow-hrtime --allow-write=tmp,typegate/tests --allow-ffi --allow-read=. --allow-net" + "test": "cd .. && deno test --trace-ops --config=typegate/deno.json --unstable --allow-run=cargo,hostname,target/debug/meta,git,python3,rm,mkdir,bash --allow-sys --allow-env --allow-hrtime --allow-write=tmp,typegate/tests --allow-ffi --allow-read=. --allow-net" }, "nodeModulesDir": false, "lock": "deno.lock", @@ -36,6 +36,7 @@ "outdent": "https://deno.land/x/outdent@v0.8.0/mod.ts", "json-schema-faker": "npm:json-schema-faker@0.5.3", "ajv": "https://esm.sh/ajv@8.12.0?pin=v131", - "@typegraph/sdk/": "../typegraph/deno/src/" + "@typegraph/sdk/": "../typegraph/deno/src/", + "test-utils/": "./tests/utils/" } } diff --git a/typegate/deno.lock b/typegate/deno.lock index 89332f52a3..3e8f6fec60 100644 --- a/typegate/deno.lock +++ b/typegate/deno.lock @@ -9,6 +9,7 @@ "npm:json-schema-faker@0.5.3": "npm:json-schema-faker@0.5.3", "npm:lodash": "npm:lodash@4.17.21", "npm:mathjs@11.11.1": "npm:mathjs@11.11.1", + "npm:pg": "npm:pg@8.11.3", "npm:validator": "npm:validator@13.11.0" }, "npm": { @@ -75,6 +76,10 @@ "sprintf-js": "sprintf-js@1.0.3" } }, + "buffer-writer@2.0.0": { + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "dependencies": {} + }, "call-me-maybe@1.0.2": { "integrity": "sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==", "dependencies": {} @@ -190,6 +195,79 @@ "format-util": "format-util@1.0.5" } }, + "packet-reader@1.0.0": { + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==", + "dependencies": {} + }, + "pg-cloudflare@1.1.1": { + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "dependencies": {} + }, + "pg-connection-string@2.6.2": { + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==", + "dependencies": {} + }, + "pg-int8@1.0.1": { + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "dependencies": {} + }, + "pg-pool@3.6.1_pg@8.11.3": { + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "dependencies": { + "pg": "pg@8.11.3" + } + }, + "pg-protocol@1.6.0": { + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==", + "dependencies": {} + }, + "pg-types@2.2.0": { + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "pg-int8@1.0.1", + "postgres-array": "postgres-array@2.0.0", + "postgres-bytea": "postgres-bytea@1.0.0", + "postgres-date": "postgres-date@1.0.7", + "postgres-interval": "postgres-interval@1.2.0" + } + }, + "pg@8.11.3": { + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "dependencies": { + "buffer-writer": "buffer-writer@2.0.0", + "packet-reader": "packet-reader@1.0.0", + "pg-cloudflare": "pg-cloudflare@1.1.1", + "pg-connection-string": "pg-connection-string@2.6.2", + "pg-pool": "pg-pool@3.6.1_pg@8.11.3", + "pg-protocol": "pg-protocol@1.6.0", + "pg-types": "pg-types@2.2.0", + "pgpass": "pgpass@1.0.5" + } + }, + "pgpass@1.0.5": { + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "split2@4.2.0" + } + }, + "postgres-array@2.0.0": { + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "dependencies": {} + }, + "postgres-bytea@1.0.0": { + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "dependencies": {} + }, + "postgres-date@1.0.7": { + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "dependencies": {} + }, + "postgres-interval@1.2.0": { + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "xtend@4.0.2" + } + }, "regenerator-runtime@0.14.0": { "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==", "dependencies": {} @@ -198,6 +276,10 @@ "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", "dependencies": {} }, + "split2@4.2.0": { + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "dependencies": {} + }, "sprintf-js@1.0.3": { "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dependencies": {} @@ -217,6 +299,10 @@ "validator@13.11.0": { "integrity": "sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ==", "dependencies": {} + }, + "xtend@4.0.2": { + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dependencies": {} } } }, @@ -525,6 +611,7 @@ "https://deno.land/std@0.202.0/streams/write_all.ts": "4cdd36256f892fe7aead46338054f6ea813a63765e87bda4c60e8c5a57d1c5c1", "https://deno.land/std@0.202.0/streams/writer_from_stream_writer.ts": "7188ce589d6179693b488b478c05137d4d15b93735ca26ec01e6e44aed8cb0c6", "https://deno.land/std@0.202.0/streams/zip_readable_streams.ts": "5639c8fea8c21d7dab6f34edcf3d08218b7e548a197f7fd79a3a995305a81e9f", + "https://deno.land/std@0.202.0/testing/asserts.ts": "b4e4b1359393aeff09e853e27901a982c685cb630df30426ed75496961931946", "https://deno.land/std@0.202.0/testing/snapshot.ts": "d53cc4ad3250e3a826df9a1a90bc19c9a92c8faa8fd508d16b5e6ce8699310ca", "https://deno.land/std@0.202.0/uuid/_common.ts": "cb1441f4df460571fc0919e1c5c217f3e7006189b703caf946604b3f791ae34d", "https://deno.land/std@0.202.0/uuid/constants.ts": "0d0e95561343da44adb4a4edbc1f04cef48b0d75288c4d1704f58743f4a50d88", diff --git a/typegate/src/runtimes/prisma/hooks/generate_schema.ts b/typegate/src/runtimes/prisma/hooks/generate_schema.ts index 1bc260903c..0ff2df866b 100644 --- a/typegate/src/runtimes/prisma/hooks/generate_schema.ts +++ b/typegate/src/runtimes/prisma/hooks/generate_schema.ts @@ -122,6 +122,10 @@ class FieldBuilder { } } + if (prop.defaultValue != null) { + tags.push(`@default(${JSON.stringify(prop.defaultValue)})`); + } + const field = new ModelField(prop.key, typeName + quant, tags); return field; } diff --git a/typegate/src/runtimes/prisma/hooks/run_migrations.ts b/typegate/src/runtimes/prisma/hooks/run_migrations.ts index a1d8ab82a9..92effd8dff 100644 --- a/typegate/src/runtimes/prisma/hooks/run_migrations.ts +++ b/typegate/src/runtimes/prisma/hooks/run_migrations.ts @@ -7,6 +7,9 @@ import { PrismaRT } from "../mod.ts"; import * as native from "native"; import { nativeResult, pluralSuffix } from "../../../utils.ts"; +const NULL_CONSTRAINT_ERROR_REGEX = + /column (?".+") of relation (?".+") contains null values/; + export const runMigrations: PushHandler = async ( typegraph, secretManager, @@ -51,7 +54,7 @@ export const runMigrations: PushHandler = async ( const { reset_reason, applied_migrations } = applyRes.Ok; if (reset_reason != null) { - response.info(`Database reset: {reset_reason}`); + response.info(`Database reset: ${reset_reason}`); } if (applied_migrations.length === 0) { response.info(`${prefix} No migration applied.`); @@ -90,7 +93,26 @@ export const runMigrations: PushHandler = async ( ); if (apply_err != null) { - response.error(apply_err); + const errors = apply_err.split(/\r?\n/).filter( + (line) => line.startsWith("ERROR: "), + ) + .map((line) => line.slice("ERROR: ".length)) + .map((err) => { + const match = NULL_CONSTRAINT_ERROR_REGEX.exec(err); + if (match != null) { + // TODO detect used for the typegraph language and write + // the message accordingly. + return `${err}: set a default value: add \`config={ "default": defaultValue }\` attribute to the type.`; + } + return err; + }); + + if (errors.length === 0) { + response.error(apply_err); + } else { + const formattedErrors = errors.map((err) => `\n- ${err}`).join(""); + response.error(`Could not apply migration: ${formattedErrors}`); + } } if (created_migration_name != null) { diff --git a/typegate/src/typegraph/types.ts b/typegate/src/typegraph/types.ts index 36e887195e..2d3028e135 100644 --- a/typegate/src/typegraph/types.ts +++ b/typegate/src/typegraph/types.ts @@ -287,6 +287,7 @@ export type Property = { injection?: ManagedInjection | null; unique: boolean; auto: boolean; + defaultValue?: any; } | { type: "relationship"; key: string; diff --git a/typegate/tests/e2e/cli/.gitignore b/typegate/tests/e2e/cli/.gitignore new file mode 100644 index 0000000000..1e0b411e72 --- /dev/null +++ b/typegate/tests/e2e/cli/.gitignore @@ -0,0 +1 @@ +/migration.py diff --git a/typegate/tests/e2e/cli_deploy_test.ts b/typegate/tests/e2e/cli/deploy_test.ts similarity index 51% rename from typegate/tests/e2e/cli_deploy_test.ts rename to typegate/tests/e2e/cli/deploy_test.ts index fde4022ee9..8d57e039a4 100644 --- a/typegate/tests/e2e/cli_deploy_test.ts +++ b/typegate/tests/e2e/cli/deploy_test.ts @@ -1,14 +1,173 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -import { gql } from "../utils/mod.ts"; +import { gql, Meta } from "test-utils/mod.ts"; +import { TestModule } from "test-utils/test_module.ts"; +import { dropSchemas, removeMigrations } from "test-utils/migrations.ts"; +import { assertStringIncludes } from "std/assert/mod.ts"; import { assertRejects } from "std/assert/mod.ts"; -import { Meta } from "../utils/mod.ts"; -import { dropSchemas, removeMigrations } from "../utils/migrations.ts"; -import { shell } from "../utils/shell.ts"; +import { shell } from "test-utils/shell.ts"; +import pg from "npm:pg"; + +const m = new TestModule(import.meta); const port = 7895; +const tgName = "migration-failure-test"; + +async function writeTypegraph(version: number | null) { + if (version == null) { + await m.shell([ + "bash", + "-c", + "cp ./templates/migration.py migration.py", + ]); + } else { + await m.shell([ + "bash", + "select.sh", + "templates/migration.py", + `${version}`, + "migration.py", + ]); + } +} + +async function deploy(noMigration = false) { + const migrationOpts = noMigration ? [] : ["--create-migration"]; + + try { + const out = await m.cli( + {}, + "deploy", + "-t", + "deploy", + "-f", + "migration.py", + "--allow-dirty", + ...migrationOpts, + "--allow-destructive", + ); + if (out.stdout.length > 0) { + console.log( + `-- deploy STDOUT start --\n${out.stdout}-- deploy STDOUT end --`, + ); + } + if (out.stderr.length > 0) { + console.log( + `-- deploy STDERR start --\n${out.stderr}-- deploy STDERR end --`, + ); + } + } catch (e) { + console.log(e.toString()); + throw e; + } +} + +async function reset() { + await removeMigrations(tgName); + + // remove the database schema + const client = new pg.Client({ + connectionString: "postgres://postgres:password@localhost:5432/db", + }); + await client.connect(); + await client.query("DROP SCHEMA IF EXISTS e2e2 CASCADE"); + await client.end(); +} + +Meta.test( + "meta deploy: fails migration for new columns without default value", + async (t) => { + await t.should("load first version of the typegraph", async () => { + await reset(); + await writeTypegraph(null); + }); + + // `deploy` must be run outside of the `should` block, + // otherwise this would fail by leaking ops. + // That is expected since it creates new engine that persists beyond the + // `should` block. + await deploy(); + + await t.should("insert records", async () => { + const e = t.getTypegraphEngine(tgName); + if (!e) { + throw new Error("typegraph not found"); + } + await gql` + mutation { + createRecord(data: {}) { + id + } + } + ` + .expectData({ + createRecord: { + id: 1, + }, + }) + .on(e); + }); + + await t.should("load second version of the typegraph", async () => { + await writeTypegraph(1); + }); + + try { + await deploy(); + } catch (e) { + assertStringIncludes( + e.message, + 'column "age" of relation "Record" contains null values: set a default value:', + ); + } + }, + { port, systemTypegraphs: true }, +); + +Meta.test( + "meta deploy: succeeds migration for new columns with default value", + async (t) => { + await t.should("load first version of the typegraph", async () => { + await reset(); + await writeTypegraph(null); + }); + + await deploy(); + + await t.should("insert records", async () => { + const e = t.getTypegraphEngine(tgName)!; + await gql` + mutation { + createRecord(data: {}) { + id + } + } + ` + .expectData({ + createRecord: { + id: 1, + }, + }) + .on(e); + }); + + await t.should("load second version of the typegraph", async () => { + await writeTypegraph(3); // int + }); + + await deploy(); + + await t.should("load third version of the typegraph", async () => { + await writeTypegraph(4); // string + }); + + await deploy(); + }, + { port, systemTypegraphs: true }, +); + Meta.test("cli:deploy - automatic migrations", async (t) => { const e = await t.engine("runtimes/prisma/prisma.py", { secrets: { diff --git a/typegate/tests/e2e/cli/select.sh b/typegate/tests/e2e/cli/select.sh new file mode 100755 index 0000000000..ce625423c9 --- /dev/null +++ b/typegate/tests/e2e/cli/select.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# select option +# usage: +# > ./select.sh