From bbdfd3555993dc81bbfdf67a220d225687a61bb8 Mon Sep 17 00:00:00 2001 From: Estifanos Bireda <77430541+destifo@users.noreply.github.com> Date: Wed, 15 May 2024 12:16:59 +0300 Subject: [PATCH] fix(SDK): Artifact upload fails when same file referred multiple times (#715) - [x] fix the bug where duplicate artifact references causing failure during artifact resolution(typegate) during runtime. - [x] add sync mode tests for Python and Deno runtime. - [x] add other edge test cases to artifact upload. - [x] test for no artifact in typegraph - [x] test for duplicate artifact reference in the same typegraph Issue: [MET-501](https://linear.app/metatypedev/issue/MET-501/bug-artifact-upload-fails-when-same-file-refered-twice) #### Migration notes _No Migrations Needed_ ... ## Summary by CodeRabbit - **New Features** - Introduced testing functionalities for Deno and Python runtimes, supporting artifact deployment and policy enforcement. - Added Python and Deno scripts for deploying Typegraphs and handling runtime configurations. - **Bug Fixes** - Improved artifact handling and deployment logic in both Python and Deno runtimes. - **Tests** - Added comprehensive test cases for Deno and Python runtimes, including scenarios for duplicate artifacts, no artifacts, and runtime synchronization. - Enhanced testing with integration of various services like Redis and S3. - **Documentation** - Updated test scripts to include detailed comments and summaries for new functionalities and test scenarios. --------- Signed-off-by: Teo Stocco Co-authored-by: Teo Stocco Co-authored-by: Teo Stocco Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../runtimes/deno/deno_duplicate_artifact.py | 68 ++ .../runtimes/deno/deno_duplicate_artifact.ts | 33 + .../tests/runtimes/deno/deno_no_artifact.py | 59 ++ .../tests/runtimes/deno/deno_sync_test.ts | 595 +++++++++++++++ typegate/tests/runtimes/deno/deno_test.ts | 96 +++ .../python/python_duplicate_artifact.py | 67 ++ .../python/python_duplicate_artifact.ts | 39 + .../runtimes/python/python_no_artifact.py | 58 ++ .../runtimes/python/python_no_artifact.ts | 34 + .../tests/runtimes/python/python_sync_test.ts | 698 ++++++++++++++++++ typegate/tests/runtimes/python/python_test.ts | 179 +++++ .../core/src/utils/postprocess/python_rt.rs | 47 +- 12 files changed, 1949 insertions(+), 24 deletions(-) create mode 100644 typegate/tests/runtimes/deno/deno_duplicate_artifact.py create mode 100644 typegate/tests/runtimes/deno/deno_duplicate_artifact.ts create mode 100644 typegate/tests/runtimes/deno/deno_no_artifact.py create mode 100644 typegate/tests/runtimes/deno/deno_sync_test.ts create mode 100644 typegate/tests/runtimes/python/python_duplicate_artifact.py create mode 100644 typegate/tests/runtimes/python/python_duplicate_artifact.ts create mode 100644 typegate/tests/runtimes/python/python_no_artifact.py create mode 100644 typegate/tests/runtimes/python/python_no_artifact.ts create mode 100644 typegate/tests/runtimes/python/python_sync_test.ts diff --git a/typegate/tests/runtimes/deno/deno_duplicate_artifact.py b/typegate/tests/runtimes/deno/deno_duplicate_artifact.py new file mode 100644 index 0000000000..132605674b --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_duplicate_artifact.py @@ -0,0 +1,68 @@ +import os +import sys + +from typegraph.gen.exports.core import ( + ArtifactResolutionConfig, + MigrationAction, + MigrationConfig, +) +from typegraph.graph.shared_types import BasicAuth +from typegraph.graph.tg_deploy import TypegraphDeployParams, tg_deploy +from typegraph.graph.typegraph import Graph +from typegraph.policy import Policy +from typegraph.runtimes.deno import DenoRuntime + +from typegraph import t, typegraph + + +@typegraph() +def deno_duplicate_artifact(g: Graph): + deno = DenoRuntime() + public = Policy.public() + + g.expose( + public, + doAddition=deno.import_( + t.struct({"a": t.float(), "b": t.float()}), + t.float(), + module="ts/dep/main.ts", + deps=["ts/dep/nested/dep.ts"], + name="doAddition", + ), + doAdditionDuplicate=deno.import_( + t.struct({"a": t.float(), "b": t.float()}), + t.float(), + module="ts/dep/main.ts", + deps=["ts/dep/nested/dep.ts"], + name="doAddition", + ), + ) + + +cwd = sys.argv[1] +PORT = sys.argv[2] +gate = f"http://localhost:{PORT}" +auth = BasicAuth("admin", "password") + +deno_tg = deno_duplicate_artifact() +deploy_result = tg_deploy( + deno_tg, + TypegraphDeployParams( + base_url=gate, + auth=auth, + typegraph_path=os.path.join(cwd, "deno_duplicate_artifact.py"), + artifacts_config=ArtifactResolutionConfig( + dir=cwd, + prefix=None, + disable_artifact_resolution=None, + codegen=None, + prisma_migration=MigrationConfig( + migration_dir="prisma-migrations", + global_action=MigrationAction(reset=False, create=True), + runtime_actions=None, + ), + ), + ), +) + +print(deploy_result.serialized) diff --git a/typegate/tests/runtimes/deno/deno_duplicate_artifact.ts b/typegate/tests/runtimes/deno/deno_duplicate_artifact.ts new file mode 100644 index 0000000000..a96219148f --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_duplicate_artifact.ts @@ -0,0 +1,33 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { Policy, t, typegraph } from "@typegraph/sdk/index.js"; +import { DenoRuntime } from "@typegraph/sdk/runtimes/deno.js"; + +export const denoDuplicateArtifact = await typegraph({ + name: "test_deno_dep", +}, (g) => { + const deno = new DenoRuntime(); + const pub = Policy.public(); + + g.expose({ + doAddition: deno.import( + t.struct({ a: t.float(), b: t.float() }), + t.float(), + { + module: "ts/dep/main.ts", + name: "doAddition", + deps: ["ts/dep/nested/dep.ts"], + }, + ).withPolicy(pub), + doAdditionDuplicate: deno.import( + t.struct({ a: t.float(), b: t.float() }), + t.float(), + { + module: "ts/dep/main.ts", + name: "doAddition", + deps: ["ts/dep/nested/dep.ts"], + }, + ).withPolicy(pub), + }); +}); diff --git a/typegate/tests/runtimes/deno/deno_no_artifact.py b/typegate/tests/runtimes/deno/deno_no_artifact.py new file mode 100644 index 0000000000..be6553ffa3 --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_no_artifact.py @@ -0,0 +1,59 @@ +import os +import sys + +from typegraph.gen.exports.core import ( + ArtifactResolutionConfig, + MigrationAction, + MigrationConfig, +) +from typegraph.graph.shared_types import BasicAuth +from typegraph.graph.tg_deploy import TypegraphDeployParams, tg_deploy +from typegraph.graph.typegraph import Graph +from typegraph.policy import Policy +from typegraph.runtimes.deno import DenoRuntime + +from typegraph import t, typegraph + + +@typegraph() +def deno_no_artifact(g: Graph): + deno = DenoRuntime() + public = Policy.public() + + g.expose( + public, + simple=deno.func( + t.struct({"a": t.float(), "b": t.float()}), + t.float(), + code="({ a, b }) => a + b", + ), + ) + + +cwd = sys.argv[1] +PORT = sys.argv[2] +gate = f"http://localhost:{PORT}" +auth = BasicAuth("admin", "password") + +deno_tg = deno_no_artifact() +deploy_result = tg_deploy( + deno_tg, + TypegraphDeployParams( + base_url=gate, + auth=auth, + typegraph_path=os.path.join(cwd, "deno_no_artifact.py"), + artifacts_config=ArtifactResolutionConfig( + dir=cwd, + prefix=None, + disable_artifact_resolution=None, + codegen=None, + prisma_migration=MigrationConfig( + migration_dir="prisma-migrations", + global_action=MigrationAction(reset=False, create=True), + runtime_actions=None, + ), + ), + ), +) + +print(deploy_result.serialized) diff --git a/typegate/tests/runtimes/deno/deno_sync_test.ts b/typegate/tests/runtimes/deno/deno_sync_test.ts new file mode 100644 index 0000000000..4325bf3b40 --- /dev/null +++ b/typegate/tests/runtimes/deno/deno_sync_test.ts @@ -0,0 +1,595 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { gql, Meta, sleep } from "../../utils/mod.ts"; +import * as path from "std/path/mod.ts"; +import { testDir } from "test-utils/dir.ts"; +import { denoDepTg } from "./deno_dep.mjs"; +import { BasicAuth, tgDeploy, tgRemove } from "@typegraph/sdk/tg_deploy.js"; +import { connect } from "redis"; +import { S3Client } from "aws-sdk/client-s3"; +import { createBucket, tryDeleteBucket } from "test-utils/s3.ts"; + +const redisKey = "typegraph"; +const redisEventKey = "typegraph_event"; + +async function cleanUp() { + using redis = await connect(syncConfig.redis); + await redis.del(redisKey); + await redis.del(redisEventKey); + + const s3 = new S3Client(syncConfig.s3); + await tryDeleteBucket(s3, syncConfig.s3Bucket); + await createBucket(s3, syncConfig.s3Bucket); + s3.destroy(); + await redis.quit(); +} + +const syncConfig = { + redis: { + hostname: "localhost", + port: 6379, + password: "password", + db: 1, + }, + s3: { + endpoint: "http://localhost:9000", + region: "local", + credentials: { + accessKeyId: "minio", + secretAccessKey: "password", + }, + forcePathStyle: true, + }, + s3Bucket: "metatype-deno-runtime-sync-test", +}; + +const cwd = path.join(testDir, "runtimes/deno"); +const auth = new BasicAuth("admin", "password"); + +const localSerializedMemo = denoDepTg.serialize({ + prismaMigration: { + globalAction: { + create: true, + reset: false, + }, + migrationDir: "prisma-migrations", + }, + dir: cwd, +}); +const reusableTgOutput = { + ...denoDepTg, + serialize: (_: any) => localSerializedMemo, +}; + +Meta.test( + { + name: "Deno runtime - Python SDK: in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deploy_deno.py", + cwd, + ); + + await t.should("work on the default worker", async () => { + await gql` + query { + add(first: 1.2, second: 2.3) + } + ` + .expectData({ + add: 3.5, + }) + .on(e); + }); + + await t.should("work on a worker runtime", async () => { + await gql` + query { + sum(numbers: [1, 2, 3, 4]) + } + ` + .expectData({ + sum: 10, + }) + .on(e); + }); + + await t.should("work with global variables in a module", async () => { + await gql` + mutation { + count + } + ` + .expectData({ + count: 1, + }) + .on(e); + + await gql` + mutation { + count + } + ` + .expectData({ + count: 2, + }) + .on(e); + }); + + await t.should("work with async function", async () => { + await gql` + query { + max(numbers: [1, 2, 3, 4]) + } + ` + .expectData({ + max: 4, + }) + .on(e); + }); + + await t.should("work with static materializer", async () => { + await gql` + query { + static { + x + } + } + ` + .expectData({ + static: { + x: [1], + }, + }) + .on(e); + }); + }, +); + +Meta.test( + { + name: "Deno runtime - Python SDK: file name reloading in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deploy_deno.py", + cwd, + ); + + await t.should("success for allowed network access", async () => { + await gql` + query { + min(numbers: [2.5, 1.2, 4, 3]) + } + ` + .expectData({ + min: 1.2, + }) + .on(e); + }); + + await t.should("work with npm packages", async () => { + await gql` + query { + log(number: 10000, base: 10) + } + ` + .expectData({ + log: 4, + }) + .on(e); + }); + }, +); + +Meta.test( + { + name: "Deno runtime - Python SDK: use local imports in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deno_dep.py", + cwd, + ); + await t.should("work for local imports", async () => { + await gql` + query { + doAddition(a: 1, b: 2) + } + ` + .expectData({ + doAddition: 3, + }) + .on(e); + }); + }, +); + +Meta.test( + { + name: "DenoRuntime - TS SDK: artifacts and deps in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (metaTest) => { + const port = metaTest.port; + const gate = `http://localhost:${port}`; + + const { serialized, typegate: _gateResponseAdd } = await tgDeploy( + reusableTgOutput, + { + baseUrl: gate, + auth, + artifactsConfig: { + prismaMigration: { + globalAction: { + create: true, + reset: false, + }, + migrationDir: "prisma-migrations", + }, + dir: cwd, + }, + typegraphPath: path.join(cwd, "deno_dep.ts"), + secrets: {}, + }, + ); + + const engine = await metaTest.engineFromDeployed(serialized); + + await metaTest.should("work after artifact upload", async () => { + await gql` + query { + doAddition(a: 1, b: 2) + } + ` + .expectData({ + doAddition: 3, + }) + .on(engine); + }); + + const { typegate: _gateResponseRem } = await tgRemove(reusableTgOutput, { + baseUrl: gate, + auth, + }); + }, +); + +// Meta.test( +// { +// name: "DenoRuntime - Python SDK: multiple typegate instances in sync mode", +// replicas: 3, +// syncConfig, +// async setup() { +// await cleanUp(); +// }, +// async teardown() { +// await cleanUp(); +// }, +// }, +// async (metaTest) => { +// const testMultipleReplica = async (instanceNumber: number) => { +// const e = await metaTest.engineFromTgDeployPython( +// "runtimes/deno/deno_dep.py", +// cwd, +// ); + +// await metaTest.should(`work on the typgate instance #${instanceNumber}`, async () => { +// await gql` +// query { +// doAddition(a: 1, b: 2) +// } +// ` +// .expectData({ +// doAddition: 3, +// }) +// .on(e); +// }); +// } + +// await testMultipleReplica(1); +// await testMultipleReplica(2); +// }, +// ); + +Meta.test( + { + name: "Deno runtime - TS SDK: file name reloading in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + const load = async (value: number) => { + Deno.env.set("DYNAMIC", path.join("dynamic", `${value}.ts`)); + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deno_reload.py", + cwd, + ); + Deno.env.delete("DYNAMIC"); + return e; + }; + + const v1 = await load(1); + await t.should("work with v1", async () => { + await gql` + query { + fire + } + ` + .expectData({ + fire: 1, + }) + .on(v1); + }); + await t.unregister(v1); + + const v2 = await load(2); + await t.should("work with v2", async () => { + await gql` + query { + fire + } + ` + .expectData({ + fire: 2, + }) + .on(v2); + }); + await t.unregister(v2); + }, +); + +Meta.test( + { + name: "Deno runtime - TS SDK: script reloading in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + const denoScript = path.join( + "typegate/tests/runtimes/deno", + "reload", + "template.ts", + ); + const originalContent = await Deno.readTextFile(denoScript); + const testReload = async (value: number) => { + try { + Deno.env.set("DYNAMIC", "reload/template.ts"); + await Deno.writeTextFile( + denoScript, + originalContent.replace('"REWRITE_ME"', `${value}`), + ); + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deno_reload.py", + cwd, + ); + await t.should(`reload with new value ${value}`, async () => { + await gql` + query { + fire + } + ` + .expectData({ + fire: value, + }) + .on(e); + }); + await t.unregister(e); + } catch (err) { + throw err; + } finally { + await Deno.writeTextFile(denoScript, originalContent); + } + }; + + await testReload(1); + await testReload(2); + }, +); + +Meta.test( + { + name: "Deno runtime - Python SDK: infinite loop or similar in sync mode", + sanitizeOps: false, + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deploy_deno.py", + cwd, + ); + + await t.should("safely fail upon stack overflow", async () => { + await gql` + query { + stackOverflow(enable: true) + } + ` + .expectErrorContains("Maximum call stack size exceeded") + .on(e); + }); + + await t.should("safely fail upon an infinite loop", async () => { + await gql` + query { + infiniteLoop(enable: true) + } + ` + .expectErrorContains("timeout exceeded") + .on(e); + }); + + const cooldownTime = 5; + console.log(`cooldown ${cooldownTime}s`); + await sleep(cooldownTime * 1000); + }, +); + +// Meta.test( +// { +// name: "Deno runtime - TS SDK: with no artifacts in sync mode", +// syncConfig, +// async setup() { +// await cleanUp(); +// }, +// async teardown() { +// await cleanUp(); +// }, +// }, +// async (t) => { +// const e = await t.engine("runtimes/deno/deno_typescript.ts"); + +// await t.should("work with no artifacts in typegrpah", async () => { +// await gql` +// query { +// hello(name: "World") +// helloFn(name: "wOrLd") +// } +// ` +// .expectData({ +// hello: "Hello World", +// helloFn: "Hello world", +// }) +// .on(e); +// }); +// } +// ); + +Meta.test( + { + name: "Deno runtime - Python SDK: with no artifacts in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deno_no_artifact.py", + cwd, + ); + + await t.should("work with no artifacts in typegrpah", async () => { + await gql` + query { + simple(a: 1, b: 20) + } + ` + .expectData({ + simple: 21, + }) + .on(e); + }); + }, +); + +// Meta.test( +// { +// name: "Deno runtime - TS SDK: with duplicate artifacts in sync mode", +// syncConfig, +// async setup() { +// await cleanUp(); +// }, +// async teardown() { +// await cleanUp(); +// }, +// }, +// async (t) => { +// const e = await t.engine("runtimes/deno/deno_duplicate_typescript.ts"); + +// await t.should("work with duplicate artifacts in typegrpah", async () => { +// await gql` +// query { +// doAddition(a: 1, b: 2) +// doAdditionDuplicate(a: 12, b: 2) +// } +// ` +// .expectData({ +// doAddition: 3, +// doAdditionDuplicate: 14, +// }) +// .on(e); +// }); +// } +// ); + +Meta.test( + { + name: "Deno runtime - Python SDK: with duplicate artifacts in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t) => { + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deno_duplicate_artifact.py", + cwd, + ); + + await t.should("work with duplicate artifacts in typegrpah", async () => { + await gql` + query { + doAddition(a: 1, b: 2) + doAdditionDuplicate(a: 12, b: 2) + } + ` + .expectData({ + doAddition: 3, + doAdditionDuplicate: 14, + }) + .on(e); + }); + }, +); diff --git a/typegate/tests/runtimes/deno/deno_test.ts b/typegate/tests/runtimes/deno/deno_test.ts index 35a40912d6..7b2482ab1e 100644 --- a/typegate/tests/runtimes/deno/deno_test.ts +++ b/typegate/tests/runtimes/deno/deno_test.ts @@ -379,3 +379,99 @@ Meta.test( await sleep(cooldownTime * 1000); }, ); + +// Meta.test( +// { +// name: "Deno runtime - TS SDK: with no artifacts in sync mode", +// }, +// async (t) => { +// const e = await t.engine("runtimes/deno/deno_typescript.ts"); + +// await t.should("work with no artifacts in typegrpah", async () => { +// await gql` +// query { +// hello(name: "World") +// helloFn(name: "wOrLd") +// } +// ` +// .expectData({ +// hello: "Hello World", +// helloFn: "Hello world", +// }) +// .on(e); +// }); +// } +// ); + +Meta.test( + { + name: "Deno runtime - Python SDK: with no artifacts in sync mode", + }, + async (t) => { + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deno_no_artifact.py", + cwd, + ); + + await t.should("work with no artifacts in typegrpah", async () => { + await gql` + query { + simple(a: 1, b: 20) + } + ` + .expectData({ + simple: 21, + }) + .on(e); + }); + }, +); + +// Meta.test( +// { +// name: "Deno runtime - TS SDK: with duplicate artifacts in sync mode", +// }, +// async (t) => { +// const e = await t.engine("runtimes/deno/deno_duplicate_typescript.ts"); + +// await t.should("work with duplicate artifacts in typegrpah", async () => { +// await gql` +// query { +// doAddition(a: 1, b: 2) +// doAdditionDuplicate(a: 12, b: 2) +// } +// ` +// .expectData({ +// doAddition: 3, +// doAdditionDuplicate: 14, +// }) +// .on(e); +// }); +// } +// ); + +Meta.test( + { + name: "Deno runtime - Python SDK: with duplicate artifacts in sync mode", + }, + async (t) => { + const e = await t.engineFromTgDeployPython( + "runtimes/deno/deno_duplicate_artifact.py", + cwd, + ); + + await t.should("work with duplicate artifacts in typegrpah", async () => { + await gql` + query { + doAddition(a: 1, b: 2) + doAdditionDuplicate(a: 12, b: 2) + } + ` + .expectData({ + doAddition: 3, + doAdditionDuplicate: 14, + }) + .on(e); + }); + }, +); diff --git a/typegate/tests/runtimes/python/python_duplicate_artifact.py b/typegate/tests/runtimes/python/python_duplicate_artifact.py new file mode 100644 index 0000000000..56854dc329 --- /dev/null +++ b/typegate/tests/runtimes/python/python_duplicate_artifact.py @@ -0,0 +1,67 @@ +import os +import sys + +from typegraph.gen.exports.core import ( + ArtifactResolutionConfig, + MigrationAction, + MigrationConfig, +) +from typegraph.graph.shared_types import BasicAuth +from typegraph.graph.tg_deploy import TypegraphDeployParams, tg_deploy +from typegraph.graph.typegraph import Graph +from typegraph.policy import Policy +from typegraph.runtimes.python import PythonRuntime + +from typegraph import t, typegraph + + +@typegraph() +def python_duplicate_artifact(g: Graph): + public = Policy.public() + python = PythonRuntime() + + g.expose( + testMod=python.import_( + t.struct({"name": t.string()}), + t.string(), + module="py/hello.py", + deps=["py/nested/dep.py"], + name="sayHello", + ).with_policy(public), + testModDuplicate=python.import_( + t.struct({"name": t.string()}), + t.string(), + module="py/hello.py", + deps=["py/nested/dep.py"], + name="sayHello", + ).with_policy(public), + ) + + +cwd = sys.argv[1] +PORT = sys.argv[2] +gate = f"http://localhost:{PORT}" +auth = BasicAuth("admin", "password") + +pytho_tg = python_duplicate_artifact() +deploy_result = tg_deploy( + pytho_tg, + TypegraphDeployParams( + base_url=gate, + auth=auth, + typegraph_path=os.path.join(cwd, "python_duplicate_artifact.py"), + artifacts_config=ArtifactResolutionConfig( + dir=cwd, + prefix=None, + disable_artifact_resolution=None, + codegen=None, + prisma_migration=MigrationConfig( + migration_dir="prisma-migrations", + global_action=MigrationAction(reset=False, create=True), + runtime_actions=None, + ), + ), + ), +) + +print(deploy_result.serialized) diff --git a/typegate/tests/runtimes/python/python_duplicate_artifact.ts b/typegate/tests/runtimes/python/python_duplicate_artifact.ts new file mode 100644 index 0000000000..18c296f3d3 --- /dev/null +++ b/typegate/tests/runtimes/python/python_duplicate_artifact.ts @@ -0,0 +1,39 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { Policy, t, typegraph } from "@typegraph/sdk/index.js"; +import { PythonRuntime } from "@typegraph/sdk/runtimes/python.js"; + +const tpe = t.struct({ + "a": t.string(), + "b": t.list(t.either([t.integer(), t.string()])), +}); + +export const tgDuplicateArtifact = await typegraph( + "python_duplicate_artifacts", + (g: any) => { + const python = new PythonRuntime(); + const pub = Policy.public(); + + g.expose({ + identityMod: python.import( + t.struct({ input: tpe }), + tpe, + { + name: "identity", + module: "py/hello.py", + deps: ["py/nested/dep.py"], + }, + ).withPolicy(pub), + identityModDuplicate: python.import( + t.struct({ input: tpe }), + tpe, + { + name: "identity", + module: "py/hello.py", + deps: ["py/nested/dep.py"], + }, + ).withPolicy(pub), + }); + }, +); diff --git a/typegate/tests/runtimes/python/python_no_artifact.py b/typegate/tests/runtimes/python/python_no_artifact.py new file mode 100644 index 0000000000..29f0eeb6af --- /dev/null +++ b/typegate/tests/runtimes/python/python_no_artifact.py @@ -0,0 +1,58 @@ +import os +import sys + +from typegraph.gen.exports.core import ( + ArtifactResolutionConfig, + MigrationAction, + MigrationConfig, +) +from typegraph.graph.shared_types import BasicAuth +from typegraph.graph.tg_deploy import TypegraphDeployParams, tg_deploy +from typegraph.graph.typegraph import Graph +from typegraph.policy import Policy +from typegraph.runtimes.python import PythonRuntime + +from typegraph import t, typegraph + + +@typegraph() +def python_no_artifact(g: Graph): + public = Policy.public() + python = PythonRuntime() + + g.expose( + test_lambda=python.from_lambda( + t.struct({"a": t.string()}), + t.string(), + lambda x: x["a"], + ).with_policy(public), + ) + + +cwd = sys.argv[1] +PORT = sys.argv[2] +gate = f"http://localhost:{PORT}" +auth = BasicAuth("admin", "password") + +pytho_tg = python_no_artifact() +deploy_result = tg_deploy( + pytho_tg, + TypegraphDeployParams( + base_url=gate, + auth=auth, + typegraph_path=os.path.join(cwd, "python_no_artifact.py"), + artifacts_config=ArtifactResolutionConfig( + dir=cwd, + prefix=None, + disable_artifact_resolution=None, + codegen=None, + prisma_migration=MigrationConfig( + migration_dir="prisma-migrations", + global_action=MigrationAction(reset=False, create=True), + runtime_actions=None, + ), + ), + ), +) + +print(deploy_result.serialized) diff --git a/typegate/tests/runtimes/python/python_no_artifact.ts b/typegate/tests/runtimes/python/python_no_artifact.ts new file mode 100644 index 0000000000..8a6048369a --- /dev/null +++ b/typegate/tests/runtimes/python/python_no_artifact.ts @@ -0,0 +1,34 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { Policy, t, typegraph } from "@typegraph/sdk/index.js"; +import { PythonRuntime } from "@typegraph/sdk/runtimes/python.js"; +import outdent from "outdent"; + +const tpe = t.struct({ + "a": t.string(), + "b": t.list(t.either([t.integer(), t.string()])), +}); + +export const tgNoArtifact = await typegraph("python_no_artifact", (g: any) => { + const python = new PythonRuntime(); + const pub = Policy.public(); + + g.expose({ + identityLambda: python.fromLambda( + t.struct({ input: tpe }), + tpe, + { code: "lambda x: x['input']" }, + ).withPolicy(pub), + identityDef: python.fromDef( + t.struct({ input: tpe }), + tpe, + { + code: outdent` + def identity(x): + return x['input'] + `, + }, + ).withPolicy(pub), + }); +}); diff --git a/typegate/tests/runtimes/python/python_sync_test.ts b/typegate/tests/runtimes/python/python_sync_test.ts new file mode 100644 index 0000000000..5a0714e858 --- /dev/null +++ b/typegate/tests/runtimes/python/python_sync_test.ts @@ -0,0 +1,698 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import { BasicAuth, tgDeploy } from "@typegraph/sdk/tg_deploy.js"; +import { gql, Meta } from "test-utils/mod.ts"; +import { testDir } from "test-utils/dir.ts"; +import * as path from "std/path/mod.ts"; +import { connect } from "redis"; +import { S3Client } from "aws-sdk/client-s3"; +import { createBucket, listObjects, tryDeleteBucket } from "test-utils/s3.ts"; +import { assert, assertEquals, assertExists } from "std/assert/mod.ts"; +import { QueryEngine } from "../../../src/engine/query_engine.ts"; +import { tg } from "./python.ts"; +// import { tgNoArtifact } from './python_no_artifact.ts'; + +const redisKey = "typegraph"; +const redisEventKey = "typegraph_event"; + +async function cleanUp() { + using redis = await connect(syncConfig.redis); + await redis.del(redisKey); + await redis.del(redisEventKey); + + const s3 = new S3Client(syncConfig.s3); + await tryDeleteBucket(s3, syncConfig.s3Bucket); + await createBucket(s3, syncConfig.s3Bucket); + s3.destroy(); + await redis.quit(); +} + +const syncConfig = { + redis: { + hostname: "localhost", + port: 6379, + password: "password", + db: 1, + }, + s3: { + endpoint: "http://localhost:9000", + region: "local", + credentials: { + accessKeyId: "minio", + secretAccessKey: "password", + }, + forcePathStyle: true, + }, + s3Bucket: "metatype-python-runtime-sync-test", +}; + +const cwd = path.join(testDir, "runtimes/python"); +const auth = new BasicAuth("admin", "password"); + +const localSerializedMemo = tg.serialize({ + prismaMigration: { + globalAction: { + create: true, + reset: false, + }, + migrationDir: "prisma-migrations", + }, + dir: cwd, +}); +const reusableTgOutput = { + ...tg, + serialize: (_: any) => localSerializedMemo, +}; + +Meta.test( + { + name: "Python Runtime typescript SDK: with Sync Config", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (metaTest: any) => { + const port = metaTest.port; + const gate = `http://localhost:${port}`; + + const { serialized, typegate: gateResponseAdd } = await tgDeploy( + reusableTgOutput, + { + baseUrl: gate, + auth, + artifactsConfig: { + prismaMigration: { + globalAction: { + create: true, + reset: false, + }, + migrationDir: "prisma-migrations", + }, + dir: cwd, + }, + typegraphPath: path.join(cwd, "python.ts"), + secrets: {}, + }, + ); + + await metaTest.should( + "work after deploying python artifacts to S3", + async () => { + const s3 = new S3Client(syncConfig.s3); + + assertExists(serialized, "serialized has a value"); + assertEquals(gateResponseAdd, { + data: { + addTypegraph: { + name: "python", + messages: [], + migrations: [], + }, + }, + }); + + const s3Objects = await listObjects(s3, syncConfig.s3Bucket); + // two objects, 2 artifacts and the typegraph + assertEquals(s3Objects?.length, 3); + + const engine = await metaTest.engineFromDeployed(serialized); + + await gql` + query { + identityDef(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + identityLambda(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + identityMod(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + } + ` + .expectData({ + identityDef: { + a: "hello", + b: [1, 2, "three"], + }, + identityLambda: { + a: "hello", + b: [1, 2, "three"], + }, + identityMod: { + a: "hello", + b: [1, 2, "three"], + }, + }) + .on(engine); + + s3.destroy(); + }, + ); + }, +); + +Meta.test( + { + name: "Python runtime: sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t: any) => { + const e = await t.engineFromTgDeployPython( + "runtimes/python/python.py", + cwd, + ); + + await t.should("work once (lambda)", async () => { + await gql` + query { + test(a: "test") + } + ` + .expectData({ + test: "test", + }) + .on(e); + }); + + await t.should("work once (def)", async () => { + await gql` + query { + testDef(a: "test") + } + ` + .expectData({ + testDef: "test", + }) + .on(e); + }); + + await t.should("work once (module)", async () => { + await gql` + query { + testMod(name: "Loyd") + } + ` + .expectData({ + testMod: "Hello Loyd", + }) + .on(e); + }); + + await t.should("return same object", async () => { + await gql` + query { + identity(input: { a: 1234, b: { c: ["one", "two", "three"] } }) { + a + b { + c + } + } + } + ` + .expectData({ + identity: { + a: 1234, + b: { c: ["one", "two", "three"] }, + }, + }) + .on(e); + }); + + await t.should("work fast enough", async () => { + const tests = [...Array(100).keys()].map((i) => + gql` + query ($a: String!) { + test(a: $a) + } + ` + .withVars({ + a: `test${i}`, + }) + .expectData({ + test: `test${i}`, + }) + .on(e) + ); + + const start = performance.now(); + await Promise.all(tests); + const end = performance.now(); + const duration = end - start; + + console.log(`duration: ${duration}ms`); + assert(duration < 800, `Python runtime was too slow: ${duration}ms`); + }); + }, +); + +Meta.test( + { + name: "Python runtime: multiple typegate instances sync mode", + replicas: 3, + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t: any) => { + const testMultipleReplica = async (instanceNumber: number) => { + const e = await t.engineFromTgDeployPython( + "runtimes/python/python.py", + cwd, + ); + + await t.should( + `work on the typgate instance #${instanceNumber}`, + async () => { + await gql` + query { + testMod(name: "Loyd") + } + ` + .expectData({ + testMod: `Hello Loyd`, + }) + .on(e); + }, + ); + }; + + await testMultipleReplica(1); + await testMultipleReplica(2); + }, +); + +Meta.test( + { + name: "Deno: def, lambda in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t: any) => { + const port = t.port; + const gate = `http://localhost:${port}`; + + const { serialized, typegate: _gateResponseAdd } = await tgDeploy( + reusableTgOutput, + { + baseUrl: gate, + auth, + artifactsConfig: { + prismaMigration: { + globalAction: { + create: true, + reset: false, + }, + migrationDir: "prisma-migrations", + }, + dir: cwd, + }, + typegraphPath: path.join(cwd, "python.ts"), + secrets: {}, + }, + ); + + const e = await t.engineFromDeployed(serialized); + + await t.should("work with def", async () => { + await gql` + query { + identityLambda(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + } + ` + .expectData({ + identityLambda: { + a: "hello", + b: [1, 2, "three"], + }, + }) + .on(e); + }); + + await t.should("work with def", async () => { + await gql` + query { + identityDef(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + } + ` + .expectData({ + identityDef: { + a: "hello", + b: [1, 2, "three"], + }, + }) + .on(e); + }); + }, +); + +Meta.test( + { + name: "Python: infinite loop or similar in sync mode", + sanitizeOps: false, + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t: any) => { + const e = await t.engineFromTgDeployPython( + "runtimes/python/python.py", + cwd, + ); + + await t.should("safely fail upon stackoverflow", async () => { + await gql` + query { + stackOverflow(enable: true) + } + ` + .expectErrorContains("maximum recursion depth exceeded") + .on(e); + }); + }, +); + +Meta.test( + { + name: "Python: typegate reloading in sync mode", + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (metaTest: any) => { + const port = metaTest.port; + const gate = `http://localhost:${port}`; + + const load = async () => { + const { serialized, typegate: _gateResponseAdd } = await tgDeploy( + reusableTgOutput, + { + baseUrl: gate, + auth, + artifactsConfig: { + prismaMigration: { + globalAction: { + create: true, + reset: false, + }, + migrationDir: "prisma-migrations", + }, + dir: cwd, + }, + typegraphPath: path.join(cwd, "python.ts"), + secrets: {}, + }, + ); + + return await metaTest.engineFromDeployed(serialized); + }; + + const runPythonOnPython = async (currentEngine: QueryEngine) => { + await gql` + query { + identityDef(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + identityLambda(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + identityMod(input: { a: "hello", b: [1, 2, "three"] }) { + a + b + } + } + ` + .expectData({ + identityDef: { + a: "hello", + b: [1, 2, "three"], + }, + identityLambda: { + a: "hello", + b: [1, 2, "three"], + }, + identityMod: { + a: "hello", + b: [1, 2, "three"], + }, + }) + .on(currentEngine); + }; + const engine = await load(); + await metaTest.should("work before typegate is reloaded", async () => { + await runPythonOnPython(engine); + }); + + // reload + const reloadedEngine = await load(); + + await metaTest.should("work after typegate is reloaded", async () => { + await runPythonOnPython(reloadedEngine); + }); + }, +); + +Meta.test( + { + name: + "PythonRuntime - Python SDK: typegraph with no artifacts in sync mode", + sanitizeOps: false, + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t: any) => { + const e = await t.engineFromTgDeployPython( + "runtimes/python/python_no_artifact.py", + cwd, + ); + + await t.should( + "work when there are no artifacts in the typegraph: python SDK, in sync mode", + async () => { + await gql` + query { + test_lambda(a: "test") + } + ` + .expectData({ + test_lambda: "test", + }) + .on(e); + }, + ); + }, +); + +// Meta.test( +// { +// name: "Python Runtime TS SDK: typegraph with no artifacts in sync mode", +// sanitizeOps: false, +// syncConfig, +// async setup() { +// await cleanUp(); +// }, +// async teardown() { +// await cleanUp(); +// }, +// }, +// async (t: any) => { +// const port = t.port; +// const gate = `http://localhost:${port}`; + +// const { serialized, typegate: _gateResponseAdd } = await tgDeploy( +// tgNoArtifact, +// { +// baseUrl: gate, +// auth, +// artifactsConfig: { +// prismaMigration: { +// globalAction: { +// create: true, +// reset: false, +// }, +// migrationDir: "prisma-migrations", +// }, +// dir: cwd, +// }, +// typegraphPath: path.join(cwd, "python_no_artifact.ts"), +// secrets: {}, +// }, +// ); + +// const e = await t.engineFromDeployed(serialized); + +// await t.should("work when there are no artifacts in the typegraph: TS SDK, in sync mode", async () => { +// await gql` +// query { +// identityDef(input: { a: "hello", b: [1, 2, "three"] }) { +// a +// b +// } +// identityLambda(input: { a: "hello", b: [1, 2, "three"] }) { +// a +// b +// } +// } +// ` +// .expectData({ +// identityDef: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// identityLambda: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// }) +// .on(e); +// }); +// }, +// ); + +Meta.test( + { + name: + "Python - Python SDK: typegraph with duplicate artifact uploads in sync mode", + sanitizeOps: false, + syncConfig, + async setup() { + await cleanUp(); + }, + async teardown() { + await cleanUp(); + }, + }, + async (t: any) => { + const e = await t.engineFromTgDeployPython( + "runtimes/python/python_duplicate_artifact.py", + cwd, + ); + + await t.should( + "work when there is duplicate artifacts uploads: Python SDK, in sync mode", + async () => { + await gql` + query { + testMod(name: "Loyd") + testModDuplicate(name: "Barney") + } + ` + .expectData({ + testMod: "Hello Loyd", + testModDuplicate: "Hello Barney", + }) + .on(e); + }, + ); + }, +); + +// Meta.test( +// { +// name: "Python Runtime - TS SDK: typegraph with duplicate artifact uploads in sync mode", +// sanitizeOps: false, +// syncConfig, +// async setup() { +// await cleanUp(); +// }, +// async teardown() { +// await cleanUp(); +// }, +// }, +// async (t: any) => { +// const port = t.port; +// const gate = `http://localhost:${port}`; + +// const { serialized, typegate: _gateResponseAdd } = await tgDeploy( +// tgDuplicateArtifact, +// { +// baseUrl: gate, +// auth, +// artifactsConfig: { +// prismaMigration: { +// globalAction: { +// create: true, +// reset: false, +// }, +// migrationDir: "prisma-migrations", +// }, +// dir: cwd, +// }, +// typegraphPath: path.join(cwd, "python_duplicate_artifact.ts"), +// secrets: {}, +// }, +// ); + +// const e = await t.engineFromDeployed(serialized); + +// await t.should("work when there is duplicate artifacts uploads: TS SDK, in sync mode", async () => { +// await gql` +// query { +// identityMod(input: { a: "hello", b: [1, 2, "three"] }) { +// a +// b +// }, +// identityModDuplicate(input: { a: "hello", b: [1, 2, "three"] }) { +// a +// b +// } +// } +// ` +// .expectData({ +// identityMod: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// identityModDuplicate: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// }) +// .on(e); +// }); +// }, +// ); diff --git a/typegate/tests/runtimes/python/python_test.ts b/typegate/tests/runtimes/python/python_test.ts index 3ae831bbb7..bff73b91df 100644 --- a/typegate/tests/runtimes/python/python_test.ts +++ b/typegate/tests/runtimes/python/python_test.ts @@ -10,6 +10,7 @@ import { testDir } from "test-utils/dir.ts"; import { tg } from "./python.ts"; import * as path from "std/path/mod.ts"; import { BasicAuth, tgDeploy } from "@typegraph/sdk/tg_deploy.js"; +// import { tgNoArtifact } from "./python_no_artifact.ts"; const cwd = path.join(testDir, "runtimes/python"); const auth = new BasicAuth("admin", "password"); @@ -434,3 +435,181 @@ Meta.test( }); }, ); + +Meta.test( + { + name: + "PythonRuntime - Python SDK: typegraph with no artifacts in sync mode", + sanitizeOps: false, + }, + async (t: any) => { + const e = await t.engineFromTgDeployPython( + "runtimes/python/python_no_artifact.py", + cwd, + ); + + await t.should( + "work when there are no artifacts in the typegraph: python SDK", + async () => { + await gql` + query { + test_lambda(a: "test") + } + ` + .expectData({ + test_lambda: "test", + }) + .on(e); + }, + ); + }, +); + +// Meta.test( +// { +// name: "Python Runtime TS SDK: typegraph with no artifacts", +// sanitizeOps: false, +// }, +// async (t: any) => { +// const port = t.port; +// const gate = `http://localhost:${port}`; + +// const { serialized, typegate: _gateResponseAdd } = await tgDeploy( +// tgNoArtifact, +// { +// baseUrl: gate, +// auth, +// artifactsConfig: { +// prismaMigration: { +// globalAction: { +// create: true, +// reset: false, +// }, +// migrationDir: "prisma-migrations", +// }, +// dir: cwd, +// }, +// typegraphPath: path.join(cwd, "python_no_artifact.ts"), +// secrets: {}, +// }, +// ); + +// const e = await t.engineFromDeployed(serialized); + +// await t.should("work when there are no artifacts in the typegraph: TS SDK", async () => { +// await gql` +// query { +// identityDef(input: { a: "hello", b: [1, 2, "three"] }) { +// a +// b +// } +// identityLambda(input: { a: "hello", b: [1, 2, "three"] }) { +// a +// b +// } +// } +// ` +// .expect({ +// identityDef: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// identityLambda: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// }) +// .on(e); +// }); +// }, +// ); + +Meta.test( + { + name: + "Python Runtime - Python SDK: typegraph with duplicate artifact uploads", + sanitizeOps: false, + }, + async (t: any) => { + const e = await t.engineFromTgDeployPython( + "runtimes/python/python_duplicate_artifact.py", + cwd, + ); + + await t.should( + "work when there is duplicate artifacts uploads: Python SDK", + async () => { + await gql` + query { + testMod(name: "Loyd") + testModDuplicate(name: "Barney") + } + ` + .expectData({ + testMod: "Hello Loyd", + testModDuplicate: "Hello Barney", + }) + .on(e); + }, + ); + }, +); + +// Meta.test( +// { +// name: "Python Runtime - TS SDK: typegraph with duplicate artifact uploads", +// sanitizeOps: false, +// }, +// async (t: any) => { +// const port = t.port; +// const gate = `http://localhost:${port}`; + +// const { serialized, typegate: _gateResponseAdd } = await tgDeploy( +// tgDuplicateArtifact, +// { +// baseUrl: gate, +// auth, +// artifactsConfig: { +// prismaMigration: { +// globalAction: { +// create: true, +// reset: false, +// }, +// migrationDir: "prisma-migrations", +// }, +// dir: cwd, +// }, +// typegraphPath: path.join(cwd, "python_duplicate_artifact.ts"), +// secrets: {}, +// }, +// ); + +// const e = await t.engineFromDeployed(serialized); + +// await t.should("work when there is duplicate artifacts uploads: TS SDK", async () => { +// await gql` +// query { +// identityMod(input: { a: "hello", b: [1, 2, "three"] }) { +// a +// b +// }, +// identityModDuplicate(input: { a: "hello", b: [1, 2, "three"] }) { +// a +// b +// } +// } +// ` +// .expectData({ +// identityMod: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// identityModDuplicate: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// }) +// .on(e); +// }); +// }, +// ); diff --git a/typegraph/core/src/utils/postprocess/python_rt.rs b/typegraph/core/src/utils/postprocess/python_rt.rs index fb6c33632c..e3dabfc76b 100644 --- a/typegraph/core/src/utils/postprocess/python_rt.rs +++ b/typegraph/core/src/utils/postprocess/python_rt.rs @@ -7,7 +7,7 @@ use common::typegraph::{ utils::{map_from_object, object_from_map}, Typegraph, }; -use std::path::PathBuf; +use std::{collections::hash_map::Entry, path::PathBuf}; use crate::utils::postprocess::PostProcessor; @@ -22,23 +22,20 @@ impl PostProcessor for PythonProcessor { let path = mat_data.python_artifact.get("path").unwrap(); let path: PathBuf = path.as_str().unwrap().into(); - if tg.meta.artifacts.contains_key(&path) { - continue; - } - - let python_module_path = fs_host::make_absolute(&path)?; + if let Entry::Vacant(entry) = tg.meta.artifacts.entry(path.clone()) { + let python_module_path = fs_host::make_absolute(&path)?; - let (module_hash, size) = fs_host::hash_file(&python_module_path)?; + let (module_hash, size) = fs_host::hash_file(&python_module_path)?; - tg.deps.push(python_module_path); - tg.meta.artifacts.insert( - path.clone(), - Artifact { + tg.deps.push(python_module_path); + entry.insert(Artifact { hash: module_hash.clone(), size, path: path.clone(), - }, - ); + }); + } + + let main_module = tg.meta.artifacts.get(&path).unwrap().clone(); let deps = mat_data.deps.clone(); let mut dep_artifacts = vec![]; @@ -46,20 +43,22 @@ impl PostProcessor for PythonProcessor { let dep_rel_path = PathBuf::from(dep); let dep_abs_path = fs_host::make_absolute(&dep_rel_path)?; - let (dep_hash, dep_size) = fs_host::hash_file(&dep_abs_path)?; - let dep_artifact = Artifact { - path: dep_rel_path.clone(), - hash: dep_hash, - size: dep_size, - }; - tg.meta.artifacts.insert(dep_rel_path, dep_artifact.clone()); - dep_artifacts.push(dep_artifact); - tg.deps.push(dep_abs_path); + if let Entry::Vacant(entry) = tg.meta.artifacts.entry(dep_rel_path.clone()) { + let (dep_hash, dep_size) = fs_host::hash_file(&dep_abs_path)?; + let dep_artifact = Artifact { + path: dep_rel_path, + hash: dep_hash, + size: dep_size, + }; + entry.insert(dep_artifact.clone()); + dep_artifacts.push(dep_artifact); + tg.deps.push(dep_abs_path); + } } mat_data.python_artifact = map_from_object(Artifact { - hash: module_hash.clone(), - size, + hash: main_module.hash.clone(), + size: main_module.size, path, }) .map_err(|e| e.to_string())?;