From c6c2e213000d6793cfd342c8de0d087f799e0d2e Mon Sep 17 00:00:00 2001 From: Hugues Chocart Date: Sat, 12 Oct 2024 17:10:56 +0200 Subject: [PATCH] fix: smart data filtering (#612) --- package-lock.json | 317 +++++++++--------- packages/backend/src/api/v1/projects/index.ts | 17 +- packages/backend/src/api/v1/runs/ingest.ts | 4 +- packages/backend/src/checks/runChecks.ts | 6 +- .../components/settings/data-warehouse.tsx | 1 + packages/frontend/pages/settings/index.tsx | 12 +- 6 files changed, 189 insertions(+), 168 deletions(-) diff --git a/package-lock.json b/package-lock.json index c04ab521..e16b2ef0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -67,6 +67,24 @@ "js-yaml": "^4.1.0" } }, + "node_modules/@apidevtools/json-schema-ref-parser/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/@apidevtools/json-schema-ref-parser/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@apidevtools/openapi-schemas": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", @@ -251,26 +269,26 @@ } }, "node_modules/@aws-sdk/client-comprehend": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-comprehend/-/client-comprehend-3.667.0.tgz", - "integrity": "sha512-bJmPS7z8ZmV93rJZFodJT4/p0w5vhYTRPIV9hidQHMDeCq+nvgC3ORKE/9/xqHVeP+pNTV24Pnv2Yw2zE7Sswg==", + "version": "3.670.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-comprehend/-/client-comprehend-3.670.0.tgz", + "integrity": "sha512-YdJ5yPDLYPPDPP0kny/zRrO5xmJKjxcNxiI6lwaClDZ+IQQ22rtEDpd8t2c3GX63uGwT7BGhUkMWwAuGz+mVdw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/client-sso-oidc": "3.667.0", - "@aws-sdk/client-sts": "3.667.0", + "@aws-sdk/client-sso-oidc": "3.670.0", + "@aws-sdk/client-sts": "3.670.0", "@aws-sdk/core": "3.667.0", - "@aws-sdk/credential-provider-node": "3.667.0", + "@aws-sdk/credential-provider-node": "3.670.0", "@aws-sdk/middleware-host-header": "3.667.0", "@aws-sdk/middleware-logger": "3.667.0", "@aws-sdk/middleware-recursion-detection": "3.667.0", - "@aws-sdk/middleware-user-agent": "3.667.0", + "@aws-sdk/middleware-user-agent": "3.669.0", "@aws-sdk/region-config-resolver": "3.667.0", "@aws-sdk/types": "3.667.0", "@aws-sdk/util-endpoints": "3.667.0", - "@aws-sdk/util-user-agent-browser": "3.667.0", - "@aws-sdk/util-user-agent-node": "3.667.0", + "@aws-sdk/util-user-agent-browser": "3.670.0", + "@aws-sdk/util-user-agent-node": "3.669.0", "@smithy/config-resolver": "^3.0.9", "@smithy/core": "^2.4.8", "@smithy/fetch-http-handler": "^3.2.9", @@ -304,9 +322,9 @@ } }, "node_modules/@aws-sdk/client-sso": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.667.0.tgz", - "integrity": "sha512-69nl3BXcAfn/zUdxpxC/5Bi4K46sMgTRFVKHkyiRFO8OQpNJDRzl6eGltESLqtf65rjWfswTDwfYrMd2+tnlDQ==", + "version": "3.670.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.670.0.tgz", + "integrity": "sha512-J+oz6uSsDvk4pimMDnKJb1wsV216zTrejvMTIL4RhUD1QPIVVOpteTdUShcjZUIZnkcJZGI+cym/SFK0kuzTpg==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", @@ -315,12 +333,12 @@ "@aws-sdk/middleware-host-header": "3.667.0", "@aws-sdk/middleware-logger": "3.667.0", "@aws-sdk/middleware-recursion-detection": "3.667.0", - "@aws-sdk/middleware-user-agent": "3.667.0", + "@aws-sdk/middleware-user-agent": "3.669.0", "@aws-sdk/region-config-resolver": "3.667.0", "@aws-sdk/types": "3.667.0", "@aws-sdk/util-endpoints": "3.667.0", - "@aws-sdk/util-user-agent-browser": "3.667.0", - "@aws-sdk/util-user-agent-node": "3.667.0", + "@aws-sdk/util-user-agent-browser": "3.670.0", + "@aws-sdk/util-user-agent-node": "3.669.0", "@smithy/config-resolver": "^3.0.9", "@smithy/core": "^2.4.8", "@smithy/fetch-http-handler": "^3.2.9", @@ -353,24 +371,24 @@ } }, "node_modules/@aws-sdk/client-sso-oidc": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.667.0.tgz", - "integrity": "sha512-ThLLR0Zn15Rk8Y6rzzVtHREZU4NAsNj9oCiQcXj4/vlBl+J0MbiTCDUJTc559O+DIMekrjusLaPIKfTZmXXuhA==", + "version": "3.670.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.670.0.tgz", + "integrity": "sha512-4qDK2L36Q4J1lfemaHHd9ZxqKRaos3STp44qPAHf/8QyX6Uk5sXgZNVO2yWM7SIEtVKwwBh/fZAsdBkGPBfZcw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.667.0", - "@aws-sdk/credential-provider-node": "3.667.0", + "@aws-sdk/credential-provider-node": "3.670.0", "@aws-sdk/middleware-host-header": "3.667.0", "@aws-sdk/middleware-logger": "3.667.0", "@aws-sdk/middleware-recursion-detection": "3.667.0", - "@aws-sdk/middleware-user-agent": "3.667.0", + "@aws-sdk/middleware-user-agent": "3.669.0", "@aws-sdk/region-config-resolver": "3.667.0", "@aws-sdk/types": "3.667.0", "@aws-sdk/util-endpoints": "3.667.0", - "@aws-sdk/util-user-agent-browser": "3.667.0", - "@aws-sdk/util-user-agent-node": "3.667.0", + "@aws-sdk/util-user-agent-browser": "3.670.0", + "@aws-sdk/util-user-agent-node": "3.669.0", "@smithy/config-resolver": "^3.0.9", "@smithy/core": "^2.4.8", "@smithy/fetch-http-handler": "^3.2.9", @@ -402,29 +420,29 @@ "node": ">=16.0.0" }, "peerDependencies": { - "@aws-sdk/client-sts": "^3.667.0" + "@aws-sdk/client-sts": "^3.670.0" } }, "node_modules/@aws-sdk/client-sts": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.667.0.tgz", - "integrity": "sha512-SnvkDDuOAwXOxzhGfvvxK8JFFVApVvXrM6+nVmSYwuHjWdI+HwNcsRC6rxZ8uHQEz2fR0a810ckUwFVfpvtIoA==", + "version": "3.670.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.670.0.tgz", + "integrity": "sha512-bExrNo8ZVWorS3cjMZKQnA2HWqDmAzcZoSN/cPVoPFNkHwdl1lzPxvcLzmhpIr48JHgKfybBjrbluDZfIYeEog==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/client-sso-oidc": "3.667.0", + "@aws-sdk/client-sso-oidc": "3.670.0", "@aws-sdk/core": "3.667.0", - "@aws-sdk/credential-provider-node": "3.667.0", + "@aws-sdk/credential-provider-node": "3.670.0", "@aws-sdk/middleware-host-header": "3.667.0", "@aws-sdk/middleware-logger": "3.667.0", "@aws-sdk/middleware-recursion-detection": "3.667.0", - "@aws-sdk/middleware-user-agent": "3.667.0", + "@aws-sdk/middleware-user-agent": "3.669.0", "@aws-sdk/region-config-resolver": "3.667.0", "@aws-sdk/types": "3.667.0", "@aws-sdk/util-endpoints": "3.667.0", - "@aws-sdk/util-user-agent-browser": "3.667.0", - "@aws-sdk/util-user-agent-node": "3.667.0", + "@aws-sdk/util-user-agent-browser": "3.670.0", + "@aws-sdk/util-user-agent-node": "3.669.0", "@smithy/config-resolver": "^3.0.9", "@smithy/core": "^2.4.8", "@smithy/fetch-http-handler": "^3.2.9", @@ -516,16 +534,16 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.667.0.tgz", - "integrity": "sha512-NiEC2Sjut57ajbWEXLFtA8YWA06ulfuaSHOP4YxoQEsL3BjMOkKS1/5eVW2o6XkKVbwt0zbDYSTlIxU5klHpbQ==", + "version": "3.670.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.670.0.tgz", + "integrity": "sha512-TB1gacUj75leaTt2JsCTzygDSIk4ksv9uZoR7VenlgFPRktyOeT+fapwIVBeB2Qg7b9uxAY2K5XkKstDZyBEEw==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "3.667.0", "@aws-sdk/credential-provider-env": "3.667.0", "@aws-sdk/credential-provider-http": "3.667.0", "@aws-sdk/credential-provider-process": "3.667.0", - "@aws-sdk/credential-provider-sso": "3.667.0", + "@aws-sdk/credential-provider-sso": "3.670.0", "@aws-sdk/credential-provider-web-identity": "3.667.0", "@aws-sdk/types": "3.667.0", "@smithy/credential-provider-imds": "^3.2.4", @@ -538,20 +556,20 @@ "node": ">=16.0.0" }, "peerDependencies": { - "@aws-sdk/client-sts": "^3.667.0" + "@aws-sdk/client-sts": "^3.670.0" } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.667.0.tgz", - "integrity": "sha512-+2oNexDvbQD9o5Xdtu1mGE25Nf2/C9KgMnkZzenbgCegw0P0YRdGrJklDF5Aag6lmb80a2vjoViuvm1ORzRAzw==", + "version": "3.670.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.670.0.tgz", + "integrity": "sha512-zwNrRYzubk4CaZ7zebeDhxsm8QtNWkbGKopZPOaZSnd5uqUGRcmx4ccVRngWUK68XDP44aEUWC8iU5Pc7btpHQ==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/credential-provider-env": "3.667.0", "@aws-sdk/credential-provider-http": "3.667.0", - "@aws-sdk/credential-provider-ini": "3.667.0", + "@aws-sdk/credential-provider-ini": "3.670.0", "@aws-sdk/credential-provider-process": "3.667.0", - "@aws-sdk/credential-provider-sso": "3.667.0", + "@aws-sdk/credential-provider-sso": "3.670.0", "@aws-sdk/credential-provider-web-identity": "3.667.0", "@aws-sdk/types": "3.667.0", "@smithy/credential-provider-imds": "^3.2.4", @@ -582,12 +600,12 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.667.0.tgz", - "integrity": "sha512-GylJquSQVWlziaEmrX38KzQTWcFL5NKht4OAj7rdo75MssC0qNVSGT+ReFrXZzKQ65eIuRVCMdsp83oNH4nzbQ==", + "version": "3.670.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.670.0.tgz", + "integrity": "sha512-5PkA8BOy4q57Vhe9AESoHKZ7vjRbElNPKjXA4qC01xY+DitClRFz4O3B9sMzFp0PHlz9nDVSXXKgq0yzF/nAag==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-sso": "3.667.0", + "@aws-sdk/client-sso": "3.670.0", "@aws-sdk/core": "3.667.0", "@aws-sdk/token-providers": "3.667.0", "@aws-sdk/types": "3.667.0", @@ -664,9 +682,9 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.667.0.tgz", - "integrity": "sha512-NJoPMV+9hpN90iZ2SgOpFmY6MJW71gGyT28kt0C68B0tBadYpT/J6WW8J7gxwdLA7d2CBwHbFGphTPWXXl+jzw==", + "version": "3.669.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.669.0.tgz", + "integrity": "sha512-K8ScPi45zjJrj5Y2gRqVsvKKQCQbvQBfYGcBw9ZOx9TTavH80bOCBjWg/GFnvs4f37tqVc1wMN2oGvcTF6HveQ==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/core": "3.667.0", @@ -758,9 +776,9 @@ } }, "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.667.0.tgz", - "integrity": "sha512-y1pKlNzNpxzddM0QSnfIfIbi3Z9LTag1VDjYyZRbEGGSVip2J00qKsET+979nRezWMyJgw5GPBQR3Y+rN+jh0Q==", + "version": "3.670.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.670.0.tgz", + "integrity": "sha512-iRynWWazqEcCKwGMcQcywKTDLdLvqts1Yx474U64I9OKQXXwhOwhXbF5CAPSRta86lkVNAVYJa/0Bsv45pNn1A==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "3.667.0", @@ -770,12 +788,12 @@ } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.667.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.667.0.tgz", - "integrity": "sha512-4OcWyWUAYRLwXMxqUqnDb/3LoassXvEJcwjiIKEa7p0JvtSa9VjCL6B8Vjw8C/N6YoqrlFEEvcN7SlwBEy74pA==", + "version": "3.669.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.669.0.tgz", + "integrity": "sha512-9jxCYrgggy2xd44ZASqI7AMiRVaSiFp+06Kg8BQSU0ijKpBJlwcsqIS8pDT/n6LxuOw2eV5ipvM2C0r1iKzrGA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "3.667.0", + "@aws-sdk/middleware-user-agent": "3.669.0", "@aws-sdk/types": "3.667.0", "@smithy/node-config-provider": "^3.1.8", "@smithy/types": "^3.5.0", @@ -1469,9 +1487,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.12.1.tgz", - "integrity": "sha512-VYrLinJNeW3idIZHTy7BsVL7bsPF9+l89WDYTDV8o1QYJzkf3BSoPXdfaz4c8DyeY12LGSaS8m8BgbT8W66h6Q==", + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.12.2.tgz", + "integrity": "sha512-bgxdZmgTrJZX50OjyVwz3+mNEnCTNkh3cIqGPWVNeW9jX6bn1ZkU80uPd+67/ZpIJIjRQ9qaHCjhavyoWYxumg==", "license": "Apache-2.0", "dependencies": { "@grpc/proto-loader": "^0.7.13", @@ -3597,9 +3615,9 @@ } }, "node_modules/@types/react-dom": { - "version": "18.3.0", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.0.tgz", - "integrity": "sha512-EhwApuTmMBmXuFOikhQLIBUn6uFg81SwLMOAUgodJF14SOBOCMdU04gDoYi0WOJJHD144TL32z4yDqCW3dnkQg==", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3619,14 +3637,15 @@ } }, "node_modules/@types/request/node_modules/form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.2.tgz", + "integrity": "sha512-GgwY0PS7DbXqajuGf4OYlsrIu3zgxD6Vvql43IBhm6MahqA5SK/7mwhtNj2AdH2z35YR34ujJ7BN+3fFC3jP5Q==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" + "mime-types": "^2.1.12", + "safe-buffer": "^5.2.1" }, "engines": { "node": ">= 0.12" @@ -3882,10 +3901,14 @@ } }, "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } }, "node_modules/arrify": { "version": "2.0.1", @@ -4327,9 +4350,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001667", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001667.tgz", - "integrity": "sha512-7LTwJjcRkzKFmtqGsibMeuXmvFDfZq/nzIjnmgCGzKKRVzjD72selLDK1oPF/Oxzmt4fNcPvTDvGqSDG4tCALw==", + "version": "1.0.30001668", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001668.tgz", + "integrity": "sha512-nWLrdxqCdblixUO+27JtGJJE/txpJlyUy5YN1u53wLZkP0emYCo5zgS6QYft7VUYR42LGgi/S5hdLZTrnyIddw==", "funding": [ { "type": "opencollective", @@ -4603,9 +4626,9 @@ } }, "node_modules/compromise": { - "version": "14.14.1", - "resolved": "https://registry.npmjs.org/compromise/-/compromise-14.14.1.tgz", - "integrity": "sha512-2N/wPyFwtDaskKorKMs2sB+KTcrkOUHSaKZkNQMo8hEcNJHLt8NdwNVsN14vnmsgjPOxctTOYt3DWVxXIwXXxA==", + "version": "14.14.2", + "resolved": "https://registry.npmjs.org/compromise/-/compromise-14.14.2.tgz", + "integrity": "sha512-g2Qe4zn8TmL7xQFR5Tx7i1txTUnzTPxhE7hDCQM+LDIfvYcriKzqH7eD2J/apUr/hRxvfKbRJ/yYXtN1cgD+Ug==", "license": "MIT", "dependencies": { "efrt": "2.7.0", @@ -5233,13 +5256,6 @@ "is-arrayish": "^0.2.1" } }, - "node_modules/error-ex/node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true, - "license": "MIT" - }, "node_modules/es-define-property": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", @@ -5549,9 +5565,9 @@ } }, "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -5845,9 +5861,9 @@ } }, "node_modules/google-auth-library": { - "version": "9.14.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.14.1.tgz", - "integrity": "sha512-Rj+PMjoNFGFTmtItH7gHfbHpGVSb3vmnGK3nwNBqxQF9NoBpttSZI/rc0WiM63ma2uGDQtYEkMHkK9U6937NiA==", + "version": "9.14.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.14.2.tgz", + "integrity": "sha512-R+FRIfk1GBo3RdlRYWPdwk8nmtVUOn6+BkDomAC46KoU8kzXzE1HLmOasSCbWUByMMAGkknVF0G5kQ69Vj7dlA==", "license": "Apache-2.0", "dependencies": { "base64-js": "^1.3.0", @@ -6297,6 +6313,12 @@ "node": ">= 12" } }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/is": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", @@ -6307,9 +6329,10 @@ } }, "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, "license": "MIT" }, "node_modules/is-binary-path": { @@ -6487,9 +6510,9 @@ } }, "node_modules/jose": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/jose/-/jose-5.9.3.tgz", - "integrity": "sha512-egLIoYSpcd+QUF+UHgobt5YzI2Pkw/H39ou9suW687MY6PmCwPmkNV/4TNjn1p2tX5xO3j0d0sq5hiYE24bSlg==", + "version": "5.9.4", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.9.4.tgz", + "integrity": "sha512-WBBl6au1qg6OHj67yCffCgFR3BADJBXN8MdRvCgJDuMv3driV2nHr7jdGvaKX9IolosAsn+M0XRArqLXUhyJHQ==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/panva" @@ -6512,12 +6535,14 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, "license": "MIT", "dependencies": { - "argparse": "^2.0.1" + "argparse": "^1.0.7", + "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -6986,9 +7011,9 @@ } }, "node_modules/magic-string": { - "version": "0.30.11", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.11.tgz", - "integrity": "sha512-+Wri9p0QHMy+545hKww7YAu5NyzF8iomPL/RQazugQ9+Ez4Ic3mERMd8ZTX5rfK944j+560ZJi8iAwgak1Ac7A==", + "version": "0.30.12", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz", + "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==", "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" @@ -7347,9 +7372,9 @@ } }, "node_modules/nan": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.20.0.tgz", - "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", + "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", "license": "MIT" }, "node_modules/nanoid": { @@ -7498,9 +7523,9 @@ } }, "node_modules/node-addon-api": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.1.0.tgz", - "integrity": "sha512-yBY+qqWSv3dWKGODD6OGE6GnTX7Q2r+4+DfpqxHSHh8x0B4EKP9+wVGLS6U/AM1vxSNNmUEuIV5EGhYwPpfOwQ==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.2.1.tgz", + "integrity": "sha512-vmEOvxwiH8tlOcv4SyE8RH34rI5/nWVaigUeAUPawC6f0+HoDthwI0vkMu4tbtsZrXq6QXFfrkhjofzKEs5tpA==", "license": "MIT", "engines": { "node": "^18 || ^20 || >= 21" @@ -8059,16 +8084,6 @@ "license": "MIT", "peer": true }, - "node_modules/openapi/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, "node_modules/openapi/node_modules/commander": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", @@ -8079,20 +8094,6 @@ "node": ">= 6" } }, - "node_modules/openapi/node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/openapi/node_modules/object-hash": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz", @@ -8103,13 +8104,6 @@ "node": ">= 6" } }, - "node_modules/openapi/node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -8712,13 +8706,6 @@ "node": ">= 10" } }, - "node_modules/pretty-format/node_modules/react-is": { - "version": "17.0.2", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", - "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", - "dev": true, - "license": "MIT" - }, "node_modules/prexit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/prexit/-/prexit-2.3.0.tgz", @@ -8761,6 +8748,12 @@ "react-is": "^16.13.1" } }, + "node_modules/prop-types/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, "node_modules/proto3-json-serializer": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-2.0.2.tgz", @@ -9003,9 +8996,10 @@ } }, "node_modules/react-is": { - "version": "16.13.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", - "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true, "license": "MIT" }, "node_modules/react-json-view-lite": { @@ -9176,15 +9170,15 @@ } }, "node_modules/recharts": { - "version": "2.12.7", - "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.12.7.tgz", - "integrity": "sha512-hlLJMhPQfv4/3NBSAyq3gzGg4h2v69RJh6KU7b3pXYNNAELs9kEoXOjbkxdXpALqKBoVmVptGfLpxdaVYqjmXQ==", + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.13.0.tgz", + "integrity": "sha512-sbfxjWQ+oLWSZEWmvbq/DFVdeRLqqA6d0CDjKx2PkxVVdoXo16jvENCE+u/x7HxOO+/fwx//nYRwb8p8X6s/lQ==", "license": "MIT", "dependencies": { "clsx": "^2.0.0", "eventemitter3": "^4.0.1", "lodash": "^4.17.21", - "react-is": "^16.10.2", + "react-is": "^18.3.1", "react-smooth": "^4.0.0", "recharts-scale": "^0.4.4", "tiny-invariant": "^1.3.1", @@ -9213,6 +9207,12 @@ "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", "license": "MIT" }, + "node_modules/recharts/node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, "node_modules/regenerator-runtime": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", @@ -9637,6 +9637,12 @@ "is-arrayish": "^0.3.1" } }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "license": "MIT" + }, "node_modules/slash": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", @@ -9735,9 +9741,10 @@ "integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==" }, "node_modules/sprintf-js": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", - "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/ssri": { diff --git a/packages/backend/src/api/v1/projects/index.ts b/packages/backend/src/api/v1/projects/index.ts index 300fb632..85328981 100644 --- a/packages/backend/src/api/v1/projects/index.ts +++ b/packages/backend/src/api/v1/projects/index.ts @@ -13,7 +13,7 @@ projects.get("/", checkAccess("projects", "read"), async (ctx: Context) => { const { orgId, userId } = ctx.state; const rows = await sql` - select + select distinct on (p.id) p.id, p.created_at, p.name, @@ -209,10 +209,17 @@ projects.post( "/:projectId/rules", checkAccess("projects", "update"), async (ctx: Context) => { - const bodySchema = z.object({ - type: z.enum(["filtering", "masking"]).default("filtering"), - filters: z.array(z.any()).optional(), - }); + const bodySchema = z.discriminatedUnion("type", [ + z.object({ + type: z.literal("filtering"), + filters: z.array(z.any()), + }), + z.object({ + type: z.literal("masking"), + filters: z.undefined().transform(() => null), + }), + ]); + const { type, filters } = bodySchema.parse(ctx.request.body); const { projectId } = ctx.params; const { userId } = ctx.state; diff --git a/packages/backend/src/api/v1/runs/ingest.ts b/packages/backend/src/api/v1/runs/ingest.ts index 9ef0f50e..b99e4af9 100644 --- a/packages/backend/src/api/v1/runs/ingest.ts +++ b/packages/backend/src/api/v1/runs/ingest.ts @@ -431,14 +431,14 @@ export async function processEventsIngestion( if (ingestionRule) { passedIngestionRule = await checkIngestionRule( - cleanedEvent, + structuredClone(cleanedEvent), ingestionRule.filters, ); } if (cleanedEvent.event === "end") { const [dbRun] = - await sql`select * from run where id = ${cleanedEvent.runId}`; + await sql`select * from run where id = ${cleanedEvent.runId || null}`; if (dbRun?.input === "__NOT_INGESTED__") { passedIngestionRule = false; } diff --git a/packages/backend/src/checks/runChecks.ts b/packages/backend/src/checks/runChecks.ts index 869ecc39..0bb0d41b 100644 --- a/packages/backend/src/checks/runChecks.ts +++ b/packages/backend/src/checks/runChecks.ts @@ -174,14 +174,14 @@ export async function checkIngestionRule(run: any, checks: CheckLogic) { if (!checks.length || (checks.length === 1 && checks[0] === "AND")) return true; - let passed = false; + let passed = true; const results: CheckResults[] = []; const logicType = checks[0]; const subChecks = checks.slice(1); if (logicType === "OR") { for (const check of subChecks) { - const res = await checkRun(run, check, false, true); + const res = await checkRun(run, check, true); results.push(res); if (res.passed) { passed = true; @@ -191,7 +191,7 @@ export async function checkIngestionRule(run: any, checks: CheckLogic) { } else { // Handle nested AND for (const check of subChecks) { - const res = await checkRun(run, check, false, true); + const res = await checkRun(run, check, true); results.push(res); passed = res.passed; if (!res.passed) break; diff --git a/packages/frontend/components/settings/data-warehouse.tsx b/packages/frontend/components/settings/data-warehouse.tsx index d894422d..6197460a 100644 --- a/packages/frontend/components/settings/data-warehouse.tsx +++ b/packages/frontend/components/settings/data-warehouse.tsx @@ -62,6 +62,7 @@ export default function DataWarehouseCard() { {PROVIDERS.map(({ name, iconSrc, settingsPath, disabled }) => ( (["AND"]); + const [checks, setChecks] = useState(["AND"]); + + useEffect(() => { + if (filteringRule?.filters) { + setChecks(filteringRule.filters); + } + }, [filteringRule]); const smartDataFilterEnabled = config.IS_SELF_HOSTED ? org.license.dataFilteringEnabled @@ -200,7 +206,7 @@ function SmartDataRule() { @@ -213,7 +219,7 @@ function SmartDataRule() { loading={addRulesLoading} style={{ float: "right" }} onClick={() => { - addRule({ type: "filtering", filters }); + addRule({ type: "filtering", filters: checks }); }} variant="full" >