From 30b5c286d4ee4037ea8f45ad3c060ed1914216e5 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 6 Feb 2024 14:27:32 +0100 Subject: [PATCH 1/2] fix(server/mongo): evolution step filters on all index columns Signed-off-by: Luka Peschke --- server/CHANGELOG.md | 3 + .../mongo_translator/steps/evolution.py | 5 +- .../mongo_translator/steps/test_evolution.py | 85 +++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 server/tests/backends/mongo_translator/steps/test_evolution.py diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index d87ca3f208..9ca4f8db4c 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Mongo: Fixed the `evolution` step when several index columns are used. The generated pipeline now filters on all of them + rather than jsut the last one. + ## [0.26.9] - 2023-10-05 - Pypika: Fixed the `todate` step by ensuring the right formatting is used for every backend diff --git a/server/src/weaverbird/backends/mongo_translator/steps/evolution.py b/server/src/weaverbird/backends/mongo_translator/steps/evolution.py index ae721701c3..c517cbaab7 100644 --- a/server/src/weaverbird/backends/mongo_translator/steps/evolution.py +++ b/server/src/weaverbird/backends/mongo_translator/steps/evolution.py @@ -82,7 +82,10 @@ def translate_evolution(step: EvolutionStep) -> list[MongoStep]: "cond": { "$and": [ {"$eq": ["$_VQB_DATE_PREV", f"$$item.{step.date_col}"]}, - {"$eq": [f"${col}", f"$$item.{col}"] for col in step.index_columns}, + *( + {"$eq": [f"${col}", f"$$item.{col}"]} + for col in step.index_columns + ), ], }, }, diff --git a/server/tests/backends/mongo_translator/steps/test_evolution.py b/server/tests/backends/mongo_translator/steps/test_evolution.py new file mode 100644 index 0000000000..2fc0beea3e --- /dev/null +++ b/server/tests/backends/mongo_translator/steps/test_evolution.py @@ -0,0 +1,85 @@ +from weaverbird.backends.mongo_translator.steps.evolution import translate_evolution +from weaverbird.pipeline.steps.evolution import EvolutionStep + + +def test_translate_evolution() -> None: + step = EvolutionStep( + dateCol="foo", + valueCol="bar", + evolutionFormat="abs", + evolutionType="vsLastDay", + indexColumns=["a", "b", "c"], + ) + + assert translate_evolution(step) == [ + { + "$addFields": { + "_VQB_DATE_PREV": { + "$dateSubtract": {"amount": 1, "startDate": "$foo", "unit": "week"} + } + } + }, + { + "$facet": { + "_VQB_COPIES_ARRAY": [ + {"$group": {"_VQB_ALL_DOCS": {"$push": "$$ROOT"}, "_id": None}} + ], + "_VQB_ORIGINALS": [{"$project": {"_id": 0}}], + } + }, + {"$unwind": "$_VQB_ORIGINALS"}, + { + "$project": { + "_VQB_ORIGINALS": { + "$mergeObjects": [ + "$_VQB_ORIGINALS", + {"$arrayElemAt": ["$_VQB_COPIES_ARRAY", 0]}, + ] + } + } + }, + {"$replaceRoot": {"newRoot": "$_VQB_ORIGINALS"}}, + { + "$addFields": { + "_VQB_ALL_DOCS": { + "$filter": { + "as": "item", + "cond": { + "$and": [ + {"$eq": ["$_VQB_DATE_PREV", "$$item.foo"]}, + {"$eq": ["$a", "$$item.a"]}, + {"$eq": ["$b", "$$item.b"]}, + {"$eq": ["$c", "$$item.c"]}, + ] + }, + "input": "$_VQB_ALL_DOCS", + } + } + } + }, + { + "$addFields": { + "_VQB_VALUE_PREV": { + "$cond": [ + {"$gt": [{"$size": "$_VQB_ALL_DOCS.bar"}, 1]}, + "Error", + {"$arrayElemAt": ["$_VQB_ALL_DOCS.bar", 0]}, + ] + } + } + }, + { + "$addFields": { + "bar_EVOL_ABS": { + "$cond": [ + {"$eq": ["$_VQB_VALUE_PREV", "Error"]}, + "Error: More than one previous " + "date found for the specified " + "index columns", + {"$subtract": ["$bar", "$_VQB_VALUE_PREV"]}, + ] + } + } + }, + {"$project": {"_VQB_ALL_DOCS": 0, "_VQB_DATE_PREV": 0, "_VQB_VALUE_PREV": 0}}, + ] From ada4ca5c59dd2cda57bf0011b49095063e6e0fe6 Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Tue, 6 Feb 2024 14:32:30 +0100 Subject: [PATCH 2/2] chore(server): v0.26.10 Signed-off-by: Luka Peschke --- server/CHANGELOG.md | 4 +++- server/pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 9ca4f8db4c..3ce07d08de 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -2,8 +2,10 @@ ## Unreleased +## [0.26.10] - 2024-02-06 + - Mongo: Fixed the `evolution` step when several index columns are used. The generated pipeline now filters on all of them - rather than jsut the last one. + rather than just the last one. ## [0.26.9] - 2023-10-05 diff --git a/server/pyproject.toml b/server/pyproject.toml index a056496aa1..808861e8b6 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "weaverbird" -version = "0.26.9" +version = "0.26.10" description = "A visual data pipeline builder with various backends" authors = ["Toucan Toco "] keywords = ["mongodb", "pandas", "sql", "data", "dataviz", "pipeline", "query", "builder"]