fix(server): accept variables in combined pipelines TCTC-7763 (#2001)

* fix(server): accept variables in combined pipelines (append) * docs(server): explain types precendence * docs(server): changerlog note on variables and combined pipelines * style(server): format
ToucanToco · Jan 26, 2024 · a9b998a · a9b998a
1 parent fe863d0
commit a9b998a
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 7 deletions.
diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Fixed
+
+- Using variables in combined pipelines was considered invalid.
+
 ## [0.41.2] - 2024-01-18
 
 ### Fixed

diff --git a/server/src/weaverbird/pipeline/steps/utils/combination.py b/server/src/weaverbird/pipeline/steps/utils/combination.py
@@ -5,7 +5,7 @@
 from pydantic import BaseModel, BeforeValidator, TypeAdapter
 
 if TYPE_CHECKING:
-    from weaverbird.pipeline.pipeline import PipelineStep, PipelineStepWithRefs
+    from weaverbird.pipeline.pipeline import PipelineStep, PipelineStepWithRefs, PipelineStepWithVariables
 
 
 class Reference(BaseModel):
@@ -55,10 +55,14 @@ def _pipelinestep_adapter() -> TypeAdapter["str | list[PipelineStep]"]:
 
 
 @cache
-def _pipelinestepwithref_adapter() -> TypeAdapter["str | list[PipelineStepWithRefs | PipelineStep]"]:
-    from weaverbird.pipeline.pipeline import PipelineStep, PipelineStepWithRefs
+def _pipelinestepwithref_adapter() -> (
+    TypeAdapter["str | list[PipelineStepWithRefs | PipelineStepWithVariables | PipelineStep]"]
+):
+    from weaverbird.pipeline.pipeline import PipelineStep, PipelineStepWithRefs, PipelineStepWithVariables
 
-    return TypeAdapter(str | list[PipelineStepWithRefs | PipelineStep])  # type: ignore[arg-type]
+    # Note: PipelineStep must appear _before_ PipelineStepWithVariables, to avoid fields that could contain variables
+    # to always be matched as strings
+    return TypeAdapter(str | list[PipelineStepWithRefs | PipelineStep | PipelineStepWithVariables])  # type: ignore[arg-type]
 
 
 def _ensure_is_pipeline_step(
@@ -72,14 +76,14 @@ def _ensure_is_pipeline_step(
 
 
 def _ensure_is_pipeline_step_with_ref(
-    v: str | list[dict] | list["PipelineStep | PipelineStepWithRefs"],
-) -> str | list["PipelineStep | PipelineStepWithRefs"]:
+    v: str | list[dict] | list["PipelineStep | PipelineStepWithVariables | PipelineStepWithRefs"],
+) -> str | list["PipelineStep | PipelineStepWithVariables | PipelineStepWithRefs"]:
     return _pipelinestepwithref_adapter().validate_python(v)
 
 
 # can be either a domain name or a complete pipeline
 PipelineWithRefsOrDomainName = Annotated[
-    str | list["PipelineStepWithRefs | PipelineStep"],
+    str | list["PipelineStepWithRefs | PipelineStepWithVariables | PipelineStep"],
     BeforeValidator(_ensure_is_pipeline_step_with_ref),
 ]
 

diff --git a/server/tests/test_pipeline.py b/server/tests/test_pipeline.py
@@ -12,6 +12,7 @@
     Pipeline,
     PipelineStep,
     PipelineStepWithVariables,
+    PipelineWithRefs,
     PipelineWithVariables,
     remove_void_conditions_from_filter_steps,
     remove_void_conditions_from_mongo_steps,
@@ -598,3 +599,33 @@ def test_remove_void_conditions_from_filter_steps_with_combinations(
     expected_steps: list[PipelineStep | PipelineStepWithVariables],
 ) -> None:
     assert remove_void_conditions_from_filter_steps(steps) == expected_steps
+
+
+def test_pipeline_with_refs_variables_and_date_validity():
+    PipelineWithRefs(
+        steps=[
+            {"domain": {"type": "ref", "uid": "83ff1fa2-d186-4a7b-a53b-47c901a076c7"}, "name": "domain"},
+            {
+                "name": "append",
+                "pipelines": [
+                    {"type": "ref", "uid": "3ecb05aa-1312-4797-bf9a-65f736bb2993"},
+                    [
+                        {"name": "domain", "domain": "foo"},
+                        {
+                            "name": "filter",
+                            "condition": {
+                                "column": "date",
+                                "operator": "from",
+                                "value": {
+                                    "date": "{{TODAY}}",
+                                    "duration": "year",
+                                    "operator": "until",
+                                    "quantity": 1,
+                                },
+                            },
+                        },
+                    ],
+                ],
+            },
+        ]
+    )