diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 9f37e5eb95..3d15fd59bb 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +### Added + +- Pypika translator: added an optional `columns` property to CustomSQL step. This column list is used when translating + the query, and avoid relying on a fake table in `table_columns`. This prevents name conflicts when joining multiple + pipelines with different CustomSQL steps. + ## [0.38.1] - 2023-10-13 ### Fixed diff --git a/server/src/weaverbird/backends/pypika_translator/translators/base.py b/server/src/weaverbird/backends/pypika_translator/translators/base.py index ab0a7e3d1f..f606449ded 100644 --- a/server/src/weaverbird/backends/pypika_translator/translators/base.py +++ b/server/src/weaverbird/backends/pypika_translator/translators/base.py @@ -227,6 +227,8 @@ def _next_step_name(self: Self) -> str: return name def _extract_columns_from_customsql_step(self: Self, *, step: "CustomSqlStep") -> list[str]: + if step.columns: + return step.columns # In case there are several provided tables, we cannot figure out which columns to use if len(self._tables_columns) != 1: raise UnknownTableColumns("Expected columns to be specified for exactly one table") diff --git a/server/src/weaverbird/pipeline/steps/customsql.py b/server/src/weaverbird/pipeline/steps/customsql.py index efcc9662d0..b18766c96d 100644 --- a/server/src/weaverbird/pipeline/steps/customsql.py +++ b/server/src/weaverbird/pipeline/steps/customsql.py @@ -8,6 +8,7 @@ class CustomSqlStep(BaseStep): name: Literal["customsql"] = "customsql" query: str + columns: list[str] | None = None @staticmethod def _strip_query(query: str) -> str: diff --git a/ui/src/components/stepforms/CustomSqlStepForm.vue b/ui/src/components/stepforms/CustomSqlStepForm.vue index 6614379483..2e4b3a4b38 100644 --- a/ui/src/components/stepforms/CustomSqlStepForm.vue +++ b/ui/src/components/stepforms/CustomSqlStepForm.vue @@ -1,3 +1,5 @@ +<!-- Unused, only supported as first step of a pipeline --> + <template> <div> <StepFormHeader diff --git a/ui/src/components/stepforms/schemas/customsql.ts b/ui/src/components/stepforms/schemas/customsql.ts index fec6403fc7..3a062dc08b 100644 --- a/ui/src/components/stepforms/schemas/customsql.ts +++ b/ui/src/components/stepforms/schemas/customsql.ts @@ -13,6 +13,14 @@ export default { title: 'Query', description: 'Write a query', }, + columns: { + description: 'Columns to select', + type: 'array', + items: { + type: 'string', + minLength: 1, + }, + }, }, required: ['name', 'query'], additionalProperties: false, diff --git a/ui/src/lib/steps.ts b/ui/src/lib/steps.ts index f84857ccbf..1f2bf04ada 100644 --- a/ui/src/lib/steps.ts +++ b/ui/src/lib/steps.ts @@ -201,6 +201,7 @@ export type CustomStep = { export type CustomSqlStep = { name: 'customsql'; query: string; + columns?: string[]; }; export type DateExtractStep = {