feat(sql): customsql step can contains a column list TCTC-7280 (#1948)

* feat(sql): customsql step can contains a column list

Avoid to rely on a __custom_sql_table__, which can be overriden if there
is multiple customsql steps

* docs: changelog note

* feat(ui): add new columns param to CustomSQLStep type

* docs: typos on changelog

server/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+### Added
+
+- Pypika translator: added an optional `columns` property to CustomSQL step. This column list is used when translating
+  the query, and avoid relying on a fake table in `table_columns`. This prevents name conflicts when joining multiple
+  pipelines with different CustomSQL steps.
+
 ## [0.38.1] - 2023-10-13
 
 ### Fixed

server/src/weaverbird/backends/pypika_translator/translators/base.py

@@ -227,6 +227,8 @@ def _next_step_name(self: Self) -> str:
         return name
 
     def _extract_columns_from_customsql_step(self: Self, *, step: "CustomSqlStep") -> list[str]:
+        if step.columns:
+            return step.columns
         # In case there are several provided tables, we cannot figure out which columns to use
         if len(self._tables_columns) != 1:
             raise UnknownTableColumns("Expected columns to be specified for exactly one table")

server/src/weaverbird/pipeline/steps/customsql.py

@@ -8,6 +8,7 @@
 class CustomSqlStep(BaseStep):
     name: Literal["customsql"] = "customsql"
     query: str
+    columns: list[str] | None = None
 
     @staticmethod
     def _strip_query(query: str) -> str:

ui/src/components/stepforms/CustomSqlStepForm.vue

@@ -1,3 +1,5 @@
+<!-- Unused, only supported as first step of a pipeline -->
+
 <template>
   <div>
     <StepFormHeader

ui/src/components/stepforms/schemas/customsql.ts

@@ -13,6 +13,14 @@ export default {
       title: 'Query',
       description: 'Write a query',
     },
+    columns: {
+      description: 'Columns to select',
+      type: 'array',
+      items: {
+        type: 'string',
+        minLength: 1,
+      },
+    },
   },
   required: ['name', 'query'],
   additionalProperties: false,

ui/src/lib/steps.ts

@@ -201,6 +201,7 @@ export type CustomStep = {
 export type CustomSqlStep = {
   name: 'customsql';
   query: string;
+  columns?: string[];
 };
 
 export type DateExtractStep = {