From 85f001982e8adf0def097bbba615c21365376639 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Fri, 31 Jan 2025 10:20:04 +0200 Subject: [PATCH 1/4] GitHub Actions Workflow updates (#7298) * Split out secrets requiring workflows * Update target * Update Cypress run command --- .github/workflows/ci.yml | 16 ++++++++-------- client/cypress/cypress.js | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 520f5c0a60..1cee14f8ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ on: push: branches: - master - pull_request_target: + pull_request: branches: - master env: @@ -60,10 +60,10 @@ jobs: mkdir -p /tmp/test-results/unit-tests docker cp tests:/app/coverage.xml ./coverage.xml docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} + # - name: Upload coverage reports to Codecov + # uses: codecov/codecov-action@v3 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} - name: Store Test Results uses: actions/upload-artifact@v4 with: @@ -134,9 +134,9 @@ jobs: COMPOSE_PROJECT_NAME: cypress CYPRESS_INSTALL_BINARY: 0 PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + # PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} + # CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} + # CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} steps: - if: github.event.pull_request.mergeable == 'false' name: Exit if PR is not mergeable diff --git a/client/cypress/cypress.js b/client/cypress/cypress.js index 320402f18d..71c2a02141 100644 --- a/client/cypress/cypress.js +++ b/client/cypress/cypress.js @@ -63,7 +63,7 @@ function runCypressCI() { CYPRESS_OPTIONS, // eslint-disable-line no-unused-vars } = process.env; - if (GITHUB_REPOSITORY === "getredash/redash") { + if (GITHUB_REPOSITORY === "getredash/redash" && process.env.CYPRESS_RECORD_KEY) { process.env.CYPRESS_OPTIONS = "--record"; } From 2776992101a93ef205daee4cc6bc68c42d77f114 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 00:33:52 +0000 Subject: [PATCH 2/4] Snapshot: 25.02.0-dev --- package.json | 2 +- pyproject.toml | 2 +- redash/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 0655dd334f..874c8d85b9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "redash-client", - "version": "25.01.0-dev", + "version": "25.02.0-dev", "description": "The frontend part of Redash.", "main": "index.js", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index b5c706a18c..be09b2b665 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ force-exclude = ''' [tool.poetry] name = "redash" -version = "25.01.0-dev" +version = "25.02.0-dev" description = "Make Your Company Data Driven. Connect to any data source, easily visualize, dashboard and share your data." authors = ["Arik Fraimovich "] # to be added to/removed from the mailing list, please reach out to Arik via the above email or Discord diff --git a/redash/__init__.py b/redash/__init__.py index 2deeaf7bb2..7d463e6995 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -14,7 +14,7 @@ from redash.destinations import import_destinations from redash.query_runner import import_query_runners -__version__ = "25.01.0-dev" +__version__ = "25.02.0-dev" if os.environ.get("REMOTE_DEBUG"): From 96ea0194e8d97115633722e67a5625eb193d7fa6 Mon Sep 17 00:00:00 2001 From: Matt Nelson Date: Tue, 4 Feb 2025 05:05:13 -0600 Subject: [PATCH 3/4] Fix errors in webex alert destination. Add formatting support for QUERY_RESULT_TABLE. (#7296) * prevent text values in payload being detected as 'set' on send. Webex send ERROR:: Object of type set is not JSON serializable Signed-off-by: Matt Nelson * add support for formatted QUERY_RESULT_TABLE in webex card Signed-off-by: Matt Nelson * don't try to send to blank destinations Signed-off-by: Matt Nelson * fix handling of the encoded QUERY_RESULTS_TABLE text Signed-off-by: Matt Nelson * re-sort imports for ruff Signed-off-by: Matt Nelson * change formatter to black Signed-off-by: Matt Nelson * Add additional tests for Webex notification handling ensure blank entries are handled for room IDs and person emails. ensure that the API is not called when no valid destinations are provided. ensure proper attachment formatting for alerts containing 2D arrays. Signed-off-by: Matt Nelson * Add test for Webex notification with 1D array handling This commit introduces a new test case to verify that the Webex notification function correctly handles a 1D array input in the alert body. The test ensures that the expected payload is constructed properly and that the requests.post method is called with the correct parameters. Signed-off-by: Matt Nelson --------- Signed-off-by: Matt Nelson --- redash/destinations/webex.py | 168 +++++++++++++++++++++------- tests/handlers/test_destinations.py | 165 ++++++++++++++++++++++++++- 2 files changed, 290 insertions(+), 43 deletions(-) diff --git a/redash/destinations/webex.py b/redash/destinations/webex.py index 16d8ed05a6..599c485e36 100644 --- a/redash/destinations/webex.py +++ b/redash/destinations/webex.py @@ -1,3 +1,5 @@ +import html +import json import logging from copy import deepcopy @@ -37,6 +39,129 @@ def api_base_url(self): @staticmethod def formatted_attachments_template(subject, description, query_link, alert_link): + # Attempt to parse the description to find a 2D array + try: + # Extract the part of the description that looks like a JSON array + start_index = description.find("[") + end_index = description.rfind("]") + 1 + json_array_str = description[start_index:end_index] + + # Decode HTML entities + json_array_str = html.unescape(json_array_str) + + # Replace single quotes with double quotes for valid JSON + json_array_str = json_array_str.replace("'", '"') + + # Load the JSON array + data_array = json.loads(json_array_str) + + # Check if it's a 2D array + if isinstance(data_array, list) and all(isinstance(i, list) for i in data_array): + # Create a table for the Adaptive Card + table_rows = [] + for row in data_array: + table_rows.append( + { + "type": "ColumnSet", + "columns": [ + {"type": "Column", "items": [{"type": "TextBlock", "text": str(item), "wrap": True}]} + for item in row + ], + } + ) + + # Create the body of the card with the table + body = ( + [ + { + "type": "TextBlock", + "text": f"{subject}", + "weight": "bolder", + "size": "medium", + "wrap": True, + }, + { + "type": "TextBlock", + "text": f"{description[:start_index]}", + "isSubtle": True, + "wrap": True, + }, + ] + + table_rows + + [ + { + "type": "TextBlock", + "text": f"Click [here]({query_link}) to check your query!", + "wrap": True, + "isSubtle": True, + }, + { + "type": "TextBlock", + "text": f"Click [here]({alert_link}) to check your alert!", + "wrap": True, + "isSubtle": True, + }, + ] + ) + else: + # Fallback to the original description if no valid 2D array is found + body = [ + { + "type": "TextBlock", + "text": f"{subject}", + "weight": "bolder", + "size": "medium", + "wrap": True, + }, + { + "type": "TextBlock", + "text": f"{description}", + "isSubtle": True, + "wrap": True, + }, + { + "type": "TextBlock", + "text": f"Click [here]({query_link}) to check your query!", + "wrap": True, + "isSubtle": True, + }, + { + "type": "TextBlock", + "text": f"Click [here]({alert_link}) to check your alert!", + "wrap": True, + "isSubtle": True, + }, + ] + except json.JSONDecodeError: + # If parsing fails, fallback to the original description + body = [ + { + "type": "TextBlock", + "text": f"{subject}", + "weight": "bolder", + "size": "medium", + "wrap": True, + }, + { + "type": "TextBlock", + "text": f"{description}", + "isSubtle": True, + "wrap": True, + }, + { + "type": "TextBlock", + "text": f"Click [here]({query_link}) to check your query!", + "wrap": True, + "isSubtle": True, + }, + { + "type": "TextBlock", + "text": f"Click [here]({alert_link}) to check your alert!", + "wrap": True, + "isSubtle": True, + }, + ] + return [ { "contentType": "application/vnd.microsoft.card.adaptive", @@ -44,44 +169,7 @@ def formatted_attachments_template(subject, description, query_link, alert_link) "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", "type": "AdaptiveCard", "version": "1.0", - "body": [ - { - "type": "ColumnSet", - "columns": [ - { - "type": "Column", - "width": 4, - "items": [ - { - "type": "TextBlock", - "text": {subject}, - "weight": "bolder", - "size": "medium", - "wrap": True, - }, - { - "type": "TextBlock", - "text": {description}, - "isSubtle": True, - "wrap": True, - }, - { - "type": "TextBlock", - "text": f"Click [here]({query_link}) to check your query!", - "wrap": True, - "isSubtle": True, - }, - { - "type": "TextBlock", - "text": f"Click [here]({alert_link}) to check your alert!", - "wrap": True, - "isSubtle": True, - }, - ], - }, - ], - } - ], + "body": body, }, } ] @@ -116,6 +204,10 @@ def notify(self, alert, query, user, new_state, app, host, metadata, options): # destinations is guaranteed to be a comma-separated string for destination_id in destinations.split(","): + destination_id = destination_id.strip() # Remove any leading or trailing whitespace + if not destination_id: # Check if the destination_id is empty or blank + continue # Skip to the next iteration if it's empty or blank + payload = deepcopy(template_payload) payload[payload_tag] = destination_id self.post_message(payload, headers) diff --git a/tests/handlers/test_destinations.py b/tests/handlers/test_destinations.py index 95fcffad09..00a52cbb00 100644 --- a/tests/handlers/test_destinations.py +++ b/tests/handlers/test_destinations.py @@ -261,15 +261,117 @@ def test_webex_notify_calls_requests_post(): alert.name = "Test Alert" alert.custom_subject = "Test custom subject" alert.custom_body = "Test custom body" + alert.render_template = mock.Mock(return_value={"Rendered": "template"}) + + query = mock.Mock() + query.id = 1 + + user = mock.Mock() + app = mock.Mock() + host = "https://localhost:5000" + options = { + "webex_bot_token": "abcd", + "to_room_ids": "1234,5678", + "to_person_emails": "example1@test.com,example2@test.com", + } + metadata = {"Scheduled": False} + + new_state = Alert.TRIGGERED_STATE + destination = Webex(options) + + with mock.patch("redash.destinations.webex.requests.post") as mock_post: + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + destination.notify(alert, query, user, new_state, app, host, metadata, options) + + query_link = f"{host}/queries/{query.id}" + alert_link = f"{host}/alerts/{alert.id}" + + expected_attachments = Webex.formatted_attachments_template( + alert.custom_subject, alert.custom_body, query_link, alert_link + ) + + expected_payload_room = { + "markdown": alert.custom_subject + "\n" + alert.custom_body, + "attachments": expected_attachments, + "roomId": "1234", + } + + expected_payload_email = { + "markdown": alert.custom_subject + "\n" + alert.custom_body, + "attachments": expected_attachments, + "toPersonEmail": "example1@test.com", + } + + # Check that requests.post was called for both roomId and toPersonEmail destinations + mock_post.assert_any_call( + destination.api_base_url, + json=expected_payload_room, + headers={"Authorization": "Bearer abcd"}, + timeout=5.0, + ) + + mock_post.assert_any_call( + destination.api_base_url, + json=expected_payload_email, + headers={"Authorization": "Bearer abcd"}, + timeout=5.0, + ) + + assert mock_response.status_code == 200 + + +def test_webex_notify_handles_blank_entries(): + alert = mock.Mock(spec_set=["id", "name", "custom_subject", "custom_body", "render_template"]) + alert.id = 1 + alert.name = "Test Alert" + alert.custom_subject = "Test custom subject" + alert.custom_body = "Test custom body" + alert.render_template = mock.Mock(return_value={"Rendered": "template"}) + + query = mock.Mock() + query.id = 1 + + user = mock.Mock() + app = mock.Mock() + host = "https://localhost:5000" + options = { + "webex_bot_token": "abcd", + "to_room_ids": "", + "to_person_emails": "", + } + metadata = {"Scheduled": False} + new_state = Alert.TRIGGERED_STATE + destination = Webex(options) + + with mock.patch("redash.destinations.webex.requests.post") as mock_post: + destination.notify(alert, query, user, new_state, app, host, metadata, options) + + # Ensure no API calls are made when destinations are blank + mock_post.assert_not_called() + + +def test_webex_notify_handles_2d_array(): + alert = mock.Mock(spec_set=["id", "name", "custom_subject", "custom_body", "render_template"]) + alert.id = 1 + alert.name = "Test Alert" + alert.custom_subject = "Test custom subject" + alert.custom_body = "Test custom body with table [['Col1', 'Col2'], ['Val1', 'Val2']]" alert.render_template = mock.Mock(return_value={"Rendered": "template"}) + query = mock.Mock() query.id = 1 user = mock.Mock() app = mock.Mock() host = "https://localhost:5000" - options = {"webex_bot_token": "abcd", "to_room_ids": "1234"} + options = { + "webex_bot_token": "abcd", + "to_room_ids": "1234", + } metadata = {"Scheduled": False} new_state = Alert.TRIGGERED_STATE @@ -277,7 +379,7 @@ def test_webex_notify_calls_requests_post(): with mock.patch("redash.destinations.webex.requests.post") as mock_post: mock_response = mock.Mock() - mock_response.status_code = 204 + mock_response.status_code = 200 mock_post.return_value = mock_response destination.notify(alert, query, user, new_state, app, host, metadata, options) @@ -285,13 +387,13 @@ def test_webex_notify_calls_requests_post(): query_link = f"{host}/queries/{query.id}" alert_link = f"{host}/alerts/{alert.id}" - formatted_attachments = Webex.formatted_attachments_template( + expected_attachments = Webex.formatted_attachments_template( alert.custom_subject, alert.custom_body, query_link, alert_link ) expected_payload = { "markdown": alert.custom_subject + "\n" + alert.custom_body, - "attachments": formatted_attachments, + "attachments": expected_attachments, "roomId": "1234", } @@ -302,7 +404,60 @@ def test_webex_notify_calls_requests_post(): timeout=5.0, ) - assert mock_response.status_code == 204 + assert mock_response.status_code == 200 + + +def test_webex_notify_handles_1d_array(): + alert = mock.Mock(spec_set=["id", "name", "custom_subject", "custom_body", "render_template"]) + alert.id = 1 + alert.name = "Test Alert" + alert.custom_subject = "Test custom subject" + alert.custom_body = "Test custom body with 1D array, however unlikely ['Col1', 'Col2']" + alert.render_template = mock.Mock(return_value={"Rendered": "template"}) + + query = mock.Mock() + query.id = 1 + + user = mock.Mock() + app = mock.Mock() + host = "https://localhost:5000" + options = { + "webex_bot_token": "abcd", + "to_room_ids": "1234", + } + metadata = {"Scheduled": False} + + new_state = Alert.TRIGGERED_STATE + destination = Webex(options) + + with mock.patch("redash.destinations.webex.requests.post") as mock_post: + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + destination.notify(alert, query, user, new_state, app, host, metadata, options) + + query_link = f"{host}/queries/{query.id}" + alert_link = f"{host}/alerts/{alert.id}" + + expected_attachments = Webex.formatted_attachments_template( + alert.custom_subject, alert.custom_body, query_link, alert_link + ) + + expected_payload = { + "markdown": alert.custom_subject + "\n" + alert.custom_body, + "attachments": expected_attachments, + "roomId": "1234", + } + + mock_post.assert_called_once_with( + destination.api_base_url, + json=expected_payload, + headers={"Authorization": "Bearer abcd"}, + timeout=5.0, + ) + + assert mock_response.status_code == 200 def test_datadog_notify_calls_requests_post(): From ec2ca6f98690d4a129ee7ff36430bbe1f2296e40 Mon Sep 17 00:00:00 2001 From: Tsuneo Yoshioka Date: Thu, 6 Feb 2025 03:25:39 +0900 Subject: [PATCH 4/4] BigQuery: show column type on Schema Browser (#7257) --- redash/query_runner/big_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redash/query_runner/big_query.py b/redash/query_runner/big_query.py index 51cf7d9542..4ccef53244 100644 --- a/redash/query_runner/big_query.py +++ b/redash/query_runner/big_query.py @@ -304,7 +304,7 @@ def get_schema(self, get_stats=False): datasets = self._get_project_datasets(project_id) query_base = """ - SELECT table_schema, table_name, field_path + SELECT table_schema, table_name, field_path, data_type FROM `{dataset_id}`.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS WHERE table_schema NOT IN ('information_schema') """ @@ -325,7 +325,7 @@ def get_schema(self, get_stats=False): table_name = "{0}.{1}".format(row["table_schema"], row["table_name"]) if table_name not in schema: schema[table_name] = {"name": table_name, "columns": []} - schema[table_name]["columns"].append(row["field_path"]) + schema[table_name]["columns"].append({"name": row["field_path"], "type": row["data_type"]}) return list(schema.values())