refactor: Update Slack and Twitter workflows to use environment varia…

…bles directly

- Removed DLT__ prefixed access token variables from Slack and Twitter workflow files.
- Updated Slack and Twitter pipeline scripts to retrieve tokens from environment variables instead of DLT secrets.
- Ensured that appropriate error handling is in place for missing environment variables.

.github/workflows/run_slack_workflow.yml

@@ -12,11 +12,9 @@ name: Run Slack Pipeline
 
 env:
   # DLT specific environment variables
-  # DLT__SECRETS__SOURCES__SLACK__ACCESS_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }}
-
-  # Optional: Add any other environment variables needed
+  # Remove the DLT__ prefixed version since we're using direct env vars
   PYTHONUNBUFFERED: "1"
 
 jobs:

.github/workflows/run_twitter.yml

@@ -12,7 +12,6 @@ name: Run Twitter Pipeline
 
 env:
   # DLT specific environment variables
-  # DLT__SECRETS__SOURCES__TWITTER__BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}
   TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}
   motherduck_token: ${{ secrets.MOTHERDUCK_TOKEN }}
 

pipeline/slack_pipeline.py

@@ -13,16 +13,17 @@
 from datetime import datetime
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
+import os
 
 @dlt.source(name="slack")
 def slack_source():
     """DLT source for Slack workspace statistics"""
     # Get token explicitly from secrets
-    access_token = dlt.secrets.get("sources.slack_pipeline.access_token")
-    if not access_token:
-        raise ValueError("Slack access token not found in secrets")
+    token = os.getenv("SLACK_BOT_TOKEN")
+    if not token:
+        raise ValueError("SLACK_BOT_TOKEN environment variable is not set")
 
-    client = WebClient(token=access_token)
+    client = WebClient(token=token)
     return slack_stats_resource(client)
 
 @dlt.resource(name="workspace_stats", write_disposition="merge", primary_key=["timestamp"])

pipeline/twitter_pipeline.py

@@ -20,7 +20,7 @@
 
 def get_twitter_headers():
     """Get Twitter API headers with bearer token authentication"""
-    access_token = dlt.secrets.get("sources.twitter.bearer_token")
+    access_token = os.getenv("TWITTER_BEARER_TOKEN")
     if not access_token:
         raise ValueError("TWITTER_BEARER_TOKEN environment variable is not set")
     return {