[SDK]improve error message when connection has scrubbed secrets & add CI for flow-as-func

.github/workflows/samples_getstarted_flowasfunction.yml

@@ -0,0 +1,52 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: samples_getstarted_flowasfunction
+on:
+  schedule:
+    - cron: "17 21 * * *" # Every day starting at 5:17 BJT
+  pull_request:
+    branches: [ main ]
+    paths: [ examples/**, .github/workflows/samples_getstarted_flowasfunction.yml ]
+  workflow_dispatch:
+
+env:
+  IS_IN_CI_PIPELINE: "true"
+
+jobs:
+  samples_notebook_ci:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Generate config.json
+        run: echo '${{ secrets.TEST_WORKSPACE_CONFIG_JSON_CANARY }}' > ${{ github.workspace }}/examples/config.json
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: Setup Python 3.9 environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Prepare requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ${{ github.workspace }}/examples/requirements.txt
+          pip install -r ${{ github.workspace }}/examples/dev_requirements.txt
+      - name: Create Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}"
+      - name: Create new Aoai Connection
+        run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY_TEST }}" api_base="${{ secrets.AOAI_API_ENDPOINT_TEST }}" name=new_ai_connection
+      - name: Test Notebook
+        working-directory: examples/tutorials/get-started
+        run: |
+          papermill -k python flow-as-function.ipynb flow-as-function.output.ipynb -p api_key secrets.AOAI_API_KEY_TEST -p api_base secrets.AOAI_API_ENDPOINT_TEST -p api_verson 2023-07-01-preview
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: artifact
+          path: examples/tutorials/get-started

examples/tutorials/get-started/flow-as-function.ipynb

@@ -39,6 +39,24 @@
     "You will need to have a connection named \"new_ai_connection\" to run flow with new connection."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# provide parameters to create connection\n",
+    "\n",
+    "conn_name = \"new_ai_connection\"\n",
+    "api_key = \"<user-input>\"\n",
+    "api_base = \"<user-input>\"\n",
+    "api_version = \"<user-input>\""
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -51,27 +69,24 @@
     "\n",
     "pf = promptflow.PFClient()\n",
     "\n",
-    "try:\n",
-    "    conn_name = \"new_ai_connection\"\n",
-    "    conn = pf.connections.get(name=conn_name)\n",
-    "    print(\"using existing connection\")\n",
-    "except:\n",
-    "    # Follow https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal to create an Azure Open AI resource.\n",
-    "    connection = AzureOpenAIConnection(\n",
-    "        name=conn_name,\n",
-    "        api_key=\"<user-input>\",\n",
-    "        api_base=\"<test_base>\",\n",
-    "        api_type=\"azure\",\n",
-    "        api_version=\"<test_version>\",\n",
-    "    )\n",
-    "\n",
-    "    # use this if you have an existing OpenAI account\n",
-    "    # connection = OpenAIConnection(\n",
-    "    #     name=conn_name,\n",
-    "    #     api_key=\"<user-input>\",\n",
-    "    # )\n",
-    "    conn = pf.connections.create_or_update(connection)\n",
-    "    print(\"successfully created connection\")\n",
+    "\n",
+    "# Follow https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal to create an Azure Open AI resource.\n",
+    "connection = AzureOpenAIConnection(\n",
+    "    name=conn_name,\n",
+    "    api_key=api_key,\n",
+    "    api_base=api_base,\n",
+    "    api_type=\"azure\",\n",
+    "    api_version=api_version,\n",
+    ")\n",
+    "\n",
+    "# use this if you have an existing OpenAI account\n",
+    "# connection = OpenAIConnection(\n",
+    "#     name=conn_name,\n",
+    "#     api_key=api_key,\n",
+    "# )\n",
+    "\n",
+    "conn = pf.connections.create_or_update(connection)\n",
+    "print(\"successfully created connection\")\n",
     "\n",
     "print(conn)"
    ]
@@ -85,8 +100,8 @@
     "f = load_flow(\n",
     "    source=flow_path,\n",
     ")\n",
-    "# need to create the connection\n",
-    "f.context.connections={\"classify_with_llm\": {\"connection\": \"new_ai_connection\"}}\n",
+    "# directly use connection created above\n",
+    "f.context.connections={\"classify_with_llm\": {\"connection\": conn}}\n",
     "\n",
     "result = f(url=sample_url)\n",
     "\n",

scripts/readme/ghactions_driver/workflow_templates/flow_as_function.yml.jinja2

@@ -26,7 +26,7 @@ steps:
   - name: Test Notebook
     working-directory: {{ gh_working_dir }}
     run: |
-      papermill -k python {{ name }}.ipynb {{ name }}.output.ipynb
+      papermill -k python {{ name }}.ipynb {{ name }}.output.ipynb -p api_key secrets.AOAI_API_KEY_TEST -p api_base secrets.AOAI_API_ENDPOINT_TEST -p api_verson 2023-07-01-preview
   - name: Upload artifact
     if: ${{ '{{' }} always() }}
     uses: actions/upload-artifact@v3

src/promptflow/promptflow/_core/connection_manager.py

@@ -15,6 +15,7 @@
 from promptflow.connections import _Connection
 from promptflow.contracts.tool import ConnectionType
 from promptflow.contracts.types import Secret
+from promptflow.exceptions import UserErrorException
 
 
 class ConnectionManager:
@@ -44,6 +45,12 @@ def _build_connections(cls, _dict: Dict[str, dict]):
         for key, connection_dict in _dict.items():
             if isinstance(connection_dict, _Connection):
                 # support directly pass connection object to executor
+                scrubbed_secrets = connection_dict._get_scrubbed_secrets()
+                if scrubbed_secrets:
+                    raise UserErrorException(
+                        f"Connection {connection_dict} contains scrubbed secrets with key {scrubbed_secrets.keys()}, "
+                        "please make sure connection has decrypted secrets to use in flow execution. "
+                    )
                 connections[key] = connection_dict
                 continue
             typ = connection_dict.get("type")

src/promptflow/promptflow/_sdk/entities/_connection.py

@@ -271,6 +271,11 @@ def _from_execution_connection_dict(cls, name, data) -> "_Connection":
             return CustomConnection(name=name, configs=configs, secrets=secrets)
         return type_cls(name=name, **value_dict)
 
+    def _get_scrubbed_secrets(self):
+        """Return the scrubbed secrets of connection."""
+        # local connection store secrets in self._secrets
+        return {key: val for key, val in self._secrets.items() if self._is_scrubbed_value(val)}
+
 
 class _StrongTypeConnection(_Connection):
     def _to_orm_object(self):

src/promptflow/promptflow/_sdk/entities/_flow.py

@@ -71,9 +71,9 @@ def __init__(
         self.environment_variables = environment_variables or {}
         self.overrides = overrides or {}
         self.streaming = streaming
-        # self.connection_provider = connection_provider
+        # TODO: introduce connection provider support
 
-    def resolve_connections(self):
+    def _resolve_connections(self):
         # resolve connections and create placeholder for connection objects
         for _, v in self.connections.items():
             if isinstance(v, dict):

src/promptflow/promptflow/_sdk/operations/_test_submitter.py

@@ -51,7 +51,7 @@ def init(self):
             tuning_node, node_variant = parse_variant(self.flow_context.variant)
         else:
             tuning_node, node_variant = None, None
-        self.flow_context.resolve_connections()
+        self.flow_context._resolve_connections()
         with variant_overwrite_context(
             flow_path=self._origin_flow.code,
             tuning_node=tuning_node,

src/promptflow/tests/sdk_cli_test/e2etests/test_flow_as_func.py

@@ -110,3 +110,11 @@ def test_flow_as_a_func_with_variant(self):
         with pytest.raises(InvalidFlowError) as e:
             f(key="a")
         assert "Variant variant_2 not found for node print_val" in str(e.value)
+
+    def test_non_scrubbed_connection(self):
+        f = load_flow(f"{FLOWS_DIR}/flow_with_custom_connection")
+        f.context.connections = {"hello_node": {"connection": CustomConnection(secrets={"k": "*****"})}}
+
+        with pytest.raises(UserErrorException) as e:
+            f(text="hello")
+        assert "please make sure connection has decrypted secrets to use in flow execution." in str(e)