update build flex flow test (#3220)

# Description

Please add an informative description that covers that changes made by
the pull request and link all relevant issues.

# All Promptflow Contribution checklist:
- [x] **The pull request does not introduce [breaking changes].**
- [ ] **CHANGELOG is updated for new features, bug fixes or other
significant changes.**
- [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).**
- [ ] **Create an issue and link to the pull request to get dedicated
review from promptflow team. Learn more: [suggested
workflow](../CONTRIBUTING.md#suggested-workflow).**

## General Guidelines and Best Practices
- [ ] Title of the pull request is clear and informative.
- [ ] There are a small number of commits, each of which have an
informative message. This means that previously merged commits do not
appear in the history of the PR. For more information on cleaning up the
commits in your PR, [see this
page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).

### Testing Guidelines
- [ ] Pull request includes test coverage for the included changes.

---------

Co-authored-by: Philip Gao <[email protected]>

src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli.py

@@ -1,3 +1,4 @@
+import filecmp
 import importlib
 import importlib.util
 import json
@@ -64,6 +65,28 @@ def run_pf_command(*args, cwd=None):
         os.chdir(origin_cwd)
 
 
+def compare_directories(dir1, dir2, ingore_path_name):
+    dir1 = Path(dir1)
+    dir2 = Path(dir2)
+    dir1_content = [item for item in dir1.iterdir() if item.name not in ingore_path_name]
+    dir2_content = [item for item in dir2.iterdir() if item.name not in ingore_path_name]
+
+    if len(dir1_content) != len(dir2_content):
+        raise Exception(f"These two folders {dir1_content} and {dir2_content} are different.")
+
+    for path1 in dir1_content:
+        path2 = dir2 / path1.name
+        if not path2.exists():
+            raise Exception(f"The path {path2} does not exist.")
+        if path1.is_file() and path2.is_file():
+            if not filecmp.cmp(path1, path2):
+                raise Exception(f"These two files {path1} and {path2} are different.")
+        elif path1.is_dir() and path2.is_dir():
+            compare_directories(path1, path2, ingore_path_name)
+        else:
+            raise Exception(f"These two path {path1} and {path2} are different.")
+
+
 @pytest.mark.usefixtures(
     "use_secrets_config_file", "recording_injection", "setup_local_connection", "install_custom_tool_pkg"
 )
@@ -1315,6 +1338,7 @@ def get_node_settings(_flow_dag_path: Path):
     def test_flex_flow_build(self):
         from promptflow._cli._pf.entry import main
 
+        origin_build = Path(f"{FLOWS_DIR}/export/flex_flow_build")
         with tempfile.TemporaryDirectory() as temp:
             temp = Path(temp)
             cmd = (
@@ -1330,18 +1354,7 @@ def test_flex_flow_build(self):
             )
             sys.argv = list(cmd)
             main()
-            assert (temp / "connections").is_dir()
-            assert (temp / "flow").is_dir()
-            assert (temp / "runit").is_dir()
-            assert (temp / "Dockerfile").is_file()
-            with open(temp / "Dockerfile", "r") as f:
-                assert r"/connections" in f.read()
-
-            origin_flow = Path(f"{EAGER_FLOWS_DIR}/chat-basic")
-            temp_flow = temp / "flow"
-            for file_path in origin_flow.rglob("*"):
-                relative_path = file_path.relative_to(origin_flow)
-                assert (temp_flow / relative_path).exists()
+            compare_directories(origin_build, temp, ("connections", ".promptflow", "__pycache__"))
 
     def test_flow_build_with_ua(self, capsys):
         with pytest.raises(SystemExit):

src/promptflow/tests/test_configs/flows/export/flex_flow_build/Dockerfile

@@ -0,0 +1,35 @@
+# syntax=docker/dockerfile:1
+FROM docker.io/continuumio/miniconda3:latest
+
+WORKDIR /
+
+COPY ./flow/requirements.txt /flow/requirements.txt
+
+# gcc is for build psutil in MacOS
+RUN apt-get update && apt-get install -y runit gcc
+
+# create conda environment
+RUN conda create -n promptflow-serve python=3.9.16 pip=23.0.1 -q -y && \
+    conda run -n promptflow-serve \
+    pip install -r /flow/requirements.txt && \
+    conda run -n promptflow-serve pip install keyrings.alt && \
+    conda run -n promptflow-serve pip install gunicorn==20.1.0 && \
+    conda run -n promptflow-serve pip install 'uvicorn>=0.27.0,<1.0.0' && \
+    conda run -n promptflow-serve pip cache purge && \
+    conda clean -a -y
+
+COPY ./flow /flow
+
+
+EXPOSE 8080
+
+COPY ./connections /connections
+
+# reset runsvdir
+RUN rm -rf /var/runit
+COPY ./runit /var/runit
+# grant permission
+RUN chmod -R +x /var/runit
+
+COPY ./start.sh /
+CMD ["bash", "./start.sh"]

src/promptflow/tests/test_configs/flows/export/flex_flow_build/README.md

@@ -0,0 +1,34 @@
+Exported Dockerfile & its dependencies are located in the same folder. The structure is as below:
+- flow: the folder contains all the flow files
+  - ...
+- connections: the folder contains yaml files to create all related connections
+  - ...
+- runit: the folder contains all the runit scripts
+  - ...
+- Dockerfile: the dockerfile to build the image
+- start.sh: the script used in `CMD` of `Dockerfile` to start the service
+- settings.json: a json file to store the settings of the docker image
+- README.md: the readme file to describe how to use the dockerfile
+
+
+Build Docker image:
+`docker build <build_output> -t <image_name>`
+
+
+**Run dag flow** with docker run by flask serving engine:
+`docker run -p 8080:8080 -e OPEN_AI_CONNECTION_API_KEY=<secret-value> -e PROMPTFLOW_WORKER_NUM=<expect-worker-num> -e PROMPTFLOW_WORKER_THREADS=<expect-thread-num-per-worker> <image_name>`
+example:
+`docker run -p 8080:8080 -e OPEN_AI_CONNECTION_API_KEY=111 -e PROMPTFLOW_WORKER_NUM=1 -e PROMPTFLOW_WORKER_THREADS=1 dag_flow_image`
+
+**Run flex flow** with docker run by flask serving engine:
+`docker run -p 8080:8080 -e PROMPTFLOW_WORKER_NUM=<expect-worker-num> -e PROMPTFLOW_WORKER_THREADS=<expect-thread-num-per-worker> -e PF_FLOW_INIT_CONFIG=<init config with json string> <image_name>`
+example:
+`docker run -p 8080:8080 -e PROMPTFLOW_WORKER_NUM=1 -e PROMPTFLOW_WORKER_THREADS=1 -e PF_FLOW_INIT_CONFIG='{"model_config": {"api_key": "111", "azure_endpoint": "https://test.openai.azure.com/", "azure_deployment": "gpt-35-turbo"}}' flex_flow_image`
+
+
+Test the endpoint:
+After start the service, you can use curl to test it: `curl http://localhost:8080/score --data <dict_data> `
+
+
+Please refer to [official doc](https://microsoft.github.io/promptflow/how-to-guides/deploy-a-flow/deploy-using-docker.html)
+for more details about how to use the exported dockerfile and scripts.

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/chat.jinja2

@@ -0,0 +1,12 @@
+system:
+You are a helpful assistant.
+
+{% for item in chat_history %}
+user:
+{{item.inputs.question}}
+assistant:
+{{item.outputs.answer}}
+{% endfor %}
+
+user:
+{{question}}

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/chat.prompty

@@ -0,0 +1,28 @@
+---
+name: Stream Chat
+description: Chat with stream enabled.
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: gpt-35-turbo
+  parameters:
+    temperature: 0.2    
+inputs: 
+  question:
+    type: string
+  chat_history:
+    type: list
+sample: sample.json
+---
+
+system:
+You are a helpful assistant.
+
+{% for item in chat_history %}
+{{item.role}}:
+{{item.content}}
+{% endfor %}
+
+user:
+{{question}}

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/data.jsonl

@@ -0,0 +1,3 @@
+{"question": "What is Prompt flow?", "chat_history":[], "statements": {"correctness": "should explain what's 'Prompt flow'", "consise": "It is a consise statement."}}
+{"question": "What is ChatGPT? Please explain with consise statement", "chat_history":[],  "statements": { "correctness": "should explain what's ChatGPT", "consise": "It is a consise statement."}}
+{"question": "How many questions did user ask?", "chat_history": [{"role": "user","content": "where is the nearest coffee shop?"},{"role": "system","content": "I'm sorry, I don't know that. Would you like me to look it up for you?"}], "statements": { "correctness": "result should be 2", "consise": "It is a consise statement."}}

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/flow.flex.yaml

@@ -0,0 +1,5 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+entry: flow:ChatFlow
+environment:
+  # image: mcr.microsoft.com/azureml/promptflow/promptflow-python
+  python_requirements_txt: requirements.txt

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/flow.py

@@ -0,0 +1,47 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from promptflow.tracing import trace
+from promptflow.core import AzureOpenAIModelConfiguration, Prompty
+
+BASE_DIR = Path(__file__).absolute().parent
+
+
+@dataclass
+class Result:
+    answer: str
+
+
+class ChatFlow:
+    def __init__(self, model_config: AzureOpenAIModelConfiguration):
+        self.model_config = model_config
+
+    @trace
+    def __call__(
+        self, question: str = "What is ChatGPT?", chat_history: list = None
+    ) -> Result:
+        """Flow entry function."""
+
+        chat_history = chat_history or []
+
+        prompty = Prompty.load(
+            source=BASE_DIR / "chat.prompty",
+            model={"configuration": self.model_config},
+        )
+
+        # output is a string
+        output = prompty(question=question, chat_history=chat_history)
+
+        return Result(answer=output)
+
+
+if __name__ == "__main__":
+    from promptflow.tracing import start_trace
+
+    start_trace()
+    config = AzureOpenAIModelConfiguration(
+        connection="open_ai_connection", azure_deployment="gpt-35-turbo"
+    )
+    flow = ChatFlow(config)
+    result = flow("What's Azure Machine Learning?", [])
+    print(result)

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/init.json

@@ -0,0 +1,6 @@
+{
+    "model_config": {
+        "connection": "open_ai_connection",
+        "azure_deployment": "gpt-35-turbo"
+    }
+}

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/paths.py

@@ -0,0 +1,6 @@
+import sys
+import pathlib
+
+# Add the path to the evaluation module
+code_path = str(pathlib.Path(__file__).parent / "../eval-checklist")
+sys.path.insert(0, code_path)

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/requirements-azure.txt

@@ -0,0 +1 @@
+promptflow-azure

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/requirements.txt

@@ -0,0 +1 @@
+promptflow[azure]>=1.10.0

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/run.yml

@@ -0,0 +1,9 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json
+flow: .
+data: data.jsonl
+init:
+  model_config:
+    connection: open_ai_connection
+    azure_deployment: gpt-35-turbo
+column_mapping:
+  question: ${data.question}

src/promptflow/tests/test_configs/flows/export/flex_flow_build/flow/sample.json

@@ -0,0 +1 @@
+{"question": "What is Prompt flow?"}

src/promptflow/tests/test_configs/flows/export/flex_flow_build/runit/promptflow-serve/finish

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "$(date -uIns) - promptflow-serve/finish $@"
+
+# stop all gunicorn processes
+echo "$(date -uIns) - Stopping all Gunicorn processes"
+pkill gunicorn
+while pgrep gunicorn >/dev/null; do
+  echo "$(date -uIns) - Gunicorn process is still running, waiting for 1s"
+  sleep 1
+done
+
+echo "$(date -uIns) - Stopped all Gunicorn processes"

src/promptflow/tests/test_configs/flows/export/flex_flow_build/runit/promptflow-serve/run

@@ -0,0 +1,17 @@
+#! /bin/bash
+
+CONDA_ENV_PATH="$(conda info --base)/envs/promptflow-serve"
+export PATH="$CONDA_ENV_PATH/bin:$PATH"
+
+WORKER_NUM=${PROMPTFLOW_WORKER_NUM:-"8"}
+WORKER_THREADS=${PROMPTFLOW_WORKER_THREADS:-"1"}
+SERVING_ENGINE=${PROMPTFLOW_SERVING_ENGINE:-"flask"}
+gunicorn_app="promptflow.core._serving.app:create_app(engine='${SERVING_ENGINE}')"
+cd /flow
+if [ "$SERVING_ENGINE" = "flask" ]; then
+    echo "start promptflow serving with worker_num: ${WORKER_NUM}, worker_threads: ${WORKER_THREADS}, app: ${gunicorn_app}"
+    gunicorn -w ${WORKER_NUM} --threads ${WORKER_THREADS} -b "0.0.0.0:8080" --timeout 300 ${gunicorn_app}
+else
+    echo "start promptflow serving with worker_num: ${WORKER_NUM}, app: ${gunicorn_app}"
+    gunicorn --worker-class uvicorn.workers.UvicornWorker -w ${WORKER_NUM} -b "0.0.0.0:8080" --timeout 300 ${gunicorn_app}
+fi

src/promptflow/tests/test_configs/flows/export/flex_flow_build/settings.json

@@ -0,0 +1 @@
+{}

src/promptflow/tests/test_configs/flows/export/flex_flow_build/start.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# stop services created by runsv and propagate SIGINT, SIGTERM to child jobs
+sv_stop() {
+    echo "$(date -uIns) - Stopping all runsv services"
+    for s in $(ls -d /var/runit/*); do
+        sv stop $s
+    done
+}
+
+# register SIGINT, SIGTERM handler
+trap sv_stop SIGINT SIGTERM
+
+# start services in background and wait all child jobs
+runsvdir /var/runit &
+wait