[trace] Move trace feature out of internal flag & improve trace experience #2767
GitHub Actions / SDK CLI Test Result [zhengfei/feature/graduate-trace](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:zhengfei/feature/graduate-trace++)
failed
Apr 12, 2024 in 0s
1 fail, 22 skipped, 569 pass in 53m 6s
Annotations
Check warning on line 0 in tests.sdk_cli_test.e2etests.test_cli.TestCli
github-actions / SDK CLI Test Result [zhengfei/feature/graduate-trace](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:zhengfei/feature/graduate-trace++)
1 out of 4 runs failed: test_flow_run_resume_from_token (tests.sdk_cli_test.e2etests.test_cli.TestCli)
artifacts/Test Results (Python 3.10) (OS ubuntu-latest)/test-results.xml [took 17s]
Raw output
AssertionError: assert 12 > 12
+ where 12 = len([{'category': 'Academic', 'evidence': 'Text content', 'line_number': 1}, {'category': 'App', 'evidence': 'Text content', 'line_number': 2}, {'category': 'App', 'evidence': 'Text content', 'line_number': 3}, {'category': 'App', 'evidence': 'Text content', 'line_number': 6}, {'category': 'Academic', 'evidence': 'Text content', 'line_number': 7}, {'category': 'App', 'evidence': 'Text content', 'line_number': 8}, ...])
+ and 12 = len([{'category': 'Academic', 'evidence': 'Text content', 'line_number': 1}, {'category': 'App', 'evidence': 'Text content', 'line_number': 2}, {'category': 'App', 'evidence': 'Text content', 'line_number': 3}, {'category': 'App', 'evidence': 'Text content', 'line_number': 6}, {'category': 'Academic', 'evidence': 'Text content', 'line_number': 7}, {'category': 'App', 'evidence': 'Text content', 'line_number': 8}, ...])
self = <sdk_cli_test.e2etests.test_cli.TestCli object at 0x7fa6128615a0>
local_client = <promptflow._sdk._pf_client.PFClient object at 0x7fa5f923d390>
def test_flow_run_resume_from_token(self, local_client) -> None:
run_id = str(uuid.uuid4())
# fetch std out
run_pf_command(
"run",
"create",
"--flow",
f"{FLOWS_DIR}/web_classification_random_fail",
"--data",
f"{FLOWS_DIR}/web_classification_random_fail/data.jsonl",
"--column-mapping",
"url='${data.url}'",
"--name",
run_id,
)
original_run = local_client.runs.get(name=run_id)
assert original_run.status == "Completed"
output_path = os.path.join(original_run.properties["output_path"], "flow_outputs", "output.jsonl")
with open(output_path, "r") as file:
original_output = [json.loads(line) for line in file]
# Since the data have 15 lines, we can assume the original run has succeeded lines in over 99% cases
original_success_count = len(original_output)
new_run_id = str(uuid.uuid4())
display_name = "test"
description = "new description"
run_pf_command(
"run",
"create",
"--resume-from",
run_id,
"--name",
new_run_id,
"--set",
f"display_name={display_name}",
f"description={description}",
"tags.A=A",
"tags.B=B",
)
resume_run = local_client.runs.get(name=new_run_id)
assert resume_run.name == new_run_id
assert resume_run.display_name == display_name
assert resume_run.description == description
assert resume_run.tags == {"A": "A", "B": "B"}
assert resume_run._resume_from == run_id
# assert new run resume from the original run
output_path = os.path.join(resume_run.properties["output_path"], "flow_outputs", "output.jsonl")
with open(output_path, "r") as file:
resume_output = [json.loads(line) for line in file]
> assert len(resume_output) > len(original_output)
E AssertionError: assert 12 > 12
E + where 12 = len([{'category': 'Academic', 'evidence': 'Text content', 'line_number': 1}, {'category': 'App', 'evidence': 'Text content', 'line_number': 2}, {'category': 'App', 'evidence': 'Text content', 'line_number': 3}, {'category': 'App', 'evidence': 'Text content', 'line_number': 6}, {'category': 'Academic', 'evidence': 'Text content', 'line_number': 7}, {'category': 'App', 'evidence': 'Text content', 'line_number': 8}, ...])
E + and 12 = len([{'category': 'Academic', 'evidence': 'Text content', 'line_number': 1}, {'category': 'App', 'evidence': 'Text content', 'line_number': 2}, {'category': 'App', 'evidence': 'Text content', 'line_number': 3}, {'category': 'App', 'evidence': 'Text content', 'line_number': 6}, {'category': 'Academic', 'evidence': 'Text content', 'line_number': 7}, {'category': 'App', 'evidence': 'Text content', 'line_number': 8}, ...])
/home/runner/work/promptflow/promptflow/src/promptflow-devkit/tests/sdk_cli_test/e2etests/test_cli.py:2312: AssertionError
Loading