[Feature][Executor] Support custom promptflow logger format

src/promptflow-core/promptflow/_utils/logger_utils.py

@@ -14,7 +14,7 @@
 from dataclasses import dataclass
 from functools import partial
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Tuple
 
 from promptflow._constants import LINE_NUMBER_WIDTH, PF_LOGGING_LEVEL
 from promptflow._utils.credential_scrubber import CredentialScrubber
@@ -29,6 +29,20 @@
 DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S %z"
 
 
+def _get_format_for_logger(default_log_format: str = None, default_date_format: str = None) -> Tuple[str, str]:
+    """
+    Get the logging format and date format for logger.
+
+    This function attempts to find the handler of the root logger with a configured formatter.
+    If such a handler is found, it returns the format and date format used by this handler.
+    This can be configured through logging.basicConfig. If no configured formatter is found,
+    it defaults to LOG_FORMAT and DATETIME_FORMAT.
+    """
+    log_format = os.environ.get("PF_LOG_FORMAT") or default_log_format or LOG_FORMAT
+    datetime_format = os.environ.get("PF_LOG_DATETIME_FORMAT") or default_date_format or DATETIME_FORMAT
+    return log_format, datetime_format
+
+
 class CredentialScrubberFormatter(logging.Formatter):
     """Formatter that scrubs credentials in logs."""
 
@@ -106,7 +120,8 @@ def __init__(self, file_path: str, formatter: Optional[logging.Formatter] = None
         self._stream_handler = self._get_stream_handler(file_path)
         if formatter is None:
             # Default formatter to scrub credentials in log message, exception and stack trace.
-            self._formatter = CredentialScrubberFormatter(fmt=LOG_FORMAT, datefmt=DATETIME_FORMAT)
+            fmt, datefmt = _get_format_for_logger()
+            self._formatter = CredentialScrubberFormatter(fmt=fmt, datefmt=datefmt)
         else:
             self._formatter = formatter
         self._stream_handler.setFormatter(self._formatter)
@@ -186,7 +201,8 @@ def get_logger(name: str) -> logging.Logger:
     logger.setLevel(get_pf_logging_level())
     logger.addHandler(FileHandlerConcurrentWrapper())
     stdout_handler = logging.StreamHandler(sys.stdout)
-    stdout_handler.setFormatter(CredentialScrubberFormatter(fmt=LOG_FORMAT, datefmt=DATETIME_FORMAT))
+    fmt, datefmt = _get_format_for_logger()
+    stdout_handler.setFormatter(CredentialScrubberFormatter(fmt=fmt, datefmt=datefmt))
     logger.addHandler(stdout_handler)
     return logger
 
@@ -406,7 +422,8 @@ def _add_handler(logger: logging.Logger, verbosity: int, target_stdout: bool = F
         # set target_stdout=True can log data into sys.stdout instead of default sys.stderr, in this way
         # logger info and python print result can be synchronized
         handler = logging.StreamHandler(stream=sys.stdout) if target_stdout else logging.StreamHandler()
-        formatter = logging.Formatter("[%(asctime)s][%(name)s][%(levelname)s] - %(message)s")
+        fmt, datefmt = _get_format_for_logger(default_log_format="[%(asctime)s][%(name)s][%(levelname)s] - %(message)s")
+        formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
         handler.setFormatter(formatter)
         handler.setLevel(verbosity)
         logger.addHandler(handler)

src/promptflow/tests/executor/e2etests/test_logs.py

@@ -1,4 +1,5 @@
 import os
+from datetime import datetime
 from pathlib import Path
 from tempfile import mkdtemp
 
@@ -277,3 +278,28 @@ def test_async_log_in_worker_thread(self):
             logs_list = ["INFO     monitor_long_running_coroutine started"]
             assert all(log in log_content for log in logs_list)
         os.environ.pop("PF_LONG_RUNNING_LOGGING_INTERVAL")
+
+    def test_change_log_format(self, monkeypatch):
+        # Change log format
+        date_format = "%Y/%m/%d %H:%M:%S"
+        log_format = "[%(asctime)s][%(name)s][%(levelname)s] - %(message)s"
+        monkeypatch.setenv("PF_LOG_FORMAT", log_format)
+        monkeypatch.setenv("PF_LOG_DATETIME_FORMAT", date_format)
+
+        logs_directory = Path(mkdtemp())
+        log_path = str(logs_directory / "flow.log")
+        with LogContext(log_path):
+            executor = FlowExecutor.create(get_yaml_file("print_input_flow"), {})
+            executor.exec_line(inputs={"text": "line_text"})
+            log_content = load_content(log_path)
+            current_date = datetime.now().strftime("%Y/%m/%d")
+            logs_list = [
+                f"[{current_date}",
+                "[execution.flow][INFO] - Start executing nodes in thread pool mode.",
+                "[execution.flow][INFO] - Executing node print_input.",
+                "[execution.flow][INFO] - Node print_input completes.",
+                "stderr> STDERR: line_text",
+            ]
+            assert all(
+                log in log_content for log in logs_list
+            ), f"Missing logs are [{[log for log in logs_list if log not in log_content]}]"