From 982b641bb6135dc8b68d738b0d8501cb1e59268a Mon Sep 17 00:00:00 2001 From: Saaketh Date: Thu, 24 Oct 2024 15:49:24 -0400 Subject: [PATCH] yo --- llmfoundry/callbacks/hf_checkpointer.py | 1 + llmfoundry/command_utils/data_prep/convert_text_to_mds.py | 1 + llmfoundry/command_utils/eval.py | 1 + llmfoundry/command_utils/train.py | 1 + scripts/inference/endpoint_generate.py | 6 +++++- scripts/misc/download_model.py | 5 +++-- 6 files changed, 12 insertions(+), 3 deletions(-) diff --git a/llmfoundry/callbacks/hf_checkpointer.py b/llmfoundry/callbacks/hf_checkpointer.py index 2c4603ea87..14ccae7191 100644 --- a/llmfoundry/callbacks/hf_checkpointer.py +++ b/llmfoundry/callbacks/hf_checkpointer.py @@ -124,6 +124,7 @@ def _register_model_with_run_id_multiprocess( logging.basicConfig( format= f'%(asctime)s: rank{dist.get_global_rank()}[%(process)d][%(threadName)s]: %(levelname)s: %(name)s: %(message)s', + force=True, ) logging.getLogger('composer').setLevel(composer_logging_level) diff --git a/llmfoundry/command_utils/data_prep/convert_text_to_mds.py b/llmfoundry/command_utils/data_prep/convert_text_to_mds.py index 11eac121d0..4aea183737 100644 --- a/llmfoundry/command_utils/data_prep/convert_text_to_mds.py +++ b/llmfoundry/command_utils/data_prep/convert_text_to_mds.py @@ -515,6 +515,7 @@ def _configure_logging(logging_level: str): logging.basicConfig( format= f'%(asctime)s: [%(process)d][%(threadName)s]: %(levelname)s: %(name)s: %(message)s', + force=True, ) logging_level = logging_level.upper() logging.getLogger('llmfoundry').setLevel(logging_level) diff --git a/llmfoundry/command_utils/eval.py b/llmfoundry/command_utils/eval.py index f25f2b5cef..472fdd52d4 100644 --- a/llmfoundry/command_utils/eval.py +++ b/llmfoundry/command_utils/eval.py @@ -272,6 +272,7 @@ def evaluate(cfg: DictConfig) -> tuple[list[Trainer], pd.DataFrame]: # 2022-06-29 11:22:26,152: rank0[822018][MainThread]: INFO: Message here format= f'%(asctime)s: rank{dist.get_global_rank()}[%(process)d][%(threadName)s]: %(levelname)s: %(name)s: %(message)s', + force=True, ) logging.getLogger('llmfoundry').setLevel( eval_config.python_log_level.upper(), diff --git a/llmfoundry/command_utils/train.py b/llmfoundry/command_utils/train.py index 2a5e95c6a1..8fb833ddde 100644 --- a/llmfoundry/command_utils/train.py +++ b/llmfoundry/command_utils/train.py @@ -227,6 +227,7 @@ def train(cfg: DictConfig) -> Trainer: # 2022-06-29 11:22:26,152: rank0[822018][MainThread]: INFO: Message here format= f'%(asctime)s: rank{dist.get_global_rank()}[%(process)d][%(threadName)s]: %(levelname)s: %(name)s: %(message)s', + force=True, ) logging.getLogger('llmfoundry').setLevel( train_cfg.python_log_level.upper(), diff --git a/scripts/inference/endpoint_generate.py b/scripts/inference/endpoint_generate.py index 9991f5093f..c42ccd9b40 100644 --- a/scripts/inference/endpoint_generate.py +++ b/scripts/inference/endpoint_generate.py @@ -25,7 +25,11 @@ from llmfoundry.utils import prompt_files as utils -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') +logging.basicConfig( + format= + f'%(asctime)s: [%(process)d][%(threadName)s]: %(levelname)s: %(name)s: %(message)s', + force=True, +) log = logging.getLogger(__name__) ENDPOINT_API_KEY_ENV: str = 'ENDPOINT_API_KEY' diff --git a/scripts/misc/download_model.py b/scripts/misc/download_model.py index 91b0c5a037..cf264b12b2 100644 --- a/scripts/misc/download_model.py +++ b/scripts/misc/download_model.py @@ -31,8 +31,9 @@ HF_TOKEN_ENV_VAR = 'HF_TOKEN' logging.basicConfig( - format=f'%(asctime)s: %(levelname)s: %(name)s: %(message)s', - level=logging.INFO, + format= + f'%(asctime)s: [%(process)d][%(threadName)s]: %(levelname)s: %(name)s: %(message)s', + force=True, ) log = logging.getLogger(__name__)