From 7e83af8667da5985bf217d394efe0667f44789f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Hag=20L=C3=B8vstad?= Date: Tue, 23 Jul 2024 13:12:41 +0200 Subject: [PATCH] fix: parser should consider actual file names when listing log files --- carbontracker/parser.py | 8 +- tests/test_parser.py | 170 +++++++++++++++++++++++----------------- 2 files changed, 102 insertions(+), 76 deletions(-) diff --git a/carbontracker/parser.py b/carbontracker/parser.py index c5d27c3..bd5ec36 100644 --- a/carbontracker/parser.py +++ b/carbontracker/parser.py @@ -319,15 +319,15 @@ def get_all_logs(log_dir): std_logs = sorted(list(filter(std_re.match, files))) if len(output_logs) != len(std_logs): # Try to remove the files with no matching output/std logs - op_fn = [f.split("_")[0] for f in output_logs] - std_fn = [f.split("_")[0] for f in std_logs] + op_fn = [f.split("_carbontracker")[0] for f in output_logs] + std_fn = [f.split("_carbontracker")[0] for f in std_logs] if len(std_logs) > len(output_logs): missing_logs = list(set(std_fn) - set(op_fn)) [std_logs.remove(f + "_carbontracker.log") for f in missing_logs] else: missing_logs = list(set(op_fn) - set(std_fn)) - [output_logs.remove(f + "carbontracker_output.log") for f in missing_logs] - ### Even after removel if then there is a mismatch, then throw the error + [output_logs.remove(f + "_carbontracker_output.log") for f in missing_logs] + ### Even after removal if then there is a mismatch, then throw the error if len(output_logs) != len(std_logs): raise exceptions.MismatchedLogFilesError( f"Found {len(output_logs)} output logs and {len(std_logs)} " diff --git a/tests/test_parser.py b/tests/test_parser.py index 5c68afd..df38c2d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -25,25 +25,27 @@ def test_get_all_logs(self, mock_getsize, mock_isfile, mock_listdir): log_dir = "/path/to/logs" self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), contents="output_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log2.log"), + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"), contents="output_log2 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content" + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + contents="std_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log2.log"), contents="std_log2 content" + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"), + contents="std_log2 content", ) mock_listdir.return_value = [ - "carbontracker_output_log1.log", - "carbontracker_output_log2.log", - "carbontracker_log1.log", - "carbontracker_log2.log", + "10151_2024-03-26T105926Z_carbontracker_output.log", + "32487_2024-06-26T141608Z_carbontracker_output.log", + "10151_2024-03-26T105926Z_carbontracker.log", + "32487_2024-06-26T141608Z_carbontracker.log", ] mock_isfile.side_effect = lambda path: path.endswith(".log") @@ -52,12 +54,12 @@ def test_get_all_logs(self, mock_getsize, mock_isfile, mock_listdir): output_logs, std_logs = parser.get_all_logs(log_dir) expected_output_logs = [ - os.path.join(log_dir, "carbontracker_output_log1.log"), - os.path.join(log_dir, "carbontracker_output_log2.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"), ] expected_std_logs = [ - os.path.join(log_dir, "carbontracker_log1.log"), - os.path.join(log_dir, "carbontracker_log2.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"), ] self.assertCountEqual(output_logs, expected_output_logs) @@ -118,25 +120,27 @@ def test_get_most_recent_logs(self, mock_getmtime, mock_isfile, mock_listdir): log_dir = "/path/to/logs" self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), contents="output_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log2.log"), + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"), contents="output_log2 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content" + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + contents="std_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log2.log"), contents="std_log2 content" + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"), + contents="std_log2 content", ) mock_listdir.return_value = [ - "carbontracker_output_log1.log", - "carbontracker_output_log2.log", - "carbontracker_log1.log", - "carbontracker_log2.log", + "10151_2024-03-26T105926Z_carbontracker_output.log", + "32487_2024-06-26T141608Z_carbontracker_output.log", + "10151_2024-03-26T105926Z_carbontracker.log", + "32487_2024-06-26T141608Z_carbontracker.log", ] mock_isfile.side_effect = lambda path: path.endswith(".log") @@ -149,8 +153,12 @@ def test_get_most_recent_logs(self, mock_getmtime, mock_isfile, mock_listdir): std_log, output_log = parser.get_most_recent_logs(log_dir) - expected_std_log = os.path.join(log_dir, "carbontracker_log2.log") - expected_output_log = os.path.join(log_dir, "carbontracker_output_log2.log") + expected_std_log = os.path.join( + log_dir, "32487_2024-06-26T141608Z_carbontracker.log" + ) + expected_output_log = os.path.join( + log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log" + ) self.assertEqual(std_log, expected_std_log) self.assertEqual(output_log, expected_output_log) @@ -205,16 +213,17 @@ def test_parse_all_logs(self, mock_isfile, mock_listdir, mock_open): log_dir = "/path/to/logs" self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), contents="output_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content" + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + contents="std_log1 content", ) mock_listdir.return_value = [ - "carbontracker_output_log1.log", - "carbontracker_log1.log", + "10151_2024-03-26T105926Z_carbontracker_output.log", + "10151_2024-03-26T105926Z_carbontracker.log", ] mock_isfile.side_effect = lambda path: path.endswith(".log") mock_open.return_value.read.return_value = "content" @@ -224,11 +233,11 @@ def test_parse_all_logs(self, mock_isfile, mock_listdir, mock_open): self.assertEqual(len(logs), 1) self.assertEqual( logs[0]["output_filename"], - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), ) self.assertEqual( logs[0]["standard_filename"], - os.path.join(log_dir, "carbontracker_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), ) @mock.patch("builtins.open", new_callable=mock.mock_open) @@ -245,11 +254,12 @@ def test_parse_logs(self, mock_get_devices, mock_isfile, mock_listdir, mock_open ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), contents="output_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log1.log"), contents=std_log_data + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + contents=std_log_data, ) mock_isfile.side_effect = lambda path: path.endswith(".log") @@ -261,8 +271,8 @@ def test_parse_logs(self, mock_get_devices, mock_isfile, mock_listdir, mock_open components = parser.parse_logs( log_dir, - os.path.join(log_dir, "carbontracker_log1.log"), - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), ) self.assertIn("gpu", components) @@ -287,48 +297,54 @@ def test_get_all_logs_mismatched_files( # Create three matching pairs of log files self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), contents="output_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content" + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + contents="std_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log2.log"), + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"), contents="output_log2 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log2.log"), contents="std_log2 content" + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"), + contents="std_log2 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log3.log"), + os.path.join(log_dir, "40793_2024-03-26T131535Z_carbontracker_output.log"), contents="output_log3 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log3.log"), contents="std_log3 content" + os.path.join(log_dir, "40793_2024-03-26T131535Z_carbontracker.log"), + contents="std_log3 content", ) # Add extra unmatched output log file self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log4.log"), + os.path.join(log_dir, "9803_2024-03-26T105836Z_carbontracker_output.log"), contents="output_log4 content", ) mock_listdir.return_value = [ - "carbontracker_output_log1.log", - "carbontracker_output_log2.log", - "carbontracker_output_log3.log", - "carbontracker_output_log4.log", - "carbontracker_log1.log", - "carbontracker_log2.log", - "carbontracker_log3.log", + "10151_2024-03-26T105926Z_carbontracker_output.log", + "32487_2024-06-26T141608Z_carbontracker_output.log", + "40793_2024-03-26T131535Z_carbontracker_output.log", + "9803_2024-03-26T105836Z_carbontracker_output.log", + "10151_2024-03-26T105926Z_carbontracker.log", + "32487_2024-06-26T141608Z_carbontracker.log", + "40793_2024-03-26T131535Z_carbontracker.log", ] mock_isfile.side_effect = lambda path: path.endswith(".log") mock_getsize.return_value = 100 - with self.assertRaises(exceptions.MismatchedLogFilesError): - parser.get_all_logs(log_dir) + expected = [os.path.join(log_dir, f) for f in mock_listdir.return_value] + self.assertTupleEqual( + parser.get_all_logs(log_dir), + (expected[:3], expected[4:]), + ) @mock.patch("os.listdir") @mock.patch("os.path.isfile") @@ -340,47 +356,53 @@ def test_get_all_logs_mismatched_files_extra_std_log( # Create three matching pairs of log files self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), contents="output_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content" + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + contents="std_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log2.log"), + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"), contents="output_log2 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log2.log"), contents="std_log2 content" + os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"), + contents="std_log2 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log3.log"), + os.path.join(log_dir, "40793_2024-03-26T131535Z_carbontracker_output.log"), contents="output_log3 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log3.log"), contents="std_log3 content" + os.path.join(log_dir, "40793_2024-03-26T131535Z_carbontracker.log"), + contents="std_log3 content", ) # Add extra unmatched std log file self.fs.create_file( - os.path.join(log_dir, "carbontracker_log4.log"), contents="std_log4 content" + os.path.join(log_dir, "9803_2024-03-26T105836Z_carbontracker.log"), + contents="std_log4 content", ) mock_listdir.return_value = [ - "carbontracker_output_log1.log", - "carbontracker_output_log2.log", - "carbontracker_output_log3.log", - "carbontracker_log1.log", - "carbontracker_log2.log", - "carbontracker_log3.log", - "carbontracker_log4.log", + "10151_2024-03-26T105926Z_carbontracker_output.log", + "32487_2024-06-26T141608Z_carbontracker_output.log", + "40793_2024-03-26T131535Z_carbontracker_output.log", + "10151_2024-03-26T105926Z_carbontracker.log", + "32487_2024-06-26T141608Z_carbontracker.log", + "40793_2024-03-26T131535Z_carbontracker.log", + "9803_2024-03-26T105836Z_carbontracker.log", ] mock_isfile.side_effect = lambda path: path.endswith(".log") mock_getsize.return_value = 100 - with self.assertRaises(exceptions.MismatchedLogFilesError): - parser.get_all_logs(log_dir) + expected = [os.path.join(log_dir, f) for f in mock_listdir.return_value] + self.assertTupleEqual( + parser.get_all_logs(log_dir), (expected[:3], expected[3:6]) + ) @mock.patch("builtins.open", new_callable=mock.mock_open) def test_get_consumption_no_equivalents(self, mock_open): @@ -553,16 +575,17 @@ def test_aggregate_consumption_actual(self, mock_isfile, mock_listdir, mock_open ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), contents=output_log_content, ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content" + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + contents="std_log1 content", ) mock_listdir.return_value = [ - "carbontracker_output_log1.log", - "carbontracker_log1.log", + "10151_2024-03-26T105926Z_carbontracker_output.log", + "10151_2024-03-26T105926Z_carbontracker.log", ] mock_isfile.side_effect = lambda path: path.endswith(".log") mock_open.return_value.read.return_value = output_log_content @@ -946,11 +969,12 @@ def test_parse_epoch_mismatch( ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"), contents="output_log1 content", ) self.fs.create_file( - os.path.join(log_dir, "carbontracker_log1.log"), contents=std_log_data + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + contents=std_log_data, ) mock_isfile.side_effect = lambda path: path.endswith(".log") @@ -963,6 +987,8 @@ def test_parse_epoch_mismatch( with self.assertRaises(exceptions.MismatchedEpochsError): components = parser.parse_logs( log_dir, - os.path.join(log_dir, "carbontracker_log1.log"), - os.path.join(log_dir, "carbontracker_output_log1.log"), + os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"), + os.path.join( + log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log" + ), )