Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix listing log files when amount of std and output logs mismatch #76

Merged
merged 1 commit into from
Aug 6, 2024
Merged

Fix listing log files when amount of std and output logs mismatch #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions carbontracker/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,15 +319,15 @@ def get_all_logs(log_dir):
std_logs = sorted(list(filter(std_re.match, files)))
if len(output_logs) != len(std_logs):
# Try to remove the files with no matching output/std logs
op_fn = [f.split("_")[0] for f in output_logs]
std_fn = [f.split("_")[0] for f in std_logs]
op_fn = [f.split("_carbontracker")[0] for f in output_logs]
std_fn = [f.split("_carbontracker")[0] for f in std_logs]
if len(std_logs) > len(output_logs):
missing_logs = list(set(std_fn) - set(op_fn))
[std_logs.remove(f + "_carbontracker.log") for f in missing_logs]
else:
missing_logs = list(set(op_fn) - set(std_fn))
[output_logs.remove(f + "carbontracker_output.log") for f in missing_logs]
### Even after removel if then there is a mismatch, then throw the error
[output_logs.remove(f + "_carbontracker_output.log") for f in missing_logs]
### Even after removal if then there is a mismatch, then throw the error
if len(output_logs) != len(std_logs):
raise exceptions.MismatchedLogFilesError(
f"Found {len(output_logs)} output logs and {len(std_logs)} "
Expand Down
170 changes: 98 additions & 72 deletions tests/test_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,27 @@ def test_get_all_logs(self, mock_getsize, mock_isfile, mock_listdir):
log_dir = "/path/to/logs"

self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
contents="output_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log2.log"),
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"),
contents="output_log2 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content"
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
contents="std_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log2.log"), contents="std_log2 content"
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"),
contents="std_log2 content",
)

mock_listdir.return_value = [
"carbontracker_output_log1.log",
"carbontracker_output_log2.log",
"carbontracker_log1.log",
"carbontracker_log2.log",
"10151_2024-03-26T105926Z_carbontracker_output.log",
"32487_2024-06-26T141608Z_carbontracker_output.log",
"10151_2024-03-26T105926Z_carbontracker.log",
"32487_2024-06-26T141608Z_carbontracker.log",
]

mock_isfile.side_effect = lambda path: path.endswith(".log")
Expand All @@ -52,12 +54,12 @@ def test_get_all_logs(self, mock_getsize, mock_isfile, mock_listdir):
output_logs, std_logs = parser.get_all_logs(log_dir)

expected_output_logs = [
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "carbontracker_output_log2.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"),
]
expected_std_logs = [
os.path.join(log_dir, "carbontracker_log1.log"),
os.path.join(log_dir, "carbontracker_log2.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"),
]

self.assertCountEqual(output_logs, expected_output_logs)
Expand Down Expand Up @@ -118,25 +120,27 @@ def test_get_most_recent_logs(self, mock_getmtime, mock_isfile, mock_listdir):
log_dir = "/path/to/logs"

self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
contents="output_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log2.log"),
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"),
contents="output_log2 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content"
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
contents="std_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log2.log"), contents="std_log2 content"
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"),
contents="std_log2 content",
)

mock_listdir.return_value = [
"carbontracker_output_log1.log",
"carbontracker_output_log2.log",
"carbontracker_log1.log",
"carbontracker_log2.log",
"10151_2024-03-26T105926Z_carbontracker_output.log",
"32487_2024-06-26T141608Z_carbontracker_output.log",
"10151_2024-03-26T105926Z_carbontracker.log",
"32487_2024-06-26T141608Z_carbontracker.log",
]

mock_isfile.side_effect = lambda path: path.endswith(".log")
Expand All @@ -149,8 +153,12 @@ def test_get_most_recent_logs(self, mock_getmtime, mock_isfile, mock_listdir):

std_log, output_log = parser.get_most_recent_logs(log_dir)

expected_std_log = os.path.join(log_dir, "carbontracker_log2.log")
expected_output_log = os.path.join(log_dir, "carbontracker_output_log2.log")
expected_std_log = os.path.join(
log_dir, "32487_2024-06-26T141608Z_carbontracker.log"
)
expected_output_log = os.path.join(
log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"
)

self.assertEqual(std_log, expected_std_log)
self.assertEqual(output_log, expected_output_log)
Expand Down Expand Up @@ -205,16 +213,17 @@ def test_parse_all_logs(self, mock_isfile, mock_listdir, mock_open):
log_dir = "/path/to/logs"

self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
contents="output_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content"
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
contents="std_log1 content",
)

mock_listdir.return_value = [
"carbontracker_output_log1.log",
"carbontracker_log1.log",
"10151_2024-03-26T105926Z_carbontracker_output.log",
"10151_2024-03-26T105926Z_carbontracker.log",
]
mock_isfile.side_effect = lambda path: path.endswith(".log")
mock_open.return_value.read.return_value = "content"
Expand All @@ -224,11 +233,11 @@ def test_parse_all_logs(self, mock_isfile, mock_listdir, mock_open):
self.assertEqual(len(logs), 1)
self.assertEqual(
logs[0]["output_filename"],
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
)
self.assertEqual(
logs[0]["standard_filename"],
os.path.join(log_dir, "carbontracker_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
)

@mock.patch("builtins.open", new_callable=mock.mock_open)
Expand All @@ -245,11 +254,12 @@ def test_parse_logs(self, mock_get_devices, mock_isfile, mock_listdir, mock_open
)

self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
contents="output_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log1.log"), contents=std_log_data
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
contents=std_log_data,
)

mock_isfile.side_effect = lambda path: path.endswith(".log")
Expand All @@ -261,8 +271,8 @@ def test_parse_logs(self, mock_get_devices, mock_isfile, mock_listdir, mock_open

components = parser.parse_logs(
log_dir,
os.path.join(log_dir, "carbontracker_log1.log"),
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
)

self.assertIn("gpu", components)
Expand All @@ -287,48 +297,54 @@ def test_get_all_logs_mismatched_files(

# Create three matching pairs of log files
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
contents="output_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content"
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
contents="std_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log2.log"),
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"),
contents="output_log2 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log2.log"), contents="std_log2 content"
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"),
contents="std_log2 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log3.log"),
os.path.join(log_dir, "40793_2024-03-26T131535Z_carbontracker_output.log"),
contents="output_log3 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log3.log"), contents="std_log3 content"
os.path.join(log_dir, "40793_2024-03-26T131535Z_carbontracker.log"),
contents="std_log3 content",
)

# Add extra unmatched output log file
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log4.log"),
os.path.join(log_dir, "9803_2024-03-26T105836Z_carbontracker_output.log"),
contents="output_log4 content",
)

mock_listdir.return_value = [
"carbontracker_output_log1.log",
"carbontracker_output_log2.log",
"carbontracker_output_log3.log",
"carbontracker_output_log4.log",
"carbontracker_log1.log",
"carbontracker_log2.log",
"carbontracker_log3.log",
"10151_2024-03-26T105926Z_carbontracker_output.log",
"32487_2024-06-26T141608Z_carbontracker_output.log",
"40793_2024-03-26T131535Z_carbontracker_output.log",
"9803_2024-03-26T105836Z_carbontracker_output.log",
"10151_2024-03-26T105926Z_carbontracker.log",
"32487_2024-06-26T141608Z_carbontracker.log",
"40793_2024-03-26T131535Z_carbontracker.log",
]

mock_isfile.side_effect = lambda path: path.endswith(".log")
mock_getsize.return_value = 100

with self.assertRaises(exceptions.MismatchedLogFilesError):
parser.get_all_logs(log_dir)
expected = [os.path.join(log_dir, f) for f in mock_listdir.return_value]
self.assertTupleEqual(
parser.get_all_logs(log_dir),
(expected[:3], expected[4:]),
)

@mock.patch("os.listdir")
@mock.patch("os.path.isfile")
Expand All @@ -340,47 +356,53 @@ def test_get_all_logs_mismatched_files_extra_std_log(

# Create three matching pairs of log files
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
contents="output_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content"
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
contents="std_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log2.log"),
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker_output.log"),
contents="output_log2 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log2.log"), contents="std_log2 content"
os.path.join(log_dir, "32487_2024-06-26T141608Z_carbontracker.log"),
contents="std_log2 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log3.log"),
os.path.join(log_dir, "40793_2024-03-26T131535Z_carbontracker_output.log"),
contents="output_log3 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log3.log"), contents="std_log3 content"
os.path.join(log_dir, "40793_2024-03-26T131535Z_carbontracker.log"),
contents="std_log3 content",
)

# Add extra unmatched std log file
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log4.log"), contents="std_log4 content"
os.path.join(log_dir, "9803_2024-03-26T105836Z_carbontracker.log"),
contents="std_log4 content",
)

mock_listdir.return_value = [
"carbontracker_output_log1.log",
"carbontracker_output_log2.log",
"carbontracker_output_log3.log",
"carbontracker_log1.log",
"carbontracker_log2.log",
"carbontracker_log3.log",
"carbontracker_log4.log",
"10151_2024-03-26T105926Z_carbontracker_output.log",
"32487_2024-06-26T141608Z_carbontracker_output.log",
"40793_2024-03-26T131535Z_carbontracker_output.log",
"10151_2024-03-26T105926Z_carbontracker.log",
"32487_2024-06-26T141608Z_carbontracker.log",
"40793_2024-03-26T131535Z_carbontracker.log",
"9803_2024-03-26T105836Z_carbontracker.log",
]

mock_isfile.side_effect = lambda path: path.endswith(".log")
mock_getsize.return_value = 100

with self.assertRaises(exceptions.MismatchedLogFilesError):
parser.get_all_logs(log_dir)
expected = [os.path.join(log_dir, f) for f in mock_listdir.return_value]
self.assertTupleEqual(
parser.get_all_logs(log_dir), (expected[:3], expected[3:6])
)

@mock.patch("builtins.open", new_callable=mock.mock_open)
def test_get_consumption_no_equivalents(self, mock_open):
Expand Down Expand Up @@ -553,16 +575,17 @@ def test_aggregate_consumption_actual(self, mock_isfile, mock_listdir, mock_open
)

self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
contents=output_log_content,
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log1.log"), contents="std_log1 content"
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
contents="std_log1 content",
)

mock_listdir.return_value = [
"carbontracker_output_log1.log",
"carbontracker_log1.log",
"10151_2024-03-26T105926Z_carbontracker_output.log",
"10151_2024-03-26T105926Z_carbontracker.log",
]
mock_isfile.side_effect = lambda path: path.endswith(".log")
mock_open.return_value.read.return_value = output_log_content
Expand Down Expand Up @@ -946,11 +969,12 @@ def test_parse_epoch_mismatch(
)

self.fs.create_file(
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"),
contents="output_log1 content",
)
self.fs.create_file(
os.path.join(log_dir, "carbontracker_log1.log"), contents=std_log_data
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
contents=std_log_data,
)

mock_isfile.side_effect = lambda path: path.endswith(".log")
Expand All @@ -963,6 +987,8 @@ def test_parse_epoch_mismatch(
with self.assertRaises(exceptions.MismatchedEpochsError):
components = parser.parse_logs(
log_dir,
os.path.join(log_dir, "carbontracker_log1.log"),
os.path.join(log_dir, "carbontracker_output_log1.log"),
os.path.join(log_dir, "10151_2024-03-26T105926Z_carbontracker.log"),
os.path.join(
log_dir, "10151_2024-03-26T105926Z_carbontracker_output.log"
),
)
Loading