diff --git a/cobaya/collection.py b/cobaya/collection.py index 8da770d6..e0c5d11c 100644 --- a/cobaya/collection.py +++ b/cobaya/collection.py @@ -96,7 +96,7 @@ def compute_temperature(logpost, logprior, loglike, check=True, extra_tolerance= """ Returns the temperature of a sample. - If ``check=True`` and the log-probabilites passed are arrays, checks consistency + If ``check=True`` and the log-probabilities passed are arrays, checks consistency of the sample temperature, and raises ``AssertionError`` if inconsistent. """ temp = (logprior + loglike) / logpost @@ -286,8 +286,7 @@ def __init__(self, model, output=None, cache_size=_default_cache_size, name=None self.reset() # If loaded, check sample weights, consistent logp sums, # and temperature (ignores the given one) - samples_loaded = len(self) > 0 - if samples_loaded: + if len(self) > 0: try: try: self.temperature = self._check_logps(extra_tolerance=False) @@ -499,8 +498,9 @@ def _check_logps(self, temperature_only=False, extra_tolerance=False): check=True, extra_tolerance=extra_tolerance) except AssertionError as excpt: raise LoggedError( - self.log, "The sample seems to have an inconsistent temperature.") \ - from excpt + self.log, "The sample seems to have an inconsistent temperature. " + "This could be due to input file truncation on the last line " + "due to crash/being killed before complete.") from excpt if not temperature_only: tols = { "rtol": 1e-4 * (10 if extra_tolerance else 1), diff --git a/cobaya/tools.py b/cobaya/tools.py index 9cb21ead..903ce6a0 100644 --- a/cobaya/tools.py +++ b/cobaya/tools.py @@ -457,6 +457,19 @@ def read_dnumber(n: Any, dim: int): return NumberWithUnits(n, "d", dtype=int, scale=dim).value +def truncate_to_end_line(file_name): + with open(file_name, "r+b") as inp: + # Find the last complete line + inp.seek(0, 2) # Go to the end of the file + pos = inp.tell() - 1 + while pos > 0 and inp.read(1) != b"\n": + pos -= 1 + inp.seek(pos, 0) + if pos > 0: + inp.seek(pos + 1, 0) + inp.truncate() + + def load_DataFrame(file_name, skip=0, root_file_name=None): """ Loads a `pandas.DataFrame` from a text file @@ -493,7 +506,17 @@ def load_DataFrame(file_name, skip=0, root_file_name=None): inp, sep=" ", header=None, names=cols, comment="#", skipinitialspace=True, skiprows=skip, index_col=False) - return data + if not data.empty: + # Check if the last row contains any NaNs + if data.iloc[-1].isna().any(): + log.warning("Last row of %s is incomplete or contains NaNs", file_name) + # If the second-to-last row exists and doesn't contain NaNs, + # delete the last row assuming this was due to crash on write + if len(data) > 1 and not data.iloc[-2].isna().any(): + data = data.iloc[:-1] + log.info(f"Saving {file_name} deleting last (in)complete line") + truncate_to_end_line(file_name) + return data def prepare_comment(comment):