Skip to content

Commit

Permalink
Updated requirements and error docstrings
Browse files Browse the repository at this point in the history
  • Loading branch information
@dwest77a
dwest77a committed Feb 9, 2024
1 parent 3875e45 commit 172b98d
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 68 deletions.
148 changes: 80 additions & 68 deletions pipeline/errors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,140 +14,152 @@ def upload_err(proj_code, groupdir, error):
with open(summ_file,'w') as f:
f.write(json.dumps(summ))

class BlacklistProjectCode(Exception):
class KerchunkException(Exception):
def __init__(self, proj_code, groupdir):
self.proj_code = proj_code
self.groupdir = groupdir
super().__init__(self.message)
if proj_code and groupdir:
self.save()
def save(self):
upload_err(self.proj_code, self.groupdir, str(self))

class BlacklistProjectCode(KerchunkException):
def __init__(self, verbose=0, proj_code=None, groupdir=None):
"""The project code you are trying to run for is on the list of project codes to ignore."""
self.message = 'Project Code listed in blacklist for bad data - will not be processed.'
super().__init__(self.message)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'BlacklistProjectCode')
def __str__(self):
return 'BlacklistProjectCode'

class MissingVariableError(Exception):
class MissingVariableError(KerchunkException):
def __init__(self, type='$', verbose=0, proj_code=None, groupdir=None):
"""A variable is missing from the environment or set of arguments."""
self.message = f'Missing variable: {type}'
super().__init__(self.message)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'MissingVariableError')
def __str__(self):
return 'MissingVariableError'

class ExpectTimeoutError(Exception):
class ExpectTimeoutError(KerchunkException):
def __init__(self, required=0, current='', verbose=0, proj_code=None, groupdir=None):
"""The process is expected to time out given timing estimates."""
self.message = f'Scan requires minimum {required} - current {current}'
super().__init__(self.message)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'ExpectTimeoutError')
def __str__(self):
return 'ExpectTimeoutError'

class ProjectCodeError(Exception):
class ProjectCodeError(KerchunkException):
def __init__(self, verbose=0, proj_code=None, groupdir=None):
"""Could not find the correct project code from the list of project codes for this run."""
self.message = f'Project Code Extraction Failed'
super().__init__(self.message)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'ProjectCodeError')
def __str__(self):
return 'ProjectCodeError'

class FilecapExceededError(Exception):
class FilecapExceededError(KerchunkException):
def __init__(self, nfiles=0, verbose=0, proj_code=None, groupdir=None):
"""During scanning, could not find suitable files within the set of files specified."""
self.message = f'Filecap exceeded: {nfiles} files attempted'
super().__init__(self.message)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'FilecapExceededError')
def __str__(self):
return 'FilecapExceededError'

class ChunkDataError(Exception):
class ChunkDataError(KerchunkException):
def __init__(self, verbose=0, proj_code=None, groupdir=None):
"""Overflow Error from pandas during decoding of chunk information, most likely caused by bad data retrieval."""
self.message = f'Decoding resulted in overflow - received chunk data contains junk (attempted 3 times)'
super().__init__(self.message)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'ChunkDataError')
def __str__(self):
return 'ChunkDataError'

class NoValidTimeSlicesError(Exception):
class NoValidTimeSlicesError(KerchunkException):
def __init__(self, message='Kerchunk', verbose=0, proj_code=None, groupdir=None):
"""Unable to find any time slices to test within the object."""
self.message = f'No valid timeslices found for {message}'
super().__init__(self.message)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'NoValidTimeSlicesError')
def __str__(self):
return 'NoValidTimeSlicesError'

class VariableMismatchError(Exception):
class VariableMismatchError(KerchunkException):
def __init__(self, missing={}, verbose=0, proj_code=None, groupdir=None):
"""During testing, variables present in the NetCDF file are not present in Kerchunk"""
self.message = f'Missing variables {missing} in Kerchunk file'
super().__init__(self.message)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'VariableMismatchError')
def __str__(self):
return 'VariableMismatchError'

class ShapeMismatchError(Exception):
class ShapeMismatchError(KerchunkException):
def __init__(self, var={}, first={}, second={}, verbose=0, proj_code=None, groupdir=None):
"""Shapes of ND arrays do not match between Kerchunk and Xarray objects - when using a subset of the Netcdf files."""
self.message = f'Kerchunk/NetCDF mismatch for variable {var} with shapes - K {first} vs N {second}'
super().__init__(self.message)
if proj_code:
self.save(proj_code, groupdir)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'ShapeMismatchError')
def __str__(self):
return 'ShapeMismatchError'

class TrueShapeValidationError(Exception):
class TrueShapeValidationError(KerchunkException):
def __init__(self, message='Kerchunk', verbose=0, proj_code=None, groupdir=None):
"""Shapes of ND arrays do not match between Kerchunk and Xarray objects - when using the complete set of files."""
self.message = f'Kerchunk/NetCDF mismatch with shapes using full dataset - check logs'
super().__init__(self.message)
if proj_code:
self.save(proj_code, groupdir)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'TrueShapeValidationError')
def __str__(self):
return 'TrueShapeValidationError'

class NoOverwriteError(Exception):
class NoOverwriteError(KerchunkException):
def __init__(self, verbose=0, proj_code=None, groupdir=None):
"""Output file already exists and the process does not have forceful overwrite (-f) set."""
self.message = 'Output file already exists and forceful overwrite not set.'
super().__init__(self.message)
if proj_code:
self.save(proj_code, groupdir)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'NoOverwriteError')
def __str__(self):
return 'NoOverwriteError'

class MissingKerchunkError(Exception):
class MissingKerchunkError(KerchunkException):
def __init__(self, message="No suitable kerchunk file found for validation.", verbose=0, proj_code=None, groupdir=None):
"""Kerchunk file not found."""
self.message = message
super().__init__(self.message)
if proj_code:
self.save(proj_code, groupdir)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'MissingKerchunkError')
def __str__(self):
return 'MissingKerchunkError'

class ValidationError(Exception):
class ValidationError(KerchunkException):
def __init__(self, message="Fatal comparison failure for Kerchunk/NetCDF", verbose=0, proj_code=None, groupdir=None):
"""One or more checks within validation have failed - most likely elementwise comparison of data."""
self.message = message
super().__init__(self.message)
if proj_code:
self.save(proj_code, groupdir)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'ValidationError')
def __str__(self):
return 'ValidationError'

class SoftfailBypassError(Exception):
class SoftfailBypassError(KerchunkException):
def __init__(self, message="Kerchunk validation failed softly with no bypass - rerun with bypass flag", verbose=0, proj_code=None, groupdir=None):
"""Validation could not be completed because some arrays only contained NaN values which cannot be compared."""
self.message = message
super().__init__(self.message)
if proj_code:
self.save(proj_code, groupdir)
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def save(self, proj_code, groupdir):
upload_err(proj_code, groupdir, 'SoftfailBypassError')
def __str__(self):
return 'SoftfailBypassError'
1 change: 1 addition & 0 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ attrs==23.1.0
certifi==2023.7.22
cftime==1.6.2
charset-normalizer==3.3.0
dask==2023.3.1
entrypoints==0.4
fasteners==0.19
frozenlist==1.4.0
Expand Down

0 comments on commit 172b98d

Please sign in to comment.