cleaned code following ruff report, except for tests

docs/conf.py

@@ -12,9 +12,6 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys
-import os
-
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the

src/mopdb/__init__.py

@@ -1 +0,0 @@
-from mopdb import *

src/mopdb/mopdb.py

@@ -19,19 +19,16 @@
 # last updated 08/04/2024
 
 import click
-import sqlite3
 import logging
 import sys
-import csv
 import json
 
 from importlib.resources import files as import_files
 from pathlib import Path
 
 from mopdb.mopdb_utils import (mapping_sql, cmorvar_sql, read_map,
-    read_map_app4, map_update_sql, create_table, write_cmor_table,
-    check_varlist, update_db) 
-from mopdb.utils import *
+    read_map_app4, create_table, write_cmor_table, update_db) 
+from mopdb.utils import (config_log, db_connect, query, delete_record)
 from mopdb.mopdb_map import (write_varlist, write_map_template,
     write_catalogue, map_variables, load_vars, get_map_obj)
 
@@ -112,7 +109,7 @@ def mopdb(ctx, debug):
     ctx.obj={}
     # set up a default value for flow if none selected for logging
     ctx.obj['debug'] = debug
-    mopdb_log = config_log(debug, logname='mopdb_log')
+    #mopdb_log = config_log(debug, logname='mopdb_log')
 
 
 @mopdb.command(name='check')
@@ -200,7 +197,7 @@ def cmor_table(ctx, dbname, fname, alias, label):
     # extract cmor_var,units,dimensions,frequency,realm,cell_methods
     var_list = []
     for v in vlist[1:]:
-        vid = (v[0], v[5], v[6])
+        #vid = (v[0], v[5], v[6])
         # This was adding variables to the table just if they didn't exists in other tables
         if v[0][:4] != 'fld_':
             if v[0] not in cmor_vars:
@@ -353,17 +350,6 @@ def map_template(ctx, fpath, match, dbname, version, alias):
         fname, vobjs, fobjs = write_varlist(conn, fpath, match, version, alias)
     if alias == '':
         alias = fname.split(".")[0]
-# also from here on it should be called by separate function I can call from intake too
-# without repeating steps
-    # read list of vars from file
-    # this should now spit out fobjs, vobjs to pass to template 
-    #with open(fname, 'r') as csvfile:
-    #    reader = csv.DictReader(csvfile, delimiter=';')
-    #    rows = list(reader)
-    #check_varlist(rows, fname)
-    # return lists of fully/partially matching variables and stash_vars 
-    # these are input_vars for calculation defined in already in mapping db
-    #parsed = map_variables(conn, rows, version)
     parsed = map_variables(conn, vobjs, version)
     # potential vars have always duplicates: 1 for each input_var
     write_map_template(conn, parsed, alias)
@@ -425,11 +411,6 @@ def write_intake(ctx, fpath, match, filelist, dbname, version, alias):
         map_file, vobjs, fobjs = load_vars(flist, indir=fpath)
     if alias == '':
         alias = fname.split(".")[0]
-    # read list of vars from file
-    #with open(fname, 'r') as csvfile:
-    #    reader = csv.DictReader(csvfile, delimiter=';')
-    #    rows = list(reader)
-    #check_varlist(rows, fname)
     # return lists of fully/partially matching variables and stash_vars 
     # these are input_vars for calculation defined in already in mapping db
     if map_file is False:
@@ -527,7 +508,7 @@ def model_vars(ctx, fpath, match, dbname, version, alias):
     if dbname == 'default':
         dbname = import_files('mopdata').joinpath('access.db')
     conn = db_connect(dbname, logname='mopdb_log')
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     fname, vobjs, fobjs = write_varlist(conn, fpath, match, version, alias)
     conn.close()
     return None

src/mopdb/mopdb_class.py

@@ -85,11 +85,6 @@ class Variable():
        and the one added by mapping.
     """ 
 
-  #  __slots__ = ('name', 'pattern', 'files', 'frequency', 'realm',
-  #      'cmor_var', 'cmor_table', 'version', 'units', 'dimensions',
-  #      'cell_methods', 'positive', 'long_name', 'standard_name',
-  #      'vtype', 'size', 'nsteps')
-
     def __init__(self, varname: str, fobj: FPattern):
         self.name = varname
         # path object
@@ -148,7 +143,7 @@ def get_match(self):
             cmor_var = self.cmor_var
         else:
             cmor_var = self.name
-        match = (self.cmor_var, self.name, '', self.frequency,
+        match = (cmor_var, self.name, '', self.frequency,
             self.realm, self.version, '', self.positive, self.units)
         return match
 

src/mopdb/mopdb_map.py

@@ -33,9 +33,9 @@
 #from access_nri_intake.source.builders import AccessEsm15Builder
 
 from mopdb.mopdb_class import FPattern, Variable, MapVariable
-from mopdb.utils import *
+from mopdb.utils import query, read_yaml
 from mopdb.mopdb_utils import (get_cell_methods, remove_duplicate,
-    get_realm, check_realm_units, get_date_pattern, check_varlist)
+    get_realm, check_realm_units, get_date_pattern)
 
 
 def get_cmorname(conn, vobj, version):
@@ -111,13 +111,12 @@ def get_file_frq(ds, fnext, int2frq):
     # so we open also next file but get only time axs
     if max_len == 1:
         if fnext is None:
-            mopdb_log.info(f"Only 1 file cannot determine frequency for: {fpattern}")
+            mopdb_log.info(f"Only 1 file with 1 tstep cannot determine frequency")
         else:
             dsnext = xr.open_dataset(fnext, decode_times = False)
             time_axs2 = [d for d in dsnext.dims if 'time' in d]
             ds = xr.concat([ds[time_axs], dsnext[time_axs2]], dim='time')
             time_axs = [d for d in ds.dims if 'time' in d]
-            time_axs_len = set(len(ds[d]) for d in time_axs)
             time_axs.sort(key=lambda x: len(ds[x]), reverse=True)
     if max_len > 0:
         for t in time_axs: 
@@ -232,7 +231,7 @@ def match_stdname(conn, vobj, stdn):
     in cmorvar table that match the standard name passed as input.
     It also return a False/True found_match boolean.
     """
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     found_match = False
     sql = f"""SELECT name FROM cmorvar where 
         standard_name='{vobj.standard_name}'"""
@@ -451,7 +450,7 @@ def write_vars(vlist, fwriter, div, conn=None, sortby='cmor_var'):
     """
     """
 
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     if len(vlist) > 0:
         if type(div) is str:
             divrow = {x:'' for x in vlist[0].attrs()}
@@ -503,14 +502,15 @@ def get_map_obj(parsed):
 def write_catalogue(conn, vobjs, fobjs, alias):
     """Write intake-esm catalogue and returns name
     """
+
     mopdb_log = logging.getLogger('mopdb_log')
     # read template json file 
     jfile = import_files('mopdata').joinpath('intake_cat_template.json')
     with open(jfile, 'r') as f:
         template = json.load(f)
     # write updated json to file
     for k,v in template.items():
-        if type(v) == str:
+        if type(v) is str:
             template[k] = v.replace("<experiment>", alias)
     jout = f"intake_{alias}.json"
     with open(jout, 'w') as f:
@@ -542,7 +542,7 @@ def write_catalogue(conn, vobjs, fobjs, alias):
 def create_file_dict(fobjs, alias):
     """
     """
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     lines = []
     for pat_obj in fobjs:
         var_list = [v.name for v in pat_obj.varlist]
@@ -574,7 +574,7 @@ def create_file_dict(fobjs, alias):
 def add_mapvars(vobjs, lines, path_list, alias):
     """
     """
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     for vobj in vobjs:
         if vobj.cmor_var != "" or vobj.standard_name != "":
             mapvar = vobj.cmor_var
@@ -598,7 +598,7 @@ def add_mapvars(vobjs, lines, path_list, alias):
 def load_vars(fname, indir=None):
     """Returns Variable and FPattern objs from varlist or map file.
     """
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     vobjs = []
     fobjs = {}
     if indir is not None:

src/mopdb/mopdb_utils.py

@@ -19,7 +19,6 @@
 # last updated 10/04/2024
 #
 
-import sqlite3
 import logging
 import sys
 import csv
@@ -28,7 +27,7 @@
 from datetime import date
 from collections import Counter
 
-from mopdb.utils import *
+from mopdb.utils import query 
 
 
 def mapping_sql():
@@ -236,7 +235,7 @@ def get_cell_methods(attrs, dims):
        `time: point`
        If `area` not specified is added at start of string as `area: `
     """
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     frqmod = ''
     val = attrs.get('cell_methods', "") 
     if 'area' not in val: 
@@ -252,7 +251,7 @@ def get_cell_methods(attrs, dims):
 
 def read_map_app4(fname):
     """Reads APP4 style mapping """
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     # old order
     #cmor_var,definable,input_vars,calculation,units,axes_mod,positive,ACCESS_ver[CM2/ESM/both],realm,notes
     var_list = []
@@ -404,7 +403,7 @@ def get_date_pattern(fname, fpattern):
     """Try to build a date range for each file pattern based
        on its filename
     """
-    mopdb_log = logging.getLogger('mopdb_log')
+    #mopdb_log = logging.getLogger('mopdb_log')
     # assign False to any character which is not a digit
     date_pattern = [True if c.isdigit() else False for c in fname]
     # assign False to fpattern

src/mopdb/utils.py

@@ -22,8 +22,6 @@
 import sqlite3
 import logging
 import os
-import csv
-import json
 import stat
 import yaml
 
@@ -57,7 +55,7 @@ def config_log(debug, logname):
     logname = f"{logname}_{day}.txt"
     flog = logging.FileHandler(logname)
     try:
-        os.chmod(logname, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO);
+        os.chmod(logname, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
     except OSError:
         pass
     flog.setLevel(flevel)
@@ -112,7 +110,7 @@ def query(conn, sql, tup=(), first=True, logname='__name__'):
     result : tuple/list(tuple)
         tuple or a list of, representing row/s returned by query 
     """
-    log = logging.getLogger(logname)
+    #log = logging.getLogger(logname)
     with conn:
         c = conn.cursor()
         c.execute(sql, tup)
@@ -127,7 +125,7 @@ def query(conn, sql, tup=(), first=True, logname='__name__'):
 def get_columns(conn, table, logname='__name__'):
     """Gets list of columns from db table
     """
-    log = logging.getLogger(logname)
+    #log = logging.getLogger(logname)
     sql = f'PRAGMA table_info({table});'
     table_data = query(conn, sql, first=False, logname=logname)
     columns = [x[1] for x in table_data]
@@ -205,6 +203,7 @@ def write_yaml(data, fname, logname='__name__'):
     try:
         with open(fname, 'w') as f:
             yaml.dump(data, f)
-    except:
-        log.error(f"Check that {data} exists and it is an object compatible with yaml")
+    except Exception as e:
+        log.error(f"Exception: {e}")
+        log.error(f"Check {data} exists and is yaml object")
     return

src/mopper/__init__.py

@@ -1 +0,0 @@
-from mopper import *

src/mopper/calculations.py

@@ -33,7 +33,6 @@
 import click
 import xarray as xr
 import os
-import yaml
 import json 
 import numpy as np
 import dask
@@ -233,7 +232,7 @@ def transAcrossLine(self, var, i_start, i_end, j_start, j_end):
                 #sum each axis apart from time (3d)
                 #trans = var.isel(yu_ocean=slice(271, 271+1), xt_ocean=slice(292, 300+1))
                 trans = var[..., j_start:j_end+1, i_start:i_end+1].sum(dim=['st_ocean', f'{y_ocean}', f'{x_ocean}']) #4D
-            except:
+            except Exception as e:
                 trans = var[..., j_start:j_end+1, i_start:i_end+1].sum(dim=[f'{y_ocean}', f'{x_ocean}']) #3D
 
             return trans
@@ -665,25 +664,6 @@ def calc_hemi_seaice_extent(self, hemi):
 
         return vout.item()
 
-
-def ocean_floor(var):
-    """Not sure.. 
-
-    Parameters
-    ----------
-    var : Xarray dataset
-        pot_temp variable
-
-    Returns
-    -------
-    vout : Xarray dataset
-        ocean floor temperature?
-    """
-    lv = (~var.isnull()).sum(dim='st_ocean') - 1
-    vout = var.take(lv, dim='st_ocean').squeeze()
-    return vout
-
-
 def maskSeaIce(var, sic):
     """Mask seaice.
 
@@ -702,7 +682,6 @@ def maskSeaIce(var, sic):
     vout = var.where(sic != 0)
     return vout
 
-
 def sithick(hi, aice):
     """Calculate seaice thickness.
 
@@ -722,7 +701,6 @@ def sithick(hi, aice):
     vout = hi / aice
     return vout
 
-
 def sisnconc(sisnthick):
     """Calculate seas ice?
 
@@ -807,7 +785,7 @@ def calc_global_ave_ocean(var, rho_dzt, area_t):
 
     try:
         vnew = var.weighted(mass).mean(dim=('st_ocean', 'yt_ocean', 'xt_ocean'), skipna=True)
-    except:
+    except Exception as e:
         vnew = var.weighted(mass[:, 0, :, :]).mean(dim=('x', 'y'), skipna=True)
 
     return vnew
@@ -1267,7 +1245,7 @@ def calc_global_ave_ocean(ctx, var, rho_dzt):
     mass = rho_dzt * area_t
     try: 
         vnew=np.average(var,axis=(1,2,3),weights=mass)
-    except: 
+    except Exception as e:
         vnew=np.average(var,axis=(1,2),weights=mass[:,0,:,:])
 
     return vnew
@@ -1437,7 +1415,7 @@ def calc_depositions(ctx, var, weight=None):
     (personal communication from M. Woodhouse)
     """
 
-    var_log = logging.getLogger(ctx.obj['var_log'])
+    #var_log = logging.getLogger(ctx.obj['var_log'])
     varlist = []
     for v in var:
         v0 = v.sel(model_theta_level_number=1).squeeze(dim='model_theta_level_number')