Skip to content

Commit

Permalink
clean up code base, add pre-commit, add dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
Zeitsperre committed Aug 31, 2023
1 parent 8b83178 commit e228211
Show file tree
Hide file tree
Showing 8 changed files with 144 additions and 121 deletions.
52 changes: 52 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
default_language_version:
python: python3

repos:
- repo: https://github.com/asottile/pyupgrade
rev: v3.10.1
hooks:
- id: pyupgrade
args: [ '--py39-plus' ]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: debug-statements
- id: mixed-line-ending
- repo: https://github.com/psf/black
rev: 23.7.0
hooks:
- id: black
args: ["--target-version", "py39"]
- repo: https://github.com/pycqa/flake8
rev: 6.1.0
hooks:
- id: flake8
args: ['--ignore=E501']
- repo: https://github.com/PyCQA/isort
rev: 5.12.0
hooks:
- id: isort
args: ['--profile', 'black']
#- repo: https://github.com/pycqa/pydocstyle
# rev: 6.1.1
# hooks:
# - id: pydocstyle
# args: ["--convention=numpy"]
- repo: meta
hooks:
- id: check-hooks-apply
- id: check-useless-excludes

ci:
autofix_commit_msg: |
[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
autofix_prs: true
autoupdate_branch: ''
autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
autoupdate_schedule: weekly
skip: [ ]
submodules: false
4 changes: 1 addition & 3 deletions LICENSE
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
Copyright 2018 United Kingdom Research and Innovation

Authors: Ag Stephens
Copyright 2018 United Kingdom Research and Innovation, Ag Stephens, and Martin Schupfner

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Expand Down
3 changes: 2 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@

setuptools>=68.0.0
xarray>=2023.3.1
3 changes: 1 addition & 2 deletions requirements_dev.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
pytest
pytest>=7.0.0
# clisops>0.8.0 # incl regridding / Grid functionalities

4 changes: 4 additions & 0 deletions roocs_grids/scripts/create_grid_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
from pathlib import Path

import xarray as xr

# FIXME: This is a circular dependency, as clisops will be listing roocs-grids as a dependency
# this must be addressed before this module can be used.
from clisops import core as clore

from roocs_grids import grids_dir, pkg_dir
Expand Down Expand Up @@ -30,6 +33,7 @@
# Create clore.Grid object
g = clore.Grid(ds)
# Drop variables and global attributes
# FIXME: This is broken in newer versions of xarray
g._drop_vars(keep_attrs=False)
# Add specified global attributes
g.ds.attrs.update(attr_dict)
Expand Down
163 changes: 74 additions & 89 deletions roocs_grids/scripts/retrieve_grids.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
#!/bin/bash
#
#
####################################################################################
#
# Short script to download the reference target grids used by the roocs regridder
Expand All @@ -11,23 +9,25 @@
#
####################################################################################



# Download function
download () {
TargetPath=$1
Link=$2
FileArr=( ${@:3} )
local TargetPath="$1"
local Link="$2"
shift 2
local FileArr=( "$@" )

if [[ ! -d "$TargetPath" ]]; then
echo "Error: the target directory '$TargetPath' does not exist!"
exit 1
fi

[[ ! -d "$TargetPath" ]] && { echo "Error: the target directory '$TargetPath' does not exist!" && exit 1 ; }
echo "... target directory '$TargetPath'..."

for file in ${FileArr[@]}
do
filebase=$( basename $file )
for file in "${FileArr[@]}"; do
filebase=$( basename "$file" )
if [[ ! -e "$TargetPath/$filebase" ]]
then
curl $Link/$file -sSf -L -o $TargetPath/$filebase && echo "... '$filebase' retrieved successfully." || echo "Error: Grid '$filebase' could not be retrieved!"
curl "$Link/$file" -sSf -L -o "$TargetPath/$filebase" && echo "... '$filebase' retrieved successfully." || echo "Error: Grid '$filebase' could not be retrieved!"
else
echo "... '$filebase' already exists. Skipping download."
fi
Expand All @@ -36,73 +36,62 @@ download () {
}



# Specify target directory
ScriptPath=$( readlink -f $0 )
targetpath=$( dirname $ScriptPath )/../grids/
ScriptPath=$(readlink -f "$0")
targetpath=$(dirname "$ScriptPath")/../grids/


# Retrieve grids from the ATLAS repository

ATLASLink=https://raw.githubusercontent.com/SantanderMetGroup/ATLAS/master/reference-grids/
declare -a ATLASGrids
ATLASGrids+=( land_sea_mask_1degree.nc4 )
ATLASGrids+=( land_sea_mask_2degree.nc4 )
ATLASGrids+=( land_sea_mask_025degree_ERA5.nc )
ATLASGrids+=( land_sea_mask_05degree.nc4 )
ATLASGrids+=( land_sea_mask_025degree_binary.nc4 )
ATLASGrids+=( land_sea_mask_1degree_binary.nc4 )
ATLASGrids+=( land_sea_mask_2degree_binary.nc4 )
#ATLASGrids+=( mountain_ranges_mask_025degree.nc4 )
#ATLASGrids+=( mountain_ranges_mask_05degree.nc4 )
#ATLASGrids+=( mountain_ranges_mask_1degree.nc4 )
#ATLASGrids+=( mountain_ranges_mask_2degree.nc4 )
#ATLASGrids+=( )



echo "Downloading ATLAS reference-grids from '$ATLASLink'..."
download $targetpath $ATLASLink ${ATLASGrids[@]}


ATLASGrids=(
land_sea_mask_1degree.nc4
land_sea_mask_2degree.nc4
land_sea_mask_025degree_ERA5.nc
land_sea_mask_05degree.nc4
land_sea_mask_025degree_binary.nc4
land_sea_mask_1degree_binary.nc4
land_sea_mask_2degree_binary.nc4
# mountain_ranges_mask_025degree.nc4
# mountain_ranges_mask_05degree.nc4
# mountain_ranges_mask_1degree.nc4
# mountain_ranges_mask_2degree.nc4
)
echo -e "\nDownloading ATLAS reference-grids from '$ATLASLink'..."
download "$targetpath" "$ATLASLink" "${ATLASGrids[@]}"
echo "... done."

# Retrieve CMIP6 target grids provided by Charles Zender

CMIP6Link=http://dust.ess.uci.edu/grids/
declare -a CMIP6Grids
CMIP6Grids+=( cmip6_72x144_scrip.20181001.nc )
CMIP6Grids+=( cmip6_145x288_scrip.20181001.nc )
CMIP6Grids+=( cmip6_180x360_scrip.20181001.nc )
CMIP6Grids+=( cmip6_241x480_scrip.20181001.nc )
CMIP6Grids+=( cmip6_361x576_scrip.20181001.nc )
CMIP6Grids+=( cmip6_720x1440_scrip.20181001.nc )
CMIP6Grids+=( cmip6_721x1440_scrip.20181001.nc )
#CMIP6Grids+=( )

CMIP6Grids=(
cmip6_72x144_scrip.20181001.nc
cmip6_145x288_scrip.20181001.nc
cmip6_180x360_scrip.20181001.nc
cmip6_241x480_scrip.20181001.nc
cmip6_361x576_scrip.20181001.nc
cmip6_720x1440_scrip.20181001.nc
cmip6_721x1440_scrip.20181001.nc
)
echo -e "\nDownloading CMIP6 reference grids from '$CMIP6Link'..."
download $targetpath $CMIP6Link ${CMIP6Grids[@]}


download "$targetpath" "$CMIP6Link" "${CMIP6Grids[@]}"
echo "... done."

# Retrieve target grids from ESGF as submitted to CMIP6

ESGFLink=http://
declare -a ESGFGrids
ESGFGrids+=( esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/HighResMIP/MPI-M/MPI-ESM1-2-XR/highresSST-present/r1i1p1f1/Amon/sfcWind/gn/v20190923/sfcWind_Amon_MPI-ESM1-2-XR_highresSST-present_r1i1p1f1_gn_195001-195012.nc )
ESGFGrids+=( esgf3.dkrz.de/thredds/fileServer/cmip6/CMIP/MPI-M/MPI-ESM1-2-LR/1pctCO2/r1i1p1f1/fx/areacella/gn/v20190710/areacella_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc
ESGFGrids=(
esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/HighResMIP/MPI-M/MPI-ESM1-2-XR/highresSST-present/r1i1p1f1/Amon/sfcWind/gn/v20190923/sfcWind_Amon_MPI-ESM1-2-XR_highresSST-present_r1i1p1f1_gn_195001-195012.nc
esgf3.dkrz.de/thredds/fileServer/cmip6/CMIP/MPI-M/MPI-ESM1-2-LR/1pctCO2/r1i1p1f1/fx/areacella/gn/v20190710/areacella_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc
esgf3.dkrz.de/thredds/fileServer/cmip6/CMIP/MPI-M/MPI-ESM1-2-LR/1pctCO2/r1i1p1f1/fx/sftlf/gn/v20190710/sftlf_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc
esgf3.dkrz.de/thredds/fileServer/cmip6/ScenarioMIP/DKRZ/MPI-ESM1-2-HR/ssp370/r3i1p1f1/fx/sftlf/gn/v20190710/sftlf_fx_MPI-ESM1-2-HR_ssp370_r3i1p1f1_gn.nc
esgf3.dkrz.de/thredds/fileServer/cmip6/ScenarioMIP/DKRZ/MPI-ESM1-2-HR/ssp245/r1i1p1f1/fx/areacella/gn/v20190710/areacella_fx_MPI-ESM1-2-HR_ssp245_r1i1p1f1_gn.nc
)
ESGFGrids+=( esgf3.dkrz.de/thredds/fileServer/cmip6/CMIP/MPI-M/MPI-ESM1-2-LR/1pctCO2/r1i1p1f1/fx/sftlf/gn/v20190710/sftlf_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc )
ESGFGrids+=( esgf3.dkrz.de/thredds/fileServer/cmip6/ScenarioMIP/DKRZ/MPI-ESM1-2-HR/ssp370/r3i1p1f1/fx/sftlf/gn/v20190710/sftlf_fx_MPI-ESM1-2-HR_ssp370_r3i1p1f1_gn.nc )
ESGFGrids+=( esgf3.dkrz.de/thredds/fileServer/cmip6/ScenarioMIP/DKRZ/MPI-ESM1-2-HR/ssp245/r1i1p1f1/fx/areacella/gn/v20190710/areacella_fx_MPI-ESM1-2-HR_ssp245_r1i1p1f1_gn.nc )

echo -e "\nDownloading ESGF target grids from '$ESGFLink'..."
download $targetpath $ESGFLink ${ESGFGrids[@]}

download "$targetpath" "$ESGFLink" "${ESGFGrids[@]}"
echo "... done."


# Apply fixes
echo -e "\nApplying fixes to the netCDF files..."
for nc in ${CMIP6Grids[@]} ${ATLASGrids[@]}; do
for nc in "${CMIP6Grids[@]}" "${ATLASGrids[@]}"; do
# 1 # Takes too long for big grids
#ncdump $targetpath/$nc > $targetpath/${nc}.cdl
# Find and replace "since NA" with "2001-01-01 00:00:00" in CDL version of file
Expand All @@ -113,58 +102,54 @@ for nc in ${CMIP6Grids[@]} ${ATLASGrids[@]}; do
#}
#rm -f $targetpath/${nc}.cdl
# 2 #
ncdump -h $targetpath/$nc | grep -q "days since NA" && {
ncatted -O -a units,time,m,c,"days since 2001-01-01 00:00:00" $targetpath/$nc && echo " ...successful for '$nc'." || echo " ...failed for '$nc'!"
ncdump -h "$targetpath/$nc" | grep -q "days since NA" && {
ncatted -O -a units,time,m,c,"days since 2001-01-01 00:00:00" "$targetpath/$nc" &&
echo " ...successful for '$nc'." || echo " ...failed for '$nc'!"
}
done
echo -e "...done!"



# Post Process ESGF Files
echo -e "\nPost-processing ESGF files..."

# Extract and merge variables
[[ ! -e $targetpath/grid_T255.nc ]] && {
mv $targetpath/sfcWind_Amon_MPI-ESM1-2-XR_highresSST-present_r1i1p1f1_gn_195001-195012.nc $targetpath/grid_T255.nc
ncks -O -h -x -v sfcWind,time,time_bnds $targetpath/grid_T255.nc $targetpath/grid_T255.nc > /dev/null
nccopy -d 9 -s $targetpath/grid_T255.nc $targetpath/grid_T255_def.nc
mv $targetpath/grid_T255_def.nc $targetpath/grid_T255.nc
[[ ! -e "$targetpath"/grid_T255.nc ]] && {
mv "$targetpath"/sfcWind_Amon_MPI-ESM1-2-XR_highresSST-present_r1i1p1f1_gn_195001-195012.nc "$targetpath"/grid_T255.nc
ncks -O -h -x -v sfcWind,time,time_bnds "$targetpath"/grid_T255.nc "$targetpath"/grid_T255.nc > /dev/null
nccopy -d 9 -s "$targetpath"/grid_T255.nc "$targetpath"/grid_T255_def.nc
mv "$targetpath"/grid_T255_def.nc "$targetpath"/grid_T255.nc
}
[[ ! -e $targetpath/grid_T127_lsm_binary.nc ]] && {
mv $targetpath/sftlf_fx_MPI-ESM1-2-HR_ssp370_r3i1p1f1_gn.nc $targetpath/grid_T127_lsm_binary.nc
ncks -A -h -v areacella $targetpath/areacella_fx_MPI-ESM1-2-HR_ssp245_r1i1p1f1_gn.nc $targetpath/grid_T127_lsm_binary.nc
nccopy -d 9 -s $targetpath/grid_T127_lsm_binary.nc $targetpath/grid_T127_lsm_binary_def.nc
mv $targetpath/grid_T127_lsm_binary_def.nc $targetpath/grid_T127_lsm_binary.nc
[[ ! -e "$targetpath"/grid_T127_lsm_binary.nc ]] && {
mv "$targetpath"/sftlf_fx_MPI-ESM1-2-HR_ssp370_r3i1p1f1_gn.nc "$targetpath"/grid_T127_lsm_binary.nc
ncks -A -h -v areacella "$targetpath"/areacella_fx_MPI-ESM1-2-HR_ssp245_r1i1p1f1_gn.nc "$targetpath"/grid_T127_lsm_binary.nc
nccopy -d 9 -s "$targetpath"/grid_T127_lsm_binary.nc "$targetpath"/grid_T127_lsm_binary_def.nc
mv "$targetpath"/grid_T127_lsm_binary_def.nc "$targetpath"/grid_T127_lsm_binary.nc
}
[[ ! -e $targetpath/grid_T63_lsm_binary.nc ]] && {
mv $targetpath/sftlf_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc $targetpath/grid_T63_lsm_binary.nc
ncks -A -h -v areacella $targetpath/areacella_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc $targetpath/grid_T63_lsm_binary.nc
nccopy -d 9 -s $targetpath/grid_T63_lsm_binary.nc $targetpath/grid_T63_lsm_binary_def.nc
mv $targetpath/grid_T63_lsm_binary_def.nc $targetpath/grid_T63_lsm_binary.nc
[[ ! -e "$targetpath"/grid_T63_lsm_binary.nc ]] && {
mv "$targetpath"/sftlf_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc "$targetpath"/grid_T63_lsm_binary.nc
ncks -A -h -v areacella $targetpath/areacella_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc "$targetpath"/grid_T63_lsm_binary.nc
nccopy -d 9 -s "$targetpath"/grid_T63_lsm_binary.nc "$targetpath"/grid_T63_lsm_binary_def.nc
mv "$targetpath"/grid_T63_lsm_binary_def.nc "$targetpath"/grid_T63_lsm_binary.nc
}

# Clean up
rm -f $targetpath/areacella_fx_MPI-ESM1-2-HR_ssp245_r1i1p1f1_gn.nc $targetpath/areacella_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc $targetpath/sfcWind_Amon_MPI-ESM1-2-XR_highresSST-present_r1i1p1f1_gn_195001-195012.nc $targetpath/sftlf_fx_MPI-ESM1-2-HR_ssp370_r3i1p1f1_gn.nc $targetpath/sftlf_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc
rm -f "$targetpath"/areacella_fx_MPI-ESM1-2-HR_ssp245_r1i1p1f1_gn.nc "$targetpath"/areacella_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc "$targetpath"/sfcWind_Amon_MPI-ESM1-2-XR_highresSST-present_r1i1p1f1_gn_195001-195012.nc "$targetpath"/sftlf_fx_MPI-ESM1-2-HR_ssp370_r3i1p1f1_gn.nc "$targetpath"/sftlf_fx_MPI-ESM1-2-LR_1pctCO2_r1i1p1f1_gn.nc

# Remove unnecessary attributes
atts=(description nco_openmp_thread_number NCO history_of_appended_files activity_id realm branch_method branch_time_in_child branch_time_in_parent contact creation_date experiment experiment_id external_variables forcing_index frequency further_info_url history initialization_index institution institution_id mip_era activity_id parent_activity_id parent_experiment_id parent_mip_era parent_source_id parent_time_units parent_variant_label physics_index product project_id realization_index references source data_specs_version source_id source_type sub_experiment_id sub_experiment table_id table_info title variable_id variant_label license cmor_version tracking_id)
vatts=(history original_name original_units comment)

for ifile in grid_T127_lsm_binary.nc grid_T63_lsm_binary.nc grid_T255.nc
do
for att in ${atts[@]}; do
ncatted -O -h -a $att,global,d,, $targetpath/$ifile
for att in "${atts[@]}"; do
ncatted -O -h -a "$att",global,d,, "$targetpath/$ifile"
done
if [[ "$ifile" != *"T255"* ]]; then
for vatt in ${vatts[@]}; do
ncatted -O -h -a $vatt,sftlf,d,, $targetpath/$ifile
ncatted -O -h -a $vatt,areacella,d,, $targetpath/$ifile
for vatt in "${vatts[@]}"; do
ncatted -O -h -a "$vatt",sftlf,d,, "$targetpath/$ifile"
ncatted -O -h -a "$vatt",areacella,d,, "$targetpath/$ifile"
done
fi
done
echo -e "...done!"



exit 0
34 changes: 9 additions & 25 deletions setup.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""The setup script."""

__author__ = "Ag Stephens"
__contact__ = "[email protected]"
__copyright__ = "Copyright 2018 United Kingdom Research and Innovation"
__license__ = "BSD - see LICENSE file in top-level package directory"
__version__ = "0.1.0"

from setuptools import find_packages
from setuptools import setup
import os

from setuptools import find_packages, setup

# One strategy for storing the overall version is to put it in the top-level
# package's __init__ but Nb. __init__.py files are not needed to declare
# packages in Python 3
Expand All @@ -38,42 +35,29 @@
# https://www.python.org/dev/peps/pep-0301/#distutils-trove-classification
classifiers=[
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"Environment :: Web Environment",
"Intended Audience :: End Users/Desktop",
"Intended Audience :: Developers",
"Intended Audience :: System Administrators",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: BSD License",
"Natural Language :: English",
"Operating System :: Microsoft :: Windows",
"Operating System :: POSIX :: Linux",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Topic :: Security",
"Topic :: Internet",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Topic :: Scientific/Engineering",
"Topic :: System :: Distributed Computing",
"Topic :: System :: Systems Administration :: Authentication/Directory",
"Topic :: Software Development :: Libraries :: Python Modules",
],
description="Grid definitions for the roocs regridder",
license=__license__,
# This qualifier can be used to selectively exclude Python versions -
# in this case early Python 2 and 3 releases
python_requires=">=3.6.0",
license_files=["LICENSE"],
python_requires=">=3.9.0",
install_requires=requirements,
long_description=_long_description,
long_description_content_type="text/x-rst",
include_package_data=True,
keywords="roocs_grids, regrid, grid, interpolate, cmip",
name="roocs_grids",
packages=find_packages(
include=["roocs_grids"]
),
packages=find_packages(include=["roocs_grids"]),
setup_requires=setup_requirements,
test_suite="tests",
tests_require=test_requirements,
Expand Down
Loading

0 comments on commit e228211

Please sign in to comment.