From ee4346193dc0cb94835cf583dea7efbeb0914979 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 27 Aug 2024 13:42:29 -0600 Subject: [PATCH] More ruff & pre-commit rules (#546) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Wei Ji <23487320+weiji14@users.noreply.github.com> --- .pre-commit-config.yaml | 8 +++++++ doc/source/conf.py | 4 +--- .../tracking/pypistats/get_pypi_stats.py | 3 ++- .../tracking/traffic/traffic_data_mgmt.py | 3 ++- doc/sphinxext/announce.py | 0 icepyx/__init__.py | 8 +++---- icepyx/conftest.py | 3 ++- icepyx/core/APIformatting.py | 1 - icepyx/core/auth.py | 3 +-- icepyx/core/exceptions.py | 4 ---- icepyx/core/granules.py | 9 +++---- icepyx/core/is2ref.py | 7 +++--- icepyx/core/query.py | 5 ++-- icepyx/core/read.py | 4 ++-- icepyx/core/spatial.py | 8 +++---- icepyx/core/temporal.py | 1 - icepyx/core/validate_inputs.py | 1 + icepyx/core/variables.py | 5 ++-- icepyx/core/visualization.py | 4 ++-- icepyx/quest/quest.py | 1 - icepyx/tests/conftest.py | 3 ++- icepyx/tests/test_APIformatting.py | 1 - icepyx/tests/test_Earthdata.py | 4 ++-- icepyx/tests/test_auth.py | 3 +-- icepyx/tests/test_behind_NSIDC_API_login.py | 2 +- icepyx/tests/test_granules.py | 5 ++-- ...2class_query.py => test_is2class_query.py} | 6 +++++ icepyx/tests/test_quest_argo.py | 2 +- icepyx/tests/test_read.py | 9 +++---- icepyx/tests/test_spatial.py | 6 ++--- icepyx/tests/test_temporal.py | 2 +- pyproject.toml | 24 +++++++++++++++++++ 32 files changed, 92 insertions(+), 57 deletions(-) mode change 100644 => 100755 doc/sphinxext/announce.py rename icepyx/tests/{is2class_query.py => test_is2class_query.py} (86%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc1236159..934cdbb97 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,9 +6,17 @@ repos: - id: check-yaml - id: check-added-large-files args: ["--maxkb=5000"] + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: debug-statements + - id: name-tests-test + args: ["--pytest-test-first"] - id: end-of-file-fixer + - id: mixed-line-ending - id: trailing-whitespace args: [--markdown-linebreak-ext=md] + - id: requirements-txt-fixer - repo: https://github.com/codespell-project/codespell rev: v2.3.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index 868af3fbc..0303f72ca 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -19,7 +19,6 @@ import icepyx - # -- Project information ----------------------------------------------------- project = "icepyx" @@ -120,10 +119,9 @@ def setup(app): # this should possibly be moved to the sphinxext directory as a standalone .py file # -- custom style for pybtex output ------------------------------------------- -from pybtex.style.formatting.unsrt import Style as UnsrtStyle - # from pybtex.style.labels.alpha import LabelStyle as AlphaLabelStyle from pybtex.plugin import register_plugin +from pybtex.style.formatting.unsrt import Style as UnsrtStyle # I seem to be unable to figure out how to control what is used for the label. It would # make sense if it were fed into this function, which then just formatted it, but I can't figure out from where diff --git a/doc/source/tracking/pypistats/get_pypi_stats.py b/doc/source/tracking/pypistats/get_pypi_stats.py index d137b99c3..78a197e8f 100644 --- a/doc/source/tracking/pypistats/get_pypi_stats.py +++ b/doc/source/tracking/pypistats/get_pypi_stats.py @@ -1,6 +1,7 @@ import os -import pypistats + import pandas as pd +import pypistats cwd = os.getcwd() diff --git a/doc/source/tracking/traffic/traffic_data_mgmt.py b/doc/source/tracking/traffic/traffic_data_mgmt.py index 0d0a6af9c..5cdf0672b 100644 --- a/doc/source/tracking/traffic/traffic_data_mgmt.py +++ b/doc/source/tracking/traffic/traffic_data_mgmt.py @@ -1,5 +1,6 @@ -import matplotlib.pyplot as plt import os + +import matplotlib.pyplot as plt import pandas as pd cwd = os.getcwd() diff --git a/doc/sphinxext/announce.py b/doc/sphinxext/announce.py old mode 100644 new mode 100755 diff --git a/icepyx/__init__.py b/icepyx/__init__.py index 40ea9e1ec..a9d61834b 100644 --- a/icepyx/__init__.py +++ b/icepyx/__init__.py @@ -1,6 +1,6 @@ -from icepyx.core.query import Query, GenQuery +from _icepyx_version import version as __version__ + +from icepyx.core.query import GenQuery, Query from icepyx.core.read import Read -from icepyx.quest.quest import Quest from icepyx.core.variables import Variables - -from _icepyx_version import version as __version__ +from icepyx.quest.quest import Quest diff --git a/icepyx/conftest.py b/icepyx/conftest.py index d90558d69..a8bae49c4 100644 --- a/icepyx/conftest.py +++ b/icepyx/conftest.py @@ -1,6 +1,7 @@ -import icepyx import pytest +import icepyx + @pytest.fixture(autouse=True) def add_ipx(doctest_namespace): diff --git a/icepyx/core/APIformatting.py b/icepyx/core/APIformatting.py index 90b7f3ac6..8761a8343 100644 --- a/icepyx/core/APIformatting.py +++ b/icepyx/core/APIformatting.py @@ -2,7 +2,6 @@ import datetime as dt - # ---------------------------------------------------------------------- # parameter-specific formatting for display # or input to a set of API parameters (CMR or NSIDC) diff --git a/icepyx/core/auth.py b/icepyx/core/auth.py index 8e263e730..9f12fbecf 100644 --- a/icepyx/core/auth.py +++ b/icepyx/core/auth.py @@ -2,6 +2,7 @@ import datetime import earthaccess + from icepyx.core.exceptions import DeprecationError @@ -10,8 +11,6 @@ class AuthenticationError(Exception): Raised when an error is encountered while authenticating Earthdata credentials """ - pass - class EarthdataAuthMixin: """ diff --git a/icepyx/core/exceptions.py b/icepyx/core/exceptions.py index d20bbfe61..94bbea768 100644 --- a/icepyx/core/exceptions.py +++ b/icepyx/core/exceptions.py @@ -3,16 +3,12 @@ class DeprecationError(Exception): Class raised for use of functionality that is no longer supported by icepyx. """ - pass - class QueryError(Exception): """ Base class for Query object exceptions """ - pass - class NsidcQueryError(QueryError): """ diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index 2c51e208b..7c748fe6f 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -1,15 +1,16 @@ import datetime -import requests -import time import io -import re import json -import numpy as np import os import pprint +import re +import time from xml.etree import ElementTree as ET import zipfile +import numpy as np +import requests + import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin import icepyx.core.exceptions diff --git a/icepyx/core/is2ref.py b/icepyx/core/is2ref.py index be3a3c8da..1cd58f10a 100644 --- a/icepyx/core/is2ref.py +++ b/icepyx/core/is2ref.py @@ -1,12 +1,11 @@ -import h5py import json -import numpy as np -import requests import warnings from xml.etree import ElementTree as ET import earthaccess - +import h5py +import numpy as np +import requests # ICESat-2 specific reference functions diff --git a/icepyx/core/query.py b/icepyx/core/query.py index ed241662b..d547a959f 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -1,6 +1,7 @@ +import pprint + import geopandas as gpd import matplotlib.pyplot as plt -import pprint import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin @@ -1131,8 +1132,8 @@ def visualize_spatial_extent( gdf = self._spatial.extent_as_gdf try: - from shapely.geometry import Polygon # noqa: F401 import geoviews as gv + from shapely.geometry import Polygon # noqa: F401 gv.extension("bokeh") diff --git a/icepyx/core/read.py b/icepyx/core/read.py index 28da685d7..b3212aef2 100644 --- a/icepyx/core/read.py +++ b/icepyx/core/read.py @@ -432,7 +432,7 @@ def _add_vars_to_ds(is2ds, ds, grp_path, wanted_groups_tiered, wanted_dict): ] # handle delta_times with 1 or more dimensions - idx_range = range(0, len(ds.delta_time.data)) + idx_range = range(len(ds.delta_time.data)) try: photon_ids = ( idx_range @@ -440,7 +440,7 @@ def _add_vars_to_ds(is2ds, ds, grp_path, wanted_groups_tiered, wanted_dict): + 1 ) except AttributeError: - photon_ids = range(0, len(ds.delta_time.data)) + photon_ids = range(len(ds.delta_time.data)) hold_delta_times = ds.delta_time.data ds = ( diff --git a/icepyx/core/spatial.py b/icepyx/core/spatial.py index 650ca06be..0bc066e78 100644 --- a/icepyx/core/spatial.py +++ b/icepyx/core/spatial.py @@ -1,10 +1,10 @@ -import geopandas as gpd -import numpy as np import os -from shapely.geometry import box, Polygon -from shapely.geometry.polygon import orient import warnings +import geopandas as gpd +import numpy as np +from shapely.geometry import Polygon, box +from shapely.geometry.polygon import orient # DevGoal: need to update the spatial_extent docstring to describe coordinate order for input diff --git a/icepyx/core/temporal.py b/icepyx/core/temporal.py index 67f59882a..c02326fbf 100644 --- a/icepyx/core/temporal.py +++ b/icepyx/core/temporal.py @@ -1,7 +1,6 @@ import datetime as dt import warnings - """ Helper functions for validation of dates """ diff --git a/icepyx/core/validate_inputs.py b/icepyx/core/validate_inputs.py index e70a8f944..db5e44d05 100644 --- a/icepyx/core/validate_inputs.py +++ b/icepyx/core/validate_inputs.py @@ -1,5 +1,6 @@ import datetime as dt import warnings + import numpy as np diff --git a/icepyx/core/variables.py b/icepyx/core/variables.py index bf4a38c66..b7b1c56b8 100644 --- a/icepyx/core/variables.py +++ b/icepyx/core/variables.py @@ -1,9 +1,10 @@ -import numpy as np import os +import numpy as np + from icepyx.core.auth import EarthdataAuthMixin -import icepyx.core.is2ref as is2ref from icepyx.core.exceptions import DeprecationError +import icepyx.core.is2ref as is2ref import icepyx.core.validate_inputs as val # DEVGOAL: use h5py to simplify some of these tasks, if possible! diff --git a/icepyx/core/visualization.py b/icepyx/core/visualization.py index 5df884e2d..0ddb9fd40 100644 --- a/icepyx/core/visualization.py +++ b/icepyx/core/visualization.py @@ -10,15 +10,15 @@ import dask.dataframe as dd import datashader as ds import holoviews as hv +from holoviews.operation.datashader import rasterize import numpy as np import pandas as pd import requests -from holoviews.operation.datashader import rasterize from tqdm import tqdm import icepyx as ipx -import icepyx.core.is2ref as is2ref import icepyx.core.granules as granules +import icepyx.core.is2ref as is2ref hv.extension("bokeh") diff --git a/icepyx/quest/quest.py b/icepyx/quest/quest.py index b61376625..c24029070 100644 --- a/icepyx/quest/quest.py +++ b/icepyx/quest/quest.py @@ -1,5 +1,4 @@ from icepyx.core.query import GenQuery, Query - from icepyx.quest.dataset_scripts.argo import Argo diff --git a/icepyx/tests/conftest.py b/icepyx/tests/conftest.py index 9ce8e4081..5bc5eb870 100644 --- a/icepyx/tests/conftest.py +++ b/icepyx/tests/conftest.py @@ -1,7 +1,8 @@ import os -import pytest from unittest import mock +import pytest + # PURPOSE: mock environmental variables @pytest.fixture(scope="session", autouse=True) diff --git a/icepyx/tests/test_APIformatting.py b/icepyx/tests/test_APIformatting.py index 6dfe98353..0c7009d10 100644 --- a/icepyx/tests/test_APIformatting.py +++ b/icepyx/tests/test_APIformatting.py @@ -2,7 +2,6 @@ import icepyx.core.APIformatting as apifmt - # DevNote: is this a situation where you'd ideally build a test class, # since you're just repeating the test function with different inputs? # Especially for the _fmt_spaital, where there's >2 tests? diff --git a/icepyx/tests/test_Earthdata.py b/icepyx/tests/test_Earthdata.py index 06b0bc150..403159e04 100644 --- a/icepyx/tests/test_Earthdata.py +++ b/icepyx/tests/test_Earthdata.py @@ -1,13 +1,13 @@ -#!/usr/bin/env python """ test icepyx.core.query.Query.earthdata_login function """ import netrc import os -import pytest import shutil +import pytest + # PURPOSE: test different authentication methods @pytest.fixture(scope="module", autouse=True) diff --git a/icepyx/tests/test_auth.py b/icepyx/tests/test_auth.py index 9b720a0f8..97d40344d 100644 --- a/icepyx/tests/test_auth.py +++ b/icepyx/tests/test_auth.py @@ -1,8 +1,7 @@ +import earthaccess import pytest import requests -import earthaccess - from icepyx.core.auth import EarthdataAuthMixin diff --git a/icepyx/tests/test_behind_NSIDC_API_login.py b/icepyx/tests/test_behind_NSIDC_API_login.py index bf77da074..b3824030e 100644 --- a/icepyx/tests/test_behind_NSIDC_API_login.py +++ b/icepyx/tests/test_behind_NSIDC_API_login.py @@ -1,11 +1,11 @@ import json import os + import pytest import icepyx as ipx import icepyx.core.is2ref as is2ref - # Misc notes and needed tests # test avail data and subsetting success for each input type # (kml, shp, list of coords, bbox) diff --git a/icepyx/tests/test_granules.py b/icepyx/tests/test_granules.py index 8b00ce91d..90e3374d9 100644 --- a/icepyx/tests/test_granules.py +++ b/icepyx/tests/test_granules.py @@ -1,11 +1,12 @@ -import pytest import re + +import pytest import responses import icepyx as ipx from icepyx.core import granules as granules -from icepyx.core.granules import Granules as Granules from icepyx.core.exceptions import NsidcQueryError +from icepyx.core.granules import Granules as Granules # @pytest.fixture # def reg_a(): diff --git a/icepyx/tests/is2class_query.py b/icepyx/tests/test_is2class_query.py similarity index 86% rename from icepyx/tests/is2class_query.py rename to icepyx/tests/test_is2class_query.py index 84c31cfa4..b05e0f654 100644 --- a/icepyx/tests/is2class_query.py +++ b/icepyx/tests/test_is2class_query.py @@ -1,5 +1,11 @@ +import pytest + import icepyx as ipx +# Skip the whole module. See: +# https://docs.pytest.org/en/stable/reference/reference.html#globalvar-pytestmark +pytestmark = pytest.mark.skip("Legacy tests -- not sure if they ever passed!") + def test_CMRparams(): reg_a = ipx.Query("ATL06", [-64, 66, -55, 72], ["2019-02-22", "2019-02-28"]) diff --git a/icepyx/tests/test_quest_argo.py b/icepyx/tests/test_quest_argo.py index f2c677e8d..e11e0cbe1 100644 --- a/icepyx/tests/test_quest_argo.py +++ b/icepyx/tests/test_quest_argo.py @@ -1,7 +1,7 @@ import os +import re import pytest -import re from icepyx.quest.quest import Quest diff --git a/icepyx/tests/test_read.py b/icepyx/tests/test_read.py index 20807c410..90efe13f9 100644 --- a/icepyx/tests/test_read.py +++ b/icepyx/tests/test_read.py @@ -29,12 +29,12 @@ def test_parse_source_no_files(): ( # check list input [ "./icepyx/core/is2ref.py", - "./icepyx/tests/is2class_query.py", + "./icepyx/tests/test_is2class_query.py", ], sorted( [ "./icepyx/core/is2ref.py", - "./icepyx/tests/is2class_query.py", + "./icepyx/tests/test_is2class_query.py", ] ), ), @@ -45,11 +45,12 @@ def test_parse_source_no_files(): ], ), ( # check filename string with glob pattern input - "./icepyx/**/is2*.py", + "./icepyx/**/*is2*.py", sorted( [ "./icepyx/core/is2ref.py", - "./icepyx/tests/is2class_query.py", + "./icepyx/tests/test_is2class_query.py", + "./icepyx/tests/test_is2ref.py", ] ), ), diff --git a/icepyx/tests/test_spatial.py b/icepyx/tests/test_spatial.py index dc71cdac9..2243c0674 100644 --- a/icepyx/tests/test_spatial.py +++ b/icepyx/tests/test_spatial.py @@ -1,13 +1,13 @@ +from pathlib import Path +import re + import geopandas as gpd import numpy as np -from pathlib import Path import pytest -import re from shapely.geometry import Polygon import icepyx.core.spatial as spat - # ######### "Bounding Box" input tests ################################################################################ # (Note that these ALSO test the @property functions for the class for bounding boxes) diff --git a/icepyx/tests/test_temporal.py b/icepyx/tests/test_temporal.py index cd24deda4..52ec947c3 100644 --- a/icepyx/tests/test_temporal.py +++ b/icepyx/tests/test_temporal.py @@ -1,9 +1,9 @@ import datetime as dt + import pytest import icepyx.core.temporal as tp - # DevNote: this test suite needs to be parameterized (and fixtured) to include ALL acceptable input types # Currently it tests most cases for lists and dicts where the input type is str # A couple of tests were added for datetime.datetime type inputs, but they are incomplete diff --git a/pyproject.toml b/pyproject.toml index 66eb7b6f9..e1ea8ed90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,12 +73,33 @@ extend-exclude = ["*.ipynb"] # docstring-code-format = true # docstring-code-line-length = "dynamic" + [tool.ruff.lint] +# DevGoal: Enable "ARG" and "B" rulesets ASAP! select = [ + # "ARG", # flake8-unused-arguments + # "B", # flake8-bugbear "C4", # flake8-comprehensions "E", # pycodestyle + # "EM", # flake8-errmsg + "EXE", # flake8-executable "F", # pyflakes + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "NPY", # NumPy specific rules + # "PD", # pandas-vet + "PGH", # pygrep-hooks + "PIE", # flake8-pie + # "PL", # pylint + # "PT", # flake8-pytest-style + # "PTH", # flake8-use-pathlib + # "RET", # flake8-return + # "RUF", # Ruff-specific "SIM", # flake8-simplify + # "T20", # flake8-print + # "UP", # pyupgrade + "YTT", # flake8-2020 ] ignore = [ # Line too long @@ -102,3 +123,6 @@ ignore = [ # Ignore too many leading '#' for block comment "*/tests/*" = ["E266"] + +[tool.ruff.lint.isort] +force-sort-within-sections = true