From bf4c70e30ce41ebf4d90d0c3c7acc365860ec196 Mon Sep 17 00:00:00 2001 From: zssherman Date: Tue, 7 Nov 2023 13:29:49 -0600 Subject: [PATCH] MNT: Update codebase with pre-commit while removing armfiles from setup.cfg. --- act/corrections/mpl.py | 4 +- act/discovery/__init__.py | 11 +- act/discovery/airnow.py | 178 ++- act/discovery/arm.py | 17 +- act/discovery/asos.py | 2 +- act/discovery/cropscape.py | 1 + act/discovery/get_airnow.py | 182 ++- act/discovery/get_arm.py | 17 +- act/discovery/get_armfiles.py | 17 +- act/discovery/get_asos.py | 2 +- act/discovery/get_cropscape.py | 3 +- act/discovery/get_neon.py | 5 +- act/discovery/get_noaapsl.py | 32 +- act/discovery/get_surfrad.py | 9 +- act/discovery/neon.py | 3 +- act/discovery/noaapsl.py | 30 +- act/discovery/surfrad.py | 7 +- act/io/__init__.py | 7 +- act/io/armfiles.py | 54 +- act/io/csvfiles.py | 6 +- act/io/icartt.py | 14 +- act/io/neon.py | 20 +- act/io/noaagml.py | 293 ++-- act/io/noaapsl.py | 593 ++++++-- act/io/sodar.py | 16 +- act/plotting/__init__.py | 5 +- act/plotting/distributiondisplay.py | 113 +- act/plotting/geodisplay.py | 5 +- act/plotting/histogramdisplay.py | 39 +- act/plotting/plot.py | 33 +- act/plotting/skewtdisplay.py | 6 +- act/plotting/timeseriesdisplay.py | 49 +- act/plotting/windrosedisplay.py | 35 +- act/qc/add_supplemental_qc.py | 33 +- act/qc/arm.py | 25 +- act/qc/bsrn_tests.py | 451 ++++-- act/qc/clean.py | 9 +- act/qc/qcfilter.py | 39 +- act/qc/qctests.py | 22 +- act/qc/sp2.py | 3 +- act/retrievals/cbh.py | 2 +- act/retrievals/doppler_lidar.py | 36 +- act/retrievals/radiation.py | 5 +- act/retrievals/sonde.py | 61 +- act/tests/sample_files.py | 18 +- act/tests/test_correct.py | 4 +- act/tests/test_discovery.py | 11 +- act/tests/test_io.py | 145 +- act/tests/test_plotting.py | 4 +- act/tests/test_qc.py | 178 ++- act/tests/test_utils.py | 13 +- act/utils/__init__.py | 26 +- act/utils/data_utils.py | 14 +- act/utils/datetime_utils.py | 6 +- act/utils/io_utils.py | 29 +- examples/discovery/plot_neon.py | 33 +- examples/io/plot_create_arm_ds.py | 2 +- examples/io/plot_sodar.py | 4 +- examples/io/plot_surfrad.py | 5 +- examples/plotting/plot_ceil.py | 6 +- examples/plotting/plot_days.py | 12 +- examples/plotting/plot_enhanced_skewt.py | 2 + examples/plotting/plot_heatmap.py | 8 +- examples/plotting/plot_hist_kwargs.py | 5 +- examples/plotting/plot_presentweathercode.py | 29 +- examples/plotting/plot_scatter.py | 51 +- examples/plotting/plot_size_distribution.py | 3 +- examples/plotting/plot_violin.py | 23 +- examples/qc/plot_qc_bsrn.py | 9 +- examples/retrievals/plot_cbh_sobel.py | 11 +- examples/templates/example_template.py | 5 +- examples/utils/plot_parse_filename.py | 32 +- examples/workflows/plot_aerioe_with_cbh.py | 60 +- examples/workflows/plot_merged_product.py | 11 +- examples/workflows/plot_multiple_dataset.py | 8 +- examples/workflows/plot_qc_transforms.py | 20 +- guides/act_cheatsheet.tex | 8 +- requirements.txt | 2 +- scripts/ads.py | 1428 ++++++++++++------ setup.cfg | 5 +- setup.py | 2 +- 81 files changed, 3157 insertions(+), 1569 deletions(-) diff --git a/act/corrections/mpl.py b/act/corrections/mpl.py index 73e2616df1..e0601817f3 100644 --- a/act/corrections/mpl.py +++ b/act/corrections/mpl.py @@ -141,8 +141,8 @@ def correct_mpl( x_data = x_data - x_ap # R-Squared Correction - co_data = co_data * height ** 2 - x_data = x_data * height ** 2 + co_data = co_data * height**2 + x_data = x_data * height**2 # Overlap Correction for j in range(ds[range_bins_var_name].size): diff --git a/act/discovery/__init__.py b/act/discovery/__init__.py index 6bc9066d4c..4579fa3133 100644 --- a/act/discovery/__init__.py +++ b/act/discovery/__init__.py @@ -8,7 +8,14 @@ __getattr__, __dir__, __all__ = lazy.attach( __name__, - submodules=['get_armfiles', 'get_cropscape', 'get_airnow', 'get_noaapsl', 'get_neon', 'get_surfrad'], + submodules=[ + 'get_armfiles', + 'get_cropscape', + 'get_airnow', + 'get_noaapsl', + 'get_neon', + 'get_surfrad', + ], submod_attrs={ 'get_arm': ['download_arm_data'], 'get_armfiles': ['download_data', 'download_arm_data', 'get_arm_doi'], @@ -17,6 +24,6 @@ 'get_cropscape': ['croptype'], 'get_noaapsl': ['download_noaa_psl_data'], 'get_neon': ['get_site_products', 'get_product_avail', 'download_neon_data'], - 'get_surfrad': ['download_surfrad'] + 'get_surfrad': ['download_surfrad'], }, ) diff --git a/act/discovery/airnow.py b/act/discovery/airnow.py index 7b458e780c..938e19a174 100644 --- a/act/discovery/airnow.py +++ b/act/discovery/airnow.py @@ -1,5 +1,5 @@ -import pandas as pd import numpy as np +import pandas as pd import xarray as xr @@ -38,25 +38,42 @@ def get_airnow_forecast(token, date, zipcode=None, latlon=None, distance=25): """ # default beginning of the query url - query_url = ('https://airnowapi.org/aq/forecast/') + query_url = 'https://airnowapi.org/aq/forecast/' # checking is either a zipcode or latlon coordinate is defined # if neither is defined then error is raised if (zipcode is None) and (latlon is None): - raise NameError("Zipcode or latlon must be defined") + raise NameError('Zipcode or latlon must be defined') if zipcode: - url = (query_url + ('zipcode/?' + 'format=text/csv' + '&zipCode=' - + str(zipcode) + '&date=' + str(date) - + '&distance=' + str(distance) - + '&API_KEY=' + str(token))) + url = query_url + ( + 'zipcode/?' + + 'format=text/csv' + + '&zipCode=' + + str(zipcode) + + '&date=' + + str(date) + + '&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) if latlon: - url = (query_url + ('latLong/?' + 'format=text/csv' - + '&latitude=' + str(latlon[0]) + '&longitude=' - + str(latlon[1]) + '&date=' + str(date) - + '&distance=' + str(distance) - + '&API_KEY=' + str(token))) + url = query_url + ( + 'latLong/?' + + 'format=text/csv' + + '&latitude=' + + str(latlon[0]) + + '&longitude=' + + str(latlon[1]) + + '&date=' + + str(date) + + '&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) df = pd.read_csv(url) @@ -103,37 +120,78 @@ def get_airnow_obs(token, date=None, zipcode=None, latlon=None, distance=25): """ # default beginning of the query url - query_url = ('https://www.airnowapi.org/aq/observation/') + query_url = 'https://www.airnowapi.org/aq/observation/' # checking is either a zipcode or latlon coordinate is defined # if neither is defined then error is raised if (zipcode is None) and (latlon is None): - raise NameError("Zipcode or latlon must be defined") + raise NameError('Zipcode or latlon must be defined') # setting the observation type to either current or historical based on the date if date is None: obs_type = 'current' if zipcode: - url = (query_url + ('zipCode/' + str(obs_type) + '/?' + 'format=text/csv' - + '&zipCode=' + str(zipcode) + '&distance=' + str(distance) - + '&API_KEY=' + str(token))) + url = query_url + ( + 'zipCode/' + + str(obs_type) + + '/?' + + 'format=text/csv' + + '&zipCode=' + + str(zipcode) + + '&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) if latlon: - url = (query_url + ('latLong/' + str(obs_type) + '/?' + 'format=text/csv' - + '&latitude=' + str(latlon[0]) - + '&longitude=' + str(latlon[1]) + '&distance=' - + str(distance) + '&API_KEY=' + str(token))) + url = query_url + ( + 'latLong/' + + str(obs_type) + + '/?' + + 'format=text/csv' + + '&latitude=' + + str(latlon[0]) + + '&longitude=' + + str(latlon[1]) + + '&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) else: obs_type = 'historical' if zipcode: - url = (query_url + ('zipCode/' + str(obs_type) + '/?' + 'format=text/csv' - + '&zipCode=' + str(zipcode) + '&date=' + str(date) - + 'T00-0000&distance=' + str(distance) + '&API_KEY=' + str(token))) + url = query_url + ( + 'zipCode/' + + str(obs_type) + + '/?' + + 'format=text/csv' + + '&zipCode=' + + str(zipcode) + + '&date=' + + str(date) + + 'T00-0000&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) if latlon: - url = (query_url + ('latLong/' + str(obs_type) + '/?' + 'format=text/csv' - + '&latitude=' + str(latlon[0]) - + '&longitude=' + str(latlon[1]) + '&date=' - + str(date) + 'T00-0000&distance=' + str(distance) - + '&API_KEY=' + str(token))) + url = query_url + ( + 'latLong/' + + str(obs_type) + + '/?' + + 'format=text/csv' + + '&latitude=' + + str(latlon[0]) + + '&longitude=' + + str(latlon[1]) + + '&date=' + + str(date) + + 'T00-0000&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) df = pd.read_csv(url) @@ -143,8 +201,9 @@ def get_airnow_obs(token, date=None, zipcode=None, latlon=None, distance=25): return ds -def get_airnow_bounded_obs(token, start_date, end_date, latlon_bnds, parameters='OZONE,PM25', data_type='B', - mon_type=0): +def get_airnow_bounded_obs( + token, start_date, end_date, latlon_bnds, parameters='OZONE,PM25', data_type='B', mon_type=0 +): """ Get AQI values or data concentrations for a specific date and time range and set of parameters within a geographic area of intrest @@ -184,16 +243,44 @@ def get_airnow_bounded_obs(token, start_date, end_date, latlon_bnds, parameters= verbose = 1 inc_raw_con = 1 - url = ('https://www.airnowapi.org/aq/data/?startDate=' + str(start_date) - + '&endDate=' + str(end_date) + '¶meters=' + str(parameters) - + '&BBOX=' + str(latlon_bnds) + '&dataType=' + str(data_type) - + '&format=text/csv' + '&verbose=' + str(verbose) - + '&monitorType=' + str(mon_type) + '&includerawconcentrations=' - + str(inc_raw_con) + '&API_KEY=' + str(token)) + url = ( + 'https://www.airnowapi.org/aq/data/?startDate=' + + str(start_date) + + '&endDate=' + + str(end_date) + + '¶meters=' + + str(parameters) + + '&BBOX=' + + str(latlon_bnds) + + '&dataType=' + + str(data_type) + + '&format=text/csv' + + '&verbose=' + + str(verbose) + + '&monitorType=' + + str(mon_type) + + '&includerawconcentrations=' + + str(inc_raw_con) + + '&API_KEY=' + + str(token) + ) # Set Column names - names = ['latitude', 'longitude', 'time', 'parameter', 'concentration', 'unit', - 'raw_concentration', 'AQI', 'category', 'site_name', 'site_agency', 'aqs_id', 'full_aqs_id'] + names = [ + 'latitude', + 'longitude', + 'time', + 'parameter', + 'concentration', + 'unit', + 'raw_concentration', + 'AQI', + 'category', + 'site_name', + 'site_agency', + 'aqs_id', + 'full_aqs_id', + ] # Read data into CSV df = pd.read_csv(url, names=names) @@ -211,12 +298,9 @@ def get_airnow_bounded_obs(token, start_date, end_date, latlon_bnds, parameters= data_vars={ 'latitude': (['sites'], latitude), 'longitude': (['sites'], longitude), - 'aqs_id': (['sites'], aqs_id) + 'aqs_id': (['sites'], aqs_id), }, - coords={ - 'time': (['time'], times), - 'sites': (['sites'], sites) - } + coords={'time': (['time'], times), 'sites': (['sites'], sites)}, ) # Set up emtpy data with nans @@ -233,7 +317,11 @@ def get_airnow_bounded_obs(token, start_date, end_date, latlon_bnds, parameters= data[v, t, s] = list(result[variables[v]])[0] atts = {'units': ''} else: - result = df.loc[(df['time'] == times[t]) & (df['site_name'] == sites[s]) & (df['parameter'] == variables[v])] + result = df.loc[ + (df['time'] == times[t]) + & (df['site_name'] == sites[s]) + & (df['parameter'] == variables[v]) + ] if len(result['concentration']) > 0: data[v, t, s] = list(result['concentration'])[0] atts = {'units': list(result['unit'])[0]} diff --git a/act/discovery/arm.py b/act/discovery/arm.py index ab831771b0..08926c2ec6 100644 --- a/act/discovery/arm.py +++ b/act/discovery/arm.py @@ -7,10 +7,11 @@ import json import os import sys -from datetime import timedelta -import requests import textwrap import warnings +from datetime import timedelta + +import requests try: from urllib.request import urlopen @@ -163,7 +164,9 @@ def download_arm_data(username, token, datastream, startdate, enddate, time=None open_bytes_file.write(data) file_names.append(output_file) # Get ARM DOI and print it out - doi = get_arm_doi(datastream, start_datetime.strftime('%Y-%m-%d'), end_datetime.strftime('%Y-%m-%d')) + doi = get_arm_doi( + datastream, start_datetime.strftime('%Y-%m-%d'), end_datetime.strftime('%Y-%m-%d') + ) print('\nIf you use these data to prepare a publication, please cite:\n') print(textwrap.fill(doi, width=80)) print('') @@ -197,13 +200,17 @@ def get_arm_doi(datastream, startdate, enddate): """ # Get the DOI information - doi_url = 'https://adc.arm.gov/citationservice/citation/datastream?id=' + datastream + '&citationType=apa' + doi_url = ( + 'https://adc.arm.gov/citationservice/citation/datastream?id=' + + datastream + + '&citationType=apa' + ) doi_url += '&startDate=' + startdate doi_url += '&endDate=' + enddate try: doi = requests.get(url=doi_url) except ValueError as err: - return "Webservice potentially down or arguments are not valid: " + err + return 'Webservice potentially down or arguments are not valid: ' + err if len(doi.text) > 0: doi = doi.json()['citation'] diff --git a/act/discovery/asos.py b/act/discovery/asos.py index 59ce415159..d1c426f14a 100644 --- a/act/discovery/asos.py +++ b/act/discovery/asos.py @@ -7,11 +7,11 @@ import time import warnings from datetime import datetime +from io import StringIO import numpy as np import pandas as pd import xarray as xr -from six import StringIO try: from urllib.request import urlopen diff --git a/act/discovery/cropscape.py b/act/discovery/cropscape.py index 07308929f9..0d1ce43d95 100644 --- a/act/discovery/cropscape.py +++ b/act/discovery/cropscape.py @@ -4,6 +4,7 @@ """ import datetime + import requests try: diff --git a/act/discovery/get_airnow.py b/act/discovery/get_airnow.py index b8b8937a8b..5d84c0ea19 100644 --- a/act/discovery/get_airnow.py +++ b/act/discovery/get_airnow.py @@ -1,13 +1,13 @@ - """ Function for getting EPA data from AirNow data portal """ -import pandas as pd +import warnings + import numpy as np +import pandas as pd import xarray as xr -import warnings def get_airnow_forecast(token, date, zipcode=None, latlon=None, distance=25): @@ -47,25 +47,42 @@ def get_airnow_forecast(token, date, zipcode=None, latlon=None, distance=25): warnings.warn(message, DeprecationWarning, 2) # default beginning of the query url - query_url = ('https://airnowapi.org/aq/forecast/') + query_url = 'https://airnowapi.org/aq/forecast/' # checking is either a zipcode or latlon coordinate is defined # if neither is defined then error is raised if (zipcode is None) and (latlon is None): - raise NameError("Zipcode or latlon must be defined") + raise NameError('Zipcode or latlon must be defined') if zipcode: - url = (query_url + ('zipcode/?' + 'format=text/csv' + '&zipCode=' - + str(zipcode) + '&date=' + str(date) - + '&distance=' + str(distance) - + '&API_KEY=' + str(token))) + url = query_url + ( + 'zipcode/?' + + 'format=text/csv' + + '&zipCode=' + + str(zipcode) + + '&date=' + + str(date) + + '&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) if latlon: - url = (query_url + ('latLong/?' + 'format=text/csv' - + '&latitude=' + str(latlon[0]) + '&longitude=' - + str(latlon[1]) + '&date=' + str(date) - + '&distance=' + str(distance) - + '&API_KEY=' + str(token))) + url = query_url + ( + 'latLong/?' + + 'format=text/csv' + + '&latitude=' + + str(latlon[0]) + + '&longitude=' + + str(latlon[1]) + + '&date=' + + str(date) + + '&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) df = pd.read_csv(url) @@ -115,37 +132,78 @@ def get_airnow_obs(token, date=None, zipcode=None, latlon=None, distance=25): warnings.warn(message, DeprecationWarning, 2) # default beginning of the query url - query_url = ('https://www.airnowapi.org/aq/observation/') + query_url = 'https://www.airnowapi.org/aq/observation/' # checking is either a zipcode or latlon coordinate is defined # if neither is defined then error is raised if (zipcode is None) and (latlon is None): - raise NameError("Zipcode or latlon must be defined") + raise NameError('Zipcode or latlon must be defined') # setting the observation type to either current or historical based on the date if date is None: obs_type = 'current' if zipcode: - url = (query_url + ('zipCode/' + str(obs_type) + '/?' + 'format=text/csv' - + '&zipCode=' + str(zipcode) + '&distance=' + str(distance) - + '&API_KEY=' + str(token))) + url = query_url + ( + 'zipCode/' + + str(obs_type) + + '/?' + + 'format=text/csv' + + '&zipCode=' + + str(zipcode) + + '&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) if latlon: - url = (query_url + ('latLong/' + str(obs_type) + '/?' + 'format=text/csv' - + '&latitude=' + str(latlon[0]) - + '&longitude=' + str(latlon[1]) + '&distance=' - + str(distance) + '&API_KEY=' + str(token))) + url = query_url + ( + 'latLong/' + + str(obs_type) + + '/?' + + 'format=text/csv' + + '&latitude=' + + str(latlon[0]) + + '&longitude=' + + str(latlon[1]) + + '&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) else: obs_type = 'historical' if zipcode: - url = (query_url + ('zipCode/' + str(obs_type) + '/?' + 'format=text/csv' - + '&zipCode=' + str(zipcode) + '&date=' + str(date) - + 'T00-0000&distance=' + str(distance) + '&API_KEY=' + str(token))) + url = query_url + ( + 'zipCode/' + + str(obs_type) + + '/?' + + 'format=text/csv' + + '&zipCode=' + + str(zipcode) + + '&date=' + + str(date) + + 'T00-0000&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) if latlon: - url = (query_url + ('latLong/' + str(obs_type) + '/?' + 'format=text/csv' - + '&latitude=' + str(latlon[0]) - + '&longitude=' + str(latlon[1]) + '&date=' - + str(date) + 'T00-0000&distance=' + str(distance) - + '&API_KEY=' + str(token))) + url = query_url + ( + 'latLong/' + + str(obs_type) + + '/?' + + 'format=text/csv' + + '&latitude=' + + str(latlon[0]) + + '&longitude=' + + str(latlon[1]) + + '&date=' + + str(date) + + 'T00-0000&distance=' + + str(distance) + + '&API_KEY=' + + str(token) + ) df = pd.read_csv(url) @@ -155,8 +213,9 @@ def get_airnow_obs(token, date=None, zipcode=None, latlon=None, distance=25): return ds -def get_airnow_bounded_obs(token, start_date, end_date, latlon_bnds, parameters='OZONE,PM25', data_type='B', - mon_type=0): +def get_airnow_bounded_obs( + token, start_date, end_date, latlon_bnds, parameters='OZONE,PM25', data_type='B', mon_type=0 +): """ Get AQI values or data concentrations for a specific date and time range and set of parameters within a geographic area of intrest @@ -199,16 +258,44 @@ def get_airnow_bounded_obs(token, start_date, end_date, latlon_bnds, parameters= verbose = 1 inc_raw_con = 1 - url = ('https://www.airnowapi.org/aq/data/?startDate=' + str(start_date) - + '&endDate=' + str(end_date) + '¶meters=' + str(parameters) - + '&BBOX=' + str(latlon_bnds) + '&dataType=' + str(data_type) - + '&format=text/csv' + '&verbose=' + str(verbose) - + '&monitorType=' + str(mon_type) + '&includerawconcentrations=' - + str(inc_raw_con) + '&API_KEY=' + str(token)) + url = ( + 'https://www.airnowapi.org/aq/data/?startDate=' + + str(start_date) + + '&endDate=' + + str(end_date) + + '¶meters=' + + str(parameters) + + '&BBOX=' + + str(latlon_bnds) + + '&dataType=' + + str(data_type) + + '&format=text/csv' + + '&verbose=' + + str(verbose) + + '&monitorType=' + + str(mon_type) + + '&includerawconcentrations=' + + str(inc_raw_con) + + '&API_KEY=' + + str(token) + ) # Set Column names - names = ['latitude', 'longitude', 'time', 'parameter', 'concentration', 'unit', - 'raw_concentration', 'AQI', 'category', 'site_name', 'site_agency', 'aqs_id', 'full_aqs_id'] + names = [ + 'latitude', + 'longitude', + 'time', + 'parameter', + 'concentration', + 'unit', + 'raw_concentration', + 'AQI', + 'category', + 'site_name', + 'site_agency', + 'aqs_id', + 'full_aqs_id', + ] # Read data into CSV df = pd.read_csv(url, names=names) @@ -226,12 +313,9 @@ def get_airnow_bounded_obs(token, start_date, end_date, latlon_bnds, parameters= data_vars={ 'latitude': (['sites'], latitude), 'longitude': (['sites'], longitude), - 'aqs_id': (['sites'], aqs_id) + 'aqs_id': (['sites'], aqs_id), }, - coords={ - 'time': (['time'], times), - 'sites': (['sites'], sites) - } + coords={'time': (['time'], times), 'sites': (['sites'], sites)}, ) # Set up emtpy data with nans @@ -248,7 +332,11 @@ def get_airnow_bounded_obs(token, start_date, end_date, latlon_bnds, parameters= data[v, t, s] = list(result[variables[v]])[0] atts = {'units': ''} else: - result = df.loc[(df['time'] == times[t]) & (df['site_name'] == sites[s]) & (df['parameter'] == variables[v])] + result = df.loc[ + (df['time'] == times[t]) + & (df['site_name'] == sites[s]) + & (df['parameter'] == variables[v]) + ] if len(result['concentration']) > 0: data[v, t, s] = list(result['concentration'])[0] atts = {'units': list(result['unit'])[0]} diff --git a/act/discovery/get_arm.py b/act/discovery/get_arm.py index 9942ad8151..f56b3b4924 100644 --- a/act/discovery/get_arm.py +++ b/act/discovery/get_arm.py @@ -7,10 +7,11 @@ import json import os import sys -from datetime import timedelta -import requests import textwrap import warnings +from datetime import timedelta + +import requests try: from urllib.request import urlopen @@ -166,7 +167,9 @@ def download_arm_data(username, token, datastream, startdate, enddate, time=None open_bytes_file.write(data) file_names.append(output_file) # Get ARM DOI and print it out - doi = get_arm_doi(datastream, start_datetime.strftime('%Y-%m-%d'), end_datetime.strftime('%Y-%m-%d')) + doi = get_arm_doi( + datastream, start_datetime.strftime('%Y-%m-%d'), end_datetime.strftime('%Y-%m-%d') + ) print('\nIf you use these data to prepare a publication, please cite:\n') print(textwrap.fill(doi, width=80)) print('') @@ -203,13 +206,17 @@ def get_arm_doi(datastream, startdate, enddate): warnings.warn(message, DeprecationWarning, 2) # Get the DOI information - doi_url = 'https://adc.arm.gov/citationservice/citation/datastream?id=' + datastream + '&citationType=apa' + doi_url = ( + 'https://adc.arm.gov/citationservice/citation/datastream?id=' + + datastream + + '&citationType=apa' + ) doi_url += '&startDate=' + startdate doi_url += '&endDate=' + enddate try: doi = requests.get(url=doi_url) except ValueError as err: - return "Webservice potentially down or arguments are not valid: " + err + return 'Webservice potentially down or arguments are not valid: ' + err if len(doi.text) > 0: doi = doi.json()['citation'] diff --git a/act/discovery/get_armfiles.py b/act/discovery/get_armfiles.py index a76b661d85..8d6a2e5d46 100644 --- a/act/discovery/get_armfiles.py +++ b/act/discovery/get_armfiles.py @@ -7,10 +7,11 @@ import json import os import sys -from datetime import timedelta -import requests import textwrap import warnings +from datetime import timedelta + +import requests try: from urllib.request import urlopen @@ -166,7 +167,9 @@ def download_data(username, token, datastream, startdate, enddate, time=None, ou open_bytes_file.write(data) file_names.append(output_file) # Get ARM DOI and print it out - doi = get_arm_doi(datastream, start_datetime.strftime('%Y-%m-%d'), end_datetime.strftime('%Y-%m-%d')) + doi = get_arm_doi( + datastream, start_datetime.strftime('%Y-%m-%d'), end_datetime.strftime('%Y-%m-%d') + ) print('\nIf you use these data to prepare a publication, please cite:\n') print(textwrap.fill(doi, width=80)) print('') @@ -203,13 +206,17 @@ def get_arm_doi(datastream, startdate, enddate): warnings.warn(message, DeprecationWarning, 2) # Get the DOI information - doi_url = 'https://adc.arm.gov/citationservice/citation/datastream?id=' + datastream + '&citationType=apa' + doi_url = ( + 'https://adc.arm.gov/citationservice/citation/datastream?id=' + + datastream + + '&citationType=apa' + ) doi_url += '&startDate=' + startdate doi_url += '&endDate=' + enddate try: doi = requests.get(url=doi_url) except ValueError as err: - return "Webservice potentially down or arguments are not valid: " + err + return 'Webservice potentially down or arguments are not valid: ' + err if len(doi.text) > 0: doi = doi.json()['citation'] diff --git a/act/discovery/get_asos.py b/act/discovery/get_asos.py index 4cad5def04..251dd8f129 100644 --- a/act/discovery/get_asos.py +++ b/act/discovery/get_asos.py @@ -7,11 +7,11 @@ import time import warnings from datetime import datetime +from io import StringIO import numpy as np import pandas as pd import xarray as xr -from six import StringIO try: from urllib.request import urlopen diff --git a/act/discovery/get_cropscape.py b/act/discovery/get_cropscape.py index 8c7e6ce6e1..72d0c24652 100644 --- a/act/discovery/get_cropscape.py +++ b/act/discovery/get_cropscape.py @@ -4,9 +4,10 @@ """ import datetime -import requests import warnings +import requests + try: from pyproj import Transformer except ImportError: diff --git a/act/discovery/get_neon.py b/act/discovery/get_neon.py index 91dfb838e4..2d63b12b3d 100644 --- a/act/discovery/get_neon.py +++ b/act/discovery/get_neon.py @@ -8,12 +8,13 @@ """ import json -import requests import os import shutil -import pandas as pd import warnings +import pandas as pd +import requests + def get_site_products(site_code, print_to_screen=False): """ diff --git a/act/discovery/get_noaapsl.py b/act/discovery/get_noaapsl.py index a76280e235..a6668a9c5e 100644 --- a/act/discovery/get_noaapsl.py +++ b/act/discovery/get_noaapsl.py @@ -3,11 +3,12 @@ """ import json -from datetime import datetime -import pandas as pd -import numpy as np import os import warnings +from datetime import datetime + +import numpy as np +import pandas as pd try: from urllib.request import urlopen @@ -15,8 +16,9 @@ from urllib import urlopen -def download_noaa_psl_data(site=None, instrument=None, startdate=None, enddate=None, - hour=None, output=None): +def download_noaa_psl_data( + site=None, instrument=None, startdate=None, enddate=None, hour=None, output=None +): """ Function to download data from the NOAA PSL Profiler Network Data Library https://psl.noaa.gov/data/obs/datadisplay/ @@ -80,9 +82,18 @@ def download_noaa_psl_data(site=None, instrument=None, startdate=None, enddate=N url = 'https://downloads.psl.noaa.gov/psd2/data/realtime/' # Set list of strings that all point to the surface meteorology dataset - met_ds = ['Pressure', 'Datalogger', 'Net Radiation', 'Temp/RH', - 'Solar Radiation', 'Tipping Bucket', 'TBRG', 'Wind Speed', - 'Wind Direction', 'Wind Speed and Direction'] + met_ds = [ + 'Pressure', + 'Datalogger', + 'Net Radiation', + 'Temp/RH', + 'Solar Radiation', + 'Tipping Bucket', + 'TBRG', + 'Wind Speed', + 'Wind Direction', + 'Wind Speed and Direction', + ] # Add to the url depending on which instrument is requested if 'Parsivel' in instrument: @@ -157,8 +168,9 @@ def download_noaa_psl_data(site=None, instrument=None, startdate=None, enddate=N # Write each file out to a file with same name as online for f in files: if hour is not None: - if (str(doy).zfill(3) + str(hour)) not in f and\ - (str(doy).zfill(3) + '.' + str(hour)) not in f: + if (str(doy).zfill(3) + str(hour)) not in f and ( + str(doy).zfill(3) + '.' + str(hour) + ) not in f: continue output_file = os.path.join(output_dir, f) try: diff --git a/act/discovery/get_surfrad.py b/act/discovery/get_surfrad.py index 1f08925b6e..7a2ea3009b 100644 --- a/act/discovery/get_surfrad.py +++ b/act/discovery/get_surfrad.py @@ -4,13 +4,14 @@ """ import json -from datetime import datetime -import pandas as pd -import numpy as np import os import re -import requests import warnings +from datetime import datetime + +import numpy as np +import pandas as pd +import requests try: from urllib.request import urlopen diff --git a/act/discovery/neon.py b/act/discovery/neon.py index cfe3eff2c1..73e2dfdaac 100644 --- a/act/discovery/neon.py +++ b/act/discovery/neon.py @@ -8,10 +8,11 @@ """ import json -import requests import os import shutil + import pandas as pd +import requests def get_neon_site_products(site_code, print_to_screen=False): diff --git a/act/discovery/noaapsl.py b/act/discovery/noaapsl.py index 30b55ff2c2..516b23bc6a 100644 --- a/act/discovery/noaapsl.py +++ b/act/discovery/noaapsl.py @@ -3,10 +3,11 @@ """ import json +import os from datetime import datetime -import pandas as pd + import numpy as np -import os +import pandas as pd try: from urllib.request import urlopen @@ -14,8 +15,9 @@ from urllib import urlopen -def download_noaa_psl_data(site=None, instrument=None, startdate=None, enddate=None, - hour=None, output=None): +def download_noaa_psl_data( + site=None, instrument=None, startdate=None, enddate=None, hour=None, output=None +): """ Function to download data from the NOAA PSL Profiler Network Data Library https://psl.noaa.gov/data/obs/datadisplay/ @@ -76,9 +78,18 @@ def download_noaa_psl_data(site=None, instrument=None, startdate=None, enddate=N url = 'https://downloads.psl.noaa.gov/psd2/data/realtime/' # Set list of strings that all point to the surface meteorology dataset - met_ds = ['Pressure', 'Datalogger', 'Net Radiation', 'Temp/RH', - 'Solar Radiation', 'Tipping Bucket', 'TBRG', 'Wind Speed', - 'Wind Direction', 'Wind Speed and Direction'] + met_ds = [ + 'Pressure', + 'Datalogger', + 'Net Radiation', + 'Temp/RH', + 'Solar Radiation', + 'Tipping Bucket', + 'TBRG', + 'Wind Speed', + 'Wind Direction', + 'Wind Speed and Direction', + ] # Add to the url depending on which instrument is requested if 'Parsivel' in instrument: @@ -153,8 +164,9 @@ def download_noaa_psl_data(site=None, instrument=None, startdate=None, enddate=N # Write each file out to a file with same name as online for f in files: if hour is not None: - if (str(doy).zfill(3) + str(hour)) not in f and\ - (str(doy).zfill(3) + '.' + str(hour)) not in f: + if (str(doy).zfill(3) + str(hour)) not in f and ( + str(doy).zfill(3) + '.' + str(hour) + ) not in f: continue output_file = os.path.join(output_dir, f) try: diff --git a/act/discovery/surfrad.py b/act/discovery/surfrad.py index c6ba6fd356..cbeeaf421c 100644 --- a/act/discovery/surfrad.py +++ b/act/discovery/surfrad.py @@ -4,11 +4,12 @@ """ import json -from datetime import datetime -import pandas as pd -import numpy as np import os import re +from datetime import datetime + +import numpy as np +import pandas as pd import requests try: diff --git a/act/io/__init__.py b/act/io/__init__.py index b5fdddefc3..8d911a2f2f 100644 --- a/act/io/__init__.py +++ b/act/io/__init__.py @@ -7,7 +7,6 @@ __getattr__, __dir__, __all__ = lazy.attach( __name__, - submodules=['armfiles', 'csvfiles', 'icartt', 'mpl', 'neon', 'noaagml', 'noaapsl', 'pysp2'], submod_attrs={ 'armfiles': [ @@ -37,9 +36,7 @@ 'read_psl_parsivel', 'read_psl_radar_fmcw_moment', ], - 'pysp2': ['read_hk_file', 'read_sp2', 'read_sp2_dat' - ], - 'sodar' : [ - 'read_mfas_sodar'] + 'pysp2': ['read_hk_file', 'read_sp2', 'read_sp2_dat'], + 'sodar': ['read_mfas_sodar'], }, ) diff --git a/act/io/armfiles.py b/act/io/armfiles.py index 9e591eae09..7f69ff87bc 100644 --- a/act/io/armfiles.py +++ b/act/io/armfiles.py @@ -5,27 +5,26 @@ """ import copy +import datetime as dt import glob import json import re -import urllib -import warnings -from pathlib import Path, PosixPath -from netCDF4 import Dataset -from os import PathLike import tarfile import tempfile +import urllib import warnings +from os import PathLike +from pathlib import Path, PosixPath -from cftime import num2date import numpy as np import xarray as xr -import datetime as dt +from cftime import num2date +from netCDF4 import Dataset import act import act.utils as utils from act.config import DEFAULT_DATASTREAM_NAME -from act.utils.io_utils import unpack_tar, unpack_gzip, cleanup_files, is_gunzip_file +from act.utils.io_utils import cleanup_files, is_gunzip_file, unpack_gzip, unpack_tar def read_netcdf( @@ -108,7 +107,6 @@ def read_netcdf( message = 'act.io.armfiles.read_netcdf will be replaced in version 2.0.0 by act.io.arm.read_arm_netcdf()' warnings.warn(message, DeprecationWarning, 2) - ds = None filenames, cleanup_temp_directory = check_if_tar_gz_file(filenames) @@ -137,7 +135,8 @@ def read_netcdf( if 'drop_variables' in kwargs.keys(): drop_variables = kwargs['drop_variables'] kwargs['drop_variables'] = keep_variables_to_drop_variables( - filenames, keep_variables, drop_variables=drop_variables) + filenames, keep_variables, drop_variables=drop_variables + ) # Create an exception tuple to use with try statements. Doing it this way # so we can add the FileNotFoundError if requested. Can add more error @@ -178,7 +177,9 @@ def read_netcdf( # If requested use base_time and time_offset to derive time. Assumes that the units # of both are in seconds and that the value is number of seconds since epoch. if use_base_time: - time = num2date(ds['base_time'].values + ds['time_offset'].values, ds['base_time'].attrs['units']) + time = num2date( + ds['base_time'].values + ds['time_offset'].values, ds['base_time'].attrs['units'] + ) time = time.astype('datetime64[ns]') # Need to use a new Dataset creation to correctly index time for use with @@ -280,10 +281,7 @@ def read_netcdf( return ds -def keep_variables_to_drop_variables( - filenames, - keep_variables, - drop_variables=None): +def keep_variables_to_drop_variables(filenames, keep_variables, drop_variables=None): """ Returns a list of variable names to exclude from reading by passing into `Xarray.open_dataset` drop_variables keyword. This can greatly help reduce @@ -347,7 +345,6 @@ def keep_variables_to_drop_variables( # Use netCDF4 library to extract the variable and dimension names. rootgrp = Dataset(filename, 'r') read_variables = list(rootgrp.variables) - dimensions = list(rootgrp.dimensions) # Loop over the variables to exclude needed coordinate dimention names. dims_to_keep = [] for var_name in keep_variables: @@ -400,7 +397,9 @@ def check_arm_standards(ds): return the_flag -def create_ds_from_arm_dod(proc, set_dims, version='', fill_value=-9999.0, scalar_fill_dim=None, local_file=False): +def create_ds_from_arm_dod( + proc, set_dims, version='', fill_value=-9999.0, scalar_fill_dim=None, local_file=False +): """ Queries the ARM DOD api and builds a dataset based on the ARM DOD and @@ -631,7 +630,9 @@ def write_netcdf( try: att_values = write_ds[var_name].attrs[attr_name] if isinstance(att_values, (list, tuple)): - att_values = [att_value.replace(' ', join_char) for att_value in att_values] + att_values = [ + att_value.replace(' ', join_char) for att_value in att_values + ] write_ds[var_name].attrs[attr_name] = ' '.join(att_values) except KeyError: @@ -759,9 +760,16 @@ def write_netcdf( pass current_time = dt.datetime.now().replace(microsecond=0) if 'history' in list(write_ds.attrs.keys()): - write_ds.attrs['history'] += ''.join(['\n', str(current_time), ' created by ACT ', str(act.__version__), - ' act.io.write.write_netcdf']) - + write_ds.attrs['history'] += ''.join( + [ + '\n', + str(current_time), + ' created by ACT ', + str(act.__version__), + ' act.io.write.write_netcdf', + ] + ) + if hasattr(write_ds, 'time_bounds') and not write_ds.time.encoding: write_ds.time.encoding.update(write_ds.time_bounds.encoding) @@ -830,7 +838,7 @@ def read_mmcr(filenames): # read it in with xarray multi_ds = [] for f in filenames: - nc = Dataset(f, "a") + nc = Dataset(f, 'a') # Change heights name to range to read appropriately to xarray if 'heights' in nc.dimensions: nc.renameDimension('heights', 'range') @@ -878,7 +886,7 @@ def read_mmcr(filenames): data=data, coords={time_name: ds['time'].values[idx], range_name: range_data[idy]}, dims=[time_name, range_name], - attrs=attrs + attrs=attrs, ) ds[new_var_name] = da diff --git a/act/io/csvfiles.py b/act/io/csvfiles.py index 32be0668b0..ab628372a9 100644 --- a/act/io/csvfiles.py +++ b/act/io/csvfiles.py @@ -10,8 +10,9 @@ from .armfiles import check_arm_standards -def read_csv(filename, sep=',', engine='python', column_names=None, skipfooter=0, ignore_index=True, **kwargs): - +def read_csv( + filename, sep=',', engine='python', column_names=None, skipfooter=0, ignore_index=True, **kwargs +): """ Returns an `xarray.Dataset` with stored data and metadata from user-defined query of CSV files. @@ -95,7 +96,6 @@ def read_csv(filename, sep=',', engine='python', column_names=None, skipfooter=0 # standard format. is_arm_file_flag = check_arm_standards(ds) if is_arm_file_flag == 0: - ds.attrs['_datastream'] = '.'.join(filename[0].split('/')[-1].split('.')[0:2]) # Add additional attributes, site, standards flag, etc... diff --git a/act/io/icartt.py b/act/io/icartt.py index 2941d29186..959c2f5558 100644 --- a/act/io/icartt.py +++ b/act/io/icartt.py @@ -12,6 +12,7 @@ try: import icartt + _ICARTT_AVAILABLE = True _format = icartt.Formats.FFI1001 except ImportError: @@ -19,8 +20,7 @@ _format = None -def read_icartt(filename, format=_format, - return_None=False, **kwargs): +def read_icartt(filename, format=_format, return_None=False, **kwargs): """ Returns `xarray.Dataset` with stored data and metadata from a user-defined @@ -56,8 +56,7 @@ def read_icartt(filename, format=_format, """ if not _ICARTT_AVAILABLE: - raise ImportError( - "ICARTT is required to use to read ICARTT files but is not installed") + raise ImportError('ICARTT is required to use to read ICARTT files but is not installed') ds = None @@ -78,8 +77,7 @@ def read_icartt(filename, format=_format, return None # If requested return None for File not found error - if (type(exception).__name__ == 'OSError' - and exception.args[0] == 'no files to open'): + if type(exception).__name__ == 'OSError' and exception.args[0] == 'no files to open': return None # Define the Uncertainty for each variable. Note it may not be calculated. @@ -106,9 +104,7 @@ def read_icartt(filename, format=_format, key2 = 'quality_flag' else: key2 = key - da = xr.DataArray(ict.data[key], - coords=dict(time=ict.times), - name=key2, dims=['time']) + da = xr.DataArray(ict.data[key], coords=dict(time=ict.times), name=key2, dims=['time']) # Assume if Uncertainity does not match the number of variables, # values were not set within the file. Needs to be string! if len(uncertainty) != len(ict.variables): diff --git a/act/io/neon.py b/act/io/neon.py index 2bdfd402ae..5d6f6e1d15 100644 --- a/act/io/neon.py +++ b/act/io/neon.py @@ -2,10 +2,12 @@ Modules for reading in NOAA PSL data. """ -import pandas as pd -import xarray as xr import datetime as dt + import numpy as np +import pandas as pd +import xarray as xr + from act.io.csvfiles import read_csv @@ -85,8 +87,18 @@ def read_neon_csv(files, variable_files=None, position_files=None): ds['lat'] = xr.DataArray(data=float(loc_df['referenceLatitude'].values[idx])) ds['lon'] = xr.DataArray(data=float(loc_df['referenceLongitude'].values[idx])) ds['alt'] = xr.DataArray(data=float(loc_df['referenceElevation'].values[idx])) - variables = ['xOffset', 'yOffset', 'zOffset', 'eastOffset', 'northOffset', - 'pitch', 'roll', 'azimuth', 'xAzimuth', 'yAzimuth'] + variables = [ + 'xOffset', + 'yOffset', + 'zOffset', + 'eastOffset', + 'northOffset', + 'pitch', + 'roll', + 'azimuth', + 'xAzimuth', + 'yAzimuth', + ] for v in variables: ds[v] = xr.DataArray(data=float(loc_df[v].values[idx])) multi_ds.append(ds) diff --git a/act/io/noaagml.py b/act/io/noaagml.py index c2f9587ef8..8992da156e 100644 --- a/act/io/noaagml.py +++ b/act/io/noaagml.py @@ -6,8 +6,8 @@ from datetime import datetime from pathlib import Path -import pandas as pd import numpy as np +import pandas as pd import xarray as xr import act @@ -47,8 +47,7 @@ def read_gml(filename, datatype=None, remove_time_vars=True, convert_missing=Tru if datatype is not None: if datatype.upper() == 'MET': - return read_gml_met( - filename, convert_missing=convert_missing, **kwargs) + return read_gml_met(filename, convert_missing=convert_missing, **kwargs) elif datatype.upper() == 'RADIATION': return read_gml_radiation( filename, @@ -59,8 +58,7 @@ def read_gml(filename, datatype=None, remove_time_vars=True, convert_missing=Tru elif datatype.upper() == 'OZONE': return read_gml_ozone(filename, **kwargs) elif datatype.upper() == 'CO2': - return read_gml_co2( - filename, convert_missing=convert_missing, **kwargs) + return read_gml_co2(filename, convert_missing=convert_missing, **kwargs) elif datatype.upper() == 'HALO': return read_gml_halo(filename, **kwargs) else: @@ -74,12 +72,10 @@ def read_gml(filename, datatype=None, remove_time_vars=True, convert_missing=Tru test_filename = str(Path(test_filename).name) if test_filename.startswith('met_') and test_filename.endswith('.txt'): - return read_gml_met( - filename, convert_missing=convert_missing, **kwargs) + return read_gml_met(filename, convert_missing=convert_missing, **kwargs) if test_filename.startswith('co2_') and test_filename.endswith('.txt'): - return read_gml_co2( - filename, convert_missing=convert_missing, **kwargs) + return read_gml_co2(filename, convert_missing=convert_missing, **kwargs) result = re.match(r'([a-z]{3})([\d]{5}).dat', test_filename) if result is not None: @@ -242,8 +238,8 @@ def read_gml_halo(filename, **kwargs): header += 1 ds = act.io.csvfiles.read_csv( - filename, sep=r'\s+', header=header, - na_values=['Nan', 'NaN', 'nan', 'NAN'], **kwargs) + filename, sep=r'\s+', header=header, na_values=['Nan', 'NaN', 'nan', 'NAN'], **kwargs + ) var_names = list(ds.data_vars) year_name, month_name, day_name, hour_name, min_name = None, None, None, None, None for var_name in var_names: @@ -258,7 +254,7 @@ def read_gml_halo(filename, **kwargs): elif var_name.endswith('min'): min_name = var_name - timestamp = np.full(ds[var_names[0]].size, np.nan, dtype="datetime64[ns]") + timestamp = np.full(ds[var_names[0]].size, np.nan, dtype='datetime64[ns]') for ii in range(0, len(timestamp)): if min_name is not None: ts = datetime( @@ -282,10 +278,9 @@ def read_gml_halo(filename, **kwargs): ds[day_name].values[ii], ) else: - ts = datetime( - ds[year_name].values[ii], ds[month_name].values[ii], 1) + ts = datetime(ds[year_name].values[ii], ds[month_name].values[ii], 1) - timestamp[ii] = np.datetime64(ts, "ns") + timestamp[ii] = np.datetime64(ts, 'ns') for var_name in [year_name, month_name, day_name, hour_name, min_name]: try: @@ -418,10 +413,9 @@ def read_gml_co2(filename=None, convert_missing=True, **kwargs): with open(test_filename) as fc: skiprows = int(fc.readline().strip().split()[-1]) - 1 - ds = act.io.csvfiles.read_csv( - filename, sep=r'\s+', skiprows=skiprows, **kwargs) + ds = act.io.csvfiles.read_csv(filename, sep=r'\s+', skiprows=skiprows, **kwargs) - timestamp = np.full(ds['year'].size, np.nan, dtype="datetime64[ns]") + timestamp = np.full(ds['year'].size, np.nan, dtype='datetime64[ns]') for ii in range(0, len(timestamp)): ts = datetime( ds['year'].values[ii], @@ -431,7 +425,7 @@ def read_gml_co2(filename=None, convert_missing=True, **kwargs): ds['minute'].values[ii], ds['second'].values[ii], ) - timestamp[ii] = np.datetime64(ts, "ns") + timestamp[ii] = np.datetime64(ts, 'ns') ds = ds.rename({'index': 'time'}) ds = ds.assign_coords(time=timestamp) @@ -538,11 +532,10 @@ def read_gml_ozone(filename=None, **kwargs): pass skiprows += 1 - ds = act.io.csvfiles.read_csv( - filename, sep=r'\s+', skiprows=skiprows, **kwargs) + ds = act.io.csvfiles.read_csv(filename, sep=r'\s+', skiprows=skiprows, **kwargs) ds.attrs['station'] = str(ds['STN'].values[0]).lower() - timestamp = np.full(ds['YEAR'].size, np.nan, dtype="datetime64[ns]") + timestamp = np.full(ds['YEAR'].size, np.nan, dtype='datetime64[ns]') for ii in range(0, len(timestamp)): ts = datetime( ds['YEAR'].values[ii], @@ -550,7 +543,7 @@ def read_gml_ozone(filename=None, **kwargs): ds['DAY'].values[ii], ds['HR'].values[ii], ) - timestamp[ii] = np.datetime64(ts, "ns") + timestamp[ii] = np.datetime64(ts, 'ns') ds = ds.rename({'index': 'time'}) ds = ds.assign_coords(time=timestamp) @@ -569,8 +562,7 @@ def read_gml_ozone(filename=None, **kwargs): return ds -def read_gml_radiation(filename=None, convert_missing=True, - remove_time_vars=True, **kwargs): +def read_gml_radiation(filename=None, convert_missing=True, remove_time_vars=True, **kwargs): """ Function to read radiation data from NOAA GML. @@ -739,19 +731,18 @@ def read_gml_radiation(filename=None, convert_missing=True, } # Add additinal column names for NOAA SPASH campaign - if str(Path(filename).name).startswith('cbc') or \ - str(Path(filename).name).startswith('ckp'): + if str(Path(filename).name).startswith('cbc') or str(Path(filename).name).startswith('ckp'): column_names['SPN1_total'] = { 'units': 'W/m^2', 'long_name': 'SPN1 total average', '_FillValue': -9999.9, - '__type': np.float32 + '__type': np.float32, } column_names['SPN1_diffuse'] = { 'units': 'W/m^2', 'long_name': 'SPN1 diffuse average', '_FillValue': -9999.9, - '__type': np.float32 + '__type': np.float32, } names = list(column_names.keys()) @@ -772,7 +763,9 @@ def read_gml_radiation(filename=None, convert_missing=True, names.insert(ii + num, 'qc_' + name) num += 1 - ds = act.io.csvfiles.read_csv(filename, sep=r'\s+', header=None, skiprows=2, column_names=names, **kwargs) + ds = act.io.csvfiles.read_csv( + filename, sep=r'\s+', header=None, skiprows=2, column_names=names, **kwargs + ) if isinstance(filename, (list, tuple)): filename = filename[0] @@ -820,7 +813,7 @@ def read_gml_radiation(filename=None, convert_missing=True, ) ds.attrs['location'] = station - timestamp = np.full(ds['year'].size, np.nan, dtype="datetime64[ns]") + timestamp = np.full(ds['year'].size, np.nan, dtype='datetime64[ns]') for ii in range(0, len(timestamp)): ts = datetime( ds['year'].values[ii], @@ -829,7 +822,7 @@ def read_gml_radiation(filename=None, convert_missing=True, ds['hour'].values[ii], ds['minute'].values[ii], ) - timestamp[ii] = np.datetime64(ts, "ns") + timestamp[ii] = np.datetime64(ts, 'ns') ds = ds.rename({'index': 'time'}) ds = ds.assign_coords(time=timestamp) @@ -995,11 +988,11 @@ def read_gml_met(filename=None, convert_missing=True, **kwargs): del column_names['minute'] ds = act.io.csvfiles.read_csv( - filename, sep=r'\s+', header=None, - column_names=column_names.keys(), **kwargs) + filename, sep=r'\s+', header=None, column_names=column_names.keys(), **kwargs + ) if ds is not None: - timestamp = np.full(ds['year'].size, np.nan, dtype="datetime64[ns]") + timestamp = np.full(ds['year'].size, np.nan, dtype='datetime64[ns]') for ii in range(0, len(timestamp)): if minutes: ts = datetime( @@ -1017,13 +1010,12 @@ def read_gml_met(filename=None, convert_missing=True, **kwargs): ds['hour'].values[ii], ) - timestamp[ii] = np.datetime64(ts, "ns") + timestamp[ii] = np.datetime64(ts, 'ns') ds = ds.rename({'index': 'time'}) ds = ds.assign_coords(time=timestamp) ds['time'].attrs['long_name'] = 'Time' for var_name, value in column_names.items(): - if value is None: del ds[var_name] else: @@ -1068,17 +1060,56 @@ def read_surfrad(filename, **kwargs): """ - names = ['year', 'jday', 'month', 'day', 'hour', 'minute', 'dec_time', - 'solar_zenith_angle', 'downwelling_global', 'qc_downwelling_global', - 'upwelling_global', 'qc_upwelling_global', 'direct_normal', 'qc_direct_normal', - 'downwelling_diffuse', 'qc_downwelling_diffuse', 'downwelling_ir', 'qc_downwelling_ir', - 'downwelling_ir_casetemp', 'qc_downwelling_ir_casetemp', 'downwelling_ir_dometemp', - 'qc_downwelling_ir_dometemp', 'upwelling_ir', 'qc_upwelling_ir', 'upwelling_ir_casetemp', - 'qc_upwelling_ir_casetemp', 'upwelling_ir_dometemp', 'qc_upwelling_ir_dometemp', - 'global_uvb', 'qc_global_uvb', 'par', 'qc_par', 'net_radiation', 'qc_net_radiation', - 'net_ir', 'qc_net_ir', 'total_net', 'qc_total_net', 'temperature', 'qc_temperature', - 'relative_humidity', 'qc_relative_humidity', 'wind_speed', 'qc_wind_speed', 'wind_direction', - 'qc_wind_direction', 'pressure', 'qc_pressure'] + names = [ + 'year', + 'jday', + 'month', + 'day', + 'hour', + 'minute', + 'dec_time', + 'solar_zenith_angle', + 'downwelling_global', + 'qc_downwelling_global', + 'upwelling_global', + 'qc_upwelling_global', + 'direct_normal', + 'qc_direct_normal', + 'downwelling_diffuse', + 'qc_downwelling_diffuse', + 'downwelling_ir', + 'qc_downwelling_ir', + 'downwelling_ir_casetemp', + 'qc_downwelling_ir_casetemp', + 'downwelling_ir_dometemp', + 'qc_downwelling_ir_dometemp', + 'upwelling_ir', + 'qc_upwelling_ir', + 'upwelling_ir_casetemp', + 'qc_upwelling_ir_casetemp', + 'upwelling_ir_dometemp', + 'qc_upwelling_ir_dometemp', + 'global_uvb', + 'qc_global_uvb', + 'par', + 'qc_par', + 'net_radiation', + 'qc_net_radiation', + 'net_ir', + 'qc_net_ir', + 'total_net', + 'qc_total_net', + 'temperature', + 'qc_temperature', + 'relative_humidity', + 'qc_relative_humidity', + 'wind_speed', + 'qc_wind_speed', + 'wind_direction', + 'qc_wind_direction', + 'pressure', + 'qc_pressure', + ] for i, f in enumerate(filename): new_df = pd.read_csv(f, names=names, skiprows=2, delimiter=r'\s+', header=None) if i == 0: @@ -1107,36 +1138,93 @@ def read_surfrad(filename, **kwargs): 'minute': {'long_name': 'Minutes', 'units': 'unitless'}, 'dec_time': {'long_name': 'Decimal time', 'units': 'unitless'}, 'solar_zenith_angle': {'long_name': 'Solar zenith angle', 'units': 'deg'}, - 'downwelling_global': {'long_name': 'Downwelling global solar', 'units': 'W m^-2', - 'standard_name': 'surface_downwelling_shortwave_flux_in_air'}, - 'upwelling_global': {'long_name': 'Upwelling global solar', 'units': 'W m^-2', - 'standard_name': 'surface_upwelling_shortwave_flux_in_air'}, - 'direct_normal': {'long_name': 'Direct normal solar', 'units': 'W m^-2', - 'standard_name': 'surface_direct_downwelling_shortwave_flux_in_air'}, - 'downwelling_diffuse': {'long_name': 'Downwelling diffuse solar', 'units': 'W m^-2', - 'standard_name': 'diffuse_downwelling_shortwave_flux_in_air'}, - 'downwelling_ir': {'long_name': 'Downwelling thermal infrared', 'units': 'W m^-2', - 'standard_name': 'net_downward_longwave_flux_in_air'}, - 'downwelling_ir_casetemp': {'long_name': 'Downwelling thermal infrared case temperature', 'units': 'K'}, - 'downwelling_ir_dometemp': {'long_name': 'Downwelling thermal infrared dome temperature', 'units': 'K'}, - 'upwelling_ir': {'long_name': 'Upwelling thermal infrared', 'units': 'W m^-2', - 'standard_name': 'net_upward_longwave_flux_in_air'}, - 'upwelling_ir_casetemp': {'long_name': 'Upwelling thermal infrared case temperature', 'units': 'K'}, - 'upwelling_ir_dometemp': {'long_name': 'Upwelling thermal infrared dome temperature', 'units': 'K'}, + 'downwelling_global': { + 'long_name': 'Downwelling global solar', + 'units': 'W m^-2', + 'standard_name': 'surface_downwelling_shortwave_flux_in_air', + }, + 'upwelling_global': { + 'long_name': 'Upwelling global solar', + 'units': 'W m^-2', + 'standard_name': 'surface_upwelling_shortwave_flux_in_air', + }, + 'direct_normal': { + 'long_name': 'Direct normal solar', + 'units': 'W m^-2', + 'standard_name': 'surface_direct_downwelling_shortwave_flux_in_air', + }, + 'downwelling_diffuse': { + 'long_name': 'Downwelling diffuse solar', + 'units': 'W m^-2', + 'standard_name': 'diffuse_downwelling_shortwave_flux_in_air', + }, + 'downwelling_ir': { + 'long_name': 'Downwelling thermal infrared', + 'units': 'W m^-2', + 'standard_name': 'net_downward_longwave_flux_in_air', + }, + 'downwelling_ir_casetemp': { + 'long_name': 'Downwelling thermal infrared case temperature', + 'units': 'K', + }, + 'downwelling_ir_dometemp': { + 'long_name': 'Downwelling thermal infrared dome temperature', + 'units': 'K', + }, + 'upwelling_ir': { + 'long_name': 'Upwelling thermal infrared', + 'units': 'W m^-2', + 'standard_name': 'net_upward_longwave_flux_in_air', + }, + 'upwelling_ir_casetemp': { + 'long_name': 'Upwelling thermal infrared case temperature', + 'units': 'K', + }, + 'upwelling_ir_dometemp': { + 'long_name': 'Upwelling thermal infrared dome temperature', + 'units': 'K', + }, 'global_uvb': {'long_name': 'Global UVB', 'units': 'milliWatts m^-2'}, - 'par': {'long_name': 'Photosynthetically active radiation', 'units': 'W m^-2', - 'standard_name': 'surface_downwelling_photosynthetic_radiative_flux_in_air'}, - 'net_radiation': {'long_name': 'Net solar (downwelling_global-upwelling_global)', 'units': 'W m^-2', - 'standard_name': 'surface_net_downward_shortwave_flux'}, - 'net_ir': {'long_name': 'Net infrared (downwelling_ir-upwelling_ir)', 'units': 'W m^-2', - 'standard_name': 'surface_net_downward_longwave_flux'}, - 'total_net': {'long_name': 'Total Net radiation (net_radiation + net_ir)', 'units': 'W m^-2'}, - 'temperature': {'long_name': '10-meter air temperature', 'units': 'degC', 'standard_name': 'air_temperature'}, - 'relative_humidity': {'long_name': 'Relative humidity', 'units': '%', 'standard_name': 'relative_humidity'}, + 'par': { + 'long_name': 'Photosynthetically active radiation', + 'units': 'W m^-2', + 'standard_name': 'surface_downwelling_photosynthetic_radiative_flux_in_air', + }, + 'net_radiation': { + 'long_name': 'Net solar (downwelling_global-upwelling_global)', + 'units': 'W m^-2', + 'standard_name': 'surface_net_downward_shortwave_flux', + }, + 'net_ir': { + 'long_name': 'Net infrared (downwelling_ir-upwelling_ir)', + 'units': 'W m^-2', + 'standard_name': 'surface_net_downward_longwave_flux', + }, + 'total_net': { + 'long_name': 'Total Net radiation (net_radiation + net_ir)', + 'units': 'W m^-2', + }, + 'temperature': { + 'long_name': '10-meter air temperature', + 'units': 'degC', + 'standard_name': 'air_temperature', + }, + 'relative_humidity': { + 'long_name': 'Relative humidity', + 'units': '%', + 'standard_name': 'relative_humidity', + }, 'wind_speed': {'long_name': 'Wind speed', 'units': 'ms^-1', 'standard_name': 'wind_speed'}, - 'wind_direction': {'long_name': 'Wind direction, clockwise from North', 'units': 'deg', - 'standard_name': 'wind_from_direction'}, - 'pressure': {'long_name': 'Station pressure', 'units': 'mb', 'standard_name': 'air_pressure'}, + 'wind_direction': { + 'long_name': 'Wind direction, clockwise from North', + 'units': 'deg', + 'standard_name': 'wind_from_direction', + }, + 'pressure': { + 'long_name': 'Station pressure', + 'units': 'mb', + 'standard_name': 'air_pressure', + }, } for v in ds: @@ -1144,23 +1232,46 @@ def read_surfrad(filename, **kwargs): ds[v].attrs = attrs[v] # Add attributes to all QC variables - qc_vars = ['downwelling_global', 'upwelling_global', 'direct_normal', 'downwelling_diffuse', - 'downwelling_ir', 'downwelling_ir_casetemp', 'downwelling_ir_dometemp', - 'upwelling_ir', 'upwelling_ir_casetemp', 'upwelling_ir_dometemp', 'global_uvb', - 'par', 'net_radiation', 'net_ir', 'total_net', 'temperature', 'relative_humidity', - 'wind_speed', 'wind_direction', 'pressure'] + qc_vars = [ + 'downwelling_global', + 'upwelling_global', + 'direct_normal', + 'downwelling_diffuse', + 'downwelling_ir', + 'downwelling_ir_casetemp', + 'downwelling_ir_dometemp', + 'upwelling_ir', + 'upwelling_ir_casetemp', + 'upwelling_ir_dometemp', + 'global_uvb', + 'par', + 'net_radiation', + 'net_ir', + 'total_net', + 'temperature', + 'relative_humidity', + 'wind_speed', + 'wind_direction', + 'pressure', + ] for v in qc_vars: - atts = {'long_name': 'Quality check results on variable: ' + v, - 'units': '1', - 'description': ''.join(['A QC flag of zero indicates that the corresponding data point is good,', - ' having passed all QC checks. A value greater than 0 indicates that', - ' the data failed one level of QC. For example, a QC value of 1 means', - ' that the recorded value is beyond a physically possible range, or it has', - ' been affected adversely in some manner to produce a knowingly bad value.', - ' A value of 2 indicates that the data value failed the second level QC check,', - ' indicating that the data value may be physically possible but should be used', - ' with scrutiny, and so on.'])} + atts = { + 'long_name': 'Quality check results on variable: ' + v, + 'units': '1', + 'description': ''.join( + [ + 'A QC flag of zero indicates that the corresponding data point is good,', + ' having passed all QC checks. A value greater than 0 indicates that', + ' the data failed one level of QC. For example, a QC value of 1 means', + ' that the recorded value is beyond a physically possible range, or it has', + ' been affected adversely in some manner to produce a knowingly bad value.', + ' A value of 2 indicates that the data value failed the second level QC check,', + ' indicating that the data value may be physically possible but should be used', + ' with scrutiny, and so on.', + ] + ), + } ds['qc_' + v].attrs = atts ds.attrs['datastream'] = 'SURFRAD Site: ' + filename[0].split('/')[-1][0:3] diff --git a/act/io/noaapsl.py b/act/io/noaapsl.py index e47b900188..850b36655e 100644 --- a/act/io/noaapsl.py +++ b/act/io/noaapsl.py @@ -2,16 +2,17 @@ Modules for reading in NOAA PSL data. """ +import datetime as dt +import re from datetime import datetime, timedelta -from os import path as ospath from itertools import groupby +from os import path as ospath + import fsspec -import yaml -import re import numpy as np import pandas as pd import xarray as xr -import datetime as dt +import yaml from act.io.csvfiles import read_csv @@ -52,17 +53,14 @@ def read_psl_wind_profiler(filepath, transpose=True): for section in sections_of_file: if section[0] != '$': list_of_datasets.append( - _parse_psl_wind_lines( - filepath, section, line_offset=start_line) + _parse_psl_wind_lines(filepath, section, line_offset=start_line) ) start_line += len(section) # Return two datasets for each mode and the merge of datasets of the # same mode. - mode_one_ds = xr.concat( - list_of_datasets[0::2], dim='time') - mode_two_ds = xr.concat( - list_of_datasets[1::2], dim='time') + mode_one_ds = xr.concat(list_of_datasets[0::2], dim='time') + mode_two_ds = xr.concat(list_of_datasets[1::2], dim='time') if transpose: mode_one_ds = mode_one_ds.transpose('HT', 'time') mode_two_ds = mode_two_ds.transpose('HT', 'time') @@ -101,8 +99,7 @@ def read_psl_wind_profiler_temperature(filepath, transpose=True): for section in sections_of_file: if section[0] != '$': list_of_datasets.append( - _parse_psl_temperature_lines( - filepath, section, line_offset=start_line) + _parse_psl_temperature_lines(filepath, section, line_offset=start_line) ) start_line += len(section) @@ -139,8 +136,7 @@ def _parse_psl_wind_lines(filepath, lines, line_offset=0): datatype, _, version = filter_list(lines[1].split(' ')) # 3 - station lat, lon, elevation - latitude, longitude, elevation = filter_list( - lines[2].split(' ')).astype(float) + latitude, longitude, elevation = filter_list(lines[2].split(' ')).astype(float) # 4 - year, month, day, hour, minute, second, utc time = parse_date_line(lines[3]) @@ -154,34 +150,42 @@ def _parse_psl_wind_lines(filepath, lines, line_offset=0): # pulse width, inner pulse period' # Values duplicate as oblique and vertical values ( - number_coherent_integrations_obl, number_coherent_integrations_vert, - number_spectral_averages_obl, number_spectral_averages_vert, - pulse_width_obl, pulse_width_vert, inner_pulse_period_obl, - inner_pulse_period_vert + number_coherent_integrations_obl, + number_coherent_integrations_vert, + number_spectral_averages_obl, + number_spectral_averages_vert, + pulse_width_obl, + pulse_width_vert, + inner_pulse_period_obl, + inner_pulse_period_vert, ) = filter_list(lines[6].split(' ')).astype(int) # 8 - full-scale doppler value, delay to first gate, number of gates, # spacing of gates. Values duplicate as oblique and vertical values. ( - full_scale_doppler_obl, full_scale_doppler_vert, + full_scale_doppler_obl, + full_scale_doppler_vert, beam_vertical_correction, - delay_first_gate_obl, delay_first_gate_vert, - number_of_gates_obl, number_of_gates_vert, - spacing_of_gates_obl, spacing_of_gates_vert + delay_first_gate_obl, + delay_first_gate_vert, + number_of_gates_obl, + number_of_gates_vert, + spacing_of_gates_obl, + spacing_of_gates_vert, ) = filter_list(lines[7].split(' ')).astype(float) # 9 - beam azimuth (degrees clockwise from north) ( - beam_azimuth1, beam_elevation1, - beam_azimuth2, beam_elevation2, - beam_azimuth3, beam_elevation3 + beam_azimuth1, + beam_elevation1, + beam_azimuth2, + beam_elevation2, + beam_azimuth3, + beam_elevation3, ) = filter_list(lines[8].split(' ')).astype(float) - beam_azimuth = np.array( - [beam_azimuth1, beam_azimuth2, beam_azimuth3], dtype='float32') - beam_elevation = np.array( - [beam_elevation1, beam_elevation2, beam_elevation3], - dtype='float32') + beam_azimuth = np.array([beam_azimuth1, beam_azimuth2, beam_azimuth3], dtype='float32') + beam_elevation = np.array([beam_elevation1, beam_elevation2, beam_elevation3], dtype='float32') # Read in the data table section using pandas df = pd.read_csv(filepath, skiprows=line_offset + 10, delim_whitespace=True) @@ -245,26 +249,21 @@ def _parse_psl_wind_lines(filepath, lines, line_offset=0): 'data_description' ] = 'https://psl.noaa.gov/data/obs/data/view_data_type_info.php?SiteID=ctd&DataOperationalID=5855&OperationalID=2371' ds.attrs['consensus_average_time'] = consensus_average_time - ds.attrs['oblique-beam_vertical_correction'] = int( - beam_vertical_correction) + ds.attrs['oblique-beam_vertical_correction'] = int(beam_vertical_correction) ds.attrs['number_of_beams'] = int(number_of_beams) ds.attrs['number_of_range_gates'] = int(number_of_range_gates) # Handle oblique and vertical attributes. ds.attrs['number_of_gates_oblique'] = int(number_of_gates_obl) ds.attrs['number_of_gates_vertical'] = int(number_of_gates_vert) - ds.attrs['number_spectral_averages_oblique'] = int( - number_spectral_averages_obl) - ds.attrs['number_spectral_averages_vertical'] = int( - number_spectral_averages_vert) + ds.attrs['number_spectral_averages_oblique'] = int(number_spectral_averages_obl) + ds.attrs['number_spectral_averages_vertical'] = int(number_spectral_averages_vert) ds.attrs['pulse_width_oblique'] = int(pulse_width_obl) ds.attrs['pulse_width_vertical'] = int(pulse_width_vert) ds.attrs['inner_pulse_period_oblique'] = int(inner_pulse_period_obl) ds.attrs['inner_pulse_period_vertical'] = int(inner_pulse_period_vert) - ds.attrs['full_scale_doppler_value_oblique'] = float( - full_scale_doppler_obl) - ds.attrs['full_scale_doppler_value_vertical'] = float( - full_scale_doppler_vert) + ds.attrs['full_scale_doppler_value_oblique'] = float(full_scale_doppler_obl) + ds.attrs['full_scale_doppler_value_vertical'] = float(full_scale_doppler_vert) ds.attrs['delay_to_first_gate_oblique'] = int(delay_first_gate_obl) ds.attrs['delay_to_first_gate_vertical'] = int(delay_first_gate_vert) ds.attrs['spacing_of_gates_oblique'] = int(spacing_of_gates_obl) @@ -298,8 +297,7 @@ def _parse_psl_temperature_lines(filepath, lines, line_offset=0): datatype, _, version = filter_list(lines[1].split(' ')) # 3 - station lat, lon, elevation - latitude, longitude, elevation = filter_list( - lines[2].split(' ')).astype(float) + latitude, longitude, elevation = filter_list(lines[2].split(' ')).astype(float) # 4 - year, month, day, hour, minute, second, utc time = parse_date_line(lines[3]) @@ -325,8 +323,7 @@ def _parse_psl_temperature_lines(filepath, lines, line_offset=0): ).astype(float) # 9 - beam azimuth (degrees clockwise from north) - beam_azimuth, beam_elevation = filter_list( - lines[8].split(' ')).astype(float) + beam_azimuth, beam_elevation = filter_list(lines[8].split(' ')).astype(float) # Read in the data table section using pandas df = pd.read_csv(filepath, skiprows=line_offset + 10, delim_whitespace=True) @@ -460,7 +457,7 @@ def read_psl_surface_met(filenames, conf_file=None): conf_file = ospath.join(ospath.dirname(__file__), 'conf', 'noaapsl_SurfaceMet.yaml') # Read configuration YAML file - with open(conf_file, "r") as fp: + with open(conf_file) as fp: try: result = yaml.load(fp, Loader=yaml.FullLoader) except AttributeError: @@ -470,18 +467,20 @@ def read_psl_surface_met(filenames, conf_file=None): try: result = result[site] except KeyError: - raise RuntimeError(f"Configuration for site '{site}' currently not available. " - "You can manually add the site configuration to a copy of " - "noaapsl_SurfaceMet.yaml and set conf_file= name of copied file " - "until the site is added.") + raise RuntimeError( + f"Configuration for site '{site}' currently not available. " + 'You can manually add the site configuration to a copy of ' + 'noaapsl_SurfaceMet.yaml and set conf_file= name of copied file ' + 'until the site is added.' + ) # Extract date and time from filename to use in extracting format from YAML file. - search_result = re.match(r"[a-z]{3}(\d{2})(\d{3})\.(\d{2})m", ospath.basename(filenames[0])) + search_result = re.match(r'[a-z]{3}(\d{2})(\d{3})\.(\d{2})m', ospath.basename(filenames[0])) yy, doy, hh = search_result.groups() if yy > '70': - yy = f"19{yy}" + yy = f'19{yy}' else: - yy = f"20{yy}" + yy = f'20{yy}' # Extract location information from configuration file. try: @@ -490,12 +489,18 @@ def read_psl_surface_met(filenames, conf_file=None): location_info = None # Loop through each date range for the site to extract the correct file format from conf file. - file_datetime = datetime.strptime(f'{yy}-01-01', '%Y-%m-%d') + timedelta(int(doy) - 1) + timedelta(hours=int(hh)) + file_datetime = ( + datetime.strptime(f'{yy}-01-01', '%Y-%m-%d') + + timedelta(int(doy) - 1) + + timedelta(hours=int(hh)) + ) for ii in result.keys(): if ii == 'info': continue - date_range = [datetime.strptime(jj, '%Y-%m-%d %H:%M:%S') for jj in result[ii]['_date_range']] + date_range = [ + datetime.strptime(jj, '%Y-%m-%d %H:%M:%S') for jj in result[ii]['_date_range'] + ] if file_datetime >= date_range[0] and file_datetime <= date_range[1]: result = result[ii] del result['_date_range'] @@ -506,14 +511,14 @@ def read_psl_surface_met(filenames, conf_file=None): # Calculate numpy datetime64 values from first 4 columns of the data file. time = np.array(ds['Year'].values - 1970, dtype='datetime64[Y]') - day = np.array(np.array(ds['J_day'].values - 1 , dtype='timedelta64[D]')) + day = np.array(np.array(ds['J_day'].values - 1, dtype='timedelta64[D]')) hourmin = ds['HoursMinutes'].values + 10000 hour = [int(str(ii)[1:3]) for ii in hourmin] hour = np.array(hour, dtype='timedelta64[h]') minute = [int(str(ii)[3:]) for ii in hourmin] minute = np.array(minute, dtype='timedelta64[m]') time = time + day + hour + minute - time = time.astype("datetime64[ns]") + time = time.astype('datetime64[ns]') # Update Dataset to use "time" coordinate and assigned calculated times ds = ds.assign_coords(index=time) ds = ds.rename(index='time') @@ -568,23 +573,136 @@ def read_psl_parsivel(files): """ # Define the names for the variables - names = ['time', 'B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B9', 'B10', 'B11', 'B12', - 'B13', 'B14', 'B15', 'B16', 'B17', 'B18', 'B19', 'B20', 'B21', 'B22', 'B23', 'B24', - 'B25', 'B26', 'B27', 'B28', 'B29', 'B30', 'B31', 'B32', 'blackout', 'good', 'bad', - 'number_detected_particles', 'precip_rate', 'precip_amount', 'precip_accumulation', - 'equivalent_radar_reflectivity', 'number_in_error', 'dirty', 'very_dirty', 'damaged', - 'laserband_amplitude', 'laserband_amplitude_stdev', 'sensor_temperature', 'sensor_temperature_stdev', - 'sensor_voltage', 'sensor_voltage_stdev', 'heating_current', 'heating_current_stdev', 'number_rain_particles', - 'number_non_rain_particles', 'number_ambiguous_particles', 'precip_type'] + names = [ + 'time', + 'B1', + 'B2', + 'B3', + 'B4', + 'B5', + 'B6', + 'B7', + 'B8', + 'B9', + 'B10', + 'B11', + 'B12', + 'B13', + 'B14', + 'B15', + 'B16', + 'B17', + 'B18', + 'B19', + 'B20', + 'B21', + 'B22', + 'B23', + 'B24', + 'B25', + 'B26', + 'B27', + 'B28', + 'B29', + 'B30', + 'B31', + 'B32', + 'blackout', + 'good', + 'bad', + 'number_detected_particles', + 'precip_rate', + 'precip_amount', + 'precip_accumulation', + 'equivalent_radar_reflectivity', + 'number_in_error', + 'dirty', + 'very_dirty', + 'damaged', + 'laserband_amplitude', + 'laserband_amplitude_stdev', + 'sensor_temperature', + 'sensor_temperature_stdev', + 'sensor_voltage', + 'sensor_voltage_stdev', + 'heating_current', + 'heating_current_stdev', + 'number_rain_particles', + 'number_non_rain_particles', + 'number_ambiguous_particles', + 'precip_type', + ] # Define the particle sizes and class width sizes based on # https://psl.noaa.gov/data/obs/data/view_data_type_info.php?SiteID=ctd&DataOperationalID=5890 - vol_equiv_diam = [0.062, 0.187, 0.312, 0.437, 0.562, 0.687, 0.812, 0.937, 1.062, 1.187, 1.375, - 1.625, 1.875, 2.125, 2.375, 2.75, 3.25, 3.75, 4.25, 4.75, 5.5, 6.5, 7.5, 8.5, - 9.5, 11.0, 13.0, 15.0, 17.0, 19.0, 21.5, 24.5] - class_size_width = [0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, - 0.250, 0.250, 0.250, 0.250, 0.250, 0.5, 0.5, 0.5, 0.5, 0.5, - 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0] + vol_equiv_diam = [ + 0.062, + 0.187, + 0.312, + 0.437, + 0.562, + 0.687, + 0.812, + 0.937, + 1.062, + 1.187, + 1.375, + 1.625, + 1.875, + 2.125, + 2.375, + 2.75, + 3.25, + 3.75, + 4.25, + 4.75, + 5.5, + 6.5, + 7.5, + 8.5, + 9.5, + 11.0, + 13.0, + 15.0, + 17.0, + 19.0, + 21.5, + 24.5, + ] + class_size_width = [ + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.125, + 0.250, + 0.250, + 0.250, + 0.250, + 0.250, + 0.5, + 0.5, + 0.5, + 0.5, + 0.5, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 3.0, + 3.0, + ] if not isinstance(files, list): files = [files] @@ -619,11 +737,17 @@ def read_psl_parsivel(files): ds = ds.rename({'index': 'time'}) long_name = 'Drop Size Distribution' attrs = {'long_name': long_name, 'units': 'count'} - da = xr.DataArray(np.transpose(dsd), dims=['time', 'particle_size'], coords=[ds['time'].values, vol_equiv_diam]) + da = xr.DataArray( + np.transpose(dsd), + dims=['time', 'particle_size'], + coords=[ds['time'].values, vol_equiv_diam], + ) ds['number_density_drops'] = da attrs = {'long_name': 'Particle class size average', 'units': 'mm'} - da = xr.DataArray(class_size_width, dims=['particle_size'], coords=[vol_equiv_diam], attrs=attrs) + da = xr.DataArray( + class_size_width, dims=['particle_size'], coords=[vol_equiv_diam], attrs=attrs + ) ds['class_size_width'] = da attrs = {'long_name': 'Class size width', 'units': 'mm'} @@ -635,31 +759,75 @@ def read_psl_parsivel(files): ds['interval_end_time'] = da # Define the attribuets and metadata and add into the DataSet - attrs = {'blackout': {'long_name': 'Number of samples excluded during PC clock sync', 'units': 'count'}, - 'good': {'long_name': 'Number of samples that passed QC checks', 'units': 'count'}, - 'bad': {'long_name': 'Number of samples that failed QC checks', 'units': 'count'}, - 'number_detected_particles': {'long_name': 'Total number of detected particles', 'units': 'count'}, - 'precip_rate': {'long_name': 'Precipitation rate', 'units': 'mm/hr'}, - 'precip_amount': {'long_name': 'Interval accumulation', 'units': 'mm'}, - 'precip_accumulation': {'long_name': 'Event accumulation', 'units': 'mm'}, - 'equivalent_radar_reflectivity': {'long_name': 'Radar Reflectivity', 'units': 'dB'}, - 'number_in_error': {'long_name': 'Number of samples that were reported dirt, very dirty, or damaged', 'units': 'count'}, - 'dirty': {'long_name': 'Laser glass is dirty but measurement is still possible', 'units': 'unitless'}, - 'very_dirty': {'long_name': 'Laser glass is dirty, partially covered no further measurements are possible', 'units': 'unitless'}, - 'damaged': {'long_name': 'Laser damaged', 'units': 'unitless'}, - 'laserband_amplitude': {'long_name': 'Average signal amplitude of the laser strip', 'units': 'unitless'}, - 'laserband_amplitude_stdev': {'long_name': 'Standard deviation of the signal amplitude of the laser strip', 'units': 'unitless'}, - 'sensor_temperature': {'long_name': 'Average sensor temperature', 'units': 'degC'}, - 'sensor_temperature_stdev': {'long_name': 'Standard deviation of sensor temperature', 'units': 'degC'}, - 'sensor_voltage': {'long_name': 'Sensor power supply voltage', 'units': 'V'}, - 'sensor_voltage_stdev': {'long_name': 'Standard deviation of the sensor power supply voltage', 'units': 'V'}, - 'heating_current': {'long_name': 'Average heating system current', 'units': 'A'}, - 'heating_current_stdev': {'long_name': 'Standard deviation of heating system current', 'units': 'A'}, - 'number_rain_particles': {'long_name': 'Number of particles detected as rain', 'units': 'unitless'}, - 'number_non_rain_particles': {'long_name': 'Number of particles detected not as rain', 'units': 'unitless'}, - 'number_ambiguous_particles': {'long_name': 'Number of particles detected as ambiguous', 'units': 'unitless'}, - 'precip_type': {'long_name': 'Precipitation type (1=rain; 2=mixed; 3=snow)', 'units': 'unitless'}, - 'number_density_drops': {'long_name': 'Drop Size Distribution', 'units': 'count'}} + attrs = { + 'blackout': { + 'long_name': 'Number of samples excluded during PC clock sync', + 'units': 'count', + }, + 'good': {'long_name': 'Number of samples that passed QC checks', 'units': 'count'}, + 'bad': {'long_name': 'Number of samples that failed QC checks', 'units': 'count'}, + 'number_detected_particles': { + 'long_name': 'Total number of detected particles', + 'units': 'count', + }, + 'precip_rate': {'long_name': 'Precipitation rate', 'units': 'mm/hr'}, + 'precip_amount': {'long_name': 'Interval accumulation', 'units': 'mm'}, + 'precip_accumulation': {'long_name': 'Event accumulation', 'units': 'mm'}, + 'equivalent_radar_reflectivity': {'long_name': 'Radar Reflectivity', 'units': 'dB'}, + 'number_in_error': { + 'long_name': 'Number of samples that were reported dirt, very dirty, or damaged', + 'units': 'count', + }, + 'dirty': { + 'long_name': 'Laser glass is dirty but measurement is still possible', + 'units': 'unitless', + }, + 'very_dirty': { + 'long_name': 'Laser glass is dirty, partially covered no further measurements are possible', + 'units': 'unitless', + }, + 'damaged': {'long_name': 'Laser damaged', 'units': 'unitless'}, + 'laserband_amplitude': { + 'long_name': 'Average signal amplitude of the laser strip', + 'units': 'unitless', + }, + 'laserband_amplitude_stdev': { + 'long_name': 'Standard deviation of the signal amplitude of the laser strip', + 'units': 'unitless', + }, + 'sensor_temperature': {'long_name': 'Average sensor temperature', 'units': 'degC'}, + 'sensor_temperature_stdev': { + 'long_name': 'Standard deviation of sensor temperature', + 'units': 'degC', + }, + 'sensor_voltage': {'long_name': 'Sensor power supply voltage', 'units': 'V'}, + 'sensor_voltage_stdev': { + 'long_name': 'Standard deviation of the sensor power supply voltage', + 'units': 'V', + }, + 'heating_current': {'long_name': 'Average heating system current', 'units': 'A'}, + 'heating_current_stdev': { + 'long_name': 'Standard deviation of heating system current', + 'units': 'A', + }, + 'number_rain_particles': { + 'long_name': 'Number of particles detected as rain', + 'units': 'unitless', + }, + 'number_non_rain_particles': { + 'long_name': 'Number of particles detected not as rain', + 'units': 'unitless', + }, + 'number_ambiguous_particles': { + 'long_name': 'Number of particles detected as ambiguous', + 'units': 'unitless', + }, + 'precip_type': { + 'long_name': 'Precipitation type (1=rain; 2=mixed; 3=snow)', + 'units': 'unitless', + }, + 'number_density_drops': {'long_name': 'Drop Size Distribution', 'units': 'count'}, + } for v in ds: if v in attrs: @@ -739,47 +907,169 @@ def _parse_psl_radar_moments(files): # Set the initial dictionary to convert to xarray dataset data = { 'site': {'dims': ['file'], 'data': [], 'attrs': {'long_name': 'NOAA site code'}}, - 'lat': {'dims': ['file'], 'data': [], 'attrs': {'long_name': 'North Latitude', 'units': 'degree_N'}}, - 'lon': {'dims': ['file'], 'data': [], 'attrs': {'long_name': 'East Longitude', 'units': 'degree_E'}}, - 'alt': {'dims': ['file'], 'data': [], 'attrs': {'long_name': 'Altitude above mean sea level', 'units': 'm'}}, - 'freq': {'dims': ['file'], 'data': [], 'attrs': {'long_name': 'Operating Frequency; Ignore for FMCW', 'units': 'Hz'}}, - 'azimuth': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Azimuth angle', 'units': 'deg'}}, - 'elevation': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Elevation angle', 'units': 'deg'}}, - 'beam_direction_code': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Beam direction code', 'units': ''}}, + 'lat': { + 'dims': ['file'], + 'data': [], + 'attrs': {'long_name': 'North Latitude', 'units': 'degree_N'}, + }, + 'lon': { + 'dims': ['file'], + 'data': [], + 'attrs': {'long_name': 'East Longitude', 'units': 'degree_E'}, + }, + 'alt': { + 'dims': ['file'], + 'data': [], + 'attrs': {'long_name': 'Altitude above mean sea level', 'units': 'm'}, + }, + 'freq': { + 'dims': ['file'], + 'data': [], + 'attrs': {'long_name': 'Operating Frequency; Ignore for FMCW', 'units': 'Hz'}, + }, + 'azimuth': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Azimuth angle', 'units': 'deg'}, + }, + 'elevation': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Elevation angle', 'units': 'deg'}, + }, + 'beam_direction_code': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Beam direction code', 'units': ''}, + }, 'year': {'dims': ['time'], 'data': [], 'attrs': {'long_name': '2-digit year', 'units': ''}}, - 'day_of_year': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Day of the year', 'units': ''}}, - 'hour': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Hour of the day', 'units': ''}}, + 'day_of_year': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Day of the year', 'units': ''}, + }, + 'hour': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Hour of the day', 'units': ''}, + }, 'minute': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Minutes', 'units': ''}}, 'second': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Seconds', 'units': ''}}, - 'interpulse_period': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Interpulse Period', 'units': 'ms'}}, - 'pulse_width': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Pulse width', 'units': 'ns'}}, - 'first_range_gate': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Range to first range gate', 'units': 'm'}}, - 'range_between_gates': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Distance between range gates', 'units': 'm'}}, - 'n_gates': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Number of range gates', 'units': 'count'}}, - 'n_coherent_integration': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Number of cohrent integration', 'units': 'count'}}, - 'n_averaged_spectra': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Number of average spectra', 'units': 'count'}}, - 'n_points_spectrum': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Number of points in spectra', 'units': 'count'}}, - 'n_code_bits': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Number of code bits', 'units': 'count'}}, - 'radial_velocity': {'dims': ['time', 'range'], 'data': [], 'attrs': {'long_name': 'Radial velocity', 'units': 'm/s'}}, - 'snr': {'dims': ['time', 'range'], 'data': [], 'attrs': {'long_name': 'Signal-to-noise ratio - not range corrected', 'units': 'dB'}}, - 'signal_power': {'dims': ['time', 'range'], 'data': [], 'attrs': {'long_name': 'Signal Power - not range corrected', 'units': 'dB'}}, - 'spectral_width': {'dims': ['time', 'range'], 'data': [], 'attrs': {'long_name': 'Spectral width', 'units': 'm/s'}}, - 'noise_amplitude': {'dims': ['time', 'range'], 'data': [], 'attrs': {'long_name': 'noise_amplitude', 'units': 'dB'}}, - 'qc_variable': {'dims': ['time', 'range'], 'data': [], 'attrs': {'long_name': 'QC Value - not used', 'units': ''}}, + 'interpulse_period': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Interpulse Period', 'units': 'ms'}, + }, + 'pulse_width': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Pulse width', 'units': 'ns'}, + }, + 'first_range_gate': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Range to first range gate', 'units': 'm'}, + }, + 'range_between_gates': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Distance between range gates', 'units': 'm'}, + }, + 'n_gates': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Number of range gates', 'units': 'count'}, + }, + 'n_coherent_integration': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Number of cohrent integration', 'units': 'count'}, + }, + 'n_averaged_spectra': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Number of average spectra', 'units': 'count'}, + }, + 'n_points_spectrum': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Number of points in spectra', 'units': 'count'}, + }, + 'n_code_bits': { + 'dims': ['time'], + 'data': [], + 'attrs': {'long_name': 'Number of code bits', 'units': 'count'}, + }, + 'radial_velocity': { + 'dims': ['time', 'range'], + 'data': [], + 'attrs': {'long_name': 'Radial velocity', 'units': 'm/s'}, + }, + 'snr': { + 'dims': ['time', 'range'], + 'data': [], + 'attrs': {'long_name': 'Signal-to-noise ratio - not range corrected', 'units': 'dB'}, + }, + 'signal_power': { + 'dims': ['time', 'range'], + 'data': [], + 'attrs': {'long_name': 'Signal Power - not range corrected', 'units': 'dB'}, + }, + 'spectral_width': { + 'dims': ['time', 'range'], + 'data': [], + 'attrs': {'long_name': 'Spectral width', 'units': 'm/s'}, + }, + 'noise_amplitude': { + 'dims': ['time', 'range'], + 'data': [], + 'attrs': {'long_name': 'noise_amplitude', 'units': 'dB'}, + }, + 'qc_variable': { + 'dims': ['time', 'range'], + 'data': [], + 'attrs': {'long_name': 'QC Value - not used', 'units': ''}, + }, 'time': {'dims': ['time'], 'data': [], 'attrs': {'long_name': 'Datetime', 'units': ''}}, 'range': {'dims': ['range'], 'data': [], 'attrs': {'long_name': 'Range', 'units': 'm'}}, - 'reflectivity_uncalibrated': {'dims': ['time', 'range'], 'data': [], 'attrs': {'long_name': 'Range', 'units': 'dB'}}, + 'reflectivity_uncalibrated': { + 'dims': ['time', 'range'], + 'data': [], + 'attrs': {'long_name': 'Range', 'units': 'dB'}, + }, } # Separate out the names as they will be accessed in different parts of the code h1_names = ['site', 'lat', 'lon', 'alt', 'freq'] - h2_names = ['azimuth', 'elevation', 'beam_direction_code', 'year', 'day_of_year', 'hour', 'minute', - 'second'] - h3_names = ['interpulse_period', 'pulse_width', 'first_range_gate', 'range_between_gates', - 'n_gates', 'n_coherent_integration', 'n_averaged_spectra', 'n_points_spectrum', - 'n_code_bits'] - names = {'radial_velocity': 0, 'snr': 1, 'signal_power': 2, 'spectral_width': 3, - 'noise_amplitude': 4, 'qc_variable': 5} + h2_names = [ + 'azimuth', + 'elevation', + 'beam_direction_code', + 'year', + 'day_of_year', + 'hour', + 'minute', + 'second', + ] + h3_names = [ + 'interpulse_period', + 'pulse_width', + 'first_range_gate', + 'range_between_gates', + 'n_gates', + 'n_coherent_integration', + 'n_averaged_spectra', + 'n_points_spectrum', + 'n_code_bits', + ] + names = { + 'radial_velocity': 0, + 'snr': 1, + 'signal_power': 2, + 'spectral_width': 3, + 'noise_amplitude': 4, + 'qc_variable': 5, + } # If file is a string, convert to list for handling. if not isinstance(files, list): @@ -793,7 +1083,7 @@ def _parse_psl_radar_moments(files): for d in df: if len(d) > 0: if d == 'lat' or d == 'lon': - data[h1_names[ctr]]['data'].append(float(d) / 100.) + data[h1_names[ctr]]['data'].append(float(d) / 100.0) else: data[h1_names[ctr]]['data'].append(d) ctr += 1 @@ -821,27 +1111,41 @@ def _parse_psl_radar_moments(files): data[h3_names[ctr]]['data'].append(d) ctr += 1 # Read in the data based on number of gates - df = pd.read_csv(f, skiprows=[0, 1, 2], nrows=int(data['n_gates']['data'][-1]) - 1, delim_whitespace=True, - names=list(names.keys())) + df = pd.read_csv( + f, + skiprows=[0, 1, 2], + nrows=int(data['n_gates']['data'][-1]) - 1, + delim_whitespace=True, + names=list(names.keys()), + ) index2 = 0 else: # Set indices for parsing data, reading 2 headers and then the columns of data index1 = ct * int(data['n_gates']['data'][-1]) index2 = index1 + int(data['n_gates']['data'][-1]) + 2 * ct + 4 - df = str(pd.read_table(f, nrows=0, skiprows=list(range(index2 - 1))).columns[0]).split(' ') + df = str( + pd.read_table(f, nrows=0, skiprows=list(range(index2 - 1))).columns[0] + ).split(' ') ctr = 0 for d in df: if len(d) > 0: data[h2_names[ctr]]['data'].append(d) ctr += 1 - df = str(pd.read_table(f, nrows=0, skiprows=list(range(index2))).columns[0]).split(' ') + df = str( + pd.read_table(f, nrows=0, skiprows=list(range(index2))).columns[0] + ).split(' ') ctr = 0 for d in df: if len(d) > 0: data[h3_names[ctr]]['data'].append(d) ctr += 1 - df = pd.read_csv(f, skiprows=list(range(index2 + 1)), nrows=int(data['n_gates']['data'][-1]) - 1, delim_whitespace=True, - names=list(names.keys())) + df = pd.read_csv( + f, + skiprows=list(range(index2 + 1)), + nrows=int(data['n_gates']['data'][-1]) - 1, + delim_whitespace=True, + names=list(names.keys()), + ) # Add data from the columns to the dictionary for n in names: @@ -849,19 +1153,24 @@ def _parse_psl_radar_moments(files): # Calculate the range based on number of gates, range to first gate and range between gates if len(data['range']['data']) == 0: - ranges = float(data['first_range_gate']['data'][-1]) + np.array(range(int(data['n_gates']['data'][-1]) - 1)) * \ - float(data['range_between_gates']['data'][-1]) + ranges = float(data['first_range_gate']['data'][-1]) + np.array( + range(int(data['n_gates']['data'][-1]) - 1) + ) * float(data['range_between_gates']['data'][-1]) data['range']['data'] = ranges # Calculate a time time = dt.datetime( - int('20' + data['year']['data'][-1]), 1, 1, int(data['hour']['data'][-1]), - int(data['minute']['data'][-1]), int(data['second']['data'][-1])) + \ - dt.timedelta(days=int(data['day_of_year']['data'][-1]) - 1) + int('20' + data['year']['data'][-1]), + 1, + 1, + int(data['hour']['data'][-1]), + int(data['minute']['data'][-1]), + int(data['second']['data'][-1]), + ) + dt.timedelta(days=int(data['day_of_year']['data'][-1]) - 1) data['time']['data'].append(time) # Range correct the snr which converts it essentially to an uncalibrated reflectivity - snr_rc = data['snr']['data'][-1] - 20. * np.log10(1. / (ranges / 1000.) ** 2) + snr_rc = data['snr']['data'][-1] - 20.0 * np.log10(1.0 / (ranges / 1000.0) ** 2) data['reflectivity_uncalibrated']['data'].append(snr_rc) except Exception as e: # Handle errors, if end of file then continue on, if something else diff --git a/act/io/sodar.py b/act/io/sodar.py index 15dc238f45..afba04474d 100644 --- a/act/io/sodar.py +++ b/act/io/sodar.py @@ -74,11 +74,9 @@ def read_mfas_sodar(filepath): # Parse data to a dataframe skipping rows that aren't data. # tmp_columns is used to removed '#' column that causes # columns to move over by one. - df = pd.read_table(filepath, - sep=r'\s+', - skiprows=skip_full_ind, - names=tmp_columns, - usecols=columns) + df = pd.read_table( + filepath, sep=r'\s+', skiprows=skip_full_ind, names=tmp_columns, usecols=columns + ) df = df[~df['W'].isin(['dir'])].reset_index(drop=True) @@ -109,9 +107,9 @@ def read_mfas_sodar(filepath): # previous version. try: mindex_coords = xr.Coordinates.from_pandas_multiindex(ind, 'Dates') - ds = ds.assign_coords(mindex_coords).unstack("Dates") + ds = ds.assign_coords(mindex_coords).unstack('Dates') except AttributeError: - ds = ds.assign(Dates=ind).unstack("Dates") + ds = ds.assign(Dates=ind).unstack('Dates') # Add file metadata. for key in file_dict.keys(): @@ -158,7 +156,7 @@ def _metadata_retrieval(lines): file_type_ind = np.argwhere(line_array == '# file type')[0][0] # Index the section of file information. - file_def = line_array[file_info_ind + 2:file_type_ind - 1] + file_def = line_array[file_info_ind + 2 : file_type_ind - 1] # Create a dictionary of file information to be plugged in later to the xarray # dataset attributes. @@ -179,7 +177,7 @@ def _metadata_retrieval(lines): data_ind = np.argwhere(line_array == '# beginning of data block')[0][0] # Index the section of variable information. - variable_def = line_array[variable_info_ind + 2 :data_ind - 1] + variable_def = line_array[variable_info_ind + 2 : data_ind - 1] # Create a dictionary of variable information to be plugged in later to the xarray # variable attributes. Skipping error code as it does not have metadata similar to diff --git a/act/plotting/__init__.py b/act/plotting/__init__.py index 4830bd667a..d5a4cab9fc 100644 --- a/act/plotting/__init__.py +++ b/act/plotting/__init__.py @@ -15,10 +15,9 @@ """ -import lazy_loader as lazy - # Load colormaps import cmweather +import lazy_loader as lazy # Eagerly load in common from . import common @@ -48,6 +47,6 @@ 'timeseriesdisplay': ['TimeSeriesDisplay'], 'windrosedisplay': ['WindRoseDisplay'], 'xsectiondisplay': ['XSectionDisplay'], - 'distributiondisplay' : ['DistributionDisplay'], + 'distributiondisplay': ['DistributionDisplay'], }, ) diff --git a/act/plotting/distributiondisplay.py b/act/plotting/distributiondisplay.py index 92baa94ff4..deec5f272e 100644 --- a/act/plotting/distributiondisplay.py +++ b/act/plotting/distributiondisplay.py @@ -2,8 +2,8 @@ import matplotlib.pyplot as plt import numpy as np -import xarray as xr import pandas as pd +import xarray as xr from ..utils import datetime_utils as dt_utils from .plot import Display @@ -178,16 +178,20 @@ def plot_stacked_bar_graph( ytitle = field if sortby_bins is None: my_hist, x_bins, y_bins = np.histogram2d( - xdata.values.flatten(), ydata.values.flatten(), density=density, + xdata.values.flatten(), + ydata.values.flatten(), + density=density, bins=bins, - **hist_kwargs) + **hist_kwargs, + ) else: my_hist, x_bins, y_bins = np.histogram2d( xdata.values.flatten(), ydata.values.flatten(), density=density, bins=[bins, sortby_bins], - **hist_kwargs) + **hist_kwargs, + ) x_inds = (x_bins[:-1] + x_bins[1:]) / 2.0 self.axes[subplot_index].bar( x_inds, @@ -205,8 +209,9 @@ def plot_stacked_bar_graph( ) self.axes[subplot_index].legend() else: - my_hist, bins = np.histogram(xdata.values.flatten(), bins=bins, - density=density, **hist_kwargs) + my_hist, bins = np.histogram( + xdata.values.flatten(), bins=bins, density=density, **hist_kwargs + ) x_inds = (bins[:-1] + bins[1:]) / 2.0 self.axes[subplot_index].bar(x_inds, my_hist) @@ -326,7 +331,9 @@ def plot_size_distribution( ) if time is not None: t = pd.Timestamp(time) - set_title += ''.join([' at ', ':'.join([str(t.hour), str(t.minute), str(t.second)])]) + set_title += ''.join( + [' at ', ':'.join([str(t.hour), str(t.minute), str(t.second)])] + ) self.axes[subplot_index].set_title(set_title) self.axes[subplot_index].step(bins.values, xdata.values, **kwargs) self.axes[subplot_index].set_xlabel(xtitle) @@ -422,15 +429,19 @@ def plot_stairstep_graph( ytitle = field if sortby_bins is None: my_hist, x_bins, y_bins = np.histogram2d( - xdata.values.flatten(), ydata.values.flatten(), bins=bins, - density=density, **hist_kwargs) + xdata.values.flatten(), + ydata.values.flatten(), + bins=bins, + density=density, + **hist_kwargs, + ) else: my_hist, x_bins, y_bins = np.histogram2d( xdata.values.flatten(), ydata.values.flatten(), density=density, bins=[bins, sortby_bins], - **hist_kwargs + **hist_kwargs, ) x_inds = (x_bins[:-1] + x_bins[1:]) / 2.0 self.axes[subplot_index].step( @@ -448,8 +459,9 @@ def plot_stairstep_graph( ) self.axes[subplot_index].legend() else: - my_hist, bins = np.histogram(xdata.values.flatten(), bins=bins, - density=density, **hist_kwargs) + my_hist, bins = np.histogram( + xdata.values.flatten(), bins=bins, density=density, **hist_kwargs + ) x_inds = (bins[:-1] + bins[1:]) / 2.0 self.axes[subplot_index].step(x_inds, my_hist, **kwargs) @@ -580,15 +592,15 @@ def plot_heatmap( if x_bins is None: my_hist, x_bins, y_bins = np.histogram2d( - xdata.values.flatten(), ydata.values.flatten(), density=density, - **hist_kwargs) + xdata.values.flatten(), ydata.values.flatten(), density=density, **hist_kwargs + ) else: my_hist, x_bins, y_bins = np.histogram2d( xdata.values.flatten(), ydata.values.flatten(), density=density, bins=[x_bins, y_bins], - **hist_kwargs + **hist_kwargs, ) # Adding in the ability to threshold the heatmaps if threshold is not None: @@ -621,7 +633,7 @@ def plot_heatmap( return return_dict - def set_ratio_line(self, subplot_index=(0, )): + def set_ratio_line(self, subplot_index=(0,)): """ Sets the 1:1 ratio line. @@ -638,16 +650,17 @@ def set_ratio_line(self, subplot_index=(0, )): ratio = np.linspace(xlims[0], xlims[-1]) self.axes[subplot_index].plot(ratio, ratio, 'k--') - def plot_scatter(self, - x_field, - y_field, - m_field=None, - dsname=None, - cbar_label=None, - set_title=None, - subplot_index=(0,), - **kwargs, - ): + def plot_scatter( + self, + x_field, + y_field, + m_field=None, + dsname=None, + cbar_label=None, + set_title=None, + subplot_index=(0,), + **kwargs, + ): """ This procedure will produce a scatter plot from 2 variables. @@ -717,11 +730,7 @@ def plot_scatter(self, self.fig.add_axes(self.axes[0]) # Display the scatter plot, pass keyword args for unspecified attributes - scc = self.axes[subplot_index].scatter(xdata, - ydata, - c=mdata, - **kwargs - ) + scc = self.axes[subplot_index].scatter(xdata, ydata, c=mdata, **kwargs) # Set Title if set_title is None: @@ -754,18 +763,19 @@ def plot_scatter(self, return self.axes[subplot_index] - def plot_violin(self, - field, - positions=None, - dsname=None, - vert=True, - showmeans=True, - showmedians=True, - showextrema=True, - subplot_index=(0,), - set_title=None, - **kwargs, - ): + def plot_violin( + self, + field, + positions=None, + dsname=None, + vert=True, + showmeans=True, + showmedians=True, + showextrema=True, + subplot_index=(0,), + set_title=None, + **kwargs, + ): """ This procedure will produce a violin plot for the selected field (or fields). @@ -828,14 +838,15 @@ def plot_violin(self, axtitle = field # Display the scatter plot, pass keyword args for unspecified attributes - scc = self.axes[subplot_index].violinplot(ndata, - positions=positions, - vert=vert, - showmeans=showmeans, - showmedians=showmedians, - showextrema=showextrema, - **kwargs - ) + scc = self.axes[subplot_index].violinplot( + ndata, + positions=positions, + vert=vert, + showmeans=showmeans, + showmedians=showmedians, + showextrema=showextrema, + **kwargs, + ) if showmeans is True: scc['cmeans'].set_edgecolor('red') scc['cmeans'].set_label('mean') diff --git a/act/plotting/geodisplay.py b/act/plotting/geodisplay.py index f26379df39..ccbd6b272e 100644 --- a/act/plotting/geodisplay.py +++ b/act/plotting/geodisplay.py @@ -214,8 +214,9 @@ def geoplot( tiler = img_tiles.Stamen(stamen) ax.add_image(tiler, tile) warnings.warn( - "Stamen is deprecated in Cartopy and in future versions of ACT, " - "please use img_tile to specify the image background. ") + 'Stamen is deprecated in Cartopy and in future versions of ACT, ' + 'please use img_tile to specify the image background. ' + ) else: if img_tile is not None: tiler = getattr(img_tiles, img_tile)(**img_tile_args) diff --git a/act/plotting/histogramdisplay.py b/act/plotting/histogramdisplay.py index 0e87083a0b..ddf4601468 100644 --- a/act/plotting/histogramdisplay.py +++ b/act/plotting/histogramdisplay.py @@ -1,9 +1,10 @@ """ Module for Histogram Plotting. """ +import warnings + import matplotlib.pyplot as plt import numpy as np import xarray as xr -import warnings from ..utils import datetime_utils as dt_utils from .plot import Display @@ -181,16 +182,20 @@ def plot_stacked_bar_graph( ytitle = field if sortby_bins is None: my_hist, x_bins, y_bins = np.histogram2d( - xdata.values.flatten(), ydata.values.flatten(), density=density, + xdata.values.flatten(), + ydata.values.flatten(), + density=density, bins=bins, - **hist_kwargs) + **hist_kwargs, + ) else: my_hist, x_bins, y_bins = np.histogram2d( xdata.values.flatten(), ydata.values.flatten(), density=density, bins=[bins, sortby_bins], - **hist_kwargs) + **hist_kwargs, + ) x_inds = (x_bins[:-1] + x_bins[1:]) / 2.0 self.axes[subplot_index].bar( x_inds, @@ -208,8 +213,9 @@ def plot_stacked_bar_graph( ) self.axes[subplot_index].legend() else: - my_hist, bins = np.histogram(xdata.values.flatten(), bins=bins, - density=density, **hist_kwargs) + my_hist, bins = np.histogram( + xdata.values.flatten(), bins=bins, density=density, **hist_kwargs + ) x_inds = (bins[:-1] + bins[1:]) / 2.0 self.axes[subplot_index].bar(x_inds, my_hist) @@ -423,15 +429,19 @@ def plot_stairstep_graph( ytitle = field if sortby_bins is None: my_hist, x_bins, y_bins = np.histogram2d( - xdata.values.flatten(), ydata.values.flatten(), bins=bins, - density=density, **hist_kwargs) + xdata.values.flatten(), + ydata.values.flatten(), + bins=bins, + density=density, + **hist_kwargs, + ) else: my_hist, x_bins, y_bins = np.histogram2d( xdata.values.flatten(), ydata.values.flatten(), density=density, bins=[bins, sortby_bins], - **hist_kwargs + **hist_kwargs, ) x_inds = (x_bins[:-1] + x_bins[1:]) / 2.0 self.axes[subplot_index].step( @@ -449,8 +459,9 @@ def plot_stairstep_graph( ) self.axes[subplot_index].legend() else: - my_hist, bins = np.histogram(xdata.values.flatten(), bins=bins, - density=density, **hist_kwargs) + my_hist, bins = np.histogram( + xdata.values.flatten(), bins=bins, density=density, **hist_kwargs + ) x_inds = (bins[:-1] + bins[1:]) / 2.0 self.axes[subplot_index].step(x_inds, my_hist, **kwargs) @@ -568,15 +579,15 @@ def plot_heatmap( if x_bins is None: my_hist, x_bins, y_bins = np.histogram2d( - xdata.values.flatten(), ydata.values.flatten(), density=density, - **hist_kwargs) + xdata.values.flatten(), ydata.values.flatten(), density=density, **hist_kwargs + ) else: my_hist, x_bins, y_bins = np.histogram2d( xdata.values.flatten(), ydata.values.flatten(), density=density, bins=[x_bins, y_bins], - **hist_kwargs + **hist_kwargs, ) x_inds = (x_bins[:-1] + x_bins[1:]) / 2.0 y_inds = (y_bins[:-1] + y_bins[1:]) / 2.0 diff --git a/act/plotting/plot.py b/act/plotting/plot.py index 76ad2693e7..26093c8e1b 100644 --- a/act/plotting/plot.py +++ b/act/plotting/plot.py @@ -3,13 +3,13 @@ """ +import inspect import warnings # Import third party libraries import matplotlib.pyplot as plt import numpy as np import xarray as xr -import inspect class Display: @@ -231,8 +231,9 @@ def assign_to_figure_axis(self, fig, ax): self.fig = fig self.axes = np.array([ax]) - def add_colorbar(self, mappable, title=None, subplot_index=(0,), pad=None, - width=None, **kwargs): + def add_colorbar( + self, mappable, title=None, subplot_index=(0,), pad=None, width=None, **kwargs + ): """ Adds a colorbar to the plot. @@ -297,7 +298,7 @@ def group_by(self, units): return DisplayGroupby(self, units) -class DisplayGroupby(object): +class DisplayGroupby: def __init__(self, display, units): """ @@ -344,8 +345,7 @@ def plot_group(self, func_name, dsname=None, **kwargs): func = getattr(self.display, func_name) if not callable(func): - raise RuntimeError("The specified string is not a function of " - "the Display object.") + raise RuntimeError('The specified string is not a function of ' 'the Display object.') subplot_shape = self.display.axes.shape i = 0 wrap_around = False @@ -364,10 +364,10 @@ def plot_group(self, func_name, dsname=None, **kwargs): else: subplot_index = (i % subplot_shape[0],) args, varargs, varkw, _, _, _, _ = inspect.getfullargspec(func) - if "subplot_index" in args: - kwargs["subplot_index"] = subplot_index - if "time_rng" in args: - kwargs["time_rng"] = (ds.time.values.min(), ds.time.values.max()) + if 'subplot_index' in args: + kwargs['subplot_index'] = subplot_index + if 'time_rng' in args: + kwargs['time_rng'] = (ds.time.values.min(), ds.time.values.max()) if num_years > 1 and self.isTimeSeriesDisplay: first_year = ds.time.dt.year[0] for yr, ds1 in ds.groupby('time.year'): @@ -377,18 +377,25 @@ def plot_group(self, func_name, dsname=None, **kwargs): days_in_year = 365 year_diff = ds1.time.dt.year - first_year time_diff = np.array( - [np.timedelta64(x * days_in_year, 'D') for x in year_diff.values]) + [np.timedelta64(x * days_in_year, 'D') for x in year_diff.values] + ) ds1['time'] = ds1.time - time_diff self.display._ds[key + '%d_%d' % (k, yr)] = ds1 func(dsname=key + '%d_%d' % (k, yr), label=str(yr), **kwargs) self.mapping[key + '%d_%d' % (k, yr)] = subplot_index - self.xlims[key + '%d_%d' % (k, yr)] = (ds1.time.values.min(), ds1.time.values.max()) + self.xlims[key + '%d_%d' % (k, yr)] = ( + ds1.time.values.min(), + ds1.time.values.max(), + ) del self.display._ds[key + '_%d' % k] else: func(dsname=key + '_%d' % k, **kwargs) self.mapping[key + '_%d' % k] = subplot_index if self.isTimeSeriesDisplay: - self.xlims[key + '_%d' % k] = (ds.time.values.min(), ds.time.values.max()) + self.xlims[key + '_%d' % k] = ( + ds.time.values.min(), + ds.time.values.max(), + ) i = i + 1 if wrap_around is False and i < np.prod(subplot_shape): diff --git a/act/plotting/skewtdisplay.py b/act/plotting/skewtdisplay.py index 2203d3cf9c..c8de24a139 100644 --- a/act/plotting/skewtdisplay.py +++ b/act/plotting/skewtdisplay.py @@ -341,7 +341,9 @@ def plot_from_u_and_v( if not all(p[i] <= p[i + 1] for i in range(len(p) - 1)): if 'time' in self._ds: self._ds[dsname][p_field] = ( - self._ds[dsname][p_field].rolling(time=smooth_p, min_periods=1, center=True).mean() + self._ds[dsname][p_field] + .rolling(time=smooth_p, min_periods=1, center=True) + .mean() ) p = self._ds[dsname][p_field] @@ -426,7 +428,7 @@ def plot_from_u_and_v( # Set Title if set_title is None: if 'time' in self._ds[dsname]: - title_time = dt_utils.numpy_to_arm_date(self._ds[dsname].time.values[0]), + title_time = (dt_utils.numpy_to_arm_date(self._ds[dsname].time.values[0]),) elif '_file_dates' in self._ds[dsname].attrs: title_time = self._ds[dsname].attrs['_file_dates'][0] else: diff --git a/act/plotting/timeseriesdisplay.py b/act/plotting/timeseriesdisplay.py index 1756635052..0ccd0d64e9 100644 --- a/act/plotting/timeseriesdisplay.py +++ b/act/plotting/timeseriesdisplay.py @@ -151,7 +151,9 @@ def day_night_background(self, dsname=None, subplot_index=(0,)): for value, name in zip(lat_lon_list, ['Latitude', 'Longitude']): if not np.isfinite(value): - warnings.warn(f"{name} value in dataset equal to '{value}' is not finite. ", RuntimeWarning) + warnings.warn( + f"{name} value in dataset equal to '{value}' is not finite. ", RuntimeWarning + ) return lat = lat_lon_list[0] @@ -215,13 +217,17 @@ def set_xrng(self, xrng, subplot_index=(0,)): # This is to catch that and expand the range so we avoid the warning. if xrng[0] == xrng[1]: if isinstance(xrng[0], np.datetime64): - print(f'\nAttempting to set xlim range to single value {xrng[0]}. ' - 'Expanding range by 2 seconds.\n') + print( + f'\nAttempting to set xlim range to single value {xrng[0]}. ' + 'Expanding range by 2 seconds.\n' + ) xrng[0] -= np.timedelta64(1, 's') xrng[1] += np.timedelta64(1, 's') elif isinstance(xrng[0], dt.datetime): - print(f'\nAttempting to set xlim range to single value {xrng[0]}. ' - 'Expanding range by 2 seconds.\n') + print( + f'\nAttempting to set xlim range to single value {xrng[0]}. ' + 'Expanding range by 2 seconds.\n' + ) xrng[0] -= dt.timedelta(seconds=1) xrng[1] += dt.timedelta(seconds=1) @@ -439,10 +445,22 @@ def plot( if cb_friendly: cmap = 'HomeyerRainbow' - assessment_overplot_category_color['Bad'] = (0.9285714285714286, 0.7130901016453677, 0.7130901016453677) - assessment_overplot_category_color['Incorrect'] = (0.9285714285714286, 0.7130901016453677, 0.7130901016453677) - assessment_overplot_category_color['Not Failing'] = (0.0, 0.4240129715562796, 0.4240129715562796), - assessment_overplot_category_color['Acceptable'] = (0.0, 0.4240129715562796, 0.4240129715562796), + assessment_overplot_category_color['Bad'] = ( + 0.9285714285714286, + 0.7130901016453677, + 0.7130901016453677, + ) + assessment_overplot_category_color['Incorrect'] = ( + 0.9285714285714286, + 0.7130901016453677, + 0.7130901016453677, + ) + assessment_overplot_category_color['Not Failing'] = ( + (0.0, 0.4240129715562796, 0.4240129715562796), + ) + assessment_overplot_category_color['Acceptable'] = ( + (0.0, 0.4240129715562796, 0.4240129715562796), + ) # Get data and dimensions data = self._ds[dsname][field] @@ -641,9 +659,7 @@ def plot( ] ) else: - date_result = search( - r'\d{4}-\d{1,2}-\d{1,2}', self._ds[dsname].time.attrs['units'] - ) + date_result = search(r'\d{4}-\d{1,2}-\d{1,2}', self._ds[dsname].time.attrs['units']) if date_result is not None: set_title = ' '.join([dsname, field, 'on', date_result.group(0)]) else: @@ -1402,8 +1418,8 @@ def qc_flag_block_plot( if cb_friendly: color_lookup['Bad'] = (0.9285714285714286, 0.7130901016453677, 0.7130901016453677) color_lookup['Incorrect'] = (0.9285714285714286, 0.7130901016453677, 0.7130901016453677) - color_lookup['Not Failing'] = (0.0, 0.4240129715562796, 0.4240129715562796), - color_lookup['Acceptable'] = (0.0, 0.4240129715562796, 0.4240129715562796), + color_lookup['Not Failing'] = ((0.0, 0.4240129715562796, 0.4240129715562796),) + color_lookup['Acceptable'] = ((0.0, 0.4240129715562796, 0.4240129715562796),) if assessment_color is not None: for asses, color in assessment_color.items(): @@ -1572,7 +1588,6 @@ def qc_flag_block_plot( ) else: - test_nums = [] for ii, assess in enumerate(flag_assessments): if assess not in color_lookup: @@ -1590,9 +1605,7 @@ def qc_flag_block_plot( # Get test number from flag_mask bitpacked number test_nums.append(parse_bit(flag_masks[ii])) # Get masked array data to use mask for finding if/where test is set - data = self._ds[dsname].qcfilter.get_masked_data( - data_field, rm_tests=test_nums[-1] - ) + data = self._ds[dsname].qcfilter.get_masked_data(data_field, rm_tests=test_nums[-1]) if np.any(data.mask): # Get time ranges from time and masked data barh_list = reduce_time_ranges( diff --git a/act/plotting/windrosedisplay.py b/act/plotting/windrosedisplay.py index 365cc1325f..3c9c6c45ea 100644 --- a/act/plotting/windrosedisplay.py +++ b/act/plotting/windrosedisplay.py @@ -347,10 +347,10 @@ def plot_data( for i, d in enumerate(dir_bins_mid): if i < len(dir_bins_mid) - 1: idx = np.where((dir_data > d) & (dir_data <= dir_bins_mid[i + 1]))[0] - bins.append(d + (dir_bins_mid[i + 1] - d) / 2.) + bins.append(d + (dir_bins_mid[i + 1] - d) / 2.0) else: - idx = np.where((dir_data > d) & (dir_data <= 360.))[0] - bins.append(d + (360. - d) / 2.) + idx = np.where((dir_data > d) & (dir_data <= 360.0))[0] + bins.append(d + (360.0 - d) / 2.0) if plot_type == 'line': if line_plot_calc == 'mean': @@ -392,8 +392,12 @@ def plot_data( ) hist = np.insert(hist, -1, hist[0], axis=0) cplot = self.axes[subplot_index].contourf( - np.deg2rad(xedges), yedges[0:-1], np.transpose(hist), - cmap=cmap, levels=clevels, **kwargs + np.deg2rad(xedges), + yedges[0:-1], + np.transpose(hist), + cmap=cmap, + levels=clevels, + **kwargs, ) plot_type_str = 'Heatmap of' cbar = self.fig.colorbar(cplot, ax=self.axes[subplot_index]) @@ -441,8 +445,13 @@ def plot_data( clevels = np.linspace(vmin, vmax, clevels) cplot = self.axes[subplot_index].contourf( - np.deg2rad(bins), spd_bins, np.transpose(mean_data), - cmap=cmap, levels=clevels, extend='both', **kwargs + np.deg2rad(bins), + spd_bins, + np.transpose(mean_data), + cmap=cmap, + levels=clevels, + extend='both', + **kwargs, ) plot_type_str = 'Mean of' cbar = self.fig.colorbar(cplot, ax=self.axes[subplot_index]) @@ -455,8 +464,8 @@ def plot_data( self.axes[subplot_index].set_theta_direction(-1) # Set Title - sdate = dt_utils.numpy_to_arm_date(self._ds[dsname].time.values[0]), - edate = dt_utils.numpy_to_arm_date(self._ds[dsname].time.values[-1]), + sdate = (dt_utils.numpy_to_arm_date(self._ds[dsname].time.values[0]),) + edate = (dt_utils.numpy_to_arm_date(self._ds[dsname].time.values[-1]),) if sdate == edate: date_str = 'on ' + sdate[0] @@ -468,13 +477,7 @@ def plot_data( units = '' if set_title is None: set_title = ' '.join( - [ - plot_type_str, - data_field + ' (' + units + ')', - 'by\n', - dir_field, - date_str - ] + [plot_type_str, data_field + ' (' + units + ')', 'by\n', dir_field, date_str] ) self.axes[subplot_index].set_title(set_title) plt.tight_layout(h_pad=1.05) diff --git a/act/qc/add_supplemental_qc.py b/act/qc/add_supplemental_qc.py index 33b6741db9..7f565affe7 100644 --- a/act/qc/add_supplemental_qc.py +++ b/act/qc/add_supplemental_qc.py @@ -1,8 +1,9 @@ -import yaml -import numpy as np +from os import environ from pathlib import Path + +import numpy as np +import yaml from dateutil import parser -from os import environ # Example of the YAML file and how to construct. # The times are set as inclusive start to inclusive end time. @@ -66,9 +67,8 @@ def read_yaml_supplemental_qc( datetime64=True, time_delim=(';', ',', '|', r'\t'), none_if_empty=True, - quiet=False + quiet=False, ): - """ Returns a dictionary converstion of YAML file for flagging data. The dictionary will contain variable names as first key, assessents as second keys containing @@ -136,7 +136,8 @@ def read_yaml_supplemental_qc( except KeyError: raise RuntimeError( 'Unable to determine datastream name from Dataset. Need to set global attribute ' - '_datastream in Dataset or provided full path to flag file.') + '_datastream in Dataset or provided full path to flag file.' + ) flag_file = list(Path(fullpath).glob(f'{datastream}.yml')) flag_file.extend(list(Path(fullpath).glob(f'{datastream}.yaml'))) @@ -164,7 +165,7 @@ def read_yaml_supplemental_qc( assessments = [ii.capitalize() for ii in assessments] # Read YAML file - with open(flag_file, "r") as fp: + with open(flag_file) as fp: try: data_dict = yaml.load(fp, Loader=yaml.FullLoader) except AttributeError: @@ -230,9 +231,8 @@ def apply_supplemental_qc( assessments=None, apply_all=True, exclude_all_variables=None, - quiet=False + quiet=False, ): - """ Apply flagging from supplemental QC file by adding new QC tests. @@ -284,7 +284,8 @@ def apply_supplemental_qc( exclude_vars.extend(exclude_all_variables) flag_dict = read_yaml_supplemental_qc( - ds, fullpath, variables=variables, assessments=assessments, quiet=quiet) + ds, fullpath, variables=variables, assessments=assessments, quiet=quiet + ) if flag_dict is None: return @@ -301,7 +302,8 @@ def apply_supplemental_qc( indexes = np.array([], dtype=np.int32) for vals in times: ind = np.argwhere( - (ds['time'].values >= vals[0]) & (ds['time'].values <= vals[1])) + (ds['time'].values >= vals[0]) & (ds['time'].values <= vals[1]) + ) if len(ind) > 0: indexes = np.append(indexes, ind) @@ -311,7 +313,8 @@ def apply_supplemental_qc( var_name, index=indexes, test_meaning=description, - test_assessment=asses_name) + test_assessment=asses_name, + ) var_name = '_all' if apply_all and var_name in flag_dict.keys(): @@ -325,7 +328,8 @@ def apply_supplemental_qc( indexes = np.array([], dtype=np.int32) for vals in times: ind = np.argwhere( - (ds['time'].values >= vals[0]) & (ds['time'].values <= vals[1])) + (ds['time'].values >= vals[0]) & (ds['time'].values <= vals[1]) + ) if ind.size > 0: indexes = np.append(indexes, np.ndarray.flatten(ind)) @@ -347,4 +351,5 @@ def apply_supplemental_qc( all_var_name, index=indexes, test_meaning=description, - test_assessment=asses_name) + test_assessment=asses_name, + ) diff --git a/act/qc/arm.py b/act/qc/arm.py index 0fb84597d4..82272456d6 100644 --- a/act/qc/arm.py +++ b/act/qc/arm.py @@ -5,9 +5,10 @@ """ import datetime as dt +import json + import numpy as np import requests -import json from act.config import DEFAULT_DATASTREAM_NAME @@ -93,8 +94,10 @@ def add_dqr_to_qc( raise ValueError('Dataset does not have datastream attribute') if datastream == DEFAULT_DATASTREAM_NAME: - raise ValueError("'datastream' name required for DQR service set to default value " - f"{datastream}. Unable to perform DQR service query.") + raise ValueError( + "'datastream' name required for DQR service set to default value " + f'{datastream}. Unable to perform DQR service query.' + ) # Clean up QC to conform to CF conventions if cleanup_qc: @@ -113,9 +116,9 @@ def add_dqr_to_qc( # Create URL url = 'https://dqr-web-service.svcs.arm.gov/dqr_full' - url += f"/{datastream}" - url += f"/{start_date}/{end_date}" - url += f"/{assessment}" + url += f'/{datastream}' + url += f'/{start_date}/{end_date}' + url += f'/{assessment}' # Call web service req = requests.get(url) @@ -163,8 +166,10 @@ def add_dqr_to_qc( } if dqr_link: - print(f"{dqr_number} - {quality_category.lower().capitalize()}: " - f"https://adc.arm.gov/ArchiveServices/DQRService?dqrid={dqr_number}") + print( + f'{dqr_number} - {quality_category.lower().capitalize()}: ' + f'https://adc.arm.gov/ArchiveServices/DQRService?dqrid={dqr_number}' + ) # Check to ensure variable is list if variable and not isinstance(variable, (list, tuple)): @@ -173,7 +178,6 @@ def add_dqr_to_qc( loc_vars = ['lat', 'lon', 'alt', 'latitude', 'longitude', 'altitude'] for key, value in dqr_results.items(): for var_name in value['variables']: - # Do not process on location variables if skip_location_vars and var_name in loc_vars: continue @@ -187,7 +191,8 @@ def add_dqr_to_qc( var_name, index=np.unique(value['index']), test_meaning=value['test_meaning'], - test_assessment=value['test_assessment']) + test_assessment=value['test_assessment'], + ) except KeyError: # Variable name not in Dataset continue diff --git a/act/qc/bsrn_tests.py b/act/qc/bsrn_tests.py index 431fefdcf3..ba6bd3fe4f 100644 --- a/act/qc/bsrn_tests.py +++ b/act/qc/bsrn_tests.py @@ -7,12 +7,13 @@ """ import warnings -import numpy as np + import dask.array as da +import numpy as np from scipy.constants import Stefan_Boltzmann -from act.utils.geo_utils import get_solar_azimuth_elevation from act.utils.data_utils import convert_units +from act.utils.geo_utils import get_solar_azimuth_elevation def _calculate_solar_parameters(ds, lat_name, lon_name, solar_constant): @@ -45,11 +46,12 @@ def _calculate_solar_parameters(ds, lat_name, lon_name, solar_constant): # Calculate solar parameters elevation, _, solar_distance = get_solar_azimuth_elevation( - latitude=latitude, longitude=longitude, time=ds['time'].values) + latitude=latitude, longitude=longitude, time=ds['time'].values + ) solar_distance = np.nanmean(solar_distance) Sa = solar_constant / solar_distance**2 - sza = 90. - elevation + sza = 90.0 - elevation return (sza, Sa) @@ -117,9 +119,8 @@ def bsrn_limits_test( solar_constant=1366, lat_name='lat', lon_name='lon', - use_dask=False + use_dask=False, ): - """ Method to apply BSRN limits test and add results to ancillary quality control variable. Need to provide variable name for each measurement for the test to be performed. If no @@ -184,75 +185,92 @@ def bsrn_limits_test( glb_LW_up_name='up_long_hemisp') """ - test_names_org = ["Physically Possible", "Extremely Rare"] + test_names_org = ['Physically Possible', 'Extremely Rare'] test = test.lower() test_names = [ii.lower() for ii in test_names_org] if test not in test_names: - raise ValueError(f"Value of '{test}' in keyword 'test' not recognized. " - f"Must a single value in options {test_names_org}") + raise ValueError( + f"Value of '{test}' in keyword 'test' not recognized. " + f'Must a single value in options {test_names_org}' + ) sza, Sa = _calculate_solar_parameters(self._ds, lat_name, lon_name, solar_constant) if test == test_names[0]: if sw_min_limit is None: - sw_min_limit = -4. + sw_min_limit = -4.0 if lw_min_dn_limit is None: - lw_min_dn_limit = 40. + lw_min_dn_limit = 40.0 if lw_min_up_limit is None: - lw_min_up_limit = 40. + lw_min_up_limit = 40.0 if lw_max_dn_limit is None: - lw_max_dn_limit = 700. + lw_max_dn_limit = 700.0 if lw_max_up_limit is None: - lw_max_up_limit = 900. + lw_max_up_limit = 900.0 elif test == test_names[1]: if sw_min_limit is None: - sw_min_limit = -2. + sw_min_limit = -2.0 if lw_min_dn_limit is None: - lw_min_dn_limit = 60. + lw_min_dn_limit = 60.0 if lw_min_up_limit is None: - lw_min_up_limit = 60. + lw_min_up_limit = 60.0 if lw_max_dn_limit is None: - lw_max_dn_limit = 500. + lw_max_dn_limit = 500.0 if lw_max_up_limit is None: - lw_max_up_limit = 700. + lw_max_up_limit = 700.0 # Global Shortwave downwelling min and max tests if gbl_SW_dn_name is not None: cos_sza = np.cos(np.radians(sza)) - cos_sza[sza > 90.] = 0. + cos_sza[sza > 90.0] = 0.0 if test == test_names[0]: - sw_max_limit = Sa * 1.5 * cos_sza**1.2 + 100. + sw_max_limit = Sa * 1.5 * cos_sza**1.2 + 100.0 elif test == test_names[1]: - sw_max_limit = Sa * 1.2 * cos_sza**1.2 + 50. + sw_max_limit = Sa * 1.2 * cos_sza**1.2 + 50.0 - index_min, index_max = _find_indexes(self._ds, gbl_SW_dn_name, sw_min_limit, sw_max_limit, use_dask) + index_min, index_max = _find_indexes( + self._ds, gbl_SW_dn_name, sw_min_limit, sw_max_limit, use_dask + ) self._ds.qcfilter.add_test( - gbl_SW_dn_name, index=index_min, test_assessment='Bad', - test_meaning=f"Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2") + gbl_SW_dn_name, + index=index_min, + test_assessment='Bad', + test_meaning=f'Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2', + ) self._ds.qcfilter.add_test( - gbl_SW_dn_name, index=index_max, test_assessment='Bad', - test_meaning=f"Value greater than BSRN {test.lower()} limit") + gbl_SW_dn_name, + index=index_max, + test_assessment='Bad', + test_meaning=f'Value greater than BSRN {test.lower()} limit', + ) # Diffuse Shortwave downwelling min and max tests if glb_diffuse_SW_dn_name is not None: with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=RuntimeWarning) if test == test_names[0]: - sw_max_limit = Sa * 0.95 * np.cos(np.radians(sza))**1.2 + 50. + sw_max_limit = Sa * 0.95 * np.cos(np.radians(sza)) ** 1.2 + 50.0 elif test == test_names[1]: - sw_max_limit = Sa * 0.75 * np.cos(np.radians(sza))**1.2 + 30. + sw_max_limit = Sa * 0.75 * np.cos(np.radians(sza)) ** 1.2 + 30.0 - index_min, index_max = _find_indexes(self._ds, glb_diffuse_SW_dn_name, sw_min_limit, - sw_max_limit, use_dask) + index_min, index_max = _find_indexes( + self._ds, glb_diffuse_SW_dn_name, sw_min_limit, sw_max_limit, use_dask + ) self._ds.qcfilter.add_test( - glb_diffuse_SW_dn_name, index=index_min, test_assessment='Bad', - test_meaning=f"Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2") + glb_diffuse_SW_dn_name, + index=index_min, + test_assessment='Bad', + test_meaning=f'Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2', + ) self._ds.qcfilter.add_test( - glb_diffuse_SW_dn_name, index=index_max, test_assessment='Bad', - test_meaning=f"Value greater than BSRN {test.lower()} limit") + glb_diffuse_SW_dn_name, + index=index_max, + test_assessment='Bad', + test_meaning=f'Value greater than BSRN {test.lower()} limit', + ) # Direct Normal Shortwave downwelling min and max tests if direct_normal_SW_dn_name is not None: @@ -261,17 +279,24 @@ def bsrn_limits_test( if test == test_names[0]: sw_max_limit = Sa elif test == test_names[1]: - sw_max_limit = Sa * 0.95 * np.cos(np.radians(sza))**0.2 + 10. + sw_max_limit = Sa * 0.95 * np.cos(np.radians(sza)) ** 0.2 + 10.0 - index_min, index_max = _find_indexes(self._ds, direct_normal_SW_dn_name, - sw_min_limit, sw_max_limit, use_dask) + index_min, index_max = _find_indexes( + self._ds, direct_normal_SW_dn_name, sw_min_limit, sw_max_limit, use_dask + ) self._ds.qcfilter.add_test( - direct_normal_SW_dn_name, index=index_min, test_assessment='Bad', - test_meaning=f"Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2") + direct_normal_SW_dn_name, + index=index_min, + test_assessment='Bad', + test_meaning=f'Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2', + ) self._ds.qcfilter.add_test( - direct_normal_SW_dn_name, index=index_max, test_assessment='Bad', - test_meaning=f"Value greater than BSRN {test.lower()} limit") + direct_normal_SW_dn_name, + index=index_max, + test_assessment='Bad', + test_meaning=f'Value greater than BSRN {test.lower()} limit', + ) # Direct Shortwave downwelling min and max tests if direct_SW_dn_name is not None: @@ -280,64 +305,92 @@ def bsrn_limits_test( if test == test_names[0]: sw_max_limit = Sa * np.cos(np.radians(sza)) elif test == test_names[1]: - sw_max_limit = Sa * 0.95 * np.cos(np.radians(sza))**1.2 + 10 + sw_max_limit = Sa * 0.95 * np.cos(np.radians(sza)) ** 1.2 + 10 - index_min, index_max = _find_indexes(self._ds, direct_SW_dn_name, - sw_min_limit, sw_max_limit, use_dask) + index_min, index_max = _find_indexes( + self._ds, direct_SW_dn_name, sw_min_limit, sw_max_limit, use_dask + ) self._ds.qcfilter.add_test( - direct_SW_dn_name, index=index_min, test_assessment='Bad', - test_meaning=f"Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2") + direct_SW_dn_name, + index=index_min, + test_assessment='Bad', + test_meaning=f'Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2', + ) self._ds.qcfilter.add_test( - direct_SW_dn_name, index=index_max, test_assessment='Bad', - test_meaning=f"Value greater than BSRN {test.lower()} limit") + direct_SW_dn_name, + index=index_max, + test_assessment='Bad', + test_meaning=f'Value greater than BSRN {test.lower()} limit', + ) # Shortwave up welling min and max tests if glb_SW_up_name is not None: with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=RuntimeWarning) if test == test_names[0]: - sw_max_limit = Sa * 1.2 * np.cos(np.radians(sza))**1.2 + 50 + sw_max_limit = Sa * 1.2 * np.cos(np.radians(sza)) ** 1.2 + 50 elif test == test_names[1]: - sw_max_limit = Sa * np.cos(np.radians(sza))**1.2 + 50 + sw_max_limit = Sa * np.cos(np.radians(sza)) ** 1.2 + 50 - index_min, index_max = _find_indexes(self._ds, glb_SW_up_name, - sw_min_limit, sw_max_limit, use_dask) + index_min, index_max = _find_indexes( + self._ds, glb_SW_up_name, sw_min_limit, sw_max_limit, use_dask + ) self._ds.qcfilter.add_test( - glb_SW_up_name, index=index_min, test_assessment='Bad', - test_meaning=f"Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2") + glb_SW_up_name, + index=index_min, + test_assessment='Bad', + test_meaning=f'Value less than BSRN {test.lower()} limit of {sw_min_limit} W/m^2', + ) self._ds.qcfilter.add_test( - glb_SW_up_name, index=index_max, test_assessment='Bad', - test_meaning=f"Value greater than BSRN {test.lower()} limit") + glb_SW_up_name, + index=index_max, + test_assessment='Bad', + test_meaning=f'Value greater than BSRN {test.lower()} limit', + ) # Longwave downwelling min and max tests if glb_LW_dn_name is not None: - index_min, index_max = _find_indexes(self._ds, glb_LW_dn_name, - lw_min_dn_limit, lw_max_dn_limit, use_dask) + index_min, index_max = _find_indexes( + self._ds, glb_LW_dn_name, lw_min_dn_limit, lw_max_dn_limit, use_dask + ) self._ds.qcfilter.add_test( - glb_LW_dn_name, index=index_min, test_assessment='Bad', - test_meaning=f"Value less than BSRN {test.lower()} limit of {lw_min_dn_limit} W/m^2") + glb_LW_dn_name, + index=index_min, + test_assessment='Bad', + test_meaning=f'Value less than BSRN {test.lower()} limit of {lw_min_dn_limit} W/m^2', + ) self._ds.qcfilter.add_test( - glb_LW_dn_name, index=index_max, test_assessment='Bad', - test_meaning=f"Value greater than BSRN {test.lower()} limit of {lw_max_dn_limit} W/m^2") + glb_LW_dn_name, + index=index_max, + test_assessment='Bad', + test_meaning=f'Value greater than BSRN {test.lower()} limit of {lw_max_dn_limit} W/m^2', + ) # Longwave upwelling min and max tests if glb_LW_up_name is not None: - index_min, index_max = _find_indexes(self._ds, glb_LW_up_name, - lw_min_up_limit, lw_max_up_limit, use_dask) + index_min, index_max = _find_indexes( + self._ds, glb_LW_up_name, lw_min_up_limit, lw_max_up_limit, use_dask + ) self._ds.qcfilter.add_test( - glb_LW_up_name, index=index_min, test_assessment='Bad', - test_meaning=f"Value less than BSRN {test.lower()} limit of {lw_min_up_limit} W/m^2") + glb_LW_up_name, + index=index_min, + test_assessment='Bad', + test_meaning=f'Value less than BSRN {test.lower()} limit of {lw_min_up_limit} W/m^2', + ) self._ds.qcfilter.add_test( - glb_LW_up_name, index=index_max, test_assessment='Bad', - test_meaning=f"Value greater than BSRN {test.lower()} limit of {lw_max_up_limit} W/m^2") + glb_LW_up_name, + index=index_max, + test_assessment='Bad', + test_meaning=f'Value greater than BSRN {test.lower()} limit of {lw_max_up_limit} W/m^2', + ) def bsrn_comparison_tests( self, @@ -352,9 +405,9 @@ def bsrn_comparison_tests( test_assessment='Indeterminate', lat_name='lat', lon_name='lon', - LWdn_lt_LWup_component=25., - LWdn_gt_LWup_component=300., - use_dask=False + LWdn_lt_LWup_component=25.0, + LWdn_gt_LWup_component=300.0, + use_dask=False, ): """ Method to apply BSRN comparison tests and add results to ancillary quality control variable. @@ -418,23 +471,36 @@ def bsrn_comparison_tests( if isinstance(test, str): test = [test] - test_options = ['Global over Sum SW Ratio', 'Diffuse Ratio', 'SW up', 'LW down to air temp', - 'LW up to air temp', 'LW down to LW up'] + test_options = [ + 'Global over Sum SW Ratio', + 'Diffuse Ratio', + 'SW up', + 'LW down to air temp', + 'LW up to air temp', + 'LW down to LW up', + ] solar_constant = 1360.8 sza, Sa = _calculate_solar_parameters(self._ds, lat_name, lon_name, solar_constant) # Ratio of Global over Sum SW if test_options[0] in test: - if gbl_SW_dn_name is None or glb_diffuse_SW_dn_name is None or direct_normal_SW_dn_name is None: - raise ValueError('Must set keywords gbl_SW_dn_name, glb_diffuse_SW_dn_name, ' - f'direct_normal_SW_dn_name for {test_options[0]} test.') + if ( + gbl_SW_dn_name is None + or glb_diffuse_SW_dn_name is None + or direct_normal_SW_dn_name is None + ): + raise ValueError( + 'Must set keywords gbl_SW_dn_name, glb_diffuse_SW_dn_name, ' + f'direct_normal_SW_dn_name for {test_options[0]} test.' + ) with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=RuntimeWarning) if use_dask and isinstance(self._ds[glb_diffuse_SW_dn_name].data, da.Array): - sum_sw_down = (self._ds[glb_diffuse_SW_dn_name].data - + self._ds[direct_normal_SW_dn_name].data * np.cos(np.radians(sza))) + sum_sw_down = self._ds[glb_diffuse_SW_dn_name].data + self._ds[ + direct_normal_SW_dn_name + ].data * np.cos(np.radians(sza)) sum_sw_down[sum_sw_down < 50] = np.nan ratio = self._ds[gbl_SW_dn_name].data / sum_sw_down index_a = sza < 75 @@ -445,8 +511,9 @@ def bsrn_comparison_tests( index_4 = da.where((ratio < 0.85) & index_b, True, False) index = (index_1 | index_2 | index_3 | index_4).compute() else: - sum_sw_down = (self._ds[glb_diffuse_SW_dn_name].values - + self._ds[direct_normal_SW_dn_name].values * np.cos(np.radians(sza))) + sum_sw_down = self._ds[glb_diffuse_SW_dn_name].values + self._ds[ + direct_normal_SW_dn_name + ].values * np.cos(np.radians(sza)) sum_sw_down[sum_sw_down < 50] = np.nan ratio = self._ds[gbl_SW_dn_name].values / sum_sw_down index_a = sza < 75 @@ -457,19 +524,33 @@ def bsrn_comparison_tests( index_4 = (ratio < 0.85) & index_b index = index_1 | index_2 | index_3 | index_4 - test_meaning = "Ratio of Global over Sum shortwave larger than expected" - self._ds.qcfilter.add_test(gbl_SW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) - self._ds.qcfilter.add_test(glb_diffuse_SW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) - self._ds.qcfilter.add_test(direct_normal_SW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) + test_meaning = 'Ratio of Global over Sum shortwave larger than expected' + self._ds.qcfilter.add_test( + gbl_SW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) + self._ds.qcfilter.add_test( + glb_diffuse_SW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) + self._ds.qcfilter.add_test( + direct_normal_SW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) # Diffuse Ratio if test_options[1] in test: if gbl_SW_dn_name is None or glb_diffuse_SW_dn_name is None: - raise ValueError('Must set keywords gbl_SW_dn_name, glb_diffuse_SW_dn_name ' - f'for {test_options[1]} test.') + raise ValueError( + 'Must set keywords gbl_SW_dn_name, glb_diffuse_SW_dn_name ' + f'for {test_options[1]} test.' + ) with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=RuntimeWarning) @@ -482,7 +563,9 @@ def bsrn_comparison_tests( index_2 = da.where((ratio >= 1.10) & index_b, True, False) index = (index_1 | index_2).compute() else: - ratio = self._ds[glb_diffuse_SW_dn_name].values / self._ds[gbl_SW_dn_name].values + ratio = ( + self._ds[glb_diffuse_SW_dn_name].values / self._ds[gbl_SW_dn_name].values + ) ratio[self._ds[gbl_SW_dn_name].values < 50] = np.nan index_a = sza < 75 index_1 = (ratio >= 1.05) & index_a @@ -490,105 +573,181 @@ def bsrn_comparison_tests( index_2 = (ratio >= 1.10) & index_b index = index_1 | index_2 - test_meaning = "Ratio of Diffuse Shortwave over Global Shortwave larger than expected" - self._ds.qcfilter.add_test(gbl_SW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) - self._ds.qcfilter.add_test(glb_diffuse_SW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) + test_meaning = 'Ratio of Diffuse Shortwave over Global Shortwave larger than expected' + self._ds.qcfilter.add_test( + gbl_SW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) + self._ds.qcfilter.add_test( + glb_diffuse_SW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) # Shortwave up comparison if test_options[2] in test: - if glb_SW_up_name is None or glb_diffuse_SW_dn_name is None or direct_normal_SW_dn_name is None: - raise ValueError('Must set keywords glb_SW_up_name, glb_diffuse_SW_dn_name, ' - f'direct_normal_SW_dn_name for {test_options[2]} test.') + if ( + glb_SW_up_name is None + or glb_diffuse_SW_dn_name is None + or direct_normal_SW_dn_name is None + ): + raise ValueError( + 'Must set keywords glb_SW_up_name, glb_diffuse_SW_dn_name, ' + f'direct_normal_SW_dn_name for {test_options[2]} test.' + ) with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=RuntimeWarning) if use_dask and isinstance(self._ds[glb_diffuse_SW_dn_name].data, da.Array): - sum_sw_down = (self._ds[glb_diffuse_SW_dn_name].data - + self._ds[direct_normal_SW_dn_name].data * np.cos(np.radians(sza))) + sum_sw_down = self._ds[glb_diffuse_SW_dn_name].data + self._ds[ + direct_normal_SW_dn_name + ].data * np.cos(np.radians(sza)) sum_sw_down[sum_sw_down < 50] = np.nan - index = da.where(self._ds[glb_SW_up_name].data > sum_sw_down, True, False).compute() + index = da.where( + self._ds[glb_SW_up_name].data > sum_sw_down, True, False + ).compute() else: - sum_sw_down = (self._ds[glb_diffuse_SW_dn_name].values - + self._ds[direct_normal_SW_dn_name].values * np.cos(np.radians(sza))) + sum_sw_down = self._ds[glb_diffuse_SW_dn_name].values + self._ds[ + direct_normal_SW_dn_name + ].values * np.cos(np.radians(sza)) sum_sw_down[sum_sw_down < 50] = np.nan index = self._ds[glb_SW_up_name].values > sum_sw_down - test_meaning = "Ratio of Shortwave Upwelling greater than Shortwave Sum" - self._ds.qcfilter.add_test(glb_SW_up_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) - self._ds.qcfilter.add_test(glb_diffuse_SW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) - self._ds.qcfilter.add_test(direct_normal_SW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) + test_meaning = 'Ratio of Shortwave Upwelling greater than Shortwave Sum' + self._ds.qcfilter.add_test( + glb_SW_up_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) + self._ds.qcfilter.add_test( + glb_diffuse_SW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) + self._ds.qcfilter.add_test( + direct_normal_SW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) # Longwave down to air temperature comparison if test_options[3] in test: if glb_LW_dn_name is None or air_temp_name is None: - raise ValueError('Must set keywords glb_LW_dn_name, air_temp_name ' - f' for {test_options[3]} test.') - - air_temp = convert_units(self._ds[air_temp_name].values, - self._ds[air_temp_name].attrs['units'], 'degK') + raise ValueError( + 'Must set keywords glb_LW_dn_name, air_temp_name ' + f' for {test_options[3]} test.' + ) + + air_temp = convert_units( + self._ds[air_temp_name].values, self._ds[air_temp_name].attrs['units'], 'degK' + ) if use_dask and isinstance(self._ds[glb_LW_dn_name].data, da.Array): air_temp = da.array(air_temp) conversion = da.array(Stefan_Boltzmann * air_temp**4) index_1 = (0.4 * conversion) > self._ds[glb_LW_dn_name].data - index_2 = (conversion + 25.) < self._ds[glb_LW_dn_name].data + index_2 = (conversion + 25.0) < self._ds[glb_LW_dn_name].data index = (index_1 | index_2).compute() else: conversion = Stefan_Boltzmann * air_temp**4 index_1 = (0.4 * conversion) > self._ds[glb_LW_dn_name].values - index_2 = (conversion + 25.) < self._ds[glb_LW_dn_name].values + index_2 = (conversion + 25.0) < self._ds[glb_LW_dn_name].values index = index_1 | index_2 - test_meaning = "Longwave downwelling comparison to air temperature out side of expected range" - self._ds.qcfilter.add_test(glb_LW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) + test_meaning = ( + 'Longwave downwelling comparison to air temperature out side of expected range' + ) + self._ds.qcfilter.add_test( + glb_LW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) # Longwave up to air temperature comparison if test_options[4] in test: if glb_LW_up_name is None or air_temp_name is None: - raise ValueError('Must set keywords glb_LW_up_name, air_temp_name ' - f'for {test_options[3]} test.') - - air_temp = convert_units(self._ds[air_temp_name].values, - self._ds[air_temp_name].attrs['units'], 'degK') + raise ValueError( + 'Must set keywords glb_LW_up_name, air_temp_name ' + f'for {test_options[3]} test.' + ) + + air_temp = convert_units( + self._ds[air_temp_name].values, self._ds[air_temp_name].attrs['units'], 'degK' + ) if use_dask and isinstance(self._ds[glb_LW_up_name].data, da.Array): air_temp = da.array(air_temp) - index_1 = (Stefan_Boltzmann * (air_temp - 15)**4) > self._ds[glb_LW_up_name].data - index_2 = (Stefan_Boltzmann * (air_temp + 25)**4) < self._ds[glb_LW_up_name].data + index_1 = (Stefan_Boltzmann * (air_temp - 15) ** 4) > self._ds[glb_LW_up_name].data + index_2 = (Stefan_Boltzmann * (air_temp + 25) ** 4) < self._ds[glb_LW_up_name].data index = (index_1 | index_2).compute() else: - index_1 = (Stefan_Boltzmann * (air_temp - 15)**4) > self._ds[glb_LW_up_name].values - index_2 = (Stefan_Boltzmann * (air_temp + 25)**4) < self._ds[glb_LW_up_name].values + index_1 = (Stefan_Boltzmann * (air_temp - 15) ** 4) > self._ds[ + glb_LW_up_name + ].values + index_2 = (Stefan_Boltzmann * (air_temp + 25) ** 4) < self._ds[ + glb_LW_up_name + ].values index = index_1 | index_2 - test_meaning = "Longwave upwelling comparison to air temperature out side of expected range" - self._ds.qcfilter.add_test(glb_LW_up_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) + test_meaning = ( + 'Longwave upwelling comparison to air temperature out side of expected range' + ) + self._ds.qcfilter.add_test( + glb_LW_up_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) # Lonwave down to longwave up comparison if test_options[5] in test: if glb_LW_dn_name is None or glb_LW_up_name is None: - raise ValueError('Must set keywords glb_LW_dn_name, glb_LW_up_name ' - f'for {test_options[3]} test.') + raise ValueError( + 'Must set keywords glb_LW_dn_name, glb_LW_up_name ' + f'for {test_options[3]} test.' + ) if use_dask and isinstance(self._ds[glb_LW_dn_name].data, da.Array): - index_1 = da.where(self._ds[glb_LW_dn_name].data - > (self._ds[glb_LW_up_name].data + LWdn_lt_LWup_component), True, False) - index_2 = da.where(self._ds[glb_LW_dn_name].data - < (self._ds[glb_LW_up_name].data - LWdn_gt_LWup_component), True, False) + index_1 = da.where( + self._ds[glb_LW_dn_name].data + > (self._ds[glb_LW_up_name].data + LWdn_lt_LWup_component), + True, + False, + ) + index_2 = da.where( + self._ds[glb_LW_dn_name].data + < (self._ds[glb_LW_up_name].data - LWdn_gt_LWup_component), + True, + False, + ) index = (index_1 | index_2).compute() else: - index_1 = self._ds[glb_LW_dn_name].values > (self._ds[glb_LW_up_name].values + LWdn_lt_LWup_component) - index_2 = self._ds[glb_LW_dn_name].values < (self._ds[glb_LW_up_name].values - LWdn_gt_LWup_component) + index_1 = self._ds[glb_LW_dn_name].values > ( + self._ds[glb_LW_up_name].values + LWdn_lt_LWup_component + ) + index_2 = self._ds[glb_LW_dn_name].values < ( + self._ds[glb_LW_up_name].values - LWdn_gt_LWup_component + ) index = index_1 | index_2 - test_meaning = "Lonwave downwelling compared to longwave upwelling outside of expected range" - self._ds.qcfilter.add_test(glb_LW_dn_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) - self._ds.qcfilter.add_test(glb_LW_up_name, index=index, test_assessment=test_assessment, - test_meaning=test_meaning) + test_meaning = ( + 'Lonwave downwelling compared to longwave upwelling outside of expected range' + ) + self._ds.qcfilter.add_test( + glb_LW_dn_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) + self._ds.qcfilter.add_test( + glb_LW_up_name, + index=index, + test_assessment=test_assessment, + test_meaning=test_meaning, + ) diff --git a/act/qc/clean.py b/act/qc/clean.py index e1a7813e58..2a8523222b 100644 --- a/act/qc/clean.py +++ b/act/qc/clean.py @@ -198,7 +198,6 @@ def handle_missing_values(self, default_missing_value=np.int32(-9999)): np.dtype('float32'), np.dtype('float64'), ]: - # Look at units variable to see if this is the stupid way some # ARM products mix data and state variables. If the units are not # in the normal list of unitless type assume this is a data variable @@ -463,10 +462,8 @@ def clean_arm_state_variables( for var in variables: flag_info = self.get_attr_info(variable=var, flag=integer_flag) if flag_info is not None: - # Add new attributes to variable for attr in ['flag_values', 'flag_meanings', 'flag_masks']: - if len(flag_info[attr]) > 0: # Only add if attribute does not exist. if attr in self._ds[var].attrs.keys() is False: @@ -596,7 +593,7 @@ def clean_arm_qc( clean_units_string=True, correct_valid_min_max=True, remove_unset_global_tests=True, - **kwargs + **kwargs, ): """ Method to clean up Xarray dataset QC variables. @@ -622,7 +619,6 @@ def clean_arm_qc( """ global_qc = self.get_attr_info() for qc_var in self.matched_qc_variables: - # Clean up units attribute from unitless to udunits '1' try: if clean_units_string and self._ds[qc_var].attrs['units'] == 'unitless': @@ -643,7 +639,6 @@ def clean_arm_qc( 'flag_values', 'flag_comments', ]: - if qc_attributes is not None and len(qc_attributes[attr]) > 0: # Only add if attribute does not exists if attr in self._ds[qc_var].attrs.keys() is False: @@ -690,7 +685,6 @@ def clean_arm_qc( flag_masks = self._ds[qc_var_name].attrs['flag_masks'] tests_to_remove = [] for ii, flag_meaning in enumerate(flag_meanings): - # Loop over usual test attribute names looking to see if they # are listed in test description. If so use that name for look up. test_attribute_limit_name = None @@ -731,7 +725,6 @@ def normalize_assessment( exclude_variables=None, qc_lookup={'Incorrect': 'Bad', 'Suspect': 'Indeterminate'}, ): - """ Method to clean up assessment terms used to be consistent between embedded QC and DQRs. diff --git a/act/qc/qcfilter.py b/act/qc/qcfilter.py index 32a762f097..70ae329493 100644 --- a/act/qc/qcfilter.py +++ b/act/qc/qcfilter.py @@ -9,7 +9,7 @@ import numpy as np import xarray as xr -from act.qc import comparison_tests, qctests, bsrn_tests +from act.qc import bsrn_tests, comparison_tests, qctests @xr.register_dataset_accessor('qcfilter') @@ -26,13 +26,7 @@ def __init__(self, ds): """initialize""" self._ds = ds - def check_for_ancillary_qc( - self, - var_name, - add_if_missing=True, - cleanup=False, - flag_type=False - ): + def check_for_ancillary_qc(self, var_name, add_if_missing=True, cleanup=False, flag_type=False): """ Method to check if a quality control variable exist in the dataset and return the quality control varible name. @@ -119,10 +113,7 @@ def check_for_ancillary_qc( return qc_var_name def create_qc_variable( - self, var_name, - flag_type=False, - flag_values_set_value=0, - qc_var_name=None + self, var_name, flag_type=False, flag_values_set_value=0, qc_var_name=None ): """ Method to create a quality control variable in the dataset. @@ -206,9 +197,7 @@ def create_qc_variable( # Update if using flag_values and don't want 0 to be default value. if flag_type and flag_values_set_value != 0: - self._ds[qc_var_name].values = self._ds[qc_var_name].values + int( - flag_values_set_value - ) + self._ds[qc_var_name].values = self._ds[qc_var_name].values + int(flag_values_set_value) # Add requried variable attributes. if flag_type: @@ -260,7 +249,6 @@ def update_ancillary_variable(self, var_name, qc_var_name=None): try: ancillary_variables = self._ds[var_name].attrs['ancillary_variables'] if qc_var_name not in ancillary_variables: - ancillary_variables = ' '.join([ancillary_variables, qc_var_name]) except KeyError: ancillary_variables = qc_var_name @@ -805,7 +793,6 @@ def get_masked_data( ma_fill_value=None, return_inverse=False, ): - """ Returns a numpy masked array containing data and mask or a numpy float array with masked values set to NaN. @@ -1030,25 +1017,25 @@ def datafilter( except KeyError: pass - print(f'No quality control variable for {var_name} found ' - f'in call to .qcfilter.datafilter()') + print( + f'No quality control variable for {var_name} found ' + f'in call to .qcfilter.datafilter()' + ) continue # Need to return data as Numpy array with NaN values. Setting the Dask array # to Numpy masked array does not work with other tools. data = self.get_masked_data( - var_name, - rm_assessments=rm_assessments, - rm_tests=rm_tests, - return_nan_array=True + var_name, rm_assessments=rm_assessments, rm_tests=rm_tests, return_nan_array=True ) # If data was orginally stored as Dask array return values to Dataset as Dask array # else set as Numpy array. try: self._ds[var_name].data = dask.array.from_array( - data, chunks=self._ds[var_name].data.chunksize) + data, chunks=self._ds[var_name].data.chunksize + ) except AttributeError: self._ds[var_name].values = data @@ -1239,12 +1226,12 @@ def parse_bit(qc_bit): raise ValueError('Must be a positive integer.') # Convert integer value to single element numpy array of type unsigned integer 64 - value = np.array([qc_bit]).astype(">u8") + value = np.array([qc_bit]).astype('>u8') # Convert value to view containing only unsigned integer 8 data type. This # is required for the numpy unpackbits function which only works with # unsigned integer 8 bit data type. - value = value.view("u1") + value = value.view('u1') # Unpack bits using numpy into array of 1 where bit is set and convert into boolean array index = np.unpackbits(value).astype(bool) diff --git a/act/qc/qctests.py b/act/qc/qctests.py index 45bfb7d179..a835a10148 100644 --- a/act/qc/qctests.py +++ b/act/qc/qctests.py @@ -9,11 +9,11 @@ import warnings import dask.array as da -from metpy.units import units -from metpy.calc import add_height_to_pressure import numpy as np import pandas as pd import xarray as xr +from metpy.calc import add_height_to_pressure +from metpy.units import units from act.utils.data_utils import convert_units, get_missing_value @@ -1353,8 +1353,7 @@ def add_iqr_test( from scikit_posthocs import outliers_iqr except ImportError: raise ImportError( - 'scikit_posthocs needs to be installed on your system to ' - 'run add_iqr_test.' + 'scikit_posthocs needs to be installed on your system to ' 'run add_iqr_test.' ) if test_meaning is None: @@ -1452,8 +1451,7 @@ def add_gesd_test( from scikit_posthocs import outliers_gesd except ImportError: raise ImportError( - 'scikit_posthocs needs to be installed on your system to ' - 'run add_gesd_test.' + 'scikit_posthocs needs to be installed on your system to ' 'run add_gesd_test.' ) if test_meaning is None: @@ -1510,7 +1508,7 @@ def add_atmospheric_pressure_test( test_number=None, flag_value=False, prepend_text=None, - use_dask=False + use_dask=False, ): """ Method to perform a limit test on atmospheric pressure data using @@ -1592,8 +1590,10 @@ def add_atmospheric_pressure_test( upper_limit = upper_limit.magnitude if test_meaning is None: - test_meaning = ('Value outside of atmospheric pressure range test range: ' - f'{round(lower_limit, 2)} to {round(upper_limit, 2)} {data_units}') + test_meaning = ( + 'Value outside of atmospheric pressure range test range: ' + f'{round(lower_limit, 2)} to {round(upper_limit, 2)} {data_units}' + ) if prepend_text is not None: test_meaning = ': '.join((prepend_text, test_meaning)) @@ -1605,7 +1605,9 @@ def add_atmospheric_pressure_test( index2 = da.where(self._ds[var_name].data > upper_limit, True, False) index = (index1 | index2).compute() else: - index = (self._ds[var_name].values > upper_limit) | (self._ds[var_name].values < lower_limit) + index = (self._ds[var_name].values > upper_limit) | ( + self._ds[var_name].values < lower_limit + ) result = self._ds.qcfilter.add_test( var_name, diff --git a/act/qc/sp2.py b/act/qc/sp2.py index 9a67123e32..526405408e 100644 --- a/act/qc/sp2.py +++ b/act/qc/sp2.py @@ -119,7 +119,8 @@ def __init__(self): 'Attempting to use SP2ParticleCriteria without' 'PySP2 installed. SP2ParticleCriteria will' 'not have any functionality besides this' - 'warning message.', RuntimeWarning + 'warning message.', + RuntimeWarning, ) diff --git a/act/retrievals/cbh.py b/act/retrievals/cbh.py index dd7aa0c04f..f68f47e52c 100644 --- a/act/retrievals/cbh.py +++ b/act/retrievals/cbh.py @@ -16,7 +16,7 @@ def generic_sobel_cbh( fill_na=None, return_thresh=False, filter_type='uniform', - edge_thresh=5., + edge_thresh=5.0, ): """ Function for calculating cloud base height from lidar/radar data diff --git a/act/retrievals/doppler_lidar.py b/act/retrievals/doppler_lidar.py index 0e5dd85154..388bdc52a7 100644 --- a/act/retrievals/doppler_lidar.py +++ b/act/retrievals/doppler_lidar.py @@ -3,6 +3,7 @@ """ import warnings + import dask import numpy as np import xarray as xr @@ -132,9 +133,16 @@ def compute_winds_from_ppi( task.append( dask.delayed(process_ppi_winds)( - time[scan_index], elevation[scan_index], azimuth[scan_index], snr[scan_index, :], - doppler[scan_index, :], rng, condition_limit, snr_threshold, remove_all_missing, - height_units + time[scan_index], + elevation[scan_index], + azimuth[scan_index], + snr[scan_index, :], + doppler[scan_index, :], + rng, + condition_limit, + snr_threshold, + remove_all_missing, + height_units, ) ) @@ -144,7 +152,9 @@ def compute_winds_from_ppi( results = [results[ii] for ii, value in enumerate(is_Dataset) if value is True] new_ds = xr.concat(results, 'time') - if isinstance(return_ds, xr.core.dataset.Dataset) and isinstance(new_ds, xr.core.dataset.Dataset): + if isinstance(return_ds, xr.core.dataset.Dataset) and isinstance( + new_ds, xr.core.dataset.Dataset + ): return_ds = xr.concat([return_ds, new_ds], dim='time') else: return_ds = new_ds @@ -152,8 +162,18 @@ def compute_winds_from_ppi( return return_ds -def process_ppi_winds(time, elevation, azimuth, snr, doppler, rng, condition_limit, - snr_threshold, remove_all_missing, height_units): +def process_ppi_winds( + time, + elevation, + azimuth, + snr, + doppler, + rng, + condition_limit, + snr_threshold, + remove_all_missing, + height_units, +): """ This function is for processing the winds using dask from the compute_winds_from_ppi function. This should not be used standalone. @@ -235,9 +255,7 @@ def process_ppi_winds(time, elevation, azimuth, snr, doppler, rng, condition_lim wdir = np.degrees(np.arctan2(u_wind, v_wind) + np.pi) wspd_err = np.sqrt((u_wind * u_err) ** 2 + (v_wind * v_err) ** 2) / wspd - wdir_err = np.degrees( - np.sqrt((u_wind * v_err) ** 2 + (v_wind * u_err) ** 2) / wspd**2 - ) + wdir_err = np.degrees(np.sqrt((u_wind * v_err) ** 2 + (v_wind * u_err) ** 2) / wspd**2) if remove_all_missing and np.isnan(wspd).all(): return np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan diff --git a/act/retrievals/radiation.py b/act/retrievals/radiation.py index 4d1964ebdd..c54e1685df 100644 --- a/act/retrievals/radiation.py +++ b/act/retrievals/radiation.py @@ -59,7 +59,7 @@ def calculate_dsh_from_dsdh_sdn( attrs={ 'long_name': 'Derived Downwelling Shortwave Hemispheric Irradiance', 'units': 'W/m^2', - } + }, ) return ds @@ -144,7 +144,6 @@ def calculate_net_radiation( dlhs='down_long_hemisp_shaded', smooth=None, ): - """ Function to calculate the net radiation from upwelling short and long-wave irradiance and @@ -207,7 +206,6 @@ def calculate_longwave_radiation( emiss_a=0.61, emiss_b=0.06, ): - """ Function to calculate longwave radiation during clear and cloudy sky conditions @@ -249,7 +247,6 @@ def calculate_longwave_radiation( """ if met_ds is not None: - T = met_ds[temperature_var] + 273.15 # C to K e = met_ds[vapor_pressure_var] * 10.0 # kpa to hpa else: diff --git a/act/retrievals/sonde.py b/act/retrievals/sonde.py index 242b7f9abb..f4994a5823 100644 --- a/act/retrievals/sonde.py +++ b/act/retrievals/sonde.py @@ -4,12 +4,13 @@ """ import warnings +from itertools import groupby +from operator import itemgetter + +import metpy.calc as mpcalc import numpy as np import pandas as pd import xarray as xr -from operator import itemgetter -from itertools import groupby -import metpy.calc as mpcalc from metpy.units import units from act.utils.data_utils import convert_to_potential_temp @@ -174,9 +175,7 @@ def calculate_stability_indicies( ds['parcel_temperature'].attrs['units'] = t_profile.units # Calculate CAPE, CIN, LCL - sbcape, sbcin = mpcalc.surface_based_cape_cin(p_sorted, - t_sorted, - td_sorted) + sbcape, sbcin = mpcalc.surface_based_cape_cin(p_sorted, t_sorted, td_sorted) lcl = mpcalc.lcl(p_sorted[0], t_sorted[0], td_sorted[0]) try: @@ -277,8 +276,14 @@ def calculate_pbl_liu_liang( """ # Preprocess the sonde data to ensure the same methods across all retrievals - ds2 = preprocess_sonde_data(ds, temperature=temperature, pressure=pressure, - height=height, smooth_height=smooth_height, base=5.) + ds2 = preprocess_sonde_data( + ds, + temperature=temperature, + pressure=pressure, + height=height, + smooth_height=smooth_height, + base=5.0, + ) pres = ds2[pressure].values wspd = ds2[windspeed].values @@ -417,7 +422,7 @@ def calculate_pbl_heffter( pressure='pres', height='alt', smooth_height=3, - base=5., + base=5.0, ): """ Function for calculating the PBL height from a radiosonde profile @@ -460,8 +465,14 @@ def calculate_pbl_heffter( """ # Preprocess the sonde data to ensure the same methods across all retrievals - ds2 = preprocess_sonde_data(ds, temperature=temperature, pressure=pressure, - height=height, smooth_height=smooth_height, base=base) + ds2 = preprocess_sonde_data( + ds, + temperature=temperature, + pressure=pressure, + height=height, + smooth_height=smooth_height, + base=base, + ) # Get data pres = ds2[pressure].values @@ -496,25 +507,25 @@ def calculate_pbl_heffter( # For each layer, calculate the difference in theta from # top and bottom of the layer. The lowest layer where the # difference is > 2 K is set as the PBL. - pbl = 0. + pbl = 0.0 theta_diff_layer = [] bottom_inversion = [] top_inversion = [] for r in ranges: - if agl[r[1]] > 4000.: + if agl[r[1]] > 4000.0: continue theta_diff = theta[r[1]] - theta[r[0]] theta_diff_layer.append(theta_diff) bottom_inversion.append(alt[r[0]]) top_inversion.append(alt[r[1]]) - if pbl == 0. and theta_diff > 2.0: + if pbl == 0.0 and theta_diff > 2.0: pbl = alt[r[0]] if len(theta_diff_layer) == 0: - pbl = -9999. + pbl = -9999.0 # If PBL is not set, set it to the layer with the max theta diff - if pbl == 0.: + if pbl == 0.0: idx = np.argmax(theta_diff_layer) pbl = bottom_inversion[idx] @@ -536,11 +547,21 @@ def calculate_pbl_heffter( ds['alt_ss'] = da atts = {'units': 'm', 'long_name': 'Bottom height of inversion layers'} - da = xr.DataArray(bottom_inversion, coords={'layers': list(range(len(bottom_inversion)))}, dims=['layers'], attrs=atts) + da = xr.DataArray( + bottom_inversion, + coords={'layers': list(range(len(bottom_inversion)))}, + dims=['layers'], + attrs=atts, + ) ds['bottom_inversion'] = da atts = {'units': 'm', 'long_name': 'Top height of inversion layers'} - da = xr.DataArray(top_inversion, coords={'layers': list(range(len(top_inversion)))}, dims=['layers'], attrs=atts) + da = xr.DataArray( + top_inversion, + coords={'layers': list(range(len(top_inversion)))}, + dims=['layers'], + attrs=atts, + ) ds['top_inversion'] = da return ds @@ -552,7 +573,7 @@ def preprocess_sonde_data( pressure='pres', height='alt', smooth_height=3, - base=5., + base=5.0, ): """ Function for processing the SONDE data for the PBL calculations. @@ -628,7 +649,7 @@ def preprocess_sonde_data( temp = ds2[temperature].values # Perform Pre-processing checks - if len(temp) == 0.: + if len(temp) == 0.0: raise ValueError('No data in profile') if np.nanmax(alt) < 1000.0: diff --git a/act/tests/sample_files.py b/act/tests/sample_files.py index 2193c416f6..1aa1f09658 100644 --- a/act/tests/sample_files.py +++ b/act/tests/sample_files.py @@ -50,7 +50,10 @@ EXAMPLE_STAMP_WILDCARD = os.path.join(DATA_PATH, 'sgpstamp*202001*.nc') EXAMPLE_NOAA_PSL = os.path.join(DATA_PATH, 'ctd21125.15w') EXAMPLE_NOAA_PSL_TEMPERATURE = os.path.join(DATA_PATH, 'ctd22187.00t.txt') -EXAMPLE_NOAA_PSL_SURFACEMET = [os.path.join(DATA_PATH, 'ayp22199.21m'), os.path.join(DATA_PATH, 'ayp22200.00m')] +EXAMPLE_NOAA_PSL_SURFACEMET = [ + os.path.join(DATA_PATH, 'ayp22199.21m'), + os.path.join(DATA_PATH, 'ayp22200.00m'), +] EXAMPLE_SP2B = os.path.join(DATA_PATH, 'mosaossp2M1.00.20191216.130601.raw.20191216x193.sp2b') EXAMPLE_INI = os.path.join(DATA_PATH, 'mosaossp2M1.00.20191216.000601.raw.20191216000000.ini') EXAMPLE_HK = os.path.join(DATA_PATH, 'mosaossp2auxM1.00.20191217.010801.raw.20191216000000.hk') @@ -58,9 +61,16 @@ EXAMPLE_CLOUDPHASE = os.path.join(DATA_PATH, 'nsacloudphaseC1.c1.20180601.000000.nc') EXAMPLE_AAF_ICARTT = os.path.join(DATA_PATH, 'AAFNAV_COR_20181104_R0.ict') EXAMPLE_MMCR = os.path.join(DATA_PATH, 'sgpmmcrC1.b1.*.cdf') -EXAMPLE_NEON = os.path.join(DATA_PATH, 'NEON.D18.BARR.DP1.00002.001.000.010.001.SAAT_1min.2022-10.expanded.20221107T205629Z.csv') -EXAMPLE_NEON_VARIABLE = os.path.join(DATA_PATH, 'NEON.D18.BARR.DP1.00002.001.variables.20221201T110553Z.csv') -EXAMPLE_NEON_POSITION = os.path.join(DATA_PATH, 'NEON.D18.BARR.DP1.00002.001.sensor_positions.20221107T205629Z.csv') +EXAMPLE_NEON = os.path.join( + DATA_PATH, + 'NEON.D18.BARR.DP1.00002.001.000.010.001.SAAT_1min.2022-10.expanded.20221107T205629Z.csv', +) +EXAMPLE_NEON_VARIABLE = os.path.join( + DATA_PATH, 'NEON.D18.BARR.DP1.00002.001.variables.20221201T110553Z.csv' +) +EXAMPLE_NEON_POSITION = os.path.join( + DATA_PATH, 'NEON.D18.BARR.DP1.00002.001.sensor_positions.20221107T205629Z.csv' +) EXAMPLE_DOD = os.path.join(DATA_PATH, 'vdis.b1') EXAMPLE_EBBR1 = os.path.join(DATA_PATH, 'sgp30ebbrE32.b1.20191125.000000.nc') EXAMPLE_EBBR2 = os.path.join(DATA_PATH, 'sgp30ebbrE32.b1.20191130.000000.nc') diff --git a/act/tests/test_correct.py b/act/tests/test_correct.py index a35fbd6b2a..c52b5a6613 100644 --- a/act/tests/test_correct.py +++ b/act/tests/test_correct.py @@ -106,9 +106,7 @@ def test_correct_rl(): ds = act.io.armfiles.read_netcdf(files) ds = act.corrections.raman_lidar.correct_rl(ds, range_normalize_log_values=True) - np.testing.assert_almost_equal( - np.max(ds['depolarization_counts_high'].values), 9.91, decimal=2 - ) + np.testing.assert_almost_equal(np.max(ds['depolarization_counts_high'].values), 9.91, decimal=2) np.testing.assert_almost_equal( np.min(ds['depolarization_counts_high'].values), -7.00, decimal=2 ) diff --git a/act/tests/test_discovery.py b/act/tests/test_discovery.py index 75b06b1b32..dd145d9dd5 100644 --- a/act/tests/test_discovery.py +++ b/act/tests/test_discovery.py @@ -263,13 +263,16 @@ def test_neon(): assert '2022-11' in result output_dir = os.path.join(os.getcwd(), site_code + '_' + product_code) - result = act.discovery.get_neon.download_neon_data(site_code, product_code, '2022-10', output_dir=output_dir) + result = act.discovery.get_neon.download_neon_data( + site_code, product_code, '2022-10', output_dir=output_dir + ) assert len(result) == 20 assert any('readme' in r for r in result) assert any('sensor_position' in r for r in result) - result = act.discovery.get_neon.download_neon_data(site_code, product_code, '2022-09', - end_date='2022-10', output_dir=output_dir) + result = act.discovery.get_neon.download_neon_data( + site_code, product_code, '2022-09', end_date='2022-10', output_dir=output_dir + ) assert len(result) == 40 assert any('readme' in r for r in result) assert any('sensor_position' in r for r in result) @@ -287,7 +290,7 @@ def test_arm_doi(): assert 'Kyrouac' in doi doi = act.discovery.get_arm_doi('test', startdate, enddate) - assert "No DOI Found" in doi + assert 'No DOI Found' in doi def test_download_surfrad(): diff --git a/act/tests/test_io.py b/act/tests/test_io.py index 0bef370abe..11a11ec138 100644 --- a/act/tests/test_io.py +++ b/act/tests/test_io.py @@ -1,9 +1,9 @@ import glob +import random +import tempfile from os import PathLike from pathlib import Path -import random from string import ascii_letters -import tempfile import fsspec import numpy as np @@ -11,7 +11,7 @@ import act import act.tests.sample_files as sample_files -from act.io import read_gml, read_psl_wind_profiler_temperature, icartt +from act.io import icartt, read_gml, read_psl_wind_profiler_temperature from act.io.noaapsl import read_psl_surface_met @@ -33,18 +33,23 @@ def test_io(): assert 'time' in ds ds = act.io.armfiles.read_netcdf([act.tests.EXAMPLE_MET_TEST2]) - assert ds['time'].values[10].astype('datetime64[ms]') == np.datetime64('2019-01-01T00:10:00', 'ms') + assert ds['time'].values[10].astype('datetime64[ms]') == np.datetime64( + '2019-01-01T00:10:00', 'ms' + ) - ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_MET1, use_base_time=True, drop_variables='time') + ds = act.io.armfiles.read_netcdf( + act.tests.EXAMPLE_MET1, use_base_time=True, drop_variables='time' + ) assert 'time' in ds assert np.issubdtype(ds['time'].dtype, np.datetime64) - assert ds['time'].values[10].astype('datetime64[ms]') == np.datetime64('2019-01-01T00:10:00', 'ms') + assert ds['time'].values[10].astype('datetime64[ms]') == np.datetime64( + '2019-01-01T00:10:00', 'ms' + ) del ds def test_keep_variables(): - var_names = [ 'temp_mean', 'rh_mean', @@ -207,8 +212,8 @@ def test_io_dod(): with np.testing.assert_raises(ValueError): ds = act.io.armfiles.create_ds_from_arm_dod('vdis.b1', {}, version='1.2') ds = act.io.armfiles.create_ds_from_arm_dod( - sample_files.EXAMPLE_DOD, dims, version=1.2, scalar_fill_dim='time', - local_file=True) + sample_files.EXAMPLE_DOD, dims, version=1.2, scalar_fill_dim='time', local_file=True + ) assert 'moment1' in ds assert len(ds['base_time'].values) == 1440 assert len(ds['drop_diameter'].values) == 50 @@ -479,9 +484,7 @@ def test_read_psl_wind_profiler(): assert test_ds_hi.dims['HT'] == 50 # test coordinates - assert ( - test_ds_low.coords['HT'][0:5] == np.array([0.151, 0.254, 0.356, 0.458, 0.561]) - ).all() + assert (test_ds_low.coords['HT'][0:5] == np.array([0.151, 0.254, 0.356, 0.458, 0.561])).all() assert ( test_ds_low.coords['time'][0:2] == np.array( @@ -497,10 +500,12 @@ def test_read_psl_wind_profiler(): assert test_ds_low.attrs['latitude'] == 34.66 assert test_ds_low.attrs['longitude'] == -87.35 assert test_ds_low.attrs['elevation'] == 187.0 - assert (test_ds_low.attrs['beam_azimuth'] == np.array( - [38.0, 38.0, 308.0], dtype='float32')).all() - assert (test_ds_low.attrs['beam_elevation'] == np.array( - [90.0, 74.7, 74.7], dtype='float32')).all() + assert ( + test_ds_low.attrs['beam_azimuth'] == np.array([38.0, 38.0, 308.0], dtype='float32') + ).all() + assert ( + test_ds_low.attrs['beam_elevation'] == np.array([90.0, 74.7, 74.7], dtype='float32') + ).all() assert test_ds_low.attrs['consensus_average_time'] == 24 assert test_ds_low.attrs['oblique-beam_vertical_correction'] == 0 assert test_ds_low.attrs['number_of_beams'] == 3 @@ -523,17 +528,13 @@ def test_read_psl_wind_profiler(): # test fields assert test_ds_low['RAD1'].shape == (4, 49) assert test_ds_hi['RAD1'].shape == (4, 50) - assert (test_ds_low['RAD1'][0, 0:5] == np.array( - [0.2, 0.1, 0.1, 0.0, -0.1])).all() - assert (test_ds_hi['RAD1'][0, 0:5] == np.array( - [0.1, 0.1, -0.1, 0.0, -0.2])).all() + assert (test_ds_low['RAD1'][0, 0:5] == np.array([0.2, 0.1, 0.1, 0.0, -0.1])).all() + assert (test_ds_hi['RAD1'][0, 0:5] == np.array([0.1, 0.1, -0.1, 0.0, -0.2])).all() assert test_ds_low['SPD'].shape == (4, 49) assert test_ds_hi['SPD'].shape == (4, 50) - assert (test_ds_low['SPD'][0, 0:5] == np.array( - [2.5, 3.3, 4.3, 4.3, 4.8])).all() - assert (test_ds_hi['SPD'][0, 0:5] == np.array( - [3.7, 4.6, 6.3, 5.2, 6.8])).all() + assert (test_ds_low['SPD'][0, 0:5] == np.array([2.5, 3.3, 4.3, 4.3, 4.8])).all() + assert (test_ds_hi['SPD'][0, 0:5] == np.array([3.7, 4.6, 6.3, 5.2, 6.8])).all() # test transpose test_ds_low, test_ds_hi = act.io.noaapsl.read_psl_wind_profiler( @@ -547,8 +548,7 @@ def test_read_psl_wind_profiler(): def test_read_psl_wind_profiler_temperature(): - ds = read_psl_wind_profiler_temperature( - act.tests.EXAMPLE_NOAA_PSL_TEMPERATURE) + ds = read_psl_wind_profiler_temperature(act.tests.EXAMPLE_NOAA_PSL_TEMPERATURE) ds.attrs['site_identifier'] == 'CTD' ds.attrs['elevation'] = 600.0 @@ -569,9 +569,11 @@ def test_read_psl_surface_met(): def test_read_psl_parsivel(): - url = ['https://downloads.psl.noaa.gov/psd2/data/realtime/DisdrometerParsivel/Stats/ctd/2022/002/ctd2200200_stats.txt', - 'https://downloads.psl.noaa.gov/psd2/data/realtime/DisdrometerParsivel/Stats/ctd/2022/002/ctd2200201_stats.txt', - 'https://downloads.psl.noaa.gov/psd2/data/realtime/DisdrometerParsivel/Stats/ctd/2022/002/ctd2200202_stats.txt'] + url = [ + 'https://downloads.psl.noaa.gov/psd2/data/realtime/DisdrometerParsivel/Stats/ctd/2022/002/ctd2200200_stats.txt', + 'https://downloads.psl.noaa.gov/psd2/data/realtime/DisdrometerParsivel/Stats/ctd/2022/002/ctd2200201_stats.txt', + 'https://downloads.psl.noaa.gov/psd2/data/realtime/DisdrometerParsivel/Stats/ctd/2022/002/ctd2200202_stats.txt', + ] ds = act.io.noaapsl.read_psl_parsivel(url) assert 'number_density_drops' in ds @@ -579,52 +581,45 @@ def test_read_psl_parsivel(): assert ds['number_density_drops'].values[10, 10] == 201 ds = act.io.noaapsl.read_psl_parsivel( - 'https://downloads.psl.noaa.gov/psd2/data/realtime/DisdrometerParsivel/Stats/ctd/2022/002/ctd2200201_stats.txt') + 'https://downloads.psl.noaa.gov/psd2/data/realtime/DisdrometerParsivel/Stats/ctd/2022/002/ctd2200201_stats.txt' + ) assert 'number_density_drops' in ds def test_read_psl_fmcw_moment(): result = act.discovery.download_noaa_psl_data( - site='kps', instrument='Radar FMCW Moment', - startdate='20220815', hour='06' + site='kps', instrument='Radar FMCW Moment', startdate='20220815', hour='06' ) ds = act.io.noaapsl.read_psl_radar_fmcw_moment([result[-1]]) assert 'range' in ds - np.testing.assert_almost_equal( - ds['reflectivity_uncalibrated'].mean(), 2.37, decimal=2) - assert ds['range'].max() == 10040. + np.testing.assert_almost_equal(ds['reflectivity_uncalibrated'].mean(), 2.37, decimal=2) + assert ds['range'].max() == 10040.0 assert len(ds['time'].values) == 115 def test_read_psl_sband_moment(): result = act.discovery.download_noaa_psl_data( - site='ctd', instrument='Radar S-band Moment', - startdate='20211225', hour='06' + site='ctd', instrument='Radar S-band Moment', startdate='20211225', hour='06' ) ds = act.io.noaapsl.read_psl_radar_sband_moment([result[-1]]) assert 'range' in ds - np.testing.assert_almost_equal( - ds['reflectivity_uncalibrated'].mean(), 1.00, decimal=2) - assert ds['range'].max() == 9997. + np.testing.assert_almost_equal(ds['reflectivity_uncalibrated'].mean(), 1.00, decimal=2) + assert ds['range'].max() == 9997.0 assert len(ds['time'].values) == 37 -@pytest.mark.skipif(not act.io.icartt._ICARTT_AVAILABLE, - reason="ICARTT is not installed.") +@pytest.mark.skipif(not act.io.icartt._ICARTT_AVAILABLE, reason='ICARTT is not installed.') def test_read_icartt(): result = act.io.icartt.read_icartt(act.tests.EXAMPLE_AAF_ICARTT) assert 'pitch' in result assert len(result['time'].values) == 14087 assert result['true_airspeed'].units == 'm/s' assert 'Revision' in result.attrs - np.testing.assert_almost_equal( - result['static_pressure'].mean(), 708.75, decimal=2) + np.testing.assert_almost_equal(result['static_pressure'].mean(), 708.75, decimal=2) def test_unpack_tar(): - with tempfile.TemporaryDirectory() as tmpdirname: - tar_file = Path(tmpdirname, 'tar_file_dir') output_dir = Path(tmpdirname, 'output_dir') tar_file.mkdir(parents=True, exist_ok=True) @@ -633,12 +628,13 @@ def test_unpack_tar(): for tar_file_name in ['test_file1.tar', 'test_file2.tar']: filenames = [] for value in range(0, 10): - filename = "".join(random.choices(list(ascii_letters), k=15)) - filename = Path(tar_file, f"{filename}.nc") + filename = ''.join(random.choices(list(ascii_letters), k=15)) + filename = Path(tar_file, f'{filename}.nc') filename.touch() filenames.append(filename) - act.utils.io_utils.pack_tar(filenames, write_filename=Path(tar_file, tar_file_name), - remove=True) + act.utils.io_utils.pack_tar( + filenames, write_filename=Path(tar_file, tar_file_name), remove=True + ) tar_files = list(tar_file.glob('*.tar')) result = act.utils.io_utils.unpack_tar(tar_files[0], write_directory=output_dir) @@ -655,7 +651,9 @@ def test_unpack_tar(): assert len(files) == 0 # Check not returing file but directory - result = act.utils.io_utils.unpack_tar(tar_files[0], write_directory=output_dir, return_files=False) + result = act.utils.io_utils.unpack_tar( + tar_files[0], write_directory=output_dir, return_files=False + ) assert isinstance(result, str) files = list(Path(result).glob('*')) assert len(files) == 10 @@ -705,24 +703,23 @@ def test_unpack_tar(): for dir_name in [tar_file, output_dir]: assert dir_name, dir_name in dir_names - filename = "".join(random.choices(list(ascii_letters), k=15)) - filename = Path(tar_file, f"{filename}.nc") + filename = ''.join(random.choices(list(ascii_letters), k=15)) + filename = Path(tar_file, f'{filename}.nc') filename.touch() result = act.utils.io_utils.pack_tar( - filename, write_filename=Path(tar_file, 'test_file_single'), remove=True) + filename, write_filename=Path(tar_file, 'test_file_single'), remove=True + ) assert Path(filename).is_file() is False assert Path(result).is_file() assert result.endswith('.tar') def test_gunzip(): - with tempfile.TemporaryDirectory() as tmpdirname: - filenames = [] for value in range(0, 10): - filename = "".join(random.choices(list(ascii_letters), k=15)) - filename = Path(tmpdirname, f"{filename}.nc") + filename = ''.join(random.choices(list(ascii_letters), k=15)) + filename = Path(tmpdirname, f'{filename}.nc') filename.touch() filenames.append(filename) @@ -752,11 +749,10 @@ def test_gunzip(): assert file.endswith('.nc') with tempfile.TemporaryDirectory() as tmpdirname: - filenames = [] for value in range(0, 10): - filename = "".join(random.choices(list(ascii_letters), k=15)) - filename = Path(tmpdirname, f"{filename}.nc") + filename = ''.join(random.choices(list(ascii_letters), k=15)) + filename = Path(tmpdirname, f'{filename}.nc') filename.touch() filenames.append(filename) @@ -768,7 +764,8 @@ def test_gunzip(): assert Path(filename).name == 'created_tarfile.tar' gzip_file = act.utils.io_utils.pack_gzip( - filename=filename, write_directory=Path(filename).parent, remove=False) + filename=filename, write_directory=Path(filename).parent, remove=False + ) files = list(Path(tmpdirname).glob('*')) assert len(files) == 2 files = list(Path(tmpdirname).glob('*gz')) @@ -776,12 +773,15 @@ def test_gunzip(): assert Path(gzip_file).name == 'created_tarfile.tar.gz' unpack_filename = act.utils.io_utils.unpack_gzip( - filename=gzip_file, write_directory=Path(filename).parent, remove=False) + filename=gzip_file, write_directory=Path(filename).parent, remove=False + ) files = list(Path(tmpdirname).glob('*')) assert len(files) == 2 assert Path(unpack_filename).name == 'created_tarfile.tar' - result = act.utils.io_utils.unpack_tar(unpack_filename, return_files=True, randomize=False, remove=True) + result = act.utils.io_utils.unpack_tar( + unpack_filename, return_files=True, randomize=False, remove=True + ) files = list(Path(Path(result[0]).parent).glob('*.nc')) assert len(result) == 10 assert len(files) == 10 @@ -792,7 +792,6 @@ def test_gunzip(): def test_read_netcdf_tarfiles(): - with tempfile.TemporaryDirectory() as tmpdirname: met_files = Path(act.tests.EXAMPLE_MET_WILDCARD) met_files = list(Path(met_files.parent).glob(met_files.name)) @@ -828,10 +827,8 @@ def test_read_mmcr(): ds = act.io.armfiles.read_mmcr(results) assert 'MeanDopplerVelocity_PR' in ds assert 'SpectralWidth_BL' in ds - np.testing.assert_almost_equal( - ds['Reflectivity_GE'].mean(), -34.62, decimal=2) - np.testing.assert_almost_equal( - ds['MeanDopplerVelocity_Receiver1'].max(), 9.98, decimal=2) + np.testing.assert_almost_equal(ds['Reflectivity_GE'].mean(), -34.62, decimal=2) + np.testing.assert_almost_equal(ds['MeanDopplerVelocity_Receiver1'].max(), 9.98, decimal=2) def test_read_neon(): @@ -845,7 +842,9 @@ def test_read_neon(): assert 'tempSingleMean' in ds assert ds['tempSingleMean'].values[0] == -0.6003 - ds = act.io.neon.read_neon_csv(data_file, variable_files=variable_file, position_files=position_file) + ds = act.io.neon.read_neon_csv( + data_file, variable_files=variable_file, position_files=position_file + ) assert ds['northOffset'].values == -5.79 assert ds['tempSingleMean'].attrs['units'] == 'celsius' assert 'lat' in ds @@ -867,11 +866,9 @@ def test_read_sodar(): assert len(ds.data_vars) == 26 assert ds['dir'].shape == (96, 58) direction = ds['dir'][0, 0:5].values - np.testing.assert_allclose( - direction, [129.9, 144.2, 147.5, 143.5, 143.0], rtol=1e-6) + np.testing.assert_allclose(direction, [129.9, 144.2, 147.5, 143.5, 143.0], rtol=1e-6) pgz = ds['PGz'][0, 0:5].values - np.testing.assert_allclose( - pgz, [4, 4, 4, 5, 5]) + np.testing.assert_allclose(pgz, [4, 4, 4, 5, 5]) assert ds['dir'].attrs['variable_name'] == 'wind direction' assert ds['dir'].attrs['symbol'] == 'dir' diff --git a/act/tests/test_plotting.py b/act/tests/test_plotting.py index b549594137..84c88983d1 100644 --- a/act/tests/test_plotting.py +++ b/act/tests/test_plotting.py @@ -412,7 +412,7 @@ def test_xsection_plot_map(): @pytest.mark.skipif(not CARTOPY_AVAILABLE, reason='Cartopy is not installed.') -@pytest.mark.mpl_image_compare(style="default", tolerance=30) +@pytest.mark.mpl_image_compare(style='default', tolerance=30) def test_geoplot(): sonde_ds = arm.read_netcdf(sample_files.EXAMPLE_SONDE1) geodisplay = GeographicPlotDisplay({'sgpsondewnpnC1.b1': sonde_ds}, figsize=(15, 8)) @@ -442,7 +442,7 @@ def test_geoplot(): @pytest.mark.skipif(not CARTOPY_AVAILABLE, reason='Cartopy is not installed.') -@pytest.mark.mpl_image_compare(style="default", tolerance=30) +@pytest.mark.mpl_image_compare(style='default', tolerance=30) def test_geoplot_tile(): sonde_ds = arm.read_netcdf(sample_files.EXAMPLE_SONDE1) geodisplay = GeographicPlotDisplay({'sgpsondewnpnC1.b1': sonde_ds}, figsize=(15, 8)) diff --git a/act/tests/test_qc.py b/act/tests/test_qc.py index bfeaaab667..16b54f1b68 100644 --- a/act/tests/test_qc.py +++ b/act/tests/test_qc.py @@ -1,33 +1,35 @@ import copy from datetime import datetime +from pathlib import Path + import dask.array as da import numpy as np import pandas as pd import pytest import xarray as xr -from pathlib import Path from act.io.armfiles import read_netcdf +from act.qc.add_supplemental_qc import apply_supplemental_qc, read_yaml_supplemental_qc from act.qc.arm import add_dqr_to_qc +from act.qc.bsrn_tests import _calculate_solar_parameters from act.qc.qcfilter import parse_bit, set_bit, unset_bit from act.qc.radiometer_tests import fft_shading_test -from act.qc.sp2 import SP2ParticleCriteria, PYSP2_AVAILABLE +from act.qc.sp2 import PYSP2_AVAILABLE, SP2ParticleCriteria from act.tests import ( + EXAMPLE_BRS, EXAMPLE_CEIL1, EXAMPLE_CO2FLX4M, + EXAMPLE_ENA_MET, EXAMPLE_MET1, + EXAMPLE_MET_YAML, EXAMPLE_METE40, EXAMPLE_MFRSR, EXAMPLE_IRT25m20s, - EXAMPLE_BRS, - EXAMPLE_MET_YAML, - EXAMPLE_ENA_MET ) -from act.qc.bsrn_tests import _calculate_solar_parameters -from act.qc.add_supplemental_qc import read_yaml_supplemental_qc, apply_supplemental_qc try: import scikit_posthocs + SCIKIT_POSTHOCS_AVAILABLE = True except ImportError: SCIKIT_POSTHOCS_AVAILABLE = False @@ -131,8 +133,14 @@ def test_arm_qc(): assert len(ds[qc_variable].attrs['flag_meanings']) == 5 # Test additional keywords - add_dqr_to_qc(ds, variable=variable, assessment='Suspect', cleanup_qc=False, - dqr_link=True, skip_location_vars=True) + add_dqr_to_qc( + ds, + variable=variable, + assessment='Suspect', + cleanup_qc=False, + dqr_link=True, + skip_location_vars=True, + ) assert len(ds[qc_variable].attrs['flag_meanings']) == 6 # Default is to normalize assessment terms. Check that we can turn off. @@ -197,9 +205,7 @@ def test_qcfilter(): data = ds.qcfilter.get_masked_data(var_name, rm_assessments='Bad') assert np.ma.count_masked(data) == len(index) - data = ds.qcfilter.get_masked_data( - var_name, rm_assessments='Suspect', return_nan_array=True - ) + data = ds.qcfilter.get_masked_data(var_name, rm_assessments='Suspect', return_nan_array=True) assert np.sum(np.isnan(data)) == len(index2) data = ds.qcfilter.get_masked_data( @@ -350,8 +356,7 @@ def test_qcfilter(): ds.close() -@pytest.mark.skipif(not SCIKIT_POSTHOCS_AVAILABLE, - reason="scikit_posthocs is not installed.") +@pytest.mark.skipif(not SCIKIT_POSTHOCS_AVAILABLE, reason='scikit_posthocs is not installed.') def test_qcfilter2(): ds = read_netcdf(EXAMPLE_IRT25m20s) var_name = 'inst_up_long_dome_resist' @@ -408,20 +413,14 @@ def test_qcfilter3(): ds[qc_var_name].values = ds[qc_var_name].values.astype(np.float32) assert ds[qc_var_name].values.dtype.kind not in np.typecodes['AllInteger'] - result = ds.qcfilter.get_qc_test_mask( - var_name=var_name, test_number=1, return_index=False - ) + result = ds.qcfilter.get_qc_test_mask(var_name=var_name, test_number=1, return_index=False) assert np.sum(result) == 100 - result = ds.qcfilter.get_qc_test_mask( - var_name=var_name, test_number=1, return_index=True - ) + result = ds.qcfilter.get_qc_test_mask(var_name=var_name, test_number=1, return_index=True) assert np.sum(result) == 4950 # Test where QC variables are not integer type ds = ds.resample(time='5min').mean(keep_attrs=True) - ds.qcfilter.add_test( - var_name, index=range(0, ds.time.size), test_meaning='Testing float' - ) + ds.qcfilter.add_test(var_name, index=range(0, ds.time.size), test_meaning='Testing float') assert np.sum(ds[qc_var_name].values) == 582 ds[qc_var_name].values = ds[qc_var_name].values.astype(np.float32) @@ -572,9 +571,7 @@ def test_qctests(): ) assert np.isclose(ds[result['qc_variable_name']].attrs['fail_equal_to'], limit_value) - result = ds.qcfilter.add_equal_to_test( - var_name, limit_value, test_assessment='Indeterminate' - ) + result = ds.qcfilter.add_equal_to_test(var_name, limit_value, test_assessment='Indeterminate') assert 'warn_equal_to' in ds[result['qc_variable_name']].attrs.keys() result = ds.qcfilter.add_equal_to_test(var_name, limit_value, use_dask=True) @@ -645,9 +642,7 @@ def test_qctests(): assert 'warn_lower_range' in ds[result['qc_variable_name']].attrs.keys() assert 'warn_upper_range' in ds[result['qc_variable_name']].attrs.keys() - result = ds.qcfilter.add_outside_test( - var_name, limit_value1, limit_value2, use_dask=True - ) + result = ds.qcfilter.add_outside_test(var_name, limit_value1, limit_value2, use_dask=True) data = ds.qcfilter.get_qc_test_mask(var_name, result['test_number'], return_index=True) assert np.sum(data) == 342254 result = ds.qcfilter.add_outside_test( @@ -767,7 +762,7 @@ def test_qctests_dos(): # persistence test data = ds[var_name].values - data[1000: 2400] = data[1000] + data[1000:2400] = data[1000] data = np.around(data, decimals=3) ds[var_name].values = data result = ds.qcfilter.add_persistence_test(var_name) @@ -816,7 +811,7 @@ def test_datafilter(): ds_filtered.qcfilter.datafilter(rm_assessments='Bad', variables=var_name) ds_2 = ds_filtered.mean() assert np.isclose(ds_2[var_name].values, 99.15, atol=0.01) - expected_var_names = sorted(list(set(data_var_names + qc_var_names) - set(['qc_' + var_name]))) + expected_var_names = sorted(list(set(data_var_names + qc_var_names) - {'qc_' + var_name})) assert sorted(list(ds_filtered.data_vars)) == expected_var_names ds_filtered = copy.deepcopy(ds) @@ -945,7 +940,6 @@ def test_clean(): def test_compare_time_series_trends(): - drop_vars = [ 'base_time', 'time_offset', @@ -1097,7 +1091,7 @@ def test_qc_speed(): assert time_diff.seconds <= 4 -@pytest.mark.skipif(not PYSP2_AVAILABLE, reason="PySP2 is not installed.") +@pytest.mark.skipif(not PYSP2_AVAILABLE, reason='PySP2 is not installed.') def test_sp2_particle_config(): particle_config_ds = SP2ParticleCriteria() assert particle_config_ds.ScatMaxPeakHt1 == 60000 @@ -1140,7 +1134,6 @@ def test_sp2_particle_config(): def test_bsrn_limits_test(): - for use_dask in [False, True]: ds = read_netcdf(EXAMPLE_BRS) var_names = list(ds.data_vars) @@ -1151,15 +1144,17 @@ def test_bsrn_limits_test(): # Add atmospheric temperature fake data ds['temp_mean'] = xr.DataArray( - data=np.full(ds.time.size, 13.5), dims=['time'], - attrs={'long_name': 'Atmospheric air temperature', 'units': 'degC'}) + data=np.full(ds.time.size, 13.5), + dims=['time'], + attrs={'long_name': 'Atmospheric air temperature', 'units': 'degC'}, + ) # Make a short direct variable since BRS does not have one ds['short_direct'] = copy.deepcopy(ds['short_direct_normal']) ds['short_direct'].attrs['ancillary_variables'] = 'qc_short_direct' ds['short_direct'].attrs['long_name'] = 'Shortwave direct irradiance, pyrheliometer' sza, Sa = _calculate_solar_parameters(ds, 'lat', 'lon', 1360.8) - ds['short_direct'].data = ds['short_direct'].data * .5 + ds['short_direct'].data = ds['short_direct'].data * 0.5 # Make up long variable since BRS does not have values ds['up_long_hemisp'].data = copy.deepcopy(ds['down_long_hemisp_shaded'].data) @@ -1219,41 +1214,55 @@ def test_bsrn_limits_test(): glb_LW_dn_name='down_long_hemisp_shaded', glb_LW_up_name='up_long_hemisp', direct_SW_dn_name='short_direct', - use_dask=use_dask) + use_dask=use_dask, + ) assert ds['qc_down_short_hemisp'].attrs['flag_masks'] == [1, 2] - assert ds['qc_down_short_hemisp'].attrs['flag_meanings'][-2] == \ - 'Value less than BSRN physically possible limit of -4.0 W/m^2' - assert ds['qc_down_short_hemisp'].attrs['flag_meanings'][-1] == \ - 'Value greater than BSRN physically possible limit' + assert ( + ds['qc_down_short_hemisp'].attrs['flag_meanings'][-2] + == 'Value less than BSRN physically possible limit of -4.0 W/m^2' + ) + assert ( + ds['qc_down_short_hemisp'].attrs['flag_meanings'][-1] + == 'Value greater than BSRN physically possible limit' + ) assert ds['qc_down_short_diffuse_hemisp'].attrs['flag_masks'] == [1, 2] assert ds['qc_down_short_diffuse_hemisp'].attrs['flag_assessments'] == ['Bad', 'Bad'] assert ds['qc_short_direct'].attrs['flag_masks'] == [1, 2] assert ds['qc_short_direct'].attrs['flag_assessments'] == ['Bad', 'Bad'] - assert ds['qc_short_direct'].attrs['flag_meanings'] == \ - ['Value less than BSRN physically possible limit of -4.0 W/m^2', - 'Value greater than BSRN physically possible limit'] + assert ds['qc_short_direct'].attrs['flag_meanings'] == [ + 'Value less than BSRN physically possible limit of -4.0 W/m^2', + 'Value greater than BSRN physically possible limit', + ] assert ds['qc_short_direct_normal'].attrs['flag_masks'] == [1, 2] - assert ds['qc_short_direct_normal'].attrs['flag_meanings'][-1] == \ - 'Value greater than BSRN physically possible limit' + assert ( + ds['qc_short_direct_normal'].attrs['flag_meanings'][-1] + == 'Value greater than BSRN physically possible limit' + ) assert ds['qc_down_short_hemisp'].attrs['flag_masks'] == [1, 2] - assert ds['qc_down_short_hemisp'].attrs['flag_meanings'][-1] == \ - 'Value greater than BSRN physically possible limit' + assert ( + ds['qc_down_short_hemisp'].attrs['flag_meanings'][-1] + == 'Value greater than BSRN physically possible limit' + ) assert ds['qc_up_short_hemisp'].attrs['flag_masks'] == [1, 2] - assert ds['qc_up_short_hemisp'].attrs['flag_meanings'][-1] == \ - 'Value greater than BSRN physically possible limit' + assert ( + ds['qc_up_short_hemisp'].attrs['flag_meanings'][-1] + == 'Value greater than BSRN physically possible limit' + ) assert ds['qc_up_long_hemisp'].attrs['flag_masks'] == [1, 2] - assert ds['qc_up_long_hemisp'].attrs['flag_meanings'][-1] == \ - 'Value greater than BSRN physically possible limit of 900.0 W/m^2' + assert ( + ds['qc_up_long_hemisp'].attrs['flag_meanings'][-1] + == 'Value greater than BSRN physically possible limit of 900.0 W/m^2' + ) ds.qcfilter.bsrn_limits_test( - test="Extremely Rare", + test='Extremely Rare', gbl_SW_dn_name='down_short_hemisp', glb_diffuse_SW_dn_name='down_short_diffuse_hemisp', direct_normal_SW_dn_name='short_direct_normal', @@ -1261,7 +1270,8 @@ def test_bsrn_limits_test(): glb_LW_dn_name='down_long_hemisp_shaded', glb_LW_up_name='up_long_hemisp', direct_SW_dn_name='short_direct', - use_dask=use_dask) + use_dask=use_dask, + ) assert ds['qc_down_short_hemisp'].attrs['flag_masks'] == [1, 2, 4, 8] assert ds['qc_down_short_diffuse_hemisp'].attrs['flag_masks'] == [1, 2, 4, 8] @@ -1270,11 +1280,15 @@ def test_bsrn_limits_test(): assert ds['qc_up_short_hemisp'].attrs['flag_masks'] == [1, 2, 4, 8] assert ds['qc_up_long_hemisp'].attrs['flag_masks'] == [1, 2, 4, 8] - assert ds['qc_up_long_hemisp'].attrs['flag_meanings'][-1] == \ - 'Value greater than BSRN extremely rare limit of 700.0 W/m^2' + assert ( + ds['qc_up_long_hemisp'].attrs['flag_meanings'][-1] + == 'Value greater than BSRN extremely rare limit of 700.0 W/m^2' + ) - assert ds['qc_down_long_hemisp_shaded'].attrs['flag_meanings'][-1] == \ - 'Value greater than BSRN extremely rare limit of 500.0 W/m^2' + assert ( + ds['qc_down_long_hemisp_shaded'].attrs['flag_meanings'][-1] + == 'Value greater than BSRN extremely rare limit of 500.0 W/m^2' + ) # down_short_hemisp result = ds.qcfilter.get_qc_test_mask('down_short_hemisp', test_number=1) @@ -1337,14 +1351,20 @@ def test_bsrn_limits_test(): assert np.sum(result) == 90 # Change data values to trip tests - ds['down_short_diffuse_hemisp'].values[0:100] = \ + ds['down_short_diffuse_hemisp'].values[0:100] = ( ds['down_short_diffuse_hemisp'].values[0:100] + 100 - ds['up_long_hemisp'].values[0:100] = \ - ds['up_long_hemisp'].values[0:100] - 200 + ) + ds['up_long_hemisp'].values[0:100] = ds['up_long_hemisp'].values[0:100] - 200 ds.qcfilter.bsrn_comparison_tests( - ['Global over Sum SW Ratio', 'Diffuse Ratio', 'SW up', 'LW down to air temp', - 'LW up to air temp', 'LW down to LW up'], + [ + 'Global over Sum SW Ratio', + 'Diffuse Ratio', + 'SW up', + 'LW down to air temp', + 'LW up to air temp', + 'LW down to LW up', + ], gbl_SW_dn_name='down_short_hemisp', glb_diffuse_SW_dn_name='down_short_diffuse_hemisp', direct_normal_SW_dn_name='short_direct_normal', @@ -1355,7 +1375,7 @@ def test_bsrn_limits_test(): test_assessment='Indeterminate', lat_name='lat', lon_name='lon', - use_dask=use_dask + use_dask=use_dask, ) # Ratio of Global over Sum SW @@ -1413,8 +1433,12 @@ def test_read_yaml_supplemental_qc(): assert isinstance(result, dict) assert len(result.keys()) == 3 - result = read_yaml_supplemental_qc(ds, Path(EXAMPLE_MET_YAML).parent, variables='temp_mean', - assessments=['Bad', 'Incorrect', 'Suspect']) + result = read_yaml_supplemental_qc( + ds, + Path(EXAMPLE_MET_YAML).parent, + variables='temp_mean', + assessments=['Bad', 'Incorrect', 'Suspect'], + ) assert len(result.keys()) == 2 assert sorted(result['temp_mean'].keys()) == ['Bad', 'Suspect'] @@ -1424,7 +1448,16 @@ def test_read_yaml_supplemental_qc(): apply_supplemental_qc(ds, EXAMPLE_MET_YAML) assert ds['qc_temp_mean'].attrs['flag_masks'] == [1, 2, 4, 8, 16, 32, 64, 128, 256] assert ds['qc_temp_mean'].attrs['flag_assessments'] == [ - 'Bad', 'Bad', 'Bad', 'Indeterminate', 'Bad', 'Bad', 'Suspect', 'Good', 'Bad'] + 'Bad', + 'Bad', + 'Bad', + 'Indeterminate', + 'Bad', + 'Bad', + 'Suspect', + 'Good', + 'Bad', + ] assert ds['qc_temp_mean'].attrs['flag_meanings'][0] == 'Value is equal to missing_value.' assert ds['qc_temp_mean'].attrs['flag_meanings'][-1] == 'Values are bad for all' assert ds['qc_temp_mean'].attrs['flag_meanings'][-2] == 'Values are good' @@ -1446,8 +1479,13 @@ def test_read_yaml_supplemental_qc(): del ds ds = read_netcdf(EXAMPLE_MET1, keep_variables=['temp_mean', 'rh_mean']) - apply_supplemental_qc(ds, Path(EXAMPLE_MET_YAML).parent, exclude_all_variables='temp_mean', - assessments='Bad', quiet=True) + apply_supplemental_qc( + ds, + Path(EXAMPLE_MET_YAML).parent, + exclude_all_variables='temp_mean', + assessments='Bad', + quiet=True, + ) assert ds['qc_rh_mean'].attrs['flag_assessments'] == ['Bad'] assert ds['qc_temp_mean'].attrs['flag_assessments'] == ['Bad', 'Bad'] assert np.sum(ds['qc_rh_mean'].values) == 124 diff --git a/act/tests/test_utils.py b/act/tests/test_utils.py index 6bb385065c..7a881b7a37 100644 --- a/act/tests/test_utils.py +++ b/act/tests/test_utils.py @@ -1,13 +1,14 @@ """ Unit tests for ACT utils module. """ import importlib +import random +import string +import tarfile import tempfile from datetime import datetime +from os import PathLike, chdir from pathlib import Path -import tarfile -from os import chdir, PathLike -import string -import random + import numpy as np import pandas as pd import pytest @@ -293,7 +294,7 @@ def test_datetime64_to_datetime(): assert time_datetime == time_datetime64_to_datetime -@pytest.mark.skipif(not PYART_AVAILABLE, reason="Py-ART is not installed.") +@pytest.mark.skipif(not PYART_AVAILABLE, reason='Py-ART is not installed.') def test_create_pyart_obj(): try: ds = act.io.mpl.read_sigma_mplv5(act.tests.EXAMPLE_SIGMA_MPLV5) @@ -392,7 +393,6 @@ def test_planck_converter(): def test_solar_azimuth_elevation(): - ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_NAV) elevation, azimuth, distance = act.utils.geo_utils.get_solar_azimuth_elevation( @@ -409,7 +409,6 @@ def test_solar_azimuth_elevation(): def test_get_sunrise_sunset_noon(): - ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_NAV) sunrise, sunset, noon = act.utils.geo_utils.get_sunrise_sunset_noon( diff --git a/act/utils/__init__.py b/act/utils/__init__.py index e7cae20635..a0978bb7ed 100644 --- a/act/utils/__init__.py +++ b/act/utils/__init__.py @@ -7,7 +7,16 @@ __getattr__, __dir__, __all__ = lazy.attach( __name__, - submodules=['data_utils', 'datetime_utils', 'geo_utils', 'inst_utils', 'io_utils', 'qc_utils', 'radiance_utils', 'ship_utils'], + submodules=[ + 'data_utils', + 'datetime_utils', + 'geo_utils', + 'inst_utils', + 'io_utils', + 'qc_utils', + 'radiance_utils', + 'ship_utils', + ], submod_attrs={ 'data_utils': [ 'ChangeUnits', @@ -29,7 +38,7 @@ 'numpy_to_arm_date', 'reduce_time_ranges', 'date_parser', - 'adjust_timestamp' + 'adjust_timestamp', ], 'geo_utils': [ 'add_solar_variable', @@ -42,12 +51,13 @@ 'qc_utils': ['calculate_dqr_times'], 'radiance_utils': ['planck_converter'], 'ship_utils': ['calc_cog_sog', 'proc_scog'], - 'io_utils': ['pack_tar', - 'unpack_tar', - 'cleanup_files', - 'is_gunzip_file', - 'pack_gzip', - 'unpack_gzip' + 'io_utils': [ + 'pack_tar', + 'unpack_tar', + 'cleanup_files', + 'is_gunzip_file', + 'pack_gzip', + 'unpack_gzip', ], }, ) diff --git a/act/utils/data_utils.py b/act/utils/data_utils.py index 66242897f7..1f5224706f 100644 --- a/act/utils/data_utils.py +++ b/act/utils/data_utils.py @@ -4,15 +4,15 @@ """ import importlib +import re import warnings +from pathlib import Path import metpy import numpy as np import pint import scipy.stats as stats import xarray as xr -from pathlib import Path -import re spec = importlib.util.find_spec('pyart') if spec is not None: @@ -106,7 +106,7 @@ def change_units( # @xr.register_dataset_accessor('utils') -class DatastreamParserARM(object): +class DatastreamParserARM: ''' Class to parse ARM datastream names or filenames into its components. Will return None for each attribute if not extracted from the filename. @@ -142,6 +142,7 @@ class DatastreamParserARM(object): ''' + def __init__(self, ds=''): ''' Constructor that initializes datastream data member and runs @@ -244,8 +245,7 @@ def datastream(self): ''' try: - return ''.join((self.__site, self.__class, self.__facility, '.', - self.__level)) + return ''.join((self.__site, self.__class, self.__facility, '.', self.__level)) except TypeError: return None @@ -301,8 +301,7 @@ def datastream_standard(self): ''' try: - return ''.join((self.site, self.datastream_class, self.facility, - '.', self.level)) + return ''.join((self.site, self.datastream_class, self.facility, '.', self.level)) except TypeError: return None @@ -970,7 +969,6 @@ def convert_to_potential_temp( temp_var_units=None, press_var_units=None, ): - """ Converts temperature to potential temperature. diff --git a/act/utils/datetime_utils.py b/act/utils/datetime_utils.py index e2e890bf9e..403de1bc18 100644 --- a/act/utils/datetime_utils.py +++ b/act/utils/datetime_utils.py @@ -55,6 +55,7 @@ def numpy_to_arm_date(_date, returnTime=False): """ from dateutil.parser._parser import ParserError + try: date = pd.to_datetime(str(_date)) if returnTime is False: @@ -261,7 +262,10 @@ def adjust_timestamp(ds, time_bounds='time_bounds', align='left', offset=None): elif align == 'right': time_start = [np.datetime64(t[1]) for t in time_bounds] elif align == 'center': - time_start = [np.datetime64(t[0]) + (np.datetime64(t[0]) - np.datetime64(t[1])) / 2. for t in time_bounds] + time_start = [ + np.datetime64(t[0]) + (np.datetime64(t[0]) - np.datetime64(t[1])) / 2.0 + for t in time_bounds + ] else: raise ValueError('Align should be set to one of [left, right, middle]') diff --git a/act/utils/io_utils.py b/act/utils/io_utils.py index c7b4dcfe7a..86d108f763 100644 --- a/act/utils/io_utils.py +++ b/act/utils/io_utils.py @@ -1,12 +1,12 @@ -from pathlib import Path -import tarfile -from os import PathLike -from shutil import rmtree -import random -import string import gzip +import random import shutil +import string +import tarfile import tempfile +from os import PathLike +from pathlib import Path +from shutil import rmtree def pack_tar(filenames, write_filename=None, write_directory=None, remove=False): @@ -55,7 +55,7 @@ def pack_tar(filenames, write_filename=None, write_directory=None, remove=False) write_filename = str(write_filename) + '.tar' write_filename = Path(write_directory, write_filename) - tar_file_handle = tarfile.open(write_filename, "w") + tar_file_handle = tarfile.open(write_filename, 'w') for filename in filenames: tar_file_handle.add(filename, arcname=Path(filename).name) @@ -68,8 +68,9 @@ def pack_tar(filenames, write_filename=None, write_directory=None, remove=False) return str(write_filename) -def unpack_tar(tar_files, write_directory=None, temp_dir=False, randomize=True, - return_files=True, remove=False): +def unpack_tar( + tar_files, write_directory=None, temp_dir=False, randomize=True, return_files=True, remove=False +): """ Unpacks TAR file contents into provided base directory @@ -129,7 +130,7 @@ def unpack_tar(tar_files, write_directory=None, temp_dir=False, randomize=True, files.extend(result) tar.close() except tarfile.ReadError: - print(f"\nCould not extract files from {tar_file}") + print(f'\nCould not extract files from {tar_file}') if return_files is False: files = str(out_dir) @@ -171,7 +172,7 @@ def cleanup_files(dirname=None, files=None): rmtree(out_dir) except Exception as error: - print("\nError removing files:", error) + print('\nError removing files:', error) def is_gunzip_file(filepath): @@ -264,12 +265,12 @@ def unpack_gzip(filename, write_directory=None, remove=False): write_filename = Path(filename).name if write_filename.endswith('.gz'): - write_filename = write_filename.replace(".gz", "") + write_filename = write_filename.replace('.gz', '') write_filename = Path(write_directory, write_filename) - with gzip.open(filename, "rb") as f_in: - with open(write_filename, "wb") as f_out: + with gzip.open(filename, 'rb') as f_in: + with open(write_filename, 'wb') as f_out: shutil.copyfileobj(f_in, f_out) if remove: diff --git a/examples/discovery/plot_neon.py b/examples/discovery/plot_neon.py index 43d9d7a61b..da1d31ebf0 100644 --- a/examples/discovery/plot_neon.py +++ b/examples/discovery/plot_neon.py @@ -8,8 +8,9 @@ """ -import os import glob +import os + import matplotlib.pyplot as plt import numpy as np @@ -33,19 +34,23 @@ # A number of files are downloaded and further explained in the readme file that's downloaded. # These are the files we will need for reading 1 minute NEON data - file = glob.glob(os.path.join( - '.', - 'BARR_DP1.00002.001', - 'NEON.D18.BARR.DP1.00002.001.000.010.001.SAAT_1min.2022-10.expanded.*.csv', - )) - variable_file = glob.glob(os.path.join( - '.', 'BARR_DP1.00002.001', 'NEON.D18.BARR.DP1.00002.001.variables.*.csv' - )) - position_file = glob.glob(os.path.join( - '.', - 'BARR_DP1.00002.001', - 'NEON.D18.BARR.DP1.00002.001.sensor_positions.*.csv', - )) + file = glob.glob( + os.path.join( + '.', + 'BARR_DP1.00002.001', + 'NEON.D18.BARR.DP1.00002.001.000.010.001.SAAT_1min.2022-10.expanded.*.csv', + ) + ) + variable_file = glob.glob( + os.path.join('.', 'BARR_DP1.00002.001', 'NEON.D18.BARR.DP1.00002.001.variables.*.csv') + ) + position_file = glob.glob( + os.path.join( + '.', + 'BARR_DP1.00002.001', + 'NEON.D18.BARR.DP1.00002.001.sensor_positions.*.csv', + ) + ) # Read in the data using the ACT reader, passing with it the variable and position files # for added information in the dataset ds2 = act.io.read_neon_csv(file, variable_files=variable_file, position_files=position_file) diff --git a/examples/io/plot_create_arm_ds.py b/examples/io/plot_create_arm_ds.py index 219ebcfee8..66cbbbbcb7 100644 --- a/examples/io/plot_create_arm_ds.py +++ b/examples/io/plot_create_arm_ds.py @@ -37,7 +37,7 @@ 'command_line': 'python plot_create_arm_ds.py', 'process_version': '1.2.3', 'history': 'Processed with Jupyter Workbench', - 'random': '1234253sdgfadf' + 'random': '1234253sdgfadf', } for a in atts: if a in ds.attrs: diff --git a/examples/io/plot_sodar.py b/examples/io/plot_sodar.py index 238d05ffa1..afe8fd20e6 100644 --- a/examples/io/plot_sodar.py +++ b/examples/io/plot_sodar.py @@ -21,8 +21,8 @@ # Create an ACT TimeSeriesDisplay. display = act.plotting.TimeSeriesDisplay( - {'Shear, Wind Direction, and Speed at ANL ATMOS': ds}, - subplot_shape=(1,), figsize=(15, 5)) + {'Shear, Wind Direction, and Speed at ANL ATMOS': ds}, subplot_shape=(1,), figsize=(15, 5) +) # Plot shear with a wind barb overlay, while using a color vision # deficiency (CVD) colormap. diff --git a/examples/io/plot_surfrad.py b/examples/io/plot_surfrad.py index 3adccdbb48..af05adef57 100644 --- a/examples/io/plot_surfrad.py +++ b/examples/io/plot_surfrad.py @@ -9,9 +9,10 @@ """ -import act import matplotlib.pyplot as plt +import act + # Easily download data from SURFRAD results = act.discovery.download_surfrad('tbl', startdate='20230601', enddate='20230602') print(results) @@ -19,7 +20,7 @@ # But it's easy enough to read form the URLs as well url = [ 'https://gml.noaa.gov/aftp/data/radiation/surfrad/Boulder_CO/2023/tbl23008.dat', - 'https://gml.noaa.gov/aftp/data/radiation/surfrad/Boulder_CO/2023/tbl23009.dat' + 'https://gml.noaa.gov/aftp/data/radiation/surfrad/Boulder_CO/2023/tbl23009.dat', ] ds = act.io.read_surfrad(url) diff --git a/examples/plotting/plot_ceil.py b/examples/plotting/plot_ceil.py index ef84721782..147c076602 100644 --- a/examples/plotting/plot_ceil.py +++ b/examples/plotting/plot_ceil.py @@ -9,7 +9,9 @@ """ import os + import matplotlib.pyplot as plt + import act # Place your username and token here @@ -21,7 +23,9 @@ ceil_ds = act.io.armfiles.read_netcdf(act.tests.sample_files.EXAMPLE_CEIL1, engine='netcdf4') else: # Example to show how easy it is to download ARM data if a username/token are set - results = act.discovery.download_data(username, token, 'sgpceilC1.b1', '2022-01-14', '2022-01-19') + results = act.discovery.download_data( + username, token, 'sgpceilC1.b1', '2022-01-14', '2022-01-19' + ) ceil_ds = act.io.armfiles.read_netcdf(results) # Adjust ceilometer data for plotting diff --git a/examples/plotting/plot_days.py b/examples/plotting/plot_days.py index 55b5bbf120..de5fb6b284 100644 --- a/examples/plotting/plot_days.py +++ b/examples/plotting/plot_days.py @@ -10,6 +10,7 @@ import matplotlib.pyplot as plt import numpy as np + import act # Read in the sample MET data @@ -18,8 +19,15 @@ # Create Plot Display display = act.plotting.WindRoseDisplay(ds, figsize=(15, 15), subplot_shape=(3, 3)) groupby = display.group_by('day') -groupby.plot_group('plot_data', None, dir_field='wdir_vec_mean', spd_field='wspd_vec_mean', - data_field='temp_mean', num_dirs=12, plot_type='line') +groupby.plot_group( + 'plot_data', + None, + dir_field='wdir_vec_mean', + spd_field='wspd_vec_mean', + data_field='temp_mean', + num_dirs=12, + plot_type='line', +) # Set theta tick markers for each axis inside display to be inside the polar axes for i in range(3): diff --git a/examples/plotting/plot_enhanced_skewt.py b/examples/plotting/plot_enhanced_skewt.py index b7a2b514a2..d0ff6e4dfb 100644 --- a/examples/plotting/plot_enhanced_skewt.py +++ b/examples/plotting/plot_enhanced_skewt.py @@ -10,10 +10,12 @@ """ import glob + import metpy import numpy as np import xarray as xr from matplotlib import pyplot as plt + import act # Read data diff --git a/examples/plotting/plot_heatmap.py b/examples/plotting/plot_heatmap.py index a3996d06c9..ef97cf037f 100644 --- a/examples/plotting/plot_heatmap.py +++ b/examples/plotting/plot_heatmap.py @@ -9,9 +9,10 @@ """ -import act import matplotlib.pyplot as plt +import act + # Read MET data in from the test data area ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_MET_WILDCARD) @@ -21,8 +22,9 @@ # Plot a heatmap and scatter plot up of RH vs Temperature # Set the number of bins for the x-axis to 25 and y to 20 title = 'Heatmap of MET RH vs Temp' -display.plot_heatmap('temp_mean', 'rh_mean', x_bins=25, y_bins=20, - threshold=0, subplot_index=(0, 0), set_title=title) +display.plot_heatmap( + 'temp_mean', 'rh_mean', x_bins=25, y_bins=20, threshold=0, subplot_index=(0, 0), set_title=title +) # Plot the scatter plot and shade by wind_speed title = 'Scatter plot of MET RH vs Temp' diff --git a/examples/plotting/plot_hist_kwargs.py b/examples/plotting/plot_hist_kwargs.py index 481198076a..794ea49022 100644 --- a/examples/plotting/plot_hist_kwargs.py +++ b/examples/plotting/plot_hist_kwargs.py @@ -8,8 +8,8 @@ Author: Zachary Sherman """ -from matplotlib import pyplot as plt import numpy as np +from matplotlib import pyplot as plt import act @@ -19,6 +19,5 @@ # Plot data hist_kwargs = {'range': (-10, 10)} histdisplay = act.plotting.HistogramDisplay(met_ds) -histdisplay.plot_stacked_bar_graph('temp_mean', bins=np.arange(-40, 40, 5), - hist_kwargs=hist_kwargs) +histdisplay.plot_stacked_bar_graph('temp_mean', bins=np.arange(-40, 40, 5), hist_kwargs=hist_kwargs) plt.show() diff --git a/examples/plotting/plot_presentweathercode.py b/examples/plotting/plot_presentweathercode.py index 1f4676f0c6..7f2e407b77 100644 --- a/examples/plotting/plot_presentweathercode.py +++ b/examples/plotting/plot_presentweathercode.py @@ -7,10 +7,9 @@ Author: Joe O'Brien """ -import numpy as np -from matplotlib.dates import DateFormatter -from matplotlib.dates import num2date import matplotlib.pyplot as plt +import numpy as np +from matplotlib.dates import DateFormatter, num2date import act @@ -19,12 +18,12 @@ # Decode the Present Weather Codes # Pass it to the function to decode it along with the variable name -ds = act.utils.inst_utils.decode_present_weather(ds, - variable='pwd_pw_code_inst') +ds = act.utils.inst_utils.decode_present_weather(ds, variable='pwd_pw_code_inst') # Calculate Precipitation Accumulation -pre_accum = act.utils.accumulate_precip(ds.where(ds.qc_tbrg_precip_total == 0), - "tbrg_precip_total").tbrg_precip_total_accumulated.compute() +pre_accum = act.utils.accumulate_precip( + ds.where(ds.qc_tbrg_precip_total == 0), 'tbrg_precip_total' +).tbrg_precip_total_accumulated.compute() # Add the Precipitation Accum to the MET DataSet ds['tbrg_accum'] = pre_accum @@ -38,13 +37,12 @@ display = act.plotting.TimeSeriesDisplay(ds) # Define the Date/Time Format -date_form = DateFormatter("%H%M UTC") +date_form = DateFormatter('%H%M UTC') # Assign the ACT display object to the matplotlib figure subplot display.assign_to_figure_axis(fig, ax) # Datastream Names are needed for plotting! -display.plot('tbrg_accum', - label='TBRG Accumualated Precip') +display.plot('tbrg_accum', label='TBRG Accumualated Precip') # Add a day/night background display.day_night_background() @@ -65,7 +63,10 @@ ndates = [num2date(x) for x in xticks] # Grab the PWD codes associated with those ticks -ncode = [ds['pwd_pw_code_inst_decoded'].sel(time=x.replace(tzinfo=None), method='nearest').data.tolist() for x in ndates] +ncode = [ + ds['pwd_pw_code_inst_decoded'].sel(time=x.replace(tzinfo=None), method='nearest').data.tolist() + for x in ndates +] pwd_code = ['\n'.join(x.split(' ')) if len(x) > 20 else x for x in ncode] # Display these select PWD codes as vertical texts along the x-axis @@ -74,11 +75,7 @@ # Plot the PWD code for i, key in enumerate(xticks): - ax.text(key, - ymin, - pwd_code[i], - rotation=90, - va='center') + ax.text(key, ymin, pwd_code[i], rotation=90, va='center') plt.subplots_adjust(bottom=0.20) diff --git a/examples/plotting/plot_scatter.py b/examples/plotting/plot_scatter.py index c221e49728..592a20049d 100644 --- a/examples/plotting/plot_scatter.py +++ b/examples/plotting/plot_scatter.py @@ -9,10 +9,10 @@ """ -import act import numpy as np - from scipy.stats.mstats import pearsonr + +import act from act.io.icartt import read_icartt # Call the read_icartt function, which supports input @@ -24,57 +24,36 @@ display = act.plotting.DistributionDisplay(ds) # Compare aircraft ground speed with indicated airspeed -display.plot_scatter('true_airspeed', - 'ground_speed', - m_field='ambient_temp', - marker='x', - cbar_label='Ambient Temperature ($^\circ$C)' - ) +display.plot_scatter( + 'true_airspeed', + 'ground_speed', + m_field='ambient_temp', + marker='x', + cbar_label=r'Ambient Temperature ($^\circ$C)', +) # Set the range of the field on the x-axis display.set_xrng((40, 140)) display.set_yrng((40, 140)) # Determine the best fit line -z = np.ma.polyfit(ds['true_airspeed'], - ds['ground_speed'], - 1 - ) +z = np.ma.polyfit(ds['true_airspeed'], ds['ground_speed'], 1) p = np.poly1d(z) # Plot the best fit line -display.axes[0].plot(ds['true_airspeed'], - p(ds['true_airspeed']), - 'r', - linewidth=2 - ) +display.axes[0].plot(ds['true_airspeed'], p(ds['true_airspeed']), 'r', linewidth=2) # Display the line equation -display.axes[0].text(45, - 135, - "y = %.3fx + (%.3f)" % (z[0], z[1]), - color='r', - fontsize=12 - ) +display.axes[0].text(45, 135, f'y = {z[0]:.3f}x + ({z[1]:.3f})', color='r', fontsize=12) # Calculate Pearson Correlation Coefficient -cc_conc = pearsonr(ds['true_airspeed'], - ds['ground_speed'] - ) +cc_conc = pearsonr(ds['true_airspeed'], ds['ground_speed']) # Display the Pearson CC -display.axes[0].text(45, - 130, - "Pearson CC: %.2f" % (cc_conc[0]), - fontsize=12 - ) +display.axes[0].text(45, 130, 'Pearson CC: %.2f' % (cc_conc[0]), fontsize=12) # Display the total number of samples -display.axes[0].text(45, - 125, - "N = %.0f" % (ds['true_airspeed'].data.shape[0]), - fontsize=12 - ) +display.axes[0].text(45, 125, 'N = %.0f' % (ds['true_airspeed'].data.shape[0]), fontsize=12) # Display the 1:1 ratio line display.set_ratio_line() diff --git a/examples/plotting/plot_size_distribution.py b/examples/plotting/plot_size_distribution.py index 0174a8d609..d84d81892d 100644 --- a/examples/plotting/plot_size_distribution.py +++ b/examples/plotting/plot_size_distribution.py @@ -11,10 +11,11 @@ """ -import act import matplotlib.pyplot as plt import numpy as np +import act + # Read CCN data in from the test data area ds = act.io.armfiles.read_netcdf(act.tests.EXAMPLE_CCN) diff --git a/examples/plotting/plot_violin.py b/examples/plotting/plot_violin.py index 0ae527314d..8ad355e90c 100644 --- a/examples/plotting/plot_violin.py +++ b/examples/plotting/plot_violin.py @@ -11,7 +11,6 @@ import matplotlib.pyplot as plt import act - from act.io.icartt import read_icartt # Call the read_icartt function, which supports input @@ -23,22 +22,20 @@ display = act.plotting.DistributionDisplay(ds) # Compare aircraft ground speed with ambient temperature -display.plot_violin('ambient_temp', - positions=[1.0], - ) +display.plot_violin( + 'ambient_temp', + positions=[1.0], +) -display.plot_violin('total_temp', - positions=[2.0], - set_title='Aircraft Temperatures 2018-11-04', - ) +display.plot_violin( + 'total_temp', + positions=[2.0], + set_title='Aircraft Temperatures 2018-11-04', +) # Update the tick information display.axes[0].set_xticks([0.5, 1, 2, 2.5]) -display.axes[0].set_xticklabels(['', - 'Ambient Air\nTemp', - 'Total\nTemperature', - ''] - ) +display.axes[0].set_xticklabels(['', 'Ambient Air\nTemp', 'Total\nTemperature', '']) # Update the y-axis label display.axes[0].set_ylabel('Temperature Observations [C]') diff --git a/examples/qc/plot_qc_bsrn.py b/examples/qc/plot_qc_bsrn.py index 2f9b721106..6e44c767b6 100644 --- a/examples/qc/plot_qc_bsrn.py +++ b/examples/qc/plot_qc_bsrn.py @@ -68,8 +68,13 @@ display = act.plotting.TimeSeriesDisplay(ds_object, figsize=(15, 10), subplot_shape=(2,)) # Plot radiation data in top plot. Add QC information to top plot. -display.plot(variable, subplot_index=(0,), day_night_background=True, assessment_overplot=True, - cb_friendly=True) +display.plot( + variable, + subplot_index=(0,), + day_night_background=True, + assessment_overplot=True, + cb_friendly=True, +) # Plot ancillary QC data in bottom plot display.qc_flag_block_plot(variable, subplot_index=(1,), cb_friendly=True) diff --git a/examples/retrievals/plot_cbh_sobel.py b/examples/retrievals/plot_cbh_sobel.py index 25fcbb2154..7bc0234564 100644 --- a/examples/retrievals/plot_cbh_sobel.py +++ b/examples/retrievals/plot_cbh_sobel.py @@ -11,16 +11,19 @@ """ import glob + +import numpy as np from matplotlib import pyplot as plt + import act -import numpy as np # Read Ceilometer data for an example file = sorted(glob.glob(act.tests.sample_files.EXAMPLE_CEIL1)) ds = act.io.armfiles.read_netcdf(file) -ds = act.retrievals.cbh.generic_sobel_cbh(ds, variable='backscatter', height_dim='range', - var_thresh=1000.0, fill_na=0.) +ds = act.retrievals.cbh.generic_sobel_cbh( + ds, variable='backscatter', height_dim='range', var_thresh=1000.0, fill_na=0.0 +) # Plot the cloud base height data display = act.plotting.TimeSeriesDisplay(ds, subplot_shape=(1, 2), figsize=(16, 6)) @@ -34,7 +37,7 @@ diff = ds['first_cbh'].values - ds['cbh_sobel_backscatter'].values -print("Average difference between ceilomter and sobel heights ", np.nanmean(diff)) +print('Average difference between ceilomter and sobel heights ', np.nanmean(diff)) ds.close() plt.show() diff --git a/examples/templates/example_template.py b/examples/templates/example_template.py index dc15a3337f..5ce68a30ca 100644 --- a/examples/templates/example_template.py +++ b/examples/templates/example_template.py @@ -1,6 +1,8 @@ # Place python module imports here, example: import os + import matplotlib.pyplot as plt + import act # Place arm username and token or example file if username and token @@ -10,8 +12,7 @@ # Download and read file or files with the IO and discovery functions # within ACT, example: -results = act.discovery.download_data( - username, token, 'sgpceilC1.b1', '2022-01-14', '2022-01-19') +results = act.discovery.download_data(username, token, 'sgpceilC1.b1', '2022-01-14', '2022-01-19') ceil_ds = act.io.armfiles.read_netcdf(results) # Plot file using the ACT display submodule, example: diff --git a/examples/utils/plot_parse_filename.py b/examples/utils/plot_parse_filename.py index c3d2dd9eba..05554741c0 100644 --- a/examples/utils/plot_parse_filename.py +++ b/examples/utils/plot_parse_filename.py @@ -17,14 +17,14 @@ # and extract the string value from the object using its properties. fn_obj = DatastreamParserARM(filename) -print(f"Site is {fn_obj.site}") -print(f"Datastream Class is {fn_obj.datastream_class}") -print(f"Facility is {fn_obj.facility}") -print(f"Level is {fn_obj.level}") -print(f"Datastream is {fn_obj.datastream}") -print(f"Date is {fn_obj.date}") -print(f"Time is {fn_obj.time}") -print(f"File extension is {fn_obj.ext}") +print(f'Site is {fn_obj.site}') +print(f'Datastream Class is {fn_obj.datastream_class}') +print(f'Facility is {fn_obj.facility}') +print(f'Level is {fn_obj.level}') +print(f'Datastream is {fn_obj.datastream}') +print(f'Date is {fn_obj.date}') +print(f'Time is {fn_obj.time}') +print(f'File extension is {fn_obj.ext}') # We can also use the parser for just the datastream part to extract the parts. # The other methods will not have a value and return None. @@ -32,11 +32,11 @@ filename = 'sgpmetE13.b1' fn_obj = DatastreamParserARM(filename) -print(f"\nSite is {fn_obj.site}") -print(f"Datastream Class is {fn_obj.datastream_class}") -print(f"Facility is {fn_obj.facility}") -print(f"Level is {fn_obj.level}") -print(f"Datastream is {fn_obj.datastream}") -print(f"Date is {fn_obj.date}") -print(f"Time is {fn_obj.time}") -print(f"File extension is {fn_obj.ext}") +print(f'\nSite is {fn_obj.site}') +print(f'Datastream Class is {fn_obj.datastream_class}') +print(f'Facility is {fn_obj.facility}') +print(f'Level is {fn_obj.level}') +print(f'Datastream is {fn_obj.datastream}') +print(f'Date is {fn_obj.date}') +print(f'Time is {fn_obj.time}') +print(f'File extension is {fn_obj.ext}') diff --git a/examples/workflows/plot_aerioe_with_cbh.py b/examples/workflows/plot_aerioe_with_cbh.py index b6ea3ff86a..36eb1465dc 100644 --- a/examples/workflows/plot_aerioe_with_cbh.py +++ b/examples/workflows/plot_aerioe_with_cbh.py @@ -9,9 +9,10 @@ """ -import matplotlib.pyplot as plt import os +import matplotlib.pyplot as plt + import act # Place your username and token here @@ -22,9 +23,13 @@ if username is None or token is None or len(username) == 0 or len(token) == 0: pass else: - results = act.discovery.download_data(username, token, 'sgpaerioe1turnC1.c1', '2022-02-11', '2022-02-11') + results = act.discovery.download_data( + username, token, 'sgpaerioe1turnC1.c1', '2022-02-11', '2022-02-11' + ) aerioe_ds = act.io.armfiles.read_netcdf(results) - results = act.discovery.download_data(username, token, 'sgpceilC1.b1', '2022-02-11', '2022-02-11') + results = act.discovery.download_data( + username, token, 'sgpceilC1.b1', '2022-02-11', '2022-02-11' + ) ceil_ds = act.io.armfiles.read_netcdf(results) # There isn't information content from the AERI above 3 km @@ -39,20 +44,47 @@ # Create a TimeSeriesDisplay object display = act.plotting.TimeSeriesDisplay( - {'AERIoe': aerioe_ds, 'Ceilometer': ceil_ds}, - subplot_shape=(2,), figsize=(20, 10) + {'AERIoe': aerioe_ds, 'Ceilometer': ceil_ds}, subplot_shape=(2,), figsize=(20, 10) ) # Plot data - display.plot('first_cbh', dsname='Ceilometer', marker='+', color='black', markeredgewidth=3, - linewidth=0, subplot_index=(0,), label='cbh') - display.plot('temperature', dsname='AERIoe', cmap='viridis', set_shading='nearest', - add_nan=True, subplot_index=(0,)) - - display.plot('first_cbh', dsname='Ceilometer', marker='+', color='black', markeredgewidth=3, - linewidth=0, subplot_index=(1,), label='cbh') - display.plot('waterVapor', dsname='AERIoe', cmap='HomeyerRainbow', set_shading='nearest', - add_nan=True, subplot_index=(1,)) + display.plot( + 'first_cbh', + dsname='Ceilometer', + marker='+', + color='black', + markeredgewidth=3, + linewidth=0, + subplot_index=(0,), + label='cbh', + ) + display.plot( + 'temperature', + dsname='AERIoe', + cmap='viridis', + set_shading='nearest', + add_nan=True, + subplot_index=(0,), + ) + + display.plot( + 'first_cbh', + dsname='Ceilometer', + marker='+', + color='black', + markeredgewidth=3, + linewidth=0, + subplot_index=(1,), + label='cbh', + ) + display.plot( + 'waterVapor', + dsname='AERIoe', + cmap='HomeyerRainbow', + set_shading='nearest', + add_nan=True, + subplot_index=(1,), + ) # If you want to save it you can # plt.savefig('sgpaerioe1turnC1.c1.20220211.png') diff --git a/examples/workflows/plot_merged_product.py b/examples/workflows/plot_merged_product.py index 69291aa702..ff670f8485 100644 --- a/examples/workflows/plot_merged_product.py +++ b/examples/workflows/plot_merged_product.py @@ -10,9 +10,10 @@ """ -import act -import xarray as xr import matplotlib.pyplot as plt +import xarray as xr + +import act # Set data files # An alternative to this is to download data from the @@ -34,7 +35,7 @@ # The ECOR and EBBR have different definitions of latent heat # flux and what is positive vs negative. Check out the ARM # Handbooks for more information -ds_ecor['lv_e'].values = ds_ecor['lv_e'].values * -1. +ds_ecor['lv_e'].values = ds_ecor['lv_e'].values * -1.0 # For example purposes, let's rename the ecor latent heat flux ds_ecor = ds_ecor.rename({'lv_e': 'latent_heat_flux_ecor'}) @@ -56,7 +57,9 @@ ds = xr.merge([ds_ecor, ds_ebbr, ds_sebs], compat='override') # Apply the QC information to set all flagged data to missing/NaN -ds.qcfilter.datafilter(del_qc_var=False, rm_assessments=['Bad', 'Incorrect', 'Indeterminate', 'Suspect']) +ds.qcfilter.datafilter( + del_qc_var=False, rm_assessments=['Bad', 'Incorrect', 'Indeterminate', 'Suspect'] +) # Plot up data from the merged dataset for each of the instruments display = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(3,)) diff --git a/examples/workflows/plot_multiple_dataset.py b/examples/workflows/plot_multiple_dataset.py index 9a0d9fbf01..ab7056b2ab 100644 --- a/examples/workflows/plot_multiple_dataset.py +++ b/examples/workflows/plot_multiple_dataset.py @@ -24,9 +24,13 @@ met_ds = act.io.armfiles.read_netcdf(act.tests.sample_files.EXAMPLE_MET1) else: # Download and read data - results = act.discovery.download_data(username, token, 'sgpceilC1.b1', '2022-01-01', '2022-01-07') + results = act.discovery.download_data( + username, token, 'sgpceilC1.b1', '2022-01-01', '2022-01-07' + ) ceil_ds = act.io.armfiles.read_netcdf(results) - results = act.discovery.download_data(username, token, 'sgpmetE13.b1', '2022-01-01', '2022-01-07') + results = act.discovery.download_data( + username, token, 'sgpmetE13.b1', '2022-01-01', '2022-01-07' + ) met_ds = act.io.armfiles.read_netcdf(results) # Read in CEIL data and correct it diff --git a/examples/workflows/plot_qc_transforms.py b/examples/workflows/plot_qc_transforms.py index 903c53da4f..b0eaf0c110 100644 --- a/examples/workflows/plot_qc_transforms.py +++ b/examples/workflows/plot_qc_transforms.py @@ -8,9 +8,10 @@ """ -import act -import xarray as xr import matplotlib.pyplot as plt +import xarray as xr + +import act # Read in some sample MFRSR data and clean up the QC ds = act.io.armfiles.read_netcdf(act.tests.sample_files.EXAMPLE_MFRSR, cleanup_qc=True) @@ -39,10 +40,17 @@ print('After: (2 5 - minute averages)', ds_5minb[variable].values[0:2]) ## Plot up the variable and qc block plot -display = act.plotting.TimeSeriesDisplay({'Original': ds, 'Average': ds_5min, 'Average_QCd': ds_5minb}, - figsize=(15, 10), subplot_shape=(2,)) +display = act.plotting.TimeSeriesDisplay( + {'Original': ds, 'Average': ds_5min, 'Average_QCd': ds_5minb}, + figsize=(15, 10), + subplot_shape=(2,), +) display.plot(variable, dsname='Original', subplot_index=(0,), day_night_background=True) -display.plot(variable, dsname='Average', subplot_index=(1,), day_night_background=True, label='No QC') -display.plot(variable, dsname='Average_QCd', subplot_index=(1,), day_night_background=True, label='QC') +display.plot( + variable, dsname='Average', subplot_index=(1,), day_night_background=True, label='No QC' +) +display.plot( + variable, dsname='Average_QCd', subplot_index=(1,), day_night_background=True, label='QC' +) plt.legend() plt.show() diff --git a/guides/act_cheatsheet.tex b/guides/act_cheatsheet.tex index 0e8dd77328..25beaf7bea 100644 --- a/guides/act_cheatsheet.tex +++ b/guides/act_cheatsheet.tex @@ -396,8 +396,8 @@ \begin{poster} { -headerborder=closed, colspacing=0.8em, bgColorOne=white, bgColorTwo=white, borderColor=lightblue, headerColorOne=black, headerColorTwo=lightblue, -headerFontColor=white, boxColorOne=white, textborder=roundedleft, eyecatcher=true, headerheight=0.06\textheight, headershape=roundedright, headerfont=\Large\bf\textsc, linewidth=2pt +headerborder=closed, colspacing=0.8em, bgColorOne=white, bgColorTwo=white, borderColor=lightblue, headerColorOne=black, headerColorTwo=lightblue, +headerFontColor=white, boxColorOne=white, textborder=roundedleft, eyecatcher=true, headerheight=0.06\textheight, headershape=roundedright, headerfont=\Large\bf\textsc, linewidth=2pt } %---------------------------------------------------------------- % Title @@ -436,7 +436,7 @@ $>$$>$$>$ display.put\_display\_in\_subplot(\\ \-\hspace{1.2cm} display, subplot\_index))\\ \-\hspace{0.2cm} $\bullet$ This will place a Display object into a specific\\ -\-\hspace{0.5cm} subplot. +\-\hspace{0.5cm} subplot. \end{tabular} \begin{tabular}{@{}ll@{}} @@ -710,4 +710,4 @@ } \end{poster} -\end{document} \ No newline at end of file +\end{document} diff --git a/requirements.txt b/requirements.txt index 72dafa725a..c8e587b93a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,4 @@ lazy_loader fsspec metpy lxml -cmweather \ No newline at end of file +cmweather diff --git a/scripts/ads.py b/scripts/ads.py index af7ec02ffb..f36a49abd1 100644 --- a/scripts/ads.py +++ b/scripts/ads.py @@ -10,17 +10,20 @@ """ import argparse -import re -import json -import glob import ast +import glob +import json import pathlib +import re + import matplotlib.pyplot as plt import numpy as np + import act try: import cartopy.crs as ccrs + CARTOPY_AVAILABLE = True except ImportError: CARTOPY_AVAILABLE = False @@ -51,8 +54,10 @@ def option_error_check(args, error_fields, check_all=False): if not value.startswith(prepend): error_fields[ii] = prepend + value - print(f"\n{pathlib.Path(__file__).name}: error: {how_many} of the arguments " - f"{' '.join(error_fields)} is requried\n") + print( + f'\n{pathlib.Path(__file__).name}: error: {how_many} of the arguments ' + f"{' '.join(error_fields)} is requried\n" + ) exit() @@ -118,7 +123,8 @@ def find_drop_vars(args): keep_vars_additional = [] for var_name in keep_vars: qc_var_name = ds.qcfilter.check_for_ancillary_qc( - var_name, add_if_missing=False, cleanup=False) + var_name, add_if_missing=False, cleanup=False + ) if qc_var_name is not None: keep_vars_additional.append(qc_var_name) @@ -142,16 +148,25 @@ def geodisplay(args): except KeyError: pass - display = act.plotting.GeographicPlotDisplay({dsname: ds}, - figsize=args.figsize) - - display.geoplot(data_field=args.field, lat_field=args.latitude, - lon_field=args.longitude, dsname=dsname, - cbar_label=args.cb_label, title=args.set_title, - plot_buffer=args.plot_buffer, stamen=args.stamen, - tile=args.tile, cartopy_feature=args.cfeatures, - cmap=args.cmap, text=args.text, gridlines=args.gridlines, - projection=args.projection, **args.kwargs) + display = act.plotting.GeographicPlotDisplay({dsname: ds}, figsize=args.figsize) + + display.geoplot( + data_field=args.field, + lat_field=args.latitude, + lon_field=args.longitude, + dsname=dsname, + cbar_label=args.cb_label, + title=args.set_title, + plot_buffer=args.plot_buffer, + stamen=args.stamen, + tile=args.tile, + cartopy_feature=args.cfeatures, + cmap=args.cmap, + text=args.text, + gridlines=args.gridlines, + projection=args.projection, + **args.kwargs, + ) plt.savefig(args.out_path) plt.show() @@ -175,26 +190,33 @@ def skewt(args): display = act.plotting.SkewTDisplay({dsname: ds}, figsize=args.figsize) if args.from_u_and_v: - display.plot_from_u_and_v(u_field=args.u_wind, v_field=args.v_wind, - p_field=args.p_field, t_field=args.t_field, - td_field=args.td_field, - subplot_index=subplot_index, - dsname=dsname, show_parcel=args.show_parcel, - p_levels_to_plot=args.plevels_plot, - shade_cape=args.shade_cape, - shade_cin=args.shade_cin, - set_title=args.set_title, - plot_barbs_kwargs=args.plot_barbs_kwargs, - plot_kwargs=args.plot_kwargs) + display.plot_from_u_and_v( + u_field=args.u_wind, + v_field=args.v_wind, + p_field=args.p_field, + t_field=args.t_field, + td_field=args.td_field, + subplot_index=subplot_index, + dsname=dsname, + show_parcel=args.show_parcel, + p_levels_to_plot=args.plevels_plot, + shade_cape=args.shade_cape, + shade_cin=args.shade_cin, + set_title=args.set_title, + plot_barbs_kwargs=args.plot_barbs_kwargs, + plot_kwargs=args.plot_kwargs, + ) if args.from_spd_and_dir: - display.plot_from_spd_and_dir(spd_field=args.spd_field, - dir_field=args.dir_field, - p_field=args.p_field, - t_field=args.t_field, - td_field=args.td_field, - dsname=dsname, - **args.kwargs) + display.plot_from_spd_and_dir( + spd_field=args.spd_field, + dir_field=args.dir_field, + p_field=args.p_field, + t_field=args.t_field, + td_field=args.td_field, + dsname=dsname, + **args.kwargs, + ) plt.savefig(args.out_path) plt.show() @@ -218,18 +240,26 @@ def xsection(args): display = act.plotting.XSectionDisplay({dsname: ds}, figsize=args.figsize) if args.plot_xsection: - display.plot_xsection(dsname=dsname, varname=args.field, - x=args.x_field, y=args.y_field, - subplot_index=subplot_index, - sel_kwargs=args.sel_kwargs, - isel_kwargs=args.isel_kwargs, **args.kwargs) + display.plot_xsection( + dsname=dsname, + varname=args.field, + x=args.x_field, + y=args.y_field, + subplot_index=subplot_index, + sel_kwargs=args.sel_kwargs, + isel_kwargs=args.isel_kwargs, + **args.kwargs, + ) if args.xsection_map: - display.plot_xsection_map(dsname=dsname, varname=args.field, - subplot_index=subplot_index, - coastlines=args.coastlines, - background=args.background, - **args.kwargs) + display.plot_xsection_map( + dsname=dsname, + varname=args.field, + subplot_index=subplot_index, + coastlines=args.coastlines, + background=args.background, + **args.kwargs, + ) plt.savefig(args.out_path) plt.show() @@ -239,7 +269,6 @@ def xsection(args): def wind_rose(args): - drop_vars = find_drop_vars(args) ds = act.io.armfiles.read_netcdf(args.file_path, drop_variables=drop_vars) @@ -253,15 +282,20 @@ def wind_rose(args): except KeyError: pass - display = act.plotting.WindRoseDisplay({dsname: ds}, - figsize=args.figsize) - - display.plot(dir_field=args.dir_field, spd_field=args.spd_field, - subplot_index=subplot_index, - dsname=dsname, cmap=args.cmap, - set_title=args.set_title, - num_dirs=args.num_dir, spd_bins=args.spd_bins, - tick_interval=args.tick_interval, **args.kwargs) + display = act.plotting.WindRoseDisplay({dsname: ds}, figsize=args.figsize) + + display.plot( + dir_field=args.dir_field, + spd_field=args.spd_field, + subplot_index=subplot_index, + dsname=dsname, + cmap=args.cmap, + set_title=args.set_title, + num_dirs=args.num_dir, + spd_bins=args.spd_bins, + tick_interval=args.tick_interval, + **args.kwargs, + ) plt.savefig(args.out_path) plt.show() plt.close(display.fig) @@ -270,7 +304,6 @@ def wind_rose(args): def timeseries(args): - drop_vars = find_drop_vars(args) ds = act.io.armfiles.read_netcdf(args.file_path, drop_variables=drop_vars) @@ -289,11 +322,19 @@ def timeseries(args): pass display = act.plotting.TimeSeriesDisplay( - {dsname: ds}, figsize=args.figsize, - subplot_shape=subplot_shape) - - options = ['plot', 'barbs_spd_dir', 'barbs_u_v', 'xsection_from_1d', - 'time_height_scatter', 'qc', 'fill_between', 'multi_panel'] + {dsname: ds}, figsize=args.figsize, subplot_shape=subplot_shape + ) + + options = [ + 'plot', + 'barbs_spd_dir', + 'barbs_u_v', + 'xsection_from_1d', + 'time_height_scatter', + 'qc', + 'fill_between', + 'multi_panel', + ] option_error_check(args, options) if args.plot: @@ -303,20 +344,27 @@ def timeseries(args): else: yrange = args.set_yrange display.plot( - field=args.field, dsname=dsname, cmap=args.cmap, - set_title=args.set_title, add_nan=args.add_nan, + field=args.field, + dsname=dsname, + cmap=args.cmap, + set_title=args.set_title, + add_nan=args.add_nan, subplot_index=subplot_index, use_var_for_y=args.var_y, day_night_background=args.day_night, invert_y_axis=args.invert_y_axis, - abs_limits=args.abs_limits, time_rng=args.time_rng, + abs_limits=args.abs_limits, + time_rng=args.time_rng, assessment_overplot=args.assessment_overplot, assessment_overplot_category=args.overplot_category, assessment_overplot_category_color=args.category_color, - force_line_plot=args.force_line_plot, labels=args.labels, - cbar_label=args.cb_label, secondary_y=args.secondary_y, + force_line_plot=args.force_line_plot, + labels=args.labels, + cbar_label=args.cb_label, + secondary_y=args.secondary_y, y_rng=yrange, - **args.kwargs) + **args.kwargs, + ) if args.barbs_spd_dir: display.plot_barbs_from_spd_dir( @@ -324,12 +372,15 @@ def timeseries(args): spd_field=args.spd_field, pres_field=args.p_field, dsname=dsname, - **args.kwargs) + **args.kwargs, + ) if args.barbs_u_v: display.plot_barbs_from_u_v( - u_field=args.u_wind, v_field=args.v_wind, - pres_field=args.p_field, dsname=dsname, + u_field=args.u_wind, + v_field=args.v_wind, + pres_field=args.p_field, + dsname=dsname, set_title=args.set_title, invert_y_axis=args.invert_y_axis, day_night_background=args.day_night, @@ -337,49 +388,61 @@ def timeseries(args): num_barbs_y=args.num_barb_y, use_var_for_y=args.var_y, subplot_index=subplot_index, - **args.kwargs) + **args.kwargs, + ) if args.xsection_from_1d: option_error_check(args, 'field') display.plot_time_height_xsection_from_1d_data( - data_field=args.field, pres_field=args.p_field, - dsname=dsname, set_title=args.set_title, + data_field=args.field, + pres_field=args.p_field, + dsname=dsname, + set_title=args.set_title, day_night_background=args.day_night, num_time_periods=args.num_time_periods, num_y_levels=args.num_y_levels, invert_y_axis=args.invert_y_axis, subplot_index=subplot_index, cbar_label=args.cb_label, - **args.kwargs) + **args.kwargs, + ) if args.time_height_scatter: option_error_check(args, 'field') display.time_height_scatter( - data_field=args.field, dsname=dsname, - cmap=args.cmap, alt_label=args.alt_label, - alt_field=args.alt_field, cb_label=args.cb_label, - **args.kwargs) + data_field=args.field, + dsname=dsname, + cmap=args.cmap, + alt_label=args.alt_label, + alt_field=args.alt_field, + cb_label=args.cb_label, + **args.kwargs, + ) if args.qc: option_error_check(args, 'field') display.qc_flag_block_plot( - data_field=args.field, dsname=dsname, + data_field=args.field, + dsname=dsname, subplot_index=subplot_index, time_rng=args.time_rng, assessment_color=args.assessment_color, - **args.kwargs) + **args.kwargs, + ) if args.fill_between: option_error_check(args, 'field') display.fill_between( - field=args.field, dsname=dsname, + field=args.field, + dsname=dsname, subplot_index=subplot_index, set_title=args.set_title, secondary_y=args.secondary_y, - **args.kwargs) + **args.kwargs, + ) if args.multi_panel: option_error_check(args, ['fields', 'plot_type'], check_all=True) @@ -387,27 +450,36 @@ def timeseries(args): for i, j, k in zip(args.fields, subplot_index, args.plot_type): if k == 'plot': display.plot( - field=i, dsname=dsname, cmap=args.cmap, - set_title=args.set_title, add_nan=args.add_nan, + field=i, + dsname=dsname, + cmap=args.cmap, + set_title=args.set_title, + add_nan=args.add_nan, subplot_index=j, use_var_for_y=args.var_y, day_night_background=args.day_night, invert_y_axis=args.invert_y_axis, - abs_limits=args.abs_limits, time_rng=args.time_rng, + abs_limits=args.abs_limits, + time_rng=args.time_rng, assessment_overplot=args.assessment_overplot, assessment_overplot_category=args.overplot_category, assessment_overplot_category_color=args.category_color, - force_line_plot=args.force_line_plot, labels=args.labels, - cbar_label=args.cb_label, secondary_y=args.secondary_y, - **args.kwargs) + force_line_plot=args.force_line_plot, + labels=args.labels, + cbar_label=args.cb_label, + secondary_y=args.secondary_y, + **args.kwargs, + ) if k == 'qc': display.qc_flag_block_plot( - data_field=i, dsname=dsname, + data_field=i, + dsname=dsname, subplot_index=j, time_rng=args.time_rng, assessment_color=args.assessment_color, - **args.kwargs) + **args.kwargs, + ) plt.savefig(args.out_path) plt.show() @@ -417,7 +489,6 @@ def timeseries(args): def histogram(args): - drop_vars = find_drop_vars(args) ds = act.io.armfiles.read_netcdf(args.file_path, drop_variables=drop_vars) @@ -433,44 +504,58 @@ def histogram(args): pass display = act.plotting.HistogramDisplay( - {dsname: ds}, figsize=args.figsize, - subplot_shape=subplot_shape) + {dsname: ds}, figsize=args.figsize, subplot_shape=subplot_shape + ) if args.stacked_bar_graph: display.plot_stacked_bar_graph( - field=args.field, dsname=dsname, - bins=args.bins, density=args.density, + field=args.field, + dsname=dsname, + bins=args.bins, + density=args.density, sortby_field=args.sortby_field, sortby_bins=args.sortby_bins, set_title=args.set_title, subplot_index=subplot_index, - **args.kwargs) + **args.kwargs, + ) if args.size_dist: display.plot_size_distribution( - field=args.field, bins=args.bin_field, - time=args.time, dsname=dsname, + field=args.field, + bins=args.bin_field, + time=args.time, + dsname=dsname, set_title=args.set_title, subplot_index=subplot_index, - **args.kwargs) + **args.kwargs, + ) if args.stairstep: display.plot_stairstep_graph( - field=args.field, dsname=dsname, - bins=args.bins, density=args.density, + field=args.field, + dsname=dsname, + bins=args.bins, + density=args.density, sortby_field=args.sortby_field, sortby_bins=args.sortby_bins, set_title=args.set_title, subplot_index=subplot_index, - **args.kwargs) + **args.kwargs, + ) if args.heatmap: display.plot_heatmap( - x_field=args.x_field, y_field=args.y_field, - dsname=dsname, x_bins=args.x_bins, - y_bins=args.y_bins, set_title=args.set_title, + x_field=args.x_field, + y_field=args.y_field, + dsname=dsname, + x_bins=args.x_bins, + y_bins=args.y_bins, + set_title=args.set_title, density=args.density, - subplot_index=subplot_index, **args.kwargs) + subplot_index=subplot_index, + **args.kwargs, + ) plt.savefig(args.out_path) plt.show() @@ -498,40 +583,51 @@ def contour(args): display = act.plotting.ContourDisplay(data, figsize=args.figsize) if args.create_contour: - display.create_contour(fields=fields, time=time, function=args.function, - grid_delta=args.grid_delta, - grid_buffer=args.grid_buffer, - subplot_index=args.subplot_index, - **args.kwargs) + display.create_contour( + fields=fields, + time=time, + function=args.function, + grid_delta=args.grid_delta, + grid_buffer=args.grid_buffer, + subplot_index=args.subplot_index, + **args.kwargs, + ) if args.contourf: - display.contourf(x=args.x, y=args.y, z=args.z, - subplot_index=args.subplot_index, - **args.kwargs) + display.contourf( + x=args.x, y=args.y, z=args.z, subplot_index=args.subplot_index, **args.kwargs + ) if args.plot_contour: - display.contour(x=args.x, y=args.y, z=args.z, - subplot_index=args.subplot_index, - **args.kwargs) + display.contour( + x=args.x, y=args.y, z=args.z, subplot_index=args.subplot_index, **args.kwargs + ) if args.vectors_spd_dir: - display.plot_vectors_from_spd_dir(fields=wind_fields, time=time, - mesh=args.mesh, function=args.function, - grid_delta=args.grid_delta, - grid_buffer=args.grid_buffer, - subplot_index=args.subplot_index, - **args.kwargs) + display.plot_vectors_from_spd_dir( + fields=wind_fields, + time=time, + mesh=args.mesh, + function=args.function, + grid_delta=args.grid_delta, + grid_buffer=args.grid_buffer, + subplot_index=args.subplot_index, + **args.kwargs, + ) if args.barbs: - display.barbs(x=args.x, y=args.y, u=args.u, v=args.v, - subplot_index=args.subplot_index, - **args.kwargs) + display.barbs( + x=args.x, y=args.y, u=args.u, v=args.v, subplot_index=args.subplot_index, **args.kwargs + ) if args.plot_station: - display.plot_station(fields=station_fields, time=time, - text_color=args.text_color, - subplot_index=args.subplot_index, - **args.kwargs) + display.plot_station( + fields=station_fields, + time=time, + text_color=args.text_color, + subplot_index=args.subplot_index, + **args.kwargs, + ) plt.savefig(args.out_path) plt.show() @@ -555,8 +651,11 @@ def convert_arg_line_to_args(line): def main(): prefix_char = '@' parser = argparse.ArgumentParser( - description=(f'Create plot from a data file. Can use command line opitons ' - f'or point to a configuration file using {prefix_char} character.')) + description=( + f'Create plot from a data file. Can use command line opitons ' + f'or point to a configuration file using {prefix_char} character.' + ) + ) # Allow user to reference a file by using the @ symbol for a specific # argument value @@ -565,336 +664,817 @@ def main(): # Update the file parsing logic to skip commented lines parser.convert_arg_line_to_args = convert_arg_line_to_args - parser.add_argument('-f', '--file_path', type=str, required=True, - help=('Required: Full path to file for creating Plot. For multiple ' - 'files use terminal syntax for matching muliple files. ' - 'For example "sgpmetE13.b1.202007*.*.nc" will match all files ' - 'for the month of July in 2020. Need to use double quotes ' - 'to stop terminal from expanding the search, and let the ' - 'python program perform search.')) + parser.add_argument( + '-f', + '--file_path', + type=str, + required=True, + help=( + 'Required: Full path to file for creating Plot. For multiple ' + 'files use terminal syntax for matching muliple files. ' + 'For example "sgpmetE13.b1.202007*.*.nc" will match all files ' + 'for the month of July in 2020. Need to use double quotes ' + 'to stop terminal from expanding the search, and let the ' + 'python program perform search.' + ), + ) out_path_default = 'image.png' - parser.add_argument('-o', '--out_path', type=str, default=out_path_default, - help=("Full path filename to use for saving image. " - "Default is '{out_path_default}'. If only a path is given " - "will use that path with image name '{out_path_default}', " - "else will use filename given.")) - parser.add_argument('-fd', '--field', type=str, default=None, - help='Name of the field to plot') - parser.add_argument('-fds', '--fields', nargs='+', - type=str, default=None, - help='Name of the fields to use to plot') - parser.add_argument('-wfs', '--wind_fields', nargs='+', - type=str, default=None, - help='Wind field names used to plot') - parser.add_argument('-sfs', '--station_fields', nargs='+', - type=str, default=None, - help='Station field names to plot sites') + parser.add_argument( + '-o', + '--out_path', + type=str, + default=out_path_default, + help=( + 'Full path filename to use for saving image. ' + "Default is '{out_path_default}'. If only a path is given " + "will use that path with image name '{out_path_default}', " + 'else will use filename given.' + ), + ) + parser.add_argument('-fd', '--field', type=str, default=None, help='Name of the field to plot') + parser.add_argument( + '-fds', + '--fields', + nargs='+', + type=str, + default=None, + help='Name of the fields to use to plot', + ) + parser.add_argument( + '-wfs', + '--wind_fields', + nargs='+', + type=str, + default=None, + help='Wind field names used to plot', + ) + parser.add_argument( + '-sfs', + '--station_fields', + nargs='+', + type=str, + default=None, + help='Station field names to plot sites', + ) default = 'lat' - parser.add_argument('-lat', '--latitude', type=str, default=default, - help=f"Name of latitude variable in file. Default is '{default}'") + parser.add_argument( + '-lat', + '--latitude', + type=str, + default=default, + help=f"Name of latitude variable in file. Default is '{default}'", + ) default = 'lon' - parser.add_argument('-lon', '--longitude', type=str, default=default, - help=f"Name of longitude variable in file. Default is '{default}'") - parser.add_argument('-xf', '--x_field', type=str, default=None, - help='Name of variable to plot on x axis') - parser.add_argument('-yf', '--y_field', type=str, default=None, - help='Name of variable to plot on y axis') - parser.add_argument('-x', type=np.array, - help='x coordinates or grid for z') - parser.add_argument('-y', type=np.array, - help='y coordinates or grid for z') - parser.add_argument('-z', type=np.array, - help='Values over which to contour') + parser.add_argument( + '-lon', + '--longitude', + type=str, + default=default, + help=f"Name of longitude variable in file. Default is '{default}'", + ) + parser.add_argument( + '-xf', '--x_field', type=str, default=None, help='Name of variable to plot on x axis' + ) + parser.add_argument( + '-yf', '--y_field', type=str, default=None, help='Name of variable to plot on y axis' + ) + parser.add_argument('-x', type=np.array, help='x coordinates or grid for z') + parser.add_argument('-y', type=np.array, help='y coordinates or grid for z') + parser.add_argument('-z', type=np.array, help='Values over which to contour') default = 'u_wind' - parser.add_argument('-u', '--u_wind', type=str, default=default, - help=f"File variable name for u_wind wind component. Default is '{default}'") + parser.add_argument( + '-u', + '--u_wind', + type=str, + default=default, + help=f"File variable name for u_wind wind component. Default is '{default}'", + ) default = 'v_wind' - parser.add_argument('-v', '--v_wind', type=str, default=default, - help=f"File variable name for v_wind wind compenent. Default is '{default}'") + parser.add_argument( + '-v', + '--v_wind', + type=str, + default=default, + help=f"File variable name for v_wind wind compenent. Default is '{default}'", + ) default = 'pres' - parser.add_argument('-pf', '--p_field', type=str, default=default, - help=f"File variable name for pressure. Default is '{default}'") + parser.add_argument( + '-pf', + '--p_field', + type=str, + default=default, + help=f"File variable name for pressure. Default is '{default}'", + ) default = 'tdry' - parser.add_argument('-tf', '--t_field', type=str, default=default, - help=f"File variable name for temperature. Default is '{default}'") + parser.add_argument( + '-tf', + '--t_field', + type=str, + default=default, + help=f"File variable name for temperature. Default is '{default}'", + ) default = 'dp' - parser.add_argument('-tdf', '--td_field', type=str, default=default, - help=f"File variable name for dewpoint temperature. Default is '{default}'") + parser.add_argument( + '-tdf', + '--td_field', + type=str, + default=default, + help=f"File variable name for dewpoint temperature. Default is '{default}'", + ) default = 'wspd' - parser.add_argument('-sf', '--spd_field', type=str, default=default, - help=f"File variable name for wind speed. Default is '{default}'") + parser.add_argument( + '-sf', + '--spd_field', + type=str, + default=default, + help=f"File variable name for wind speed. Default is '{default}'", + ) default = 'deg' - parser.add_argument('-df', '--dir_field', type=str, default=default, - help=f"File variable name for wind direction. Default is '{default}'") - parser.add_argument('-al', '--alt_label', type=str, default=None, - help='Altitude axis label') + parser.add_argument( + '-df', + '--dir_field', + type=str, + default=default, + help=f"File variable name for wind direction. Default is '{default}'", + ) + parser.add_argument('-al', '--alt_label', type=str, default=None, help='Altitude axis label') default = 'alt' - parser.add_argument('-af', '--alt_field', type=str, default=default, - help=f"File variable name for altitude. Default is '{default}'") + parser.add_argument( + '-af', + '--alt_field', + type=str, + default=default, + help=f"File variable name for altitude. Default is '{default}'", + ) global _default_dsname _default_dsname = 'act_datastream' - parser.add_argument('-ds', '--dsname', type=str, default=_default_dsname, - help=f"Name of datastream to plot. Default is '{_default_dsname}'") + parser.add_argument( + '-ds', + '--dsname', + type=str, + default=_default_dsname, + help=f"Name of datastream to plot. Default is '{_default_dsname}'", + ) default = '(0, )' - parser.add_argument('-si', '--subplot_index', type=ast.literal_eval, - default=default, - help=f'Index of the subplot via tuple syntax. ' - f'Example for two plots is "(0,), (1,)". ' - f"Default is '{default}'") - default = (1, ) - parser.add_argument('-ss', '--subplot_shape', nargs='+', type=int, - default=default, - help=(f'The number of (rows, columns) ' - f'for the subplots in the display. ' - f'Default is {default}')) + parser.add_argument( + '-si', + '--subplot_index', + type=ast.literal_eval, + default=default, + help=f'Index of the subplot via tuple syntax. ' + f'Example for two plots is "(0,), (1,)". ' + f"Default is '{default}'", + ) + default = (1,) + parser.add_argument( + '-ss', + '--subplot_shape', + nargs='+', + type=int, + default=default, + help=( + f'The number of (rows, columns) ' + f'for the subplots in the display. ' + f'Default is {default}' + ), + ) plot_type_options = ['plot', 'qc'] - parser.add_argument('-pt', '--plot_type', nargs='+', type=str, - help=f'Type of plot to make. Current options include: ' - f'{plot_type_options}') - parser.add_argument('-vy', '--var_y', type=str, default=None, - help=('Set this to the name of a data variable in ' - 'the Dataset to use as the y-axis variable ' - 'instead of the default dimension.')) - parser.add_argument('-plp', '--plevels_plot', - type=np.array, default=None, - help='Pressure levels to plot the wind barbs on.') - parser.add_argument('-cbl', '--cb_label', type=str, default=None, - help='Colorbar label to use') - parser.add_argument('-st', '--set_title', type=str, default=None, - help='Title for the plot') + parser.add_argument( + '-pt', + '--plot_type', + nargs='+', + type=str, + help=f'Type of plot to make. Current options include: ' f'{plot_type_options}', + ) + parser.add_argument( + '-vy', + '--var_y', + type=str, + default=None, + help=( + 'Set this to the name of a data variable in ' + 'the Dataset to use as the y-axis variable ' + 'instead of the default dimension.' + ), + ) + parser.add_argument( + '-plp', + '--plevels_plot', + type=np.array, + default=None, + help='Pressure levels to plot the wind barbs on.', + ) + parser.add_argument('-cbl', '--cb_label', type=str, default=None, help='Colorbar label to use') + parser.add_argument('-st', '--set_title', type=str, default=None, help='Title for the plot') default = 0.08 - parser.add_argument('-pb', '--plot_buffer', type=float, default=default, - help=(f'Buffer to add around data on plot in lat ' - f'and lon dimension. Default is {default}')) + parser.add_argument( + '-pb', + '--plot_buffer', + type=float, + default=default, + help=( + f'Buffer to add around data on plot in lat ' f'and lon dimension. Default is {default}' + ), + ) default = 'terrain-background' - parser.add_argument('-sm', '--stamen', type=str, default=default, - help=f"Dataset to use for background image. Default is '{default}'") + parser.add_argument( + '-sm', + '--stamen', + type=str, + default=default, + help=f"Dataset to use for background image. Default is '{default}'", + ) default = 8 - parser.add_argument('-tl', '--tile', type=int, default=default, - help=f'Tile zoom to use with background image. Default is {default}') - parser.add_argument('-cfs', '--cfeatures', nargs='+', type=str, default=None, - help='Cartopy feature to add to plot') - parser.add_argument('-txt', '--text', type=json.loads, default=None, - help=('Dictionary of {text:[lon,lat]} to add to plot. ' - 'Can have more than one set of text to add.')) + parser.add_argument( + '-tl', + '--tile', + type=int, + default=default, + help=f'Tile zoom to use with background image. Default is {default}', + ) + parser.add_argument( + '-cfs', + '--cfeatures', + nargs='+', + type=str, + default=None, + help='Cartopy feature to add to plot', + ) + parser.add_argument( + '-txt', + '--text', + type=json.loads, + default=None, + help=( + 'Dictionary of {text:[lon,lat]} to add to plot. ' + 'Can have more than one set of text to add.' + ), + ) default = 'rainbow' - parser.add_argument('-cm', '--cmap', default=default, - help=f"colormap to use. Defaut is '{default}'") - parser.add_argument('-abl', '--abs_limits', nargs='+', type=float, - default=(None, None), - help=('Sets the bounds on plot limits even if data ' - 'values exceed those limits. Y axis limits. Default is no limits.')) - parser.add_argument('-tr', '--time_rng', nargs='+', type=float, default=None, - help=('List or tuple with (min,max) values to set the ' - 'x-axis range limits')) + parser.add_argument( + '-cm', '--cmap', default=default, help=f"colormap to use. Defaut is '{default}'" + ) + parser.add_argument( + '-abl', + '--abs_limits', + nargs='+', + type=float, + default=(None, None), + help=( + 'Sets the bounds on plot limits even if data ' + 'values exceed those limits. Y axis limits. Default is no limits.' + ), + ) + parser.add_argument( + '-tr', + '--time_rng', + nargs='+', + type=float, + default=None, + help=('List or tuple with (min,max) values to set the ' 'x-axis range limits'), + ) default = 20 - parser.add_argument('-nd', '--num_dir', type=int, default=default, - help=(f'Number of directions to splot the wind rose into. ' - f'Default is {default}')) - parser.add_argument('-sb', '--spd_bins', nargs='+', type=float, default=None, - help='Bin boundaries to sort the wind speeds into') + parser.add_argument( + '-nd', + '--num_dir', + type=int, + default=default, + help=(f'Number of directions to splot the wind rose into. ' f'Default is {default}'), + ) + parser.add_argument( + '-sb', + '--spd_bins', + nargs='+', + type=float, + default=None, + help='Bin boundaries to sort the wind speeds into', + ) default = 3 - parser.add_argument('-ti', '--tick_interval', type=int, default=default, - help=(f'Interval (in percentage) for the ticks ' - f'on the radial axis. Default is {default}')) - parser.add_argument('-ac', '--assessment_color', type=json.loads, - default=None, - help=('dictionary lookup to override default ' - 'assessment to color')) + parser.add_argument( + '-ti', + '--tick_interval', + type=int, + default=default, + help=( + f'Interval (in percentage) for the ticks ' f'on the radial axis. Default is {default}' + ), + ) + parser.add_argument( + '-ac', + '--assessment_color', + type=json.loads, + default=None, + help=('dictionary lookup to override default ' 'assessment to color'), + ) default = False - parser.add_argument('-ao', '--assessment_overplot', - default=default, action='store_true', - help=(f'Option to overplot quality control colored ' - f'symbols over plotted data using ' - f'flag_assessment categories. Default is {default}')) - default = {'Incorrect': ['Bad', 'Incorrect'], - 'Suspect': ['Indeterminate', 'Suspect']} - parser.add_argument('-oc', '--overplot_category', type=json.loads, default=default, - help=(f'Look up to categorize assessments into groups. ' - f'This allows using multiple terms for the same ' - f'quality control level of failure. ' - f'Also allows adding more to the defaults. Default is {default}')) + parser.add_argument( + '-ao', + '--assessment_overplot', + default=default, + action='store_true', + help=( + f'Option to overplot quality control colored ' + f'symbols over plotted data using ' + f'flag_assessment categories. Default is {default}' + ), + ) + default = {'Incorrect': ['Bad', 'Incorrect'], 'Suspect': ['Indeterminate', 'Suspect']} + parser.add_argument( + '-oc', + '--overplot_category', + type=json.loads, + default=default, + help=( + f'Look up to categorize assessments into groups. ' + f'This allows using multiple terms for the same ' + f'quality control level of failure. ' + f'Also allows adding more to the defaults. Default is {default}' + ), + ) default = {'Incorrect': 'red', 'Suspect': 'orange'} - parser.add_argument('-co', '--category_color', type=json.loads, - default=default, - help=(f'Lookup to match overplot category color to ' - f'assessment grouping. Default is {default}')) - parser.add_argument('-flp', '--force_line_plot', default=False, - action='store_true', - help='Option to plot 2D data as 1D line plots') - parser.add_argument('-l', '--labels', nargs='+', default=False, - type=str, - help=('Option to overwrite the legend labels. ' - 'Must have same dimensions as number of ' - 'lines plottes.')) - parser.add_argument('-sy', '--secondary_y', default=False, action='store_true', - help='Option to plot on secondary y axis') + parser.add_argument( + '-co', + '--category_color', + type=json.loads, + default=default, + help=( + f'Lookup to match overplot category color to ' + f'assessment grouping. Default is {default}' + ), + ) + parser.add_argument( + '-flp', + '--force_line_plot', + default=False, + action='store_true', + help='Option to plot 2D data as 1D line plots', + ) + parser.add_argument( + '-l', + '--labels', + nargs='+', + default=False, + type=str, + help=( + 'Option to overwrite the legend labels. ' + 'Must have same dimensions as number of ' + 'lines plottes.' + ), + ) + parser.add_argument( + '-sy', + '--secondary_y', + default=False, + action='store_true', + help='Option to plot on secondary y axis', + ) if CARTOPY_AVAILABLE: default = ccrs.PlateCarree() - parser.add_argument('-prj', '--projection', type=str, - default=default, - help=f"Projection to use on plot. Default is {default}") + parser.add_argument( + '-prj', + '--projection', + type=str, + default=default, + help=f'Projection to use on plot. Default is {default}', + ) default = 20 - parser.add_argument('-bx', '--num_barb_x', type=int, default=default, - help=f'Number of wind barbs to plot in the x axis. Default is {default}') + parser.add_argument( + '-bx', + '--num_barb_x', + type=int, + default=default, + help=f'Number of wind barbs to plot in the x axis. Default is {default}', + ) default = 20 - parser.add_argument('-by', '--num_barb_y', type=int, default=default, - help=f"Number of wind barbs to plot in the y axis. Default is {default}") + parser.add_argument( + '-by', + '--num_barb_y', + type=int, + default=default, + help=f'Number of wind barbs to plot in the y axis. Default is {default}', + ) default = 20 - parser.add_argument('-tp', '--num_time_periods', type=int, default=default, - help=f'Set how many time periods. Default is {default}') - parser.add_argument('-bn', '--bins', nargs='+', type=int, default=None, - help='histogram bin boundaries to use') - parser.add_argument('-bf', '--bin_field', type=str, default=None, - help=('name of the field that stores the ' - 'bins for the spectra')) - parser.add_argument('-xb', '--x_bins', nargs='+', type=int, default=None, - help='Histogram bin boundaries to use for x axis variable') - parser.add_argument('-yb', '--y_bins', nargs='+', type=int, default=None, - help='Histogram bin boundaries to use for y axis variable') - parser.add_argument('-t', '--time', type=str, default=None, - help='Time period to be plotted') - parser.add_argument('-sbf', '--sortby_field', type=str, default=None, - help='Sort histograms by a given field parameter') - parser.add_argument('-sbb', '--sortby_bins', nargs='+', type=int, - default=None, - help='Bins to sort the histograms by') + parser.add_argument( + '-tp', + '--num_time_periods', + type=int, + default=default, + help=f'Set how many time periods. Default is {default}', + ) + parser.add_argument( + '-bn', '--bins', nargs='+', type=int, default=None, help='histogram bin boundaries to use' + ) + parser.add_argument( + '-bf', + '--bin_field', + type=str, + default=None, + help=('name of the field that stores the ' 'bins for the spectra'), + ) + parser.add_argument( + '-xb', + '--x_bins', + nargs='+', + type=int, + default=None, + help='Histogram bin boundaries to use for x axis variable', + ) + parser.add_argument( + '-yb', + '--y_bins', + nargs='+', + type=int, + default=None, + help='Histogram bin boundaries to use for y axis variable', + ) + parser.add_argument('-t', '--time', type=str, default=None, help='Time period to be plotted') + parser.add_argument( + '-sbf', + '--sortby_field', + type=str, + default=None, + help='Sort histograms by a given field parameter', + ) + parser.add_argument( + '-sbb', + '--sortby_bins', + nargs='+', + type=int, + default=None, + help='Bins to sort the histograms by', + ) default = 20 - parser.add_argument('-nyl', '--num_y_levels', type=int, default=default, - help=f'Number of levels in the y axis to use. Default is {default}') - parser.add_argument('-sk', '--sel_kwargs', type=json.loads, default=None, - help=('The keyword arguments to pass into ' - ':py:func:`xarray.DataArray.sel`')) - parser.add_argument('-ik', '--isel_kwargs', type=json.loads, default=None, - help=('The keyword arguments to pass into ' - ':py:func:`xarray.DataArray.sel`')) + parser.add_argument( + '-nyl', + '--num_y_levels', + type=int, + default=default, + help=f'Number of levels in the y axis to use. Default is {default}', + ) + parser.add_argument( + '-sk', + '--sel_kwargs', + type=json.loads, + default=None, + help=('The keyword arguments to pass into ' ':py:func:`xarray.DataArray.sel`'), + ) + parser.add_argument( + '-ik', + '--isel_kwargs', + type=json.loads, + default=None, + help=('The keyword arguments to pass into ' ':py:func:`xarray.DataArray.sel`'), + ) default = 'cubic' - parser.add_argument('-fn', '--function', type=str, default=default, - help=(f'Defaults to cubic function for interpolation. ' - f'See scipy.interpolate.Rbf for additional options. ' - f'Default is {default}')) + parser.add_argument( + '-fn', + '--function', + type=str, + default=default, + help=( + f'Defaults to cubic function for interpolation. ' + f'See scipy.interpolate.Rbf for additional options. ' + f'Default is {default}' + ), + ) default = 0.1 - parser.add_argument('-gb', '--grid_buffer', type=float, default=default, - help=f'Buffer to apply to grid. Default is {default}') + parser.add_argument( + '-gb', + '--grid_buffer', + type=float, + default=default, + help=f'Buffer to apply to grid. Default is {default}', + ) default = (0.01, 0.01) - parser.add_argument('-gd', '--grid_delta', nargs='+', - type=float, default=default, - help=f'X and Y deltas for creating grid. Default is {default}') - parser.add_argument('-fg', '--figsize', nargs='+', type=float, - default=None, - help='Width and height in inches of figure') + parser.add_argument( + '-gd', + '--grid_delta', + nargs='+', + type=float, + default=default, + help=f'X and Y deltas for creating grid. Default is {default}', + ) + parser.add_argument( + '-fg', + '--figsize', + nargs='+', + type=float, + default=None, + help='Width and height in inches of figure', + ) default = 'white' - parser.add_argument('-tc', '--text_color', type=str, default=default, - help=f"Color of text. Default is '{default}'") - parser.add_argument('-kwargs', type=json.loads, default=dict(), - help='keyword arguments to use in plotting function') - parser.add_argument('-pk', '--plot_kwargs', type=json.loads, default=dict(), - help=("Additional keyword arguments to pass " - "into MetPy's SkewT.plot")) - parser.add_argument('-pbk', '--plot_barbs_kwargs', type=json.loads, - default=dict(), - help=("Additional keyword arguments to pass " - "into MetPy's SkewT.plot_barbs")) + parser.add_argument( + '-tc', + '--text_color', + type=str, + default=default, + help=f"Color of text. Default is '{default}'", + ) + parser.add_argument( + '-kwargs', + type=json.loads, + default=dict(), + help='keyword arguments to use in plotting function', + ) + parser.add_argument( + '-pk', + '--plot_kwargs', + type=json.loads, + default=dict(), + help=('Additional keyword arguments to pass ' "into MetPy's SkewT.plot"), + ) + parser.add_argument( + '-pbk', + '--plot_barbs_kwargs', + type=json.loads, + default=dict(), + help=('Additional keyword arguments to pass ' "into MetPy's SkewT.plot_barbs"), + ) default = True - parser.add_argument('-cu', '--cleanup', default=default, action='store_false', - help=f'Turn off standard methods for obj cleanup. Default is {default}') - parser.add_argument('-gl', '--gridlines', default=False, action='store_true', - help='Use latitude and lingitude gridlines.') - parser.add_argument('-cl', '--coastlines', default=False, action='store_true', - help='Plot coastlines on geographical map') - parser.add_argument('-bg', '--background', default=False, action='store_true', - help='Plot a stock image background') - parser.add_argument('-nan', '--add_nan', default=False, action='store_true', - help='Fill in data gaps with NaNs') - parser.add_argument('-dn', '--day_night', default=False, action='store_true', - help=("Fill in color coded background according " - "to time of day.")) - parser.add_argument('-yr', '--set_yrange', default=None, nargs=2, - help=("Set the yrange for the specific plot")) - parser.add_argument('-iya', '--invert_y_axis', default=False, - action='store_true', - help='Invert y axis') - parser.add_argument('-sp', '--show_parcel', default=False, action='store_true', - help='set to true to plot the parcel path.') - parser.add_argument('-cape', '--shade_cape', default=False, - action='store_true', - help='set to true to shade regions of cape.') - parser.add_argument('-cin', '--shade_cin', default=False, action='store_true', - help='set to true to shade regions of cin.') - parser.add_argument('-d', '--density', default=False, action='store_true', - help='Plot a p.d.f. instead of a frequency histogram') - parser.add_argument('-m', '--mesh', default=False, action='store_true', - help=('Set to True to interpolate u and v to ' - 'grid and create wind barbs')) - parser.add_argument('-uv', '--from_u_and_v', default=False, action='store_true', - help='Create SkewTPLot with u and v wind') - parser.add_argument('-sd', '--from_spd_and_dir', default=False, action='store_true', - help='Create SkewTPlot with wind speed and direction') - parser.add_argument('-px', '--plot_xsection', default=False, action='store_true', - help='plots a cross section whose x and y coordinates') - parser.add_argument('-pxm', '--xsection_map', default=False, action='store_true', - help='plots a cross section of 2D data on a geographical map') - parser.add_argument('-p', '--plot', default=False, action='store_true', - help='Makes a time series plot') - parser.add_argument('-mp', '--multi_panel', default=False, - action='store_true', - help='Makes a 2 panel timeseries plot') - parser.add_argument('-qc', '--qc', default=False, action='store_true', - help='Create time series plot of embedded quality control values') - parser.add_argument('-fb', '--fill_between', default=False, action='store_true', - help='makes a fill betweem plot based on matplotlib') - parser.add_argument('-bsd', '--barbs_spd_dir', default=False, action='store_true', - help=('Makes time series plot of wind barbs ' - 'using wind speed and dir.')) - parser.add_argument('-buv', '--barbs_u_v', default=False, action='store_true', - help=('Makes time series plot of wind barbs ' - 'using u and v wind components.')) - parser.add_argument('-pxs', '--xsection_from_1d', default=False, - action='store_true', - help='Will plot a time-height cross section from 1D dataset') - parser.add_argument('-ths', '--time_height_scatter', - default=False, action='store_true', - help='Create a scatter time series plot') - parser.add_argument('-sbg', '--stacked_bar_graph', - default=False, action='store_true', - help='Create stacked bar graph histogram') - parser.add_argument('-psd', '--size_dist', default=False, action='store_true', - help='Plots a stairstep plot of size distribution') - parser.add_argument('-sg', '--stairstep', default=False, action='store_true', - help='Plots stairstep plot of a histogram') - parser.add_argument('-hm', '--heatmap', default=False, action='store_true', - help='Plot a heatmap histogram from 2 variables') - parser.add_argument('-cc', '--create_contour', default=False, action='store_true', - help='Extracts, grids, and creates a contour plot') - parser.add_argument('-cf', '--contourf', default=False, action='store_true', - help=('Base function for filled contours if user ' - 'already has data gridded')) - parser.add_argument('-ct', '--plot_contour', default=False, action='store_true', - help=('Base function for contours if user ' - 'already has data gridded')) - parser.add_argument('-vsd', '--vectors_spd_dir', default=False, action='store_true', - help='Extracts, grids, and creates a contour plot.') - parser.add_argument('-b', '--barbs', default=False, action='store_true', - help='Base function for wind barbs.') - parser.add_argument('-ps', '--plot_station', default=False, action='store_true', - help='Extracts, grids, and creates a contour plot') + parser.add_argument( + '-cu', + '--cleanup', + default=default, + action='store_false', + help=f'Turn off standard methods for obj cleanup. Default is {default}', + ) + parser.add_argument( + '-gl', + '--gridlines', + default=False, + action='store_true', + help='Use latitude and lingitude gridlines.', + ) + parser.add_argument( + '-cl', + '--coastlines', + default=False, + action='store_true', + help='Plot coastlines on geographical map', + ) + parser.add_argument( + '-bg', + '--background', + default=False, + action='store_true', + help='Plot a stock image background', + ) + parser.add_argument( + '-nan', '--add_nan', default=False, action='store_true', help='Fill in data gaps with NaNs' + ) + parser.add_argument( + '-dn', + '--day_night', + default=False, + action='store_true', + help=('Fill in color coded background according ' 'to time of day.'), + ) + parser.add_argument( + '-yr', '--set_yrange', default=None, nargs=2, help=('Set the yrange for the specific plot') + ) + parser.add_argument( + '-iya', '--invert_y_axis', default=False, action='store_true', help='Invert y axis' + ) + parser.add_argument( + '-sp', + '--show_parcel', + default=False, + action='store_true', + help='set to true to plot the parcel path.', + ) + parser.add_argument( + '-cape', + '--shade_cape', + default=False, + action='store_true', + help='set to true to shade regions of cape.', + ) + parser.add_argument( + '-cin', + '--shade_cin', + default=False, + action='store_true', + help='set to true to shade regions of cin.', + ) + parser.add_argument( + '-d', + '--density', + default=False, + action='store_true', + help='Plot a p.d.f. instead of a frequency histogram', + ) + parser.add_argument( + '-m', + '--mesh', + default=False, + action='store_true', + help=('Set to True to interpolate u and v to ' 'grid and create wind barbs'), + ) + parser.add_argument( + '-uv', + '--from_u_and_v', + default=False, + action='store_true', + help='Create SkewTPLot with u and v wind', + ) + parser.add_argument( + '-sd', + '--from_spd_and_dir', + default=False, + action='store_true', + help='Create SkewTPlot with wind speed and direction', + ) + parser.add_argument( + '-px', + '--plot_xsection', + default=False, + action='store_true', + help='plots a cross section whose x and y coordinates', + ) + parser.add_argument( + '-pxm', + '--xsection_map', + default=False, + action='store_true', + help='plots a cross section of 2D data on a geographical map', + ) + parser.add_argument( + '-p', '--plot', default=False, action='store_true', help='Makes a time series plot' + ) + parser.add_argument( + '-mp', + '--multi_panel', + default=False, + action='store_true', + help='Makes a 2 panel timeseries plot', + ) + parser.add_argument( + '-qc', + '--qc', + default=False, + action='store_true', + help='Create time series plot of embedded quality control values', + ) + parser.add_argument( + '-fb', + '--fill_between', + default=False, + action='store_true', + help='makes a fill betweem plot based on matplotlib', + ) + parser.add_argument( + '-bsd', + '--barbs_spd_dir', + default=False, + action='store_true', + help=('Makes time series plot of wind barbs ' 'using wind speed and dir.'), + ) + parser.add_argument( + '-buv', + '--barbs_u_v', + default=False, + action='store_true', + help=('Makes time series plot of wind barbs ' 'using u and v wind components.'), + ) + parser.add_argument( + '-pxs', + '--xsection_from_1d', + default=False, + action='store_true', + help='Will plot a time-height cross section from 1D dataset', + ) + parser.add_argument( + '-ths', + '--time_height_scatter', + default=False, + action='store_true', + help='Create a scatter time series plot', + ) + parser.add_argument( + '-sbg', + '--stacked_bar_graph', + default=False, + action='store_true', + help='Create stacked bar graph histogram', + ) + parser.add_argument( + '-psd', + '--size_dist', + default=False, + action='store_true', + help='Plots a stairstep plot of size distribution', + ) + parser.add_argument( + '-sg', + '--stairstep', + default=False, + action='store_true', + help='Plots stairstep plot of a histogram', + ) + parser.add_argument( + '-hm', + '--heatmap', + default=False, + action='store_true', + help='Plot a heatmap histogram from 2 variables', + ) + parser.add_argument( + '-cc', + '--create_contour', + default=False, + action='store_true', + help='Extracts, grids, and creates a contour plot', + ) + parser.add_argument( + '-cf', + '--contourf', + default=False, + action='store_true', + help=('Base function for filled contours if user ' 'already has data gridded'), + ) + parser.add_argument( + '-ct', + '--plot_contour', + default=False, + action='store_true', + help=('Base function for contours if user ' 'already has data gridded'), + ) + parser.add_argument( + '-vsd', + '--vectors_spd_dir', + default=False, + action='store_true', + help='Extracts, grids, and creates a contour plot.', + ) + parser.add_argument( + '-b', '--barbs', default=False, action='store_true', help='Base function for wind barbs.' + ) + parser.add_argument( + '-ps', + '--plot_station', + default=False, + action='store_true', + help='Extracts, grids, and creates a contour plot', + ) # The mutually exclusive but one requried group group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-gp', '--geodisplay', dest='action', action='store_const', - const=geodisplay, help='Set to genereate a geographic plot') - group.add_argument('-skt', '--skewt', dest='action', action='store_const', - const=skewt, help='Set to genereate a skew-t plot') - group.add_argument('-xs', '--xsection', dest='action', action='store_const', - const=xsection, help='Set to genereate a XSection plot') - group.add_argument('-wr', '--wind_rose', dest='action', action='store_const', - const=wind_rose, help='Set to genereate a wind rose plot') - group.add_argument('-ts', '--timeseries', dest='action', action='store_const', - const=timeseries, help='Set to genereate a timeseries plot') - group.add_argument('-c', '--contour', dest='action', action='store_const', - const=contour, help='Set to genereate a contour plot') - group.add_argument('-hs', '--histogram', dest='action', action='store_const', - const=histogram, help='Set to genereate a histogram plot') + group.add_argument( + '-gp', + '--geodisplay', + dest='action', + action='store_const', + const=geodisplay, + help='Set to genereate a geographic plot', + ) + group.add_argument( + '-skt', + '--skewt', + dest='action', + action='store_const', + const=skewt, + help='Set to genereate a skew-t plot', + ) + group.add_argument( + '-xs', + '--xsection', + dest='action', + action='store_const', + const=xsection, + help='Set to genereate a XSection plot', + ) + group.add_argument( + '-wr', + '--wind_rose', + dest='action', + action='store_const', + const=wind_rose, + help='Set to genereate a wind rose plot', + ) + group.add_argument( + '-ts', + '--timeseries', + dest='action', + action='store_const', + const=timeseries, + help='Set to genereate a timeseries plot', + ) + group.add_argument( + '-c', + '--contour', + dest='action', + action='store_const', + const=contour, + help='Set to genereate a contour plot', + ) + group.add_argument( + '-hs', + '--histogram', + dest='action', + action='store_const', + const=histogram, + help='Set to genereate a histogram plot', + ) args = parser.parse_args() diff --git a/setup.cfg b/setup.cfg index 58b0e8d9d6..28c95c6344 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,9 @@ [flake8] -exclude = act/io/armfiles.py docs *__init__.py* setup.cfg +exclude = docs *__init__.py* setup.cfg act/tests/data/ ignore = E203,E266,E501,W503,E722,E402,C901,E731,F401 max-line-length = 100 max-complexity = 18 -extend-exclude = act/io/armfiles.py docs *__init__.py* +extend-exclude = docs *__init__.py* extend-ignore = E203,E266,E501,W503,E722,E402,C901,E731,F401 [isort] @@ -18,7 +18,6 @@ line_length=100 skip= docs/source/conf.py setup.py - act/io/armfiles.py [tool:pytest] addopts = --cov=./ --cov-report=xml --verbose diff --git a/setup.py b/setup.py index 3e6c4bff89..71404a0979 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ entry_points={'console_scripts': []}, include_package_data=True, package_data={'act': []}, - scripts=glob.glob("scripts/*"), + scripts=glob.glob('scripts/*'), install_requires=requirements, license='BSD (3-clause)', classifiers=[