Add 2024 processing scripts

afsc-gap-products · Sep 19, 2024 · 926cb8b · 926cb8b
1 parent 2db5b55
commit 926cb8b
Show file tree

Hide file tree

Showing 5 changed files with 566 additions and 0 deletions.
diff --git a/processing_files/2024/2024_AI.R b/processing_files/2024/2024_AI.R
@@ -0,0 +1,137 @@
+# Generate data products
+
+library(gapctd)
+
+# Replace this with paths to cast rds files that should be included in the data product.
+cast_files <- c(
+  list.files(path = "C:/Users/sean.rohan/Work/afsc/WIP/2024_AKP_8101/final_cnv/", full.names = TRUE),
+  list.files(path = "C:/Users/sean.rohan/Work/afsc/WIP/2024_AKP_8106/final_cnv/", full.names = TRUE),
+  list.files(path = "C:/Users/sean.rohan/Work/afsc/WIP/2024_OEX_8103/final_cnv/", full.names = TRUE),
+  list.files(path = "C:/Users/sean.rohan/Work/afsc/WIP/2024_OEX_8104/final_cnv/", full.names = TRUE)
+)
+
+# Replace this with paths to metadata rds files that should be included in the data product.
+metadata_files <- c(
+  list.files(path = "C:/Users/sean.rohan/Work/afsc/WIP/2024_AKP_8101/metadata/", full.names = TRUE),
+  list.files(path = "C:/Users/sean.rohan/Work/afsc/WIP/2024_AKP_8106/metadata/", full.names = TRUE),
+  list.files(path = "C:/Users/sean.rohan/Work/afsc/WIP/2024_OEX_8103/metadata/", full.names = TRUE),
+  list.files(path = "C:/Users/sean.rohan/Work/afsc/WIP/2024_OEX_8104/metadata/", full.names = TRUE)
+)
+
+
+
+# Update this section with the relevant metadata
+year <- 2024
+region <- "AI"
+dataset_name <- "Water temperature, salinity, and others taken by trawl-mounted CTD from chartered fishing vessels Alaska Provider and Ocean Explorer during the 2024 Alaska Fisheries Science Center Aleutian Islands Biennial Summer Bottom Trawl Survey"
+cruise_name <- "2022 Aleutian Islands Bottom Trawl Survey"
+ctd_team <- "Bethany Riggle, Nicole Charriere, Nate Raring" # Do not list yourself!
+creator_name <- "Sean Rohan" # your name
+creator_email <- "[email protected]" # your email
+dataset_citation <- "Rohan, S.K., Riggle, B.,  Charriere, N.E., O'Leary, C.A., Raring, N.W. (2024). Water temperature, salinity, and others taken by trawl-mounted CTD from chartered fishing vessels Alaska Provider and Ocean Explorer during the 2022-2024 Alaska Fisheries Science Center Aleutian Islands Biennial Summer Bottom Trawl Surveys (NCEI Accession 0286095). [indicate subset used]. NOAA National Centers for Environmental Information. Dataset. https://doi.org/10.25921/zha7-tt11. Accessed [date]."
+dataset_doi <- "10.25921/zha7-tt11"
+ctd_unit <- "Sea-Bird SBE19plus V2 SeaCAT"
+references <- "Rohan, S. K., Charriere, N. E., Riggle, B., O’Leary, C. A., and Raring, N. W. 2023. A flexible approach for processing data collected using trawl-mounted CTDs during Alaska bottom-trawl surveys. U.S. Dep. Commer., NOAA Tech. Memo. NMFS-AFSC-475, 43 p. https://doi.org/10.25923/8ape-q461"
+creator_institution <- "NOAA Alaska Fisheries Science Center"
+processing_info <- paste0("CTD data processed using gapctd ", packageVersion(pkg = "gapctd"))
+ncei_accession_number <- ""
+publisher_url <- "https://github.com/afsc-gap-products/gapctd"
+auxiliary_sensors <- c("Sea-Bird SBE43 dissolved oxygen sensor")
+text_output_columns <- c("vessel", 
+                   "cruise", 
+                   "haul", 
+                   "stationid", 
+                   "serial_number", 
+                   "cast_direction", 
+                   "timeS", 
+                   "depth", 
+                   "pressure", 
+                   "temperature", 
+                   "conductivity", 
+                   "salinity", 
+                   "sound_speed", 
+                   "oxygen",
+                   "flag")
+
+column_descriptions <- c(
+  "vessel" = "vessel: AFSC/RACE vessel code (148: Ocean Explorer, 176: Alaska Provider)",
+  "cruise" = "cruise: AFSC/RACE cruise code",
+  "haul" = "haul: AFSC/RACE haul number",
+  "stationid" = "stationid: AFSC/RACE station code",
+  "serial_number" = "serial_number: Primary instrument serial number",
+  "cast_direction" = "cast_directon: Cast direction",
+  "datetime" = "datetime: date and time in Alaska Daylight Time [UTC-9:00]",
+  "depth" = "depth: depth [m], down is positive",
+  "pressure" = "pressure: pressure, strain gauge [db]",
+  "conductivity" = "conductivity: conductivity [S/m]",
+  "temperature" = "temperature: temperature [ITS-90, degrees C]",
+  "salinity" = "salinity: salinity [PSS-78]",
+  "sound_speed" = "sound_speed: Chen-Millero sound speed [m/s]",
+  "oxygen" = "oxygen: dissolved oxygen [ml/l]",
+  "flag" = "flag: data quality flag"
+)
+
+# Create a netCDF file with cast data and metadata in the working directory.
+gapctd::make_oce_ncdf(
+  cast_files = cast_files,
+  metadata_files = metadata_files,
+  output_file = here::here("data", paste0("GAPCTD_", year, "_", region, ".nc")),
+  global_attributes = list(title = dataset_name, 
+                           references = references,
+                           id = dataset_doi,
+                           cdm_data_type = "Point",
+                           cruise = cruise_name,
+                           institution = creator_institution,
+                           contributor_name = ctd_team,
+                           creator_name = creator_name,
+                           creator_institution = creator_institution,
+                           creator_email = creator_email,
+                           publisher = creator_institution,
+                           publisher_type = "institution",
+                           publisher_url = publisher_url,
+                           geospatial_bounds_crs = "EPSG:4326",
+                           license = "http://www.usa.gov/publicdomain/label/1.0/",
+                           metadata_link = "",
+                           instrument = "CTD",
+                           Conventions = c("CF-1.8"),
+                           standard_name_vocabulary = "CF Standard Name Table v79",
+                           source = processing_info 
+  )
+)
+
+
+# Create an rds file with cast data in the working directory
+make_oce_table(cast_files = cast_files,
+               output_file = here::here("data", paste0("GAPCTD_", year, "_", region, ".rds")),
+               precision = c(temperature = 4,
+                             conductivity = 6,
+                             salinity = 4,
+                             absolute_salinity = 4,
+                             sound_speed = 3,
+                             density = 3,
+                             pressure = 3,
+                             oxygen = 4,
+                             velocity = 3))
+
+make_text_table(x = readRDS(here::here("data", 
+                                       paste0("GAPCTD_", 
+                                              year, "_", 
+                                              region, 
+                                              ".rds")))[text_output_columns] |>
+                  dplyr::rename(datetime = timeS), 
+                            output_file = here::here("data", paste0("GAPCTD_", year, "_", region)),
+                            column_descriptions = column_descriptions, 
+                            ctd_unit = ctd_unit, 
+                            auxiliary_sensors = auxiliary_sensors,
+                            dataset_name = dataset_name, 
+                            cruise_name = cruise_name, 
+                            creator_name = creator_name, 
+                            creator_email = creator_email,
+                            creator_institution = creator_institution,
+                            collaborators = ctd_team, 
+                            references = references, 
+                            dataset_doi = dataset_doi, 
+                            ncei_accession_number = ncei_accession_number, 
+                            processing_info = processing_info, 
+                            publisher_url = publisher_url)
+
diff --git a/processing_files/2024/2024_AKK.R b/processing_files/2024/2024_AKK.R
@@ -0,0 +1,90 @@
+# This CTD was deployed with a dissolved oxygen sensors on external voltage channels 0
+
+library(gapctd)
+
+vessel <- 162
+region <- "BS"
+cruise <- 202401
+processing_method <- "gapctd"
+ctd_dir <- "G:/RACE_CTD/data/2024/ebs/v162_8102"
+
+
+# 1. Setup -----------------------------------------------------------------------------------------
+
+# Setup directory and retrieve haul data. Running this more than once will remove all files from the working directory and require reprocessing from scratch.
+# gapctd:::setup_gapctd_directory(processing_method = processing_method,
+#                                 ctd_dir = ctd_dir,
+#                                 use_sbedp_to_convert = FALSE)
+
+
+# 2. Run gapctd ------------------------------------------------------------------------------------
+
+# Wrapper function around automated data processing steps that are used to create deployment rds files containing upcast, downcast, and bottom data.
+# Performs the following tasks:
+# - Retrieves haul data from racebase and race_data; writes haul data to /output/. 
+# - Splits deployment data into downcast, bottom, and upcast; writes file to /data/split/
+# - Runs four versions of the data processing (Typical, Typical CTM, T-s Area, and S Path distance) on deployment data. 
+# - Saves processed deployment data to /output/[processing_method] directory as lists. Filename suffix and metadata in the file describe the processing method (Example: files ending in '_typical.rds' were processed using the Typical method.
+# - Processes deployment files to create metadata file that includes bottom variables for each deployment; saved to file in /metadata/ directory.
+# - Checks files for data flags moves data with flagged files to /bad_cnv/
+
+gapctd::wrapper_run_gapctd(cnv_dir_path = here::here("cnv"),
+                           processing_method = processing_method,
+                           haul_df = NULL,
+                           vessel = vessel,
+                           cruise = cruise,
+                           channel = NULL)
+
+
+#3. Select best method -----------------------------------------------------------------------------
+
+# Review profile plots from data processing using four methods; choose the best method.
+# Files chosen as the best will be renamed to end with "_best.rds"
+
+gapctd::select_best_method(
+  rds_dir_path = here::here("output", processing_method))
+
+
+# 4. Align oxygen ----------------------------------------------------------------------------------
+
+# Test oxygen alignment offsets ranging from 2-7 seconds
+# Candidate aligned oxygen data will be saved to /output/align_oxygen/*_ox_align.rds
+gapctd:::wrapper_align_oxygen()
+
+
+# 5. Select best oxygen alignment ------------------------------------------------------------------
+
+# Review profiles of oxygen with different alignments from step 4 and select the best one.
+# Data are reprocessed using run_gapctd(). Profiles written to /output/gapctd/*_best_oxygen.rds
+
+gapctd:::select_best_oxygen_method()
+
+
+# 6. Flag and interpolate --------------------------------------------------------------------------
+
+# Visually inspect, flag, and interpolate
+# Outputs written to /output/gapctd/ in rds files ending with "_qc.rds"
+
+gapctd::wrapper_flag_interpolate(rds_dir_path = here::here("output", processing_method),
+                                 in_pattern = "_best_oxygen.rds",
+                                 review = c("density", "salinity", "oxygen"))
+
+
+# 7. Review ----------------------------------------------------------------------------------------
+
+# Review profiles plots and select the casts that pass quality control checks.
+
+gapctd::review_profiles(rds_dir_path = here::here("output", processing_method),
+                        threshold = -1e-5,
+                        in_pattern = "_qc.rds",
+                        out_pattern = "_final_ts.rds")
+
+gapctd::review_oxygen_ph_profiles(rds_dir_path = here::here("output", processing_method),
+                                  in_pattern = "_final_ts.rds",
+                                  out_pattern = "_final.rds")
+
+
+# 8. Finalize --------------------------------------------------------------------------------------
+
+# Move final profile data to /final_cnv/
+gapctd::finalize_data(rds_dir_path = here::here("output", processing_method))
diff --git a/processing_files/2024/2024_AKK_L1.R b/processing_files/2024/2024_AKK_L1.R
@@ -0,0 +1,90 @@
+# This CTD was deployed with a dissolved oxygen sensors on external voltage channels 0
+
+library(gapctd)
+
+vessel <- 162
+region <- "BS"
+cruise <- 202401
+processing_method <- "gapctd"
+ctd_dir <- "G:/RACE_CTD/data/2024/ebs/v162_8102_L1"
+
+
+# 1. Setup -----------------------------------------------------------------------------------------
+
+# Setup directory and retrieve haul data. Running this more than once will remove all files from the working directory and require reprocessing from scratch.
+gapctd:::setup_gapctd_directory(processing_method = processing_method,
+                                ctd_dir = ctd_dir,
+                                use_sbedp_to_convert = FALSE)
+
+
+# 2. Run gapctd ------------------------------------------------------------------------------------
+
+# Wrapper function around automated data processing steps that are used to create deployment rds files containing upcast, downcast, and bottom data.
+# Performs the following tasks:
+# - Retrieves haul data from racebase and race_data; writes haul data to /output/. 
+# - Splits deployment data into downcast, bottom, and upcast; writes file to /data/split/
+# - Runs four versions of the data processing (Typical, Typical CTM, T-s Area, and S Path distance) on deployment data. 
+# - Saves processed deployment data to /output/[processing_method] directory as lists. Filename suffix and metadata in the file describe the processing method (Example: files ending in '_typical.rds' were processed using the Typical method.
+# - Processes deployment files to create metadata file that includes bottom variables for each deployment; saved to file in /metadata/ directory.
+# - Checks files for data flags moves data with flagged files to /bad_cnv/
+
+gapctd::wrapper_run_gapctd(cnv_dir_path = here::here("cnv"),
+                           processing_method = processing_method,
+                           haul_df = NULL,
+                           vessel = vessel,
+                           cruise = cruise,
+                           channel = NULL)
+
+
+#3. Select best method -----------------------------------------------------------------------------
+
+# Review profile plots from data processing using four methods; choose the best method.
+# Files chosen as the best will be renamed to end with "_best.rds"
+
+gapctd::select_best_method(
+  rds_dir_path = here::here("output", processing_method))
+
+
+# 4. Align oxygen ----------------------------------------------------------------------------------
+
+# Test oxygen alignment offsets ranging from 2-7 seconds
+# Candidate aligned oxygen data will be saved to /output/align_oxygen/*_ox_align.rds
+gapctd:::wrapper_align_oxygen()
+
+
+# 5. Select best oxygen alignment ------------------------------------------------------------------
+
+# Review profiles of oxygen with different alignments from step 4 and select the best one.
+# Data are reprocessed using run_gapctd(). Profiles written to /output/gapctd/*_best_oxygen.rds
+
+gapctd:::select_best_oxygen_method()
+
+
+# 6. Flag and interpolate --------------------------------------------------------------------------
+
+# Visually inspect, flag, and interpolate
+# Outputs written to /output/gapctd/ in rds files ending with "_qc.rds"
+
+gapctd::wrapper_flag_interpolate(rds_dir_path = here::here("output", processing_method),
+                                 in_pattern = "_best_oxygen.rds",
+                                 review = c("density", "salinity", "oxygen"))
+
+
+# 7. Review ----------------------------------------------------------------------------------------
+
+# Review profiles plots and select the casts that pass quality control checks.
+
+gapctd::review_profiles(rds_dir_path = here::here("output", processing_method),
+                        threshold = -1e-5,
+                        in_pattern = "_qc.rds",
+                        out_pattern = "_final_ts.rds")
+
+gapctd::review_oxygen_ph_profiles(rds_dir_path = here::here("output", processing_method),
+                                  in_pattern = "_final_ts.rds",
+                                  out_pattern = "_final.rds")
+
+
+# 8. Finalize --------------------------------------------------------------------------------------
+
+# Move final profile data to /final_cnv/
+gapctd::finalize_data(rds_dir_path = here::here("output", processing_method))