diff --git a/retriever/lib/engine.py b/retriever/lib/engine.py index 4d152a8b4..9b172178c 100644 --- a/retriever/lib/engine.py +++ b/retriever/lib/engine.py @@ -737,7 +737,6 @@ def get_input(self): if not os.path.exists(self.opts['data_dir']): os.makedirs(self.opts['data_dir']) - def insert_data_from_archive(self, url, filenames): """Insert data from files located in an online archive. This function extracts the file, inserts the data, and deletes the file if raw data diff --git a/scripts/veg_plots_sdl.json b/scripts/veg_plots_sdl.json index 5f97453dc..db4d2918f 100644 --- a/scripts/veg_plots_sdl.json +++ b/scripts/veg_plots_sdl.json @@ -1,6 +1,7 @@ { "citation": "Susana Rodriguez-Buritica, Helen Raichle, Robert H. Webb, Raymond M. Turner, and D. Lawrence Venable. 2013. One hundred and six years of population and community dynamics of Sonoran Desert Laboratory perennials. Ecology 94:976.", "description": "The data set constitutes all information associated with the Spalding-Shreve permanent vegetation plots from 1906 through 2012, which is the longest-running plant monitoring program in the world.", + "homepage": "https://figshare.com/articles/Data_Paper_Data_Paper/3555756", "keywords": [ "plants" ], @@ -13,7 +14,90 @@ ] }, "name": "Plots", - "schema": {}, + "schema": { + "fields": [ + { + "name": "plot", + "type": "char" + }, + { + "name": "area_m2", + "type": "char" + }, + { + "name": "source_area", + "type": "char" + }, + { + "name": "elevation_masl", + "type": "char" + }, + { + "name": "slope_direction", + "type": "char" + }, + { + "name": "slope_angle", + "type": "char" + }, + { + "name": "slope_description", + "type": "char" + }, + { + "name": "source_slope", + "type": "char" + }, + { + "name": "status", + "type": "char" + }, + { + "name": "originator", + "type": "char" + }, + { + "name": "date_established", + "type": "int" + }, + { + "name": "location_orgnotes", + "type": "char" + }, + { + "name": "criteria", + "type": "char" + }, + { + "name": "photo_stake", + "type": "char" + }, + { + "name": "substrate", + "type": "char" + }, + { + "name": "source_substrate", + "type": "char" + }, + { + "name": "notes_disturbance", + "type": "char" + }, + { + "name": "num_photos", + "type": "int" + }, + { + "name": "repeat_photo", + "type": "char" + }, + { + "name": "num_stakes", + "type": "char" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625330" }, { @@ -23,7 +107,50 @@ ] }, "name": "Plot_corners", - "schema": {}, + "schema": { + "fields": [ + { + "name": "plot", + "type": "char" + }, + { + "name": "vertex", + "type": "char" + }, + { + "name": "latitude", + "type": "char" + }, + { + "name": "longitude", + "type": "char" + }, + { + "name": "elevation", + "type": "char" + }, + { + "name": "source_proj", + "type": "char" + }, + { + "name": "accuracy", + "type": "char" + }, + { + "name": "gps_id", + "type": "char" + }, + { + "name": "easting_utm", + "type": "double" + }, + { + "name": "northing_utm", + "type": "double" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625333" }, { @@ -33,7 +160,54 @@ ] }, "name": "Species", - "schema": {}, + "schema": { + "fields": [ + { + "name": "code", + "type": "char" + }, + { + "name": "plants_code", + "type": "char" + }, + { + "name": "reportedname", + "type": "char" + }, + { + "name": "acceptedname", + "type": "char" + }, + { + "name": "authority", + "type": "char" + }, + { + "name": "family", + "type": "char" + }, + { + "name": "common_name", + "type": "char" + }, + { + "name": "habit", + "type": "char" + }, + { + "name": "duration", + "type": "char" + }, + { + "name": "enumeration", + "type": "char" + }, + { + "name": "comment", + "type": "char" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625336" }, { @@ -43,7 +217,66 @@ ] }, "name": "Seedling_counts", - "schema": {}, + "schema": { + "fields": [ + { + "name": "plot", + "type": "char" + }, + { + "name": "year", + "type": "int" + }, + { + "name": "startdate", + "type": "char" + }, + { + "name": "enddate", + "type": "char" + }, + { + "name": "row", + "type": "int" + }, + { + "name": "col", + "type": "int" + }, + { + "name": "plant", + "type": "char" + }, + { + "name": "number", + "type": "int" + }, + { + "name": "upper_limit", + "type": "int" + }, + { + "name": "unit", + "type": "char" + }, + { + "name": "dimension", + "type": "char" + }, + { + "name": "present", + "type": "char" + }, + { + "name": "mapped", + "type": "char" + }, + { + "name": "digitized", + "type": "char" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625339" }, { @@ -53,7 +286,50 @@ ] }, "name": "Count1906", - "schema": {}, + "schema": { + "fields": [ + { + "name": "plot", + "type": "int" + }, + { + "name": "code", + "type": "char" + }, + { + "name": "cat", + "type": "char" + }, + { + "name": "num", + "type": "int" + }, + { + "name": "height_m", + "type": "double" + }, + { + "name": "joints", + "type": "int" + }, + { + "name": "notes", + "type": "char" + }, + { + "name": "abund", + "type": "char" + }, + { + "name": "distr", + "type": "char" + }, + { + "name": "name", + "type": "char" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625342" }, { @@ -63,7 +339,30 @@ ] }, "name": "SMCover", - "schema": {}, + "schema": { + "fields": [ + { + "name": "code", + "type": "char" + }, + { + "name": "plot", + "type": "char" + }, + { + "name": "year", + "type": "int" + }, + { + "name": "coverns", + "type": "double" + }, + { + "name": "cover", + "type": "double" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625345" }, { @@ -73,7 +372,30 @@ ] }, "name": "SMDensity", - "schema": {}, + "schema": { + "fields": [ + { + "name": "code", + "type": "char" + }, + { + "name": "plot", + "type": "char" + }, + { + "name": "year", + "type": "int" + }, + { + "name": "countns", + "type": "int" + }, + { + "name": "count", + "type": "int" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625348" }, { @@ -84,7 +406,58 @@ ] }, "name": "Stake_info", - "schema": {}, + "schema": { + "fields": [ + { + "name": "stkphnum", + "type": "char" + }, + { + "name": "plot", + "type": "char" + }, + { + "name": "location", + "type": "char" + }, + { + "name": "easting_utm", + "type": "double" + }, + { + "name": "northing_utm", + "type": "double" + }, + { + "name": "lat_deg", + "type": "double" + }, + { + "name": "long_deg", + "type": "double" + }, + { + "name": "elev_m", + "type": "int" + }, + { + "name": "depository", + "type": "char" + }, + { + "name": "original_number", + "type": "char" + }, + { + "name": "no_photos", + "type": "int" + }, + { + "name": "marker", + "type": "char" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625351" }, { @@ -94,12 +467,87 @@ ] }, "name": "Photo_info", - "schema": {}, + "schema": { + "fields": [ + { + "name": "ordered", + "type": "int" + }, + { + "name": "stkphnum", + "type": "char" + }, + { + "name": "record_date", + "type": "char" + }, + { + "name": "photographer", + "type": "char" + }, + { + "name": "time", + "type": "char" + }, + { + "name": "camera", + "type": "char" + }, + { + "name": "lenstype", + "type": "char" + }, + { + "name": "focaldist", + "type": "char" + }, + { + "name": "lensnote", + "type": "char" + }, + { + "name": "azimuth", + "type": "char" + }, + { + "name": "verttilt", + "type": "char" + }, + { + "name": "tiltdeg", + "type": "char" + }, + { + "name": "camheight", + "type": "char" + }, + { + "name": "bwneg", + "type": "char" + }, + { + "name": "colorneg", + "type": "char" + }, + { + "name": "colorpos", + "type": "char" + }, + { + "name": "filter", + "type": "char" + }, + { + "name": "digital", + "type": "int" + } + ] + }, "url": "https://ndownloader.figshare.com/files/5625354" } ], "retriever": "True", "retriever_minimum_version": "2.0.dev", "title": "Sonoran Desert Lab perennials vegetation plots", - "version": "1.2.1" + "version": "2.0.0" } diff --git a/test/test_integration.py b/test/test_integration.py index 68560e94e..cedb72076 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -401,6 +401,47 @@ 'expect_out': ['aa,bb,c_c', '1,2,3', '4,5,6'] } +underscore_csv = { + 'name': 'underscore_csv', + 'raw_data': ['a,b,c', + '1,2,y', + '4,5,2_0'], + 'script': {"name": "underscore_csv", + "resources": [ + {"dialect": {"do_not_bulk_insert": "True"}, + "name": "underscore_csv", + "schema": { + "fields": [ + { + "name": "record_id", + "type": "pk-auto" + }, + { + "name": "a", + "type": "int" + }, + { + "name": "b", + "type": "int" + }, + { + "name": "c", + "type": "char" + } + ] + }, + "url": "http://example.com/underscore_csv.txt" + } + ], + "retriever": "True", + "retriever_minimum_version": "2.0.dev", + "version": "1.0.0", + "urls": {"underscore_csv": "http://example.com/underscore_csv.txt"} + }, + 'expect_out': ['record_id,a,b,c', '1,1,2,y', '2,4,5,2_0'] +} + + tests = [ simple_csv, comma_delimiter, @@ -412,7 +453,9 @@ autopk_crosstab, skip_csv, extra_newline, - change_header_values] + change_header_values, + underscore_csv +] # Create a tuple of all test scripts with their expected values test_parameters = [(test, test['expect_out']) for test in tests] diff --git a/version.txt b/version.txt index 33ca335f8..e69b851a2 100644 --- a/version.txt +++ b/version.txt @@ -89,7 +89,7 @@ tree_canopy_geometries.json,1.0.1 tree_demog_wghats.py,1.3.2 turtle_offspring_nesting.json,1.0.2 usgs_elevation.py,1.0.0 -veg_plots_sdl.json,1.2.1 +veg_plots_sdl.json,2.0.0 vertnet.py,1.4.2 vertnet_amphibians.py,1.1.2 vertnet_birds.py,1.1.2