From 8b53286eda4d4851ade0ed346c476e8ac978407c Mon Sep 17 00:00:00 2001 From: henrykironde Date: Thu, 25 Apr 2024 07:11:46 -0500 Subject: [PATCH 1/2] Limit field names to ten characters Co-authored-by: Ethan White --- Snakefile | 4 ++-- combine_nests.py | 4 ++-- everglades_workflow.sh | 2 +- nest_detection.py | 27 ++++++++++++++------------- process_nests.py | 18 +++++++++--------- upload_mapbox.py | 2 +- 6 files changed, 29 insertions(+), 28 deletions(-) diff --git a/Snakefile b/Snakefile index d1e4565..a95d8c3 100644 --- a/Snakefile +++ b/Snakefile @@ -38,7 +38,7 @@ rule all: zip, site=SITES, year=YEARS, flight=FLIGHTS), expand(f"{working_dir}/processed_nests/{{year}}/{{site}}/{{site}}_{{year}}_processed_nests.shp", zip, site=SITES, year=YEARS), - expand(f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles", + expand(f"{working_dir}/mapbox/last_uploaded/{{year}}/{{site}}/{{flight}}.mbtiles", zip, site=SITES, year=YEARS, flight=FLIGHTS) @@ -131,7 +131,7 @@ rule upload_mapbox: input: f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles" output: - f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles" + touch(f"{working_dir}/mapbox/last_uploaded/{{year}}/{{site}}/{{flight}}.mbtiles") conda: "EvergladesTools" shell: diff --git a/combine_nests.py b/combine_nests.py index e94783f..2cb9d35 100644 --- a/combine_nests.py +++ b/combine_nests.py @@ -31,8 +31,8 @@ def load_shapefile(x): 'last_obs': 'str', 'num_obs': 'int', 'species': 'str', - 'sum_top1_s': 'float', - 'num_obs_to': 'int', + 'sum_top1': 'float', + 'num_top1': 'int', 'bird_match': 'str' }) shp["site"] = get_site(x) diff --git a/everglades_workflow.sh b/everglades_workflow.sh index c47d4da..0e94aad 100644 --- a/everglades_workflow.sh +++ b/everglades_workflow.sh @@ -4,7 +4,7 @@ #SBATCH --mail-type=FAIL #SBATCH --gpus=a100:4 #SBATCH --cpus-per-task=60 -#SBATCH --mem=1200gb +#SBATCH --mem=600gb #SBATCH --time=80:00:00 #SBATCH --partition=gpu #SBATCH --output=/blue/ewhite/everglades/EvergladesTools/logs/everglades_workflow.out diff --git a/nest_detection.py b/nest_detection.py index ad5ef2e..677277c 100644 --- a/nest_detection.py +++ b/nest_detection.py @@ -68,12 +68,12 @@ def compare_site(gdf): # add target info to match row = geopandas.GeoDataFrame(pd.DataFrame(row).transpose(), crs=matches.crs) matches = geopandas.GeoDataFrame(pd.concat([matches, row], ignore_index=True)) - matches["target_index"] = index + matches["target_ind"] = index matches = matches.rename(columns={ - "xmin": "matched_xmin", - "max": "matched_xmax", - "ymin": "matched_ymin", - "ymax": "matched_ymax" + "xmin": "match_xmin", + "xmax": "match_xmax", + "ymin": "match_ymin", + "ymax": "match_ymax" }) results.append(matches) @@ -82,8 +82,8 @@ def compare_site(gdf): results = pd.concat(results) else: results = pd.DataFrame(columns=[ - 'matched_xmin', 'matched_ymin', 'xmax', 'matched_ymax', 'label', 'score', 'image_path', 'Date', 'bird_id', - 'target_index', 'geometry' + 'match_xmin', 'match_ymin', 'match_xmax', 'match_ymax', 'label', 'score', 'image_path', 'Site', 'Date', + 'Year', 'event', 'file_posts', 'bird_id', 'target_ind' ]) return results @@ -102,19 +102,20 @@ def detect_nests(bird_detection_file, year, site, savedir): schema = { "geometry": "Polygon", "properties": { - 'matched_xmin': 'float', - 'matched_ymin': 'float', - 'xmax': 'float', - 'matched_ymax': 'float', + 'match_xmin': 'float', + 'match_ymin': 'float', + 'match_xmax': 'float', + 'match_ymax': 'float', 'label': 'str', 'score': 'float', 'image_path': 'str', 'Site': 'str', 'Date': 'str', 'Year': 'str', - 'bird_id': 'int', 'event': 'str', - 'target_index': 'int' + 'file_posts': 'str', + 'bird_id': 'int', + 'target_ind': 'int' } } if not results.empty: diff --git a/process_nests.py b/process_nests.py index 8cbf896..66277ce 100644 --- a/process_nests.py +++ b/process_nests.py @@ -55,13 +55,13 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections= top_score_data = summed_scores[summed_scores['sum'] == max(summed_scores['sum'])].reset_index() nest_info = nest_data.groupby(['Site', 'Year', 'target_ind']).agg({ 'Date': ['min', 'max', 'count'], - 'matched_xm': ['mean'], - 'matched_ym': ['mean'], - 'xmax': ['mean'], - 'matched__1': ['mean'] + 'match_xmin': ['mean'], + 'match_ymin': ['mean'], + 'match_xmax': ['mean'], + 'match_ymax': ['mean'] }).reset_index() - xmean = (nest_info['matched_xm']['mean'][0] + nest_info['xmax']['mean']) / 2 - ymean = (nest_info['matched_ym']['mean'][0] + nest_info['matched__1']['mean']) / 2 + xmean = (nest_info['match_xmin']['mean'][0] + nest_info['match_xmax']['mean']) / 2 + ymean = (nest_info['match_ymin']['mean'][0] + nest_info['match_ymax']['mean']) / 2 bird_match = ",".join([str(x) for x in nest_data["bird_id"]]) nests.append([ target_ind, nest_info['Site'][0], nest_info['Year'][0], xmean[0], ymean[0], nest_info['Date']['min'][0], @@ -77,7 +77,7 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections= nests = pd.DataFrame(nests, columns=[ 'nest_id', 'Site', 'Year', 'xmean', 'ymean', 'first_obs', 'last_obs', 'num_obs', - 'species', 'sum_top1_score', 'num_obs_top1', 'bird_match' + 'species', 'sum_top1', 'num_top1', 'bird_match' ]) nests_shp = geopandas.GeoDataFrame(nests, geometry=geopandas.points_from_xy(nests.xmean, nests.ymean)) nests_shp.crs = nests_data.crs @@ -95,8 +95,8 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections= 'last_obs': 'str', 'num_obs': 'int', 'species': 'str', - 'sum_top1_score': 'float', - 'num_obs_top1': 'int', + 'sum_top1': 'float', + 'num_top1': 'int', 'bird_match': 'str' } } diff --git a/upload_mapbox.py b/upload_mapbox.py index faeea70..88c303e 100644 --- a/upload_mapbox.py +++ b/upload_mapbox.py @@ -56,7 +56,7 @@ def retrieve_upload_status(self, upload_id): def get_credentials(): """Get credentials from mapbox.ini""" - with open("/blue/ewhite/everglades/mapbox/mapbox.ini", "rb") as f: + with open("/blue/ewhite/everglades/mapbox.ini", "rb") as f: toml_dict = tomli.load(f) access_token = toml_dict['mapbox']['access-token'] return access_token From a123183a21217a20972384ccff9b4cf176f0a922 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Thu, 25 Apr 2024 07:39:04 -0500 Subject: [PATCH 2/2] Update github actions --- .github/workflows/python-package-conda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index fd1d35e..2366cbe 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -30,11 +30,11 @@ jobs: - uses: mamba-org/provision-with-micromamba@v15 with: environment-name: Zooniverse - environment-file: Zooniverse/dev_environment.yml + environment-file: dev_environment.yml python-version: "${{ matrix.python-version }}" cache-env: true cache-downloads: true - name: "Run tests" run: | - yapf -d --recursive ./Zooniverse/ --style=.style.yapf 2>&1 + yapf -d ./*.py --style=.style.yapf 2>&1