diff --git a/Zooniverse/.DS_Store b/Zooniverse/.DS_Store new file mode 100644 index 00000000..d2822dcf Binary files /dev/null and b/Zooniverse/.DS_Store differ diff --git a/Zooniverse/__init__.py b/Zooniverse/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/Zooniverse/__pycache__/__init__.cpython-36.pyc b/Zooniverse/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 00000000..2e35f74b Binary files /dev/null and b/Zooniverse/__pycache__/__init__.cpython-36.pyc differ diff --git a/Zooniverse/__pycache__/__init__.cpython-37.pyc b/Zooniverse/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 00000000..3d8541c9 Binary files /dev/null and b/Zooniverse/__pycache__/__init__.cpython-37.pyc differ diff --git a/Zooniverse/__pycache__/tile_raster.cpython-36.pyc b/Zooniverse/__pycache__/tile_raster.cpython-36.pyc new file mode 100644 index 00000000..99cdd185 Binary files /dev/null and b/Zooniverse/__pycache__/tile_raster.cpython-36.pyc differ diff --git a/Zooniverse/__pycache__/tile_raster.cpython-37.pyc b/Zooniverse/__pycache__/tile_raster.cpython-37.pyc new file mode 100644 index 00000000..20f61e75 Binary files /dev/null and b/Zooniverse/__pycache__/tile_raster.cpython-37.pyc differ diff --git a/Zooniverse/cron.txt b/Zooniverse/cron.txt new file mode 100644 index 00000000..d3bf781c --- /dev/null +++ b/Zooniverse/cron.txt @@ -0,0 +1,14 @@ +# Cron job for weekly runs on the HiPerGator +# This is currently running under ethanwhite's account on daemon2 +# To recreate copy, paste, and modify this text into the editor resulting +# from `crontab -e` +# WEECOLOGYDEPLOYGITHUBPAT needs to be replaced with the appropriate GITHUB +# Personal Access Token +# cron jobs on HiPerGator run from a blank environment so we need to set both +# basic environmental variables (HOME and PATH) and also the GitHub tokens +# because they are not loaded into the cronjob environment + +HOME=/home/b.weinstein/ +PATH=/opt/slurm/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/bin:/home/b.weinstein/bin +MAILTO=ben.weinstein@weecology.org +15 15 * * * sbatch ${HOME}/EvergladesWadingBird/Zooniverse/submit_job.sbatch >> ${HOME}/logs/cron.log 2>&1 \ No newline at end of file diff --git a/Zooniverse/manifest.py b/Zooniverse/manifest.py index c6b686f0..b4f1b374 100644 --- a/Zooniverse/manifest.py +++ b/Zooniverse/manifest.py @@ -116,7 +116,7 @@ def main(path, everglades_watch, save_dir="/orange/ewhite/everglades/Zooniverse/ assert os.path.exists(saved_file) uploaded = uploaded.path.append(pd.Series({"path":saved_file}),ignore_index=True) except Exception as e: - print("{} failed with exception".format(path)) + print("{} failed with exception {e}".format(path, e)) #Overwrite uploaded manifest uploaded.to_csv("uploaded.csv") \ No newline at end of file diff --git a/Zooniverse/submit_job.sbatch b/Zooniverse/submit_job.sbatch index 5ebd345a..699eae5e 100644 --- a/Zooniverse/submit_job.sbatch +++ b/Zooniverse/submit_job.sbatch @@ -16,4 +16,4 @@ source activate everglades rclone sync everglades:"WadingBird2020Deliverables" /orange/ewhite/everglades/WadingBirds2020/ #Upload -python manifest.py \ No newline at end of file +python /home/b.weinstein/EvergladesWadingBird/Zooniverse/manifest.py \ No newline at end of file diff --git a/Zooniverse/tests/output/SOAP_046_0.tif b/Zooniverse/tests/output/SOAP_046_0.tif new file mode 100644 index 00000000..84480fc5 Binary files /dev/null and b/Zooniverse/tests/output/SOAP_046_0.tif differ diff --git a/Zooniverse/tests/output/SOAP_046_1.tif b/Zooniverse/tests/output/SOAP_046_1.tif new file mode 100644 index 00000000..4ea08e27 Binary files /dev/null and b/Zooniverse/tests/output/SOAP_046_1.tif differ diff --git a/Zooniverse/tests/output/SOAP_046_2.tif b/Zooniverse/tests/output/SOAP_046_2.tif new file mode 100644 index 00000000..0e51739c Binary files /dev/null and b/Zooniverse/tests/output/SOAP_046_2.tif differ diff --git a/Zooniverse/tests/output/SOAP_046_3.tif b/Zooniverse/tests/output/SOAP_046_3.tif new file mode 100644 index 00000000..9a69f852 Binary files /dev/null and b/Zooniverse/tests/output/SOAP_046_3.tif differ diff --git a/Zooniverse/tests/submit_test.sbatch b/Zooniverse/tests/submit_test.sbatch new file mode 100644 index 00000000..7234661e --- /dev/null +++ b/Zooniverse/tests/submit_test.sbatch @@ -0,0 +1,17 @@ +#test connect in sbatch +#!/bin/bash +#SBATCH --job-name=Everglades_rclone # Job name +#SBATCH --mail-type=END # Mail events +#SBATCH --mail-user=benweinstein2010@gmail.com # Where to send mail +#SBATCH --account=ewhite +#SBATCH --nodes=1 # Number of MPI ranks +#SBATCH --cpus-per-task=1 +#SBATCH --mem=10GB +#SBATCH --time=72:00:00 #Time limit hrs:min:sec +#SBATCH --output=/home/b.weinstein/logs/rclone_%j.out # Standard output and error log +#SBATCH --error=/home/b.weinstein/logs/rclone_%j.err + +#comet debug + +#Sync data +python tests/test_connect.py \ No newline at end of file diff --git a/Zooniverse/zooniverse.wpu b/Zooniverse/zooniverse.wpu index 228c50cc..75ce77e1 100644 --- a/Zooniverse/zooniverse.wpu +++ b/Zooniverse/zooniverse.wpu @@ -6,8 +6,8 @@ [user attributes] debug.exceptions-ignored = {loc('../../../miniconda3/envs/crowns/lib/python3.7/site-packages/_pytest/config/__init__.py'): {1130: True}} guimgr.overall-gui-state = {'windowing-policy': 'combined-window', - 'windows': [{'name': 'P17d1M9PDrDDNBOIKLr7d7agln'\ - 'JSlCRG', + 'windows': [{'name': 'LfPD5D2a8ynAK2sybEe3c4A0IQ'\ + 'zTXQPX', 'size-state': '', 'type': 'dock', 'view': {'area': 'tall', @@ -149,42 +149,22 @@ guimgr.overall-gui-state = {'windowing-policy': 'combined-window', 'wide', 1, {})], - 'primary_view_state': {'editor_states': ({'bookmarks': ([[loc('manifest.py'), - {'attrib-starts': [], - 'first-line': 0L, - 'folded-linenos': [], - 'sel-line': 3L, - 'sel-line-start': 48L, - 'selection_end': 68L, - 'selection_start': 48L, - 'zoom': 0L}, - 1584653319.210618]], - 1), - 'current-loc': loc('submit_job.sbatch'), + 'primary_view_state': {'editor_states': ({'bookmarks': ([], + 0), + 'current-loc': loc('manifest.py'), 'editor-state-list': [(loc('manifest.py'), {'attrib-starts': [], - 'first-line': 0L, + 'first-line': 113L, 'folded-linenos': [], - 'sel-line': 3L, - 'sel-line-start': 48L, - 'selection_end': 68L, - 'selection_start': 48L, - 'zoom': 0L}), - (loc('submit_job.sbatch'), - {'attrib-starts': [], - 'first-line': 0L, - 'folded-linenos': [], - 'sel-line': 10L, - 'sel-line-start': 451L, - 'selection_end': 469L, - 'selection_start': 469L, + 'sel-line': 121L, + 'sel-line-start': 3768L, + 'selection_end': 3803L, + 'selection_start': 3803L, 'zoom': 0L})], 'has-focus': True, 'locked': False}, - [loc('manifest.py'), - loc('submit_job.sbatch')]), - 'open_files': [u'manifest.py', - u'submit_job.sbatch']}, + [loc('manifest.py')]), + 'open_files': [u'manifest.py']}, 'saved_notebook_display': None, 'split_percents': {0: 0.5}, 'splits': 2, @@ -195,12 +175,12 @@ guimgr.overall-gui-state = {'windowing-policy': 'combined-window', 'splits': 2, 'tab_location': 'left', 'user_data': {}}, - 'window-alloc': (0, + 'window-alloc': (20, 23, - 1418, + 1414, 877)}]} -guimgr.recent-documents = [loc('submit_job.sbatch'), - loc('manifest.py')] +guimgr.recent-documents = [loc('manifest.py'), + loc('submit_job.sbatch')] guimgr.visual-state = {loc('upload.sh'): {'attrib-starts': [], 'first-line': 0L, 'folded-linenos': [],