run_and_upload.sh

#!/bin/bash
#########################################################
################ DO NOT EDIT THIS FILE ##################
#########################################################

set -eof

echo "Starting Job..."; echo;

RUN_DATE=$(date '+%Y-%m-%d_%H%M%S')
JSON_FILE=$RUN_DATE.json
S3_URI=$1

echo "Date: $RUN_DATE"
echo "Filename: $JSON_FILE"
echo "S3 URI: $S3_URI"; echo;

echo "Testing AWS CLI..."
touch empty.test
aws s3 cp empty.test $1
echo "AWS CLI test successful."; echo;

echo "Starting Xvfb..."; echo;
Xvfb :99 -screen 0 1280x1024x24 &

echo "Running scraper with filename $JSON_FILE..."
python -m scraper $JSON_FILE
echo "Scrape complete with final output $(pwd)/$JSON_FILE."

echo;
echo "########################################"
echo "Summary"
echo "########################################"
echo "Lines: $(wc -l < ${JSON_FILE})"
echo "Bytes: $(wc -c < ${JSON_FILE})"

echo;
echo "########################################"
echo "Sample: First 10 lines"
echo "########################################"
head -n 10 $JSON_FILE

echo;
echo "########################################"
echo "Sample: Last 10 lines"
echo "########################################"
tail -n 10 $JSON_FILE

echo;
echo "Uploading $JSON_FILE to $S3_URI..."
aws s3 cp $JSON_FILE $S3_URI
echo "Succesfully uploaded $S3_URI."

echo;
echo "Job Complete."