diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml
index e678d9929..e5d3489ba 100644
--- a/.github/workflows/cd.yaml
+++ b/.github/workflows/cd.yaml
@@ -53,3 +53,20 @@ jobs:
platforms: linux/amd64
push: true
tags: ${{ steps.image-tags.outputs.result }}
+ docs:
+ name: Deploy docs
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ # "ref" specifies the branch to check out.
+ # "github.event.release.target_commitish" is a global variable and specifies the branch the release targeted
+ ref: ${{ github.event.release.target_commitish }}
+
+ # Deploy docs
+ - name: Deploy docs
+ uses: mhausenblas/mkdocs-deploy-gh-pages@1.26
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ CONFIG_FILE: mkdocs.yml
+ EXTRA_PACKAGES: build-base
\ No newline at end of file
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 9331f65b6..62b1b9d27 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -70,7 +70,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- tox-env: [test, check-migrations]
+ tox-env: [test, check-migrations, builddocs]
services:
postgres:
image: postgis/postgis:latest
diff --git a/.github/workflows/publish-node.yaml b/.github/workflows/publish-node.yaml
deleted file mode 100644
index 9a4952832..000000000
--- a/.github/workflows/publish-node.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-name: Publish Node packages
-on:
- push:
- branches:
- - "main"
- paths:
- - "tileserver/**"
-jobs:
- publish:
- runs-on: ubuntu-latest
- permissions:
- contents: read
- packages: write
- defaults:
- run:
- working-directory: tileserver/
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-node@v4
- with:
- node-version: 20
- registry-url: 'https://npm.pkg.github.com'
- scope: ${{ github.repository_owner }}
- - run: npm ci
- - run: npm publish
- env:
- NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/README.md b/README.md
index 2826255e9..97d50abe4 100644
--- a/README.md
+++ b/README.md
@@ -30,9 +30,9 @@ RDWATCH (**R**esonantGeo**D**ata **WATCH**) is a modern web application that off
## Documentation
-* [Development](https://github.com/ResonantGeoData/RD-WATCH/blob/main/DEVELOPMENT.md)
+* [Getting Started](https://ResonantGeoData.github.io/RD-WATCH/DevDocs/GettingStarted/)
-* [Data Ingestion](https://github.com/ResonantGeoData/RD-WATCH/blob/main/INGESTION.md)
+* [Data Ingestion](https://ResonantGeoData.github.io/RD-WATCH/DevDocs/IngestingData/)
## Related Work
diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml
index 422ff0d16..f74cdad95 100644
--- a/docker-compose.override.yaml
+++ b/docker-compose.override.yaml
@@ -1,5 +1,6 @@
services:
django:
+ platform: linux/amd64
build:
dockerfile: Dockerfile
target: dev
@@ -33,6 +34,7 @@ services:
- minio
celery_worker:
+ platform: linux/amd64
build:
dockerfile: Dockerfile
target: dev
@@ -114,23 +116,5 @@ services:
# Run the npm development script
npm run dev
"
- vectorserver:
- image: node:20
- container_name: rd-watch-vectorserver
- working_dir: /usr/tileserver
- environment:
- RDWATCH_POSTGRESQL_URI: "${RDWATCH_POSTGRESQL_URI-postgresql://rdwatch:secretkey@postgresql:5432/rdwatch}"
- PORT: "8001"
- RDWATCH_REDIS_URI: "${RDWATHC_REDIS_URI-redis://redis:6379}"
- RDWATCH_POSTGRESQL_SCORING_URI: "${RDWATCH_POSTGRESQL_SCORING_URI-postgresql://scoring:secretkey@scoredb:5432/scoring}"
- RDWATCH_SUPPRESS_VECTOR_TILE_LOGGING: "true" # prevent console logging of information for dev server
- volumes:
- - ./tileserver:/usr/tileserver
- ports:
- - "8001:8001"
- # required for the git display of the commit hash
- command: >
- sh -c "npm install && npm run dev"
-
volumes:
celery-SAM-model:
diff --git a/docs/DevDocs/Architecture.md b/docs/DevDocs/Architecture.md
new file mode 100644
index 000000000..19b6bddf2
--- /dev/null
+++ b/docs/DevDocs/Architecture.md
@@ -0,0 +1,58 @@
+# Architecture
+
+## Overview
+
+The basic RDWATCH systems is based on utilizing docker containers to create a full contained application for local development and deployment.
+
+- Django is used as the back-end ORM with Postgres/PostGIS as a database. There are multiple Django apps. One for base RDWATCH and one for the Scoring data.
+- Celery is used to execute and manage long running tasks.
+- MinIO/S3 is used as object storage for cropped/chipped satellite images.
+- Vue based front-end utilizing Vuetify for components and Maplibre for vector tile rendering
+
+## Django Overview
+
+### Core
+
+This is the base application for RDWATCH. All endpoints that don't contain `api/v1/scoring/*` will utilize the base Core Application.
+This is also the mode location for all satellite image storage and downloading progress.
+
+### Scoring
+
+When the Environment Variable `RDWATCH_POSTGRESQL_SCORING_URI` is set the system expects to be connected to a second Database. This database is based of the T&E metrics scoring database. The RDWATCH team doesn't have direct control over the schema of this database so any data that is modified or stored should be placed in the RDWATCH database (This includes data like Stored Satellite Images or downloading progress).
+
+For reading from the database Django Models are created but the `managed` property in the `Meta` class is set to False. This allows for reading from the database using similar django querysets and orm logic while not having django actively manage the system. This means that when the DB changes the models need to be updated.
+
+The standard `rdwatch/core/views` endpoints for visualizing model-runs, sites, siteobservations are all mirrored in the Scoring application so they can access information directly from the scoring database.
+
+### Tasks
+
+Each app **core** and **scoring** has it's own tasks used in celery. These are tasks that will download GeoJSON for an entire model run as well as downloading satellite images. All information regarding satellite images are stored in the core rdwatch database because this project doesn't have access to modify the **scoring** database.
+
+## Stack Links
+
+### Django
+
+A single Django application (`rdwatch`) for the backend. Source code is in the ["rdatch"](https://github.com/ResonantGeoData/RD-WATCH/tree/main/rdwatch) folder.
+
+- [Django](https://docs.djangoproject.com/) with [GeoDjango](https://docs.djangoproject.com/en/5.0/ref/contrib/gis/)
+- [Django Ninja](https://django-ninja.dev/)
+- [Poetry](https://python-poetry.org/docs/) for dependency management
+
+### Vue
+
+The Vue-based SPA frontend. Source code is in the ["vue"](https://github.com/ResonantGeoData/RD-WATCH/tree/phase-ii/vue) folder.
+
+- [Vue 3](https://vuejs.org/guide/introduction.html)
+- [Vuetify](https://vuetifyjs.com/en/)
+- [MapLibre GL JS](https://maplibre.org/maplibre-gl-js-docs/api/)
+- [npm](https://docs.npmjs.com/) for dependency management
+
+### Services
+
+Services the application requires.
+
+- [NGINX Unit](https://unit.nginx.org/): serves both the Django application backend and the bundled static assets from the Vue frontend
+- [PostgreSQL](https://www.postgresql.org/docs/) and [PostGIS](http://www.postgis.net/documentation/): the RDWATCH database
+- [Redis](https://redis.io/docs/): caching and a job queue for celery
+- [MinIO/S3](https://min.io/): storage for satellite images for faster browsing
+- [Celery](https://docs.celeryq.dev/en/stable/getting-started/introduction.html): long running tasks for image chipping, downloading and compressing geoJSONs.
diff --git a/docs/DevDocs/GettingStarted.md b/docs/DevDocs/GettingStarted.md
new file mode 100644
index 000000000..3f3d719f1
--- /dev/null
+++ b/docs/DevDocs/GettingStarted.md
@@ -0,0 +1,154 @@
+# Getting Started
+
+This document gives an overview of the code contained in this monorepo and the recommended development setup.
+
+## Develop with Docker (recommended)
+This is the simplest configuration for developers to start with.
+
+1. Make a copy of `template.env` and call it `.env`.
+2. Set the environment variables in `.env`.
+3. Run `docker compose up` to start the Django development server and Celery worker, plus all backing services
+ like PostGIS, Redis, RabbitMQ, etc.
+4. Run `docker compose run --rm django poetry run django-admin migrate` to apply database migrations.
+5. Run `docker compose run --rm django poetry run django-admin loaddata lookups` to initialize your database with required data.
+7. Optionally, create an account for the Django admin (http://localhost:8000/admin) by running `docker compose run --rm django poetry --directory django run django-admin createsuperuser`
+8. If running the docker compose by default a client development server should be started at http://localhost:8080/
+9. On first login you will be redirected to the `Adminstrator` page. This is for logging in. Afterwards you should be able to redirect back to either http://localhost:8080/ or http://localhost:3000/. The deployed version will automatically redirect.
+9. If doing local Client Development, start the client development server:
+ ```sh
+ cd vue
+ npm install
+ npm run dev
+ ```
+ The server will be started at http://localhost:3000 as to not conflict with the docker compose development service
+ When finished, use `Ctrl+C`
+
+## Develop Natively
+This configuration still uses Docker to run attached services in the background,
+but allows developers to run Python code on their native system.
+
+### Initial Setup
+1. Make a copy of `template.env` and call it `.env`.
+2. Set the environment variables in `.env`.
+3. Run `docker compose -f ./docker-compose.yaml up -d`
+4. Install Python 3.11
+5. Install
+ [`psycopg2` build prerequisites](https://www.psycopg.org/docs/install.html#build-prerequisites)
+6. Install Poetry
+7. Run `poetry --directory django install`
+8. Run the following command to configure your environment: `source ./dev/export-env.sh dev/.env.docker-compose-native ./dev/export-env.sh .env`
+10. Optionally, create an account for the Django admin (http://localhost:8000/admin) by running `poetry --directory django run django-admin createsuperuser`
+
+### Run Application
+1. Ensure `docker compose -f ./docker-compose.yaml up -d` is still active
+2. Run:
+ 1. `source ./dev/export-env.sh dev/.env.docker-compose-native`
+ 2. `source ./dev/export-env.sh .env`
+ 3. `poetry run --directory django django/src/manage.py migrate`
+ 4. `poetry run --directory django django/src/manage.py loaddata lookups`
+ 5. `poetry run --directory django django/src/manage.py runserver`
+3. Run in a separate terminal:
+ 1. `source ./dev/export-env.sh`
+ 2. `poetry run --directory django celery --app rdwatch.celery worker --loglevel INFO --without-heartbeat`
+4. Run in another separate terminal:
+ 1. `source ./dev/export-env.sh`
+ 2. `poetry run --directory django celery --app rdwatch.celery beat --loglevel INFO`
+5. When finished, run `docker compose stop`
+6. To destroy the stack and start fresh, run `docker compose down`
+ 1. Note: this command does not destroy docker volumes, such as those associated with the postgresql and minio services. To destroy those as well, run `docker compose down -v`.
+
+### A note on database migrations
+Note that database migrations are _not_ run automatically. Anytime a new migration is introduced, you must run the following command to apply it:
+
+`poetry --directory django run django-admin migrate`
+
+## Type support for ".vue" imports in VS Code
+
+Enable ["takeover mode"](https://github.com/johnsoncodehk/volar/discussions/471) for Volar.
+
+1. Disable built-in TypeScript extension:
+ 1. Open the Command Palette (⌘⇧P or CtrlShiftP) and run `>Extensions: Show Built-in Extensions` command
+ 2. Find "TypeScript and JavaScript Language Features", right click and select "Disable (Workspace)"
+2. Reload VS Code
+
+## Stack
+
+The key software used to build the application.
+
+### Django
+
+A single Django application (`rdwatch`) for the backend. Source code is in the ["django"](https://github.com/ResonantGeoData/RD-WATCH/tree/phase-ii/django) folder.
+
+- [Django 4](https://docs.djangoproject.com/en/4.1/contents/) with [GeoDjango](https://docs.djangoproject.com/en/4.0/ref/contrib/gis/)
+- [Django Ninja](https://django-ninja.dev/)
+- [Poetry](https://python-poetry.org/docs/) for dependency management
+
+### Vue
+
+The Vue-based SPA frontend. Source code is in the ["vue"](https://github.com/ResonantGeoData/RD-WATCH/tree/phase-ii/vue) folder.
+
+- [Vue 3](https://vuejs.org/guide/introduction.html)
+- [Vuetify](https://vuetifyjs.com/en/)
+- [MapLibre GL JS](https://maplibre.org/maplibre-gl-js-docs/api/)
+- [npm](https://docs.npmjs.com/) for dependency management
+
+### Services
+
+Services the application requires.
+
+- [NGINX Unit](https://unit.nginx.org/): serves both the backend and the bundled static assets ([Dockerfile](https://github.com/ResonantGeoData/RD-WATCH/blob/phase-ii/Dockerfile))
+- [PostgreSQL 14](https://www.postgresql.org/docs/14/index.html) and [PostGIS 3.2](http://www.postgis.net/documentation/): data warehouse ([Dockerfile](https://github.com/ResonantGeoData/RD-WATCH/blob/phase-ii/docker/services/postgresql/Dockerfile))
+- [Redis 7](https://redis.io/docs/): caching (and maybe in the future as a job queue) ([Dockerfile](https://github.com/ResonantGeoData/RD-WATCH/blob/phase-ii/docker/services/redis/Dockerfile))
+
+### Ingesting Data
+
+#### Loading Ground Truth Data
+
+Within the ./scripts directory is a python script named `loadGroundTruth.py`. This file can be used in conjunction with the ground truth annotaitons located in the annotation Repo:
+[Annotation Repo](https://smartgitlab.com/TE/annotations)
+Running a command like `python loadGroundTruth.py ~/AnnotationRepoLocation --skip_region` will load all of the annotations for the ground truth while skipping the regions.
+
+
+#### Loading Single Model Runs
+Within the ./scripts directory is a python script named `loadModelRuns.py`. This can be used to load a folder filled with geojson data into the system by using a command like:
+
+```
+python loadModelRuns.py 'KR_0001' "./site_models/KR_R001_*.geojson" --title Test_Eval_12 --performer_shortcode 'KIT' --eval_num 12 --eval_run_num 0
+```
+Within this python file at the top is the rgd_endpoint variable which needs to be set to the server URL and port for where RGD is hosted. By default this assumes running locally with `http://localhost:8000`
+Be sure that the system is up and running before running the commands.
+The above command will load the data in the site_models/KR_R001 files and give it the title 'Test_Eval_12'. The eval_num and eval_run_num aren't required unless the scoring database is going to be connected to the system. Within the script there is
+
+### Scoring
+
+The [Metrics and Test Framework](https://smartgitlab.com/TE/metrics-and-test-framework#creating-a-metrics-database) can be used in addition with RGD to display scores from results.
+In development mode a scoring Database is automatically initialized at URI: `postgresql+psycopg2://scoring:secretkey@localhost:5433/scoring`
+
+To score data:
+- Clone the [Metrics and Test Framework](https://smartgitlab.com/TE/metrics-and-test-framework) repo.
+- In the Metrics and Test Framework repo:
+ - Copy the `alembic_example.ini` to `alembic.ini` and set the `sqlalchemy.url = postgresql+psycopg2://scoring:secretkey@localhost:5433/scoring`
+ - Run `pip install -e .` to install the metrics-and-test-framework package
+ - Run `alembic upgrade head` to initialize the scoring database schema
+ - Execute the scoring code from inside the metrics and test framework:
+```
+ python -m iarpa_smart_metrics.run_evaluation \
+ --roi KR_R001 \
+ --gt_dir ../annotations/site_models/ \
+ --rm_dir ../KR_R001/region_models/ \
+ --sm_dir ../KR_R001/site_models/ \
+ --output_dir ../KR_R001/output \
+ --eval_num 12 \
+ --eval_run_num 0 \
+ --performer kit \
+ --no-viz \
+ --no-viz-detection-table \
+ --no-viz-comparison-table \
+ --no-viz-associate-metrics \
+ --no-viz-activity-metrics \
+ --sequestered_id KR_R001 \
+ --db_conn_str postgresql+psycopg2://scoring:secretkey@localhost:5433/scoring
+```
+- the rm_dir and sm_dir shgould be your test annotaitons.
+- gt annotations can be retrieved from the [Annotation Repo](https://smartgitlab.com/TE/annotations)
+- be sure to set the val_num and eval_run_num and remember them when ingesting data into RGD. The region, eval_num, eval_run_num and performer are used to connect data loaded in RGD to the scoring data.
diff --git a/docs/DevDocs/IngestingData.md b/docs/DevDocs/IngestingData.md
new file mode 100644
index 000000000..2d17beb00
--- /dev/null
+++ b/docs/DevDocs/IngestingData.md
@@ -0,0 +1,172 @@
+### Ingesting Data
+
+## RDWATCH_MODEL_RUN_API_KEY
+Check the `./dev/.env.docker-compose` environment file or the `.env` file in the root of the repository for the RDWATCH_MODEL_RUN_API_KEY.
+This key is a special key used for services outside of the standard Django login to be able to push data into the system.
+The key will be used in the scripts and in the headers for pushing data into the system.
+
+
+## Loading Ground Truth Data
+Within the ./scripts directory is a python script named `loadGroundTruth.py`. This file can be used in conjunction with the ground truth annotaitons located in the annotation Repo:
+[Annotation Repo](https://smartgitlab.com/TE/annotations)
+Running a command like `python loadGroundTruth.py ~/AnnotationRepoLocation --rgd-api-key {RDWATCH_MODEL_RUN_API_KEY}` will load all of the annotations for the ground truth while skipping the regions.
+
+
+## Loading Single Model Runs
+Within the ./scripts directory is a python script named `loadModelRuns.py`. This can be used to load a folder filled with geojson data into the system by using a command like:
+
+```
+python loadModelRuns.py 'KR_0001' "./site_models/KR_R001_*.geojson" --title Test_Eval_12 --performer_shortcode 'KIT' --eval_num 12 --eval_run_num 0 --rgd-api-key {RDWATCH_MODEL_RUN_API_KEY}
+```
+Within this python file at the top is the rgd_endpoint variable which needs to be set to the server URL and port for where RGD is hosted. By default this assumes running locally with `http://localhost:8000`
+Be sure that the system is up and running before running the commands.
+The above command will load the data in the site_models/KR_R001 files and give it the title 'Test_Eval_12'. The eval_num and eval_run_num aren't required unless the scoring database is going to be connected to the system. Within the script there is
+
+## Scoring
+
+The [Metrics and Test Framework](https://smartgitlab.com/TE/metrics-and-test-framework#creating-a-metrics-database) can be used in addition with RGD to display scores from results.
+In development mode a scoring Database is automatically initialized at URI: `postgresql+psycopg2://scoring:secretkey@localhost:5433/scoring`
+
+To score data:
+- Clone the [Metrics and Test Framework](https://smartgitlab.com/TE/metrics-and-test-framework) repo.
+- In the Metrics and Test Framework repo:
+ - Copy the `alembic_example.ini` to `alembic.ini` and set the `sqlalchemy.url = postgresql+psycopg2://scoring:secretkey@localhost:5433/scoring`
+ - Run `pip install -e .` to install the metrics-and-test-framework package
+ - Run `alembic upgrade head` to initialize the scoring database schema
+ - Execute the scoring code from inside the metrics and test framework:
+```
+ python -m iarpa_smart_metrics.run_evaluation \
+ --roi KR_R001 \
+ --gt_dir ../annotations/site_models/ \
+ --rm_dir ../KR_R001/region_models/ \
+ --sm_dir ../KR_R001/site_models/ \
+ --output_dir ../KR_R001/output \
+ --eval_num 12 \
+ --eval_run_num 0 \
+ --performer kit \
+ --no-viz \
+ --no-viz-detection-table \
+ --no-viz-comparison-table \
+ --no-viz-associate-metrics \
+ --no-viz-activity-metrics \
+ --sequestered_id KR_R001 \
+ --db_conn_str postgresql+psycopg2://scoring:secretkey@localhost:5433/scoring
+```
+- the rm_dir and sm_dir shgould be your test annotaitons.
+- gt annotations can be retrieved from the [Annotation Repo](https://smartgitlab.com/TE/annotations)
+- be sure to set the val_num and eval_run_num and remember them when ingesting data into RGD. The region, eval_num, eval_run_num and performer are used to connect data loaded in RGD to the scoring data.
+
+## Manually Loading
+
+### Create a `model-run`
+
+A `model-run` is a grouping of site evaluations. This grouping can contain outputs of a machine learning model, ground truth outputs, etc. In order to ingest data, it must be associated with a `model-run`.
+
+You can view and create `model-runs` on the `/api/model-runs` endpoint.
+
+- GET `/api/model-runs`: list all
+- GET `/api/model-runs/{id}`: retrieve a single instance
+- POST `/api/model-runs`: create an instance
+
+Prior to creating a model run, you may have to create a performer to associate it with. RD-WATCH comes pre-configured with some performers by default; you can send a request to the `/api/performers/` endpoint to check the available performers:
+
+```bash
+$ curl https://some.rgd.host/api/performers/
+```
+
+To create a new performer, you can make a separate POST request to the API.
+The following JSON is an example of data to be used to create a `performer`:
+
+```jsonc
+// performer.json
+{
+ "team_name": "Kitware",
+ "short_code": "KIT"
+}
+```
+
+To create this performer:
+
+```bash
+$ curl \
+ -H "Content-Type: application/json" \
+ -H "X-RDWATCH-API-KEY: {RDWATCH_MODEL_RUN_API_KEY}" \
+ -X POST \
+ -d @performer.json \
+ https://some.rgd.host/api/performers/
+```
+
+Once you've ensured the desired performer exists, you can create a model run.
+The following JSON is an example of data to be used to create a `model-run`:
+
+```jsonc
+//model-run.json
+{
+ // must be a valid performer short-code
+ "performer": "KIT",
+ // must be a valid region string
+ "region": "KW_R001",
+ // a human-readable title
+ "title": "Ingestion 3.2",
+ // number of hours after upload when this model run should be automatically deleted.
+ // exclude this field if you want the model run to remain in the database permanently.
+ "expiration_time": 2,
+ // can be any JSON that helps keep track of what this model-run is
+ "parameters": {
+ "any": "data"
+ }
+}
+```
+
+To create this `model-run`:
+
+```bash
+$ curl \
+ -H "Content-Type: application/json" \
+ -H "X-RDWATCH-API-KEY: {RDWATCH_MODEL_RUN_API_KEY}" \
+ -X POST \
+ -d @model-run.json \
+ https://some.rgd.host/api/model-runs/
+```
+
+You'll get the newly created `model-run` as a response:
+
+```json
+{
+ "id": 12,
+ "title": "Ingestion 3.2",
+ "region": null,
+ "performer": {
+ "id": 7,
+ "team_name": "Kitware",
+ "short_code": "KIT"
+ },
+ "parameters": {
+ "any": "data"
+ },
+ "numsites": 0,
+ "score": null,
+ "timestamp": null,
+ "timerange": null,
+ "bbox": null,
+ "created": "",
+ "expiration_time": "01:00:00"
+}
+```
+
+## Add data to a `model-run`
+
+You can `POST` a [Site Model Specification](https://smartgitlab.com/TE/standards/-/wikis/Site-Model-Specification) JSON to the endpoint `/api/model-runs/{id}/site-model/` or a [Region Model Specification](https://smartgitlab.com/TE/standards/-/wikis/Region-Model-Specification) JSON to the endpoint `/api/model-runs/{id}/region-model/`.
+
+Following the above example, lets POST a [Site Model Specification](https://smartgitlab.com/TE/standards/-/wikis/Site-Model-Specification) JSON file in the current working directory named "site.json" to the newly created `model-run`:
+
+```bash
+$ curl \
+ -H "Content-Type: application/json" \
+ -H "X-RDWATCH-API-KEY: {RDWATCH_MODEL_RUN_API_KEY}" \
+ -X POST \
+ -d @site.json \
+ https://some.rgd.host/api/model-runs/12/site-model/
+```
+
+Ensure the JSON correctly validates against the [Site Model Specification](https://smartgitlab.com/TE/standards/-/wikis/Site-Model-Specification). While many validation errors are reported, a malformed JSON will not report helpful errors. For example, the specification mandates each 'Observation' feature must include a `current_phase` string, but some data in the wild is not compliant with this and instead includes `"current_phase": null`. This is a malformed JSON and will not able to be parsed.
diff --git a/docs/DevDocs/SatelliteImages.md b/docs/DevDocs/SatelliteImages.md
new file mode 100644
index 000000000..803ec76e0
--- /dev/null
+++ b/docs/DevDocs/SatelliteImages.md
@@ -0,0 +1,89 @@
+# Satellite Image Retrieval
+
+The RDWATCH system relies on a [STAC API](https://stacspec.org/en) to query for visual satellite images stored in the [COG (Cloud Optimized GeoTIFF)](https://www.cogeo.org/) format. After searching for these images based on a spatial and temporal filter it will download them for display. These COG files are stored in S3 buckets and the STAC API provides the S3 URL to the file.
+
+## Collections
+
+RD-WATCH will query the STAC server for 4 sources:
+
+- WV: WorldView
+- S2: Sentinel 2
+- L8: Landsat 8
+- PL: Planet Labs
+
+WorldView is treated differently from the rest of Satellite Sources. WorldView can be pansharpened if there exists additional imagery data. This allows a higher resolution image to be displayed when compared to the other Sources.
+
+There is an environment variable `RDWATCH_ACCENTURE_VERSION` that is used to determine the collections that are searched when querying the STAC server.
+
+## Code Layout
+
+The [**pystac_client**](https://github.com/stac-utils/pystac-client) library is utilized to make queries to the STAC server.
+
+Within the **./rdwatch/core/utils** directory contains helper tools for Querying the STAC server and processing the resulting images.
+
+In the root of that folder is a `stac_search.py` which is used for S2,L8,PL image sources.
+The Collections to search for images are specified [here](https://github.com/ResonantGeoData/RD-WATCH/blob/58ab08d3d5ef7905295b9cec53b0e7a73c0dd5d6/rdwatch/core/utils/stac_search.py#L59-L62)
+
+```
+if settings.ACCENTURE_VERSION is not None:
+ COLLECTIONS['S2'].append(f'ta1-s2-acc-{settings.ACCENTURE_VERSION}')
+ COLLECTIONS['L8'].append(f'ta1-ls-acc-{settings.ACCENTURE_VERSION}')
+ COLLECTIONS['PL'].append(f'ta1-pd-acc-{settings.ACCENTURE_VERSION}')
+```
+
+### Querying STAC
+
+Within **./rdwatch/core/utils/satellite_bands.py** file is the main function `get_bands`. This function performs a STAC Query for a time range and a bounding box utilizing **pystac**. The results are typically verbose so it is paginated with 100 results at a time. It iterates over these times and returns all 'visual' images that match the query. If the results match a visual band image they are returned as a `Band` class. This `Band` class contains the following information:
+
+- **constellation**: source of image (S2, L8, PL)
+- **spectrum**: Information about the spectrum the band captures (μm)
+- **level**: processing level
+- **timestamp**: date of the satellite image
+- **bbox**: bounding box associated with the image
+- **uri**: Typically contains the S3 URL for the COG that contains the image
+- **cloudcover**: optional value that will extract metadata to indicate the amount of cloud cover in the image
+- **collection**: The collection this image was retrieved from
+
+### WorldView Differences
+
+Within the **./rdwatch/core/utils** folder there is a **worldview_processed** folder. It contains tools to create a pansharpened version of worldview images by downloading extra data.
+
+Instead of having a `Band` class for worldview there is a `WorldViewCapture` class in the base and a `WorldViewProcessedCapture`.
+The older base **worldview** folder will probably be removed as deprecated
+
+`WorldViewProcessedCapture` is similar to the `Band` class but contains an optional `panuri` property to indicate an additional S3 location of the extra image to create a pansharpened image.
+
+the `get_captures` function in **worldview_process/satellite_captures.py** is similar to `get_bands` except there is a secondary process to attempt to find the `panuri` if available.
+
+#### Pansharpening
+
+For worldview if there is a secondary panchromatic image found it will use the [riotiler](https://cogeotiff.github.io/rio-tiler/) `pansharpening_brovy` function to increase the resolution of the image.
+
+## Region Satellite Images
+
+When viewing an entire region images can be turned on for the wole region. It defaults to S2 as the source but WV can also be used.
+It follows the following process:
+
+1. A search for the region bounding box and a time range of 2010 to now will be search for the source images (either S2 or WV)
+1. After a list of images are found the client is provided with this list.
+1. When the satellite image is turned on the client will request the image closest to the current time in the slider.
+1. This then uses the image list to grab the URI for this image directly and use it for serving tiles. The URI is sent to the back-end along with the x/y/z tile. The riotiler then uses range requests to S3 to get the requested tile from S3 and provide it to the client. This can be seen `raster_tile.py` file for both the default `utils` and the `worldview_processed` folders.
+
+This process can be a bit slow when switching between satellite images. Once an image is loaded it is cached so subsequent loads become faster but the caching isn't that large for the system.
+
+## Site Satellite Image Chipping
+
+Instead of downloading and using a tile server to view images for a whole region there is a task (in each App: core/scoring) that will chip/crop the images for individual sites and save them in a Object Store like S3/MinIO.
+
+Within **(core/scoring)/tasks/__init__.py** there are two functions called `generate_site_images_for_evaluation_run` and `generate_site_images`. The `generatie_site_images_for_evaluation_run` calls `generate_site_images` for each site in a Model Run. The core function that is eventually called is `get_siteobservations_images`. This does the process of STAC Querying for images, processing them and eventually creating chips that are loaded into S3/MinIO.
+
+## Image Chipping Parameters
+
+- *site_id: UUID4* - The UUID for the Site to download
+- *constellation:=['WV']* - Watch satellite image sources to download. Can be WV, S2, L8, PL or any combination
+- *force=False* - Forces re-downloading images. Useful if you change the filters (overrideDates, dayRange, noData) or if you believe the STAC Query has updated since last time you downloading images.
+- *dayRange=14* - S2, L8, PL imagery can have dense temporal images. I.E there may be 5-6 pictures in a week and the changes between them aren't significant. This parameter will prevent images from downloading if an image already exists in the *dayRange*.
+- *noData=50* - Filter to remove images that report having > *noData*% of noData in them. It's intended to remove majority black images from the images downloaded.
+- *overrideDates: None | list[datetime, datetime] = None* - if set to `None` it will utilize the site time range for downloading images +/- 30 days to add a buffer. If the site time range is null it will use 20130101 to the present time. The *overrideDates* can be used to extend or reduce the time range when downloading images.
+- *scale: Literal['default', 'bits'] | list[int] = 'default'* - THe image bit scaling for the brightness levels can be adjusted herre. The Default scaling is 0-10,000. The `list[int]` allows for two custom values. The `bits` option will use the 2% lows and 98% highs to adjust the scaling.
+- *bboxScale: float = BboxScaleDefault* - a number that will be used to scale up the boundingbox of the area around the site downloaded. I.E. a value of 1.2 will ad 20% to the bounding box of polygon when downloading the image. If the site image source is now WorldView (WV) it will estimate the real world size and if the height or width are under 1000 meters it will add whatever is needed to get the size of the bbox to 1000 meters. This is done because of the lower resolution of S2,L8,PL. The more context helps identifying features in the image.
diff --git a/docs/UserGuide/LayerManager.md b/docs/UserGuide/LayerManager.md
new file mode 100644
index 000000000..6f6f8e615
--- /dev/null
+++ b/docs/UserGuide/LayerManager.md
@@ -0,0 +1,36 @@
+# Layer Manager
+
+The layer manager is located at the top of the map. It is a series of buttons that allow for toggling on/off different layers as well as adding regions/sites/observations in additional modes.
+
+## Ground Truth
+
+Both the **Site** and **Site Observation** buttons will contain and option to toggle on/off the Ground Truth. For the **RDWATCH** database the ground truth is determined by loading up the latest GroundTruth tagged model run for that specific Region. Using the **Scoring** Database it will utilize the GroundTruth that the model run was scored against. The toggling of GroundTruth will only be enabled for both instances if GroundTruth is found for the region or model run.
+
+## Site Observations / Sites
+
+### Site Observations
+This is the lowest level of polygons in the RDWATCH system. A Site Observation is a Polygon associated with a specific instance in time. Clicking on the button will toggle on a selected Model Run's Site Observations and associated Ground Truth Observations if they are found. Hovering over the Site Observation will allow for toggling on/off either the model run site observations or the ground truth obsersvations individually.
+
+### Sites
+Everything mentioned above is also applicable for sites. Sites are different because they are a polygon that are associated with a Date Range instead of a specific date.
+
+### Time Limits
+
+Sites have an additional option called **Time Limits**. By default the Site display will persist a site for all time after it is initially displayed. I.E if the global time slider is at date 20240704 and a Site has a time range of 20200101-20220101 it will display this site. Turning on **Time Limits** will hide a site if the current time is outside of the time range of the site.
+
+## Region
+
+Toggles on/off the polygon Region if it is available
+
+### Region Editing (RDWATCH Database)
+
+- **Region Deletion**: If you are the owner of a custom user region you can delete the region. **ONLY IF** the region doesn't have any associated model runs with it.
+- **Download Region**: Downloads the GeoJSON for the Region
+- **Add Region**: Allows for adding a custom Region by a user. The region can be drawn by the user and then given a unique name. This region can be used in the future to add sites or possible run SMARTFLOW on a region in the future. Regions that are generated by users can be set to public so everyone can see it, or private to prevent other users from seeing the region.
+
+## Scoring (Scoring Database)
+
+While using the Scoring Database two additional views can be displayed:
+
+- **Simple**: A simple view of the scoring results. When this view is turned on, all other layers are disabled. The Map Legend will update to indicate what colors mean in this mode.
+- **Detailed**: A simple view of the scoring results. When this view is turned on, all other layers are disabled. The Map Legend will update to indicate what colors mean in this mode.
diff --git a/docs/UserGuide/LeftSidePanel.md b/docs/UserGuide/LeftSidePanel.md
new file mode 100644
index 000000000..dfa5224b5
--- /dev/null
+++ b/docs/UserGuide/LeftSidePanel.md
@@ -0,0 +1,68 @@
+
+# Model Run List (Left Side Panel)
+
+## Mode Selection
+
+Select between Analyst and Annotator Mode.
+If the database is not **RDWATCH** it will display that the Source is **Scoring**. The Database selection is persistent so reloading the page won't reset the source. The source database can be set in the **Settings**.
+
+## Timeline Slider
+
+The Model Runs contain Sites and Site Observations that vary over time. This time slider is a global setting that indicates the current time selection so the proper Site Observations and Satellite images are displayed.
+
+The Timeline Slider start/end date are set auomatically based on the list of model runs being displayed, or the model runs that are selected. Any other tools that adjust the current time will be reflected in this global time slider.
+
+## Global Map/Model Run Settings
+
+There is a row of icons that are used to adjust settings in the system.
+
+- **==:material-road:== Base Map Visibility**: Turns on/off the base map (roads, boundaries, land vs water). This is useful to create a screenshot without giving spatial clues as to the location of the polygons being visualized.
+- **==:material-satellite-variant:== Satellite Images**: This button will download and cache the satellite timestamps initially and then allow toggling on/off region base satellite images. The first time this is clicked there is a warning because the system will query the STAC API to find all satellite images for the region during the time period currently displayed in the **Time Slider**. After this is complete a display of the number of Satellite Images will be displayed next to the Number of Model Runs below the ModelRun filter. If there are images found the icon will change to **==:material-image:==** indicating that the satellite image can be turned on. Once a satellite image is turned on the location where the number of satellite images were displayed will change to the closest timestamp of satellite image that matches the current global time set in the **Time Slider**
+- **==:material-format-text:== Format Text**: This turns on text status labels for the Site and Site Observations in the Map. It is disabled by default because the density of labels can make it difficult to see the polygons.
+- **==:material-map-legend:== Map Legend**: Toggles on/off the map legend. The map legend changes based on the type of visible annotations
+- **==:material-check-decagram:== Ground Truth**: By default the ModelRun list only contains user runs. This will show all of the GroundTruth Model runs. GroundTruth model runs are indicated as GroundTruth when they are ingested into RDWATCH. GroundTruth Model Runs, Sites and Site Observations are all indicated by the following icon: ==:material-check-decagram:==
+- **:material-settings:== Settings**: Opens the Settings panel to adjust settings for the system. See the **Settings** section to get more information
+
+## ModelRun Filters
+
+ - **Performer Filter**: this will filter model runs based on the performer
+ - **Region Filter**: filters model runs by their associated region. Regions that are user created and that are public will be included at the top of the list. All other Regions will be in alphebetical order. The region filter will be added to the URL and can be copied and pasted. If pasted into a browser it will open with the filter pre-selected.
+
+### Scoring Filters
+
+When connected to the Scoring Database there are additional filters that are displayed
+
+- **Mode Filter**: Filters the scoring results based on modes (**batch** or **incremental**)
+- **Evaluation Filter**: When scoring a model run they are all tied to an **Evaluation Number**. This filter will adjust the results base don this value.
+
+## Model Run Information
+
+Directly below the filters is small area that includes two pieces of information
+
+-**# of Model Runs** - A small tag that indicates the number of model runs that the current filter shows
+-**# of SatelliteImages / Current Satellite Image Timestamp** - If the satellite images are turned on or if there are cached a list of a satellite images this will display either the total number of satellite images found for the region. If the image is currently turned on it will display the current timestamp for the satellite image.
+
+## Model Run Card
+
+Each model run card in the list can be selected by either clicking on the card or the checkbox for the card. Once clicked on, the camera will zoom to the region for the model run and indicate it is selected by filling in the checkbox and highlight the model run card.
+
+### Model Run Card Information
+
+- **Title**: a descriptive title for the model run is located at the top
+- **Region**: Associated region for the model run.
+- **Date Coverage**: Analyzes the Sites in the listing and displays the Date Coverage for the Model Run
+- **Last Updated**: The timestamp of the model run being added or updated
+
+- **Model Run TimeSlider**: A slider that is bound to the start/end date based on the **Date Coverage**. When moved this will adjust the global time slider.
+
+- **==:material-map-marker-outline:== Site Number**: Number of Sites in the Model Run
+- **==:material-license:== Average Score**: Indication of the average score of the sites in the Model Run
+- **Scoring Database Only**
+ - **==:material-checkbox-multiple-blank:== Batch Mode**: Using scoring database this indicates batch mode
+ - **==:material--trending-up:==**: ?Using scoring database this indicates incremental mode
+
+### Model Run Actions
+
+- **RDWATCH Database Only**
+ - **==:material-download-box-outline:== Download GeoJSON**: Downloads a Zip file that contains all of the geoJSON for the sites that are displayed.
+- **==:material-image:== Satellite Image Downloading**: Opens a dialog to begin downloading Satellite Images for all of the Sites in the model run.
diff --git a/docs/UserGuide/UsingRDWatch.md b/docs/UserGuide/UsingRDWatch.md
new file mode 100644
index 000000000..6b7acbd95
--- /dev/null
+++ b/docs/UserGuide/UsingRDWatch.md
@@ -0,0 +1,11 @@
+# Using RDWATCH
+
+RDWATCH has the two main modes: **Analyst** and **Annotator**, as well as either being connected to the **RDWATCH** or **Scoring** database. While there are these different modes, most of the interface is shared with minor changes between the modes and the source database. This page will give a high level overview of the main UI elements that are shared.
+
+## [Model Run List (Left Side Panel)](LeftSidePanel.md)
+
+The Left Side panel contains tools for selecting modes, adjusting the global Timeline, Filtering Model Runs and providing a list of Model Runs to select.
+
+## [Layer Manager](LayerManager.md)
+
+The LayerManager manages what polygons are displayed on the map and filters based on time/coloring or other items.
diff --git a/docs/UserGuide/Vocabulary.md b/docs/UserGuide/Vocabulary.md
new file mode 100644
index 000000000..fc13d906d
--- /dev/null
+++ b/docs/UserGuide/Vocabulary.md
@@ -0,0 +1,50 @@
+# Vocabulary and Terms
+
+RDWATCH and the underlying SMART project have numerous Domain specific language and data structures. This page helps define common terms.
+
+## Background
+
+RDWATCH is meant to display geometry (polygons/points) in a geospatial and temporal context with annotations on an interactive map. Geometry can exist for a single point in time (SiteObservation) or for a time interval (Site). The original purpose for RDWATCH was to compare Construction states (Site Preparation, Active Construction, Post Construction) for polygons with a GroundTruth. This was to inspect the accuracy of algorithms that are meant to detect these states from Satellite Images.
+
+## Analyst Mode
+
+This mode is made for reviewing the results of algorithms and comparing them with ground truth
+
+## Annotator Mode
+
+This mode allows for modification of Sites and Site Observations. This includes modifying the *Status*, *Time/Time Range* or the *Polygon*. These sites/site Observations can be approved or rejected and the geoJSON can be exported.
+
+## Data Hierarchy
+
+- *Region*: A polygon indicating an area on the globe where Model Runs can be located
+- *Performer*: A Name for an organization/individual/group that creates a Model Run.
+- *GroundTruth*: A Model Run in a Region that contains Groundtruth Sites and Site Observations for comparison with non GroundTruth Model Runs
+- *Model Run*: a collection of Sites that contain Site Observations. ModelRuns are identified with a *title* and have a *Region* as well as a *Performer*. Model Runs are filtered by their Region, Performer or if they are *GroundTruth*
+- *Site*: A Point or Polygon that extends over a Range of time. The Site also has it's own State annotation that indicates information about the Site (positive, negative, ignore, syse,_conmfirmed). It has a start date and an end date. These dates can be `null` if the Start Date and End Date are unknown. These dates are used in RDWATCH to filter and display the Sites. A Site is a parent to zero or more Site Observations.
+- *Site Observation*: A Point or Polygon that is associated with a specific point in time. It has a date as well as a state (Site Preparation, Active Construction, Post Construction). A Site Observation is always associated with a Site.
+
+## Satellite Images
+
+The RDWATCH system relies on a [STAC API](https://stacspec.org/en) to query for visual satellite images stored in the [COG (Cloud Optimized GeoTIFF)](https://www.cogeo.org/) format. After searching for these images based on a spatial and temporal filter it will download them for display. These COG files are stored in S3 buckets and the STAC API provides the S3 URL to the file.
+
+### Region Satellite Mode
+
+This mode will convert a specificif COG image into a tile server so that a whole region can be displayed at once. This defaults to utilizing Sentinel 2 (S2) for the image source. In the settings it can be modified to WorldView (WV) but the download would take more time.
+
+### Satellite Site Chipping
+
+There is an option to download individual images around the region of a Site Polygon. This still uses the STAC API to get the COG files, but then takes those files and crops them to an area around the polygon and stores them in MinIO/S3 for faster access when compared to accessing a COG.
+
+### Satellite Sources
+
+- Simple Sources
+ - **S2: Sentinel 2**
+ - **L8: LandSat 8**
+ - **PL: PlanetLabs**
+ - These simple sources are lower resolution and may have a higher frequency of images across a given time range. This is why in the download settings there is a **Day Limit** that can be used to limit it so only one image is downloaded every X days.
+- PanSharpened
+ - **WV**: WorldView
+ - Pansharpening is a post process that occurs if there are multiple images that can be referenced. This combines the images and creates a higher resolution image.
+ - The day limits aren't applied to WorldView images because they aren't as frequent as the other sources and having the higher resolution is beneficial.
+
+## Future SmartFlow Documentation
diff --git a/docs/images/General/Kitware-Logo-Stacked.png b/docs/images/General/Kitware-Logo-Stacked.png
new file mode 100644
index 000000000..627ee727c
Binary files /dev/null and b/docs/images/General/Kitware-Logo-Stacked.png differ
diff --git a/docs/images/General/Kitware-Mark.png b/docs/images/General/Kitware-Mark.png
new file mode 100644
index 000000000..67ce8d0fa
Binary files /dev/null and b/docs/images/General/Kitware-Mark.png differ
diff --git a/docs/images/General/Kitware_Logo_HighRes_thumbnail.jpg b/docs/images/General/Kitware_Logo_HighRes_thumbnail.jpg
new file mode 100644
index 000000000..e8f251b48
Binary files /dev/null and b/docs/images/General/Kitware_Logo_HighRes_thumbnail.jpg differ
diff --git a/docs/images/General/logo.svg b/docs/images/General/logo.svg
new file mode 100644
index 000000000..51f897a3b
--- /dev/null
+++ b/docs/images/General/logo.svg
@@ -0,0 +1,17 @@
+
diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 000000000..aee0bc0c8
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,56 @@
+# RD-WATCH Documentation
+
+