Staticbuild v1.3 fixes (#697)

* Fix proxy and redirects for globus auth on docker compose

* Various lint and test fixes

* Update yarn.lock

* Some changes made to try and get globus transfers working.

* Update docs to match new config (#707)

* Updated the local config file used for local testing. The new file .local_config, contains the configuration used when running Metagrid locally. Updated the manage_metagrid script to run Metagrid from the single docker-compose file. Updated some settings to use llnl instead of ornl for local development and testing. Removed some unused environment variable files after after testing that they aren't used. Left the default variables in place because Metagrid fails to build when several of the defaults aren't set, still working to troubeshoot why. Globus transfer steps in the front-end are working, however transfers seem to fail and need to be resolved.

* Switch from CRA to Vite and clean up tests (#692)

* Switch from CRA to Vite and clean up tests

* Fixed issue with loading screen continuing even after cancel button click. Updated local test config. Globus transfers still not working.

---------

Co-authored-by: Zach Price <[email protected]>
Co-authored-by: downiec <[email protected]>

* Fix proxy and redirects for globus auth on docker compose

* Various lint and test fixes

* Update yarn.lock

* Some changes made to try and get globus transfers working.

* Update docs to match new config
Auto generate configuration documentation with pre-commit hooks

* Revert yarn.lock, fix more tests

* Includes some updates and minor fixes so that everything builds and runs correctly. Removed the GLOBUS_REDIRECT setting and did some cleanup of the manage_metagrid.sh functions.

* Updated the test files to no longer use the act() functions, as they are deprecated. Cleaned up and fixed some failing tests.

* Fix esgsearch json tests

* Minor test fixes to see if tests will pass on github (they pass locally).

---------

Co-authored-by: downiec <[email protected]>
Co-authored-by: Zach Price <[email protected]>

---------

Co-authored-by: Zach Price <[email protected]>
Co-authored-by: bstrdsmkr <[email protected]>

.github/workflows/backend.yml

@@ -10,22 +10,17 @@ defaults:
     working-directory: backend
 
 env:
-  DJANGO_SETTINGS_MODULE: metagrid.config.local
-  DOMAIN_NAME: http://localhost:8000
-  DJANGO_SECURE_SSL_REDIRECT: False
-  CORS_ORIGIN_WHITELIST: http://localhost:3000
-  KEYCLOAK_URL: https://esgf-login.ceda.ac.uk/
-  KEYCLOAK_REALM: esgf
-  KEYCLOAK_CLIENT_ID: metagrid-localhost
-  DATABASE_URL: pgsql://postgres:postgres@localhost:5432/postgres
-  REACT_APP_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
-  REACT_APP_WGET_API_URL: https://esgf-node.llnl.gov/esg-search/wget
-  REACT_APP_ESGF_NODE_STATUS_URL: https://aims2.llnl.gov/metagrid-backend/proxy/status
-  REACT_APP_ESGF_SOLR_URL: https://esgf-fedtest.llnl.gov/solr
-  DJANGO_LOGIN_REDIRECT_URL: http://localhost:3000/search
-  DJANGO_LOGOUT_REDIRECT_URL: http://localhost:3000/search
-  GLOBUS_CLIENT_KEY: ${{ secrets.GLOBUS_KEY }}
-  GLOBUS_CLIENT_SECRET: ${{ secrets.GLOBUS_SECRET }}
+  DJANGO_SETTINGS_MODULE: config.settings
+  PGHOST: localhost
+  PGPASSWORD: postgres
+  PGUSER: postgres
+  METAGRID_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
+  METAGRID_WGET_URL: https://esgf-node.llnl.gov/esg-search/wget
+  METAGRID_STATUS_URL: https://esgf-node.llnl.gov/proxy/status
+  METAGRID_SOLR_URL: https://esgf-node.llnl.gov/esg-search
+  METAGRID_SOCIAL_AUTH_GLOBUS_KEY: ${{ secrets.GLOBUS_KEY }}
+  METAGRID_SOCIAL_AUTH_GLOBUS_SECRET: ${{ secrets.GLOBUS_SECRET }}
+  METAGRID_GLOBUS_CLIENT_ID: 21982de0-2b7a-4ba8-bef5-5606ae098201
 
 jobs:
   build:

.gitignore

@@ -183,6 +183,7 @@ tags
 .env
 .envs/*
 !.envs/.local/
+docker-compose.*-overlay.yml
 
 # Documentation artifacts
 docs/site

.pre-commit-config.yaml

@@ -1,44 +1,81 @@
 exclude: "docs|node_modules|migrations|.git|.tox"
-default_stages: [commit]
+default_stages: [ commit ]
 fail_fast: true
 
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-      - id: check-yaml
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+  - id: trailing-whitespace
+  - id: end-of-file-fixer
+  - id: check-yaml
 
-  # Back-end
-  # ------------------------------------------------------------------------------
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
-    hooks:
-      - id: flake8
-        args: ["--config=backend/setup.cfg"]
-        additional_dependencies: [flake8-isort]
+# Back-end
+# ------------------------------------------------------------------------------
+- repo: https://github.com/pycqa/flake8
+  rev: 7.1.1
+  hooks:
+  - id: flake8
+    args: [ "--config=backend/setup.cfg" ]
+    additional_dependencies: [ flake8-isort ]
 
-  - repo: https://github.com/psf/black
-    rev: 24.10.0
-    hooks:
-      - id: black
-        args: ["--config=backend/pyproject.toml"]
+- repo: https://github.com/psf/black
+  rev: 24.10.0
+  hooks:
+  - id: black
+    args: [ "--config=backend/pyproject.toml" ]
 
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.12.0
-    hooks:
-      - id: mypy
-        additional_dependencies: ["types-requests"]
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.12.0
+  hooks:
+  - id: mypy
+    additional_dependencies: [ "types-requests" ]
 
-  # Front-end
-  # ------------------------------------------------------------------------------
-  # Runs the local node `lint` script to avoid dependencies being out of sync with `frontend/package.json`
-  # https://github.com/pre-commit/pre-commit/issues/945
-  - repo: local
-    hooks:
-      - id: eslint_and_tsc
-        name: eslint_and_tsc
-        language: system
-        files: .+(js|jsx|ts|tsx|json)$
-        entry: bash -c 'cd frontend && yarn precommit'
+- repo: https://github.com/radeklat/settings-doc
+  rev: '4.3.1'
+  hooks:
+  - id: settings-doc-markdown
+    name: Generate backend settings markdown
+    language: system
+    entry: env PYTHONPATH=$PYTHONPATH:backend settings-doc generate --output-format markdown
+    args:
+    - '--class'
+    - 'config.settings.site_specific.MetagridBackendSettings'
+    - '--update'
+    - 'docs/docs/users/configurable_environment_variables.md'
+    - '--between'
+    - '<!-- start generated backend settings markdown -->'
+    - '<!-- end generated backend settings markdown -->'
+    - '--templates'
+    - 'docs/custom_templates'
+    - '--heading-offset'
+    - '3'
+
+  - id: settings-doc-markdown
+    name: Generate frontend settings markdown
+    language: system
+    entry: env PYTHONPATH=$PYTHONPATH:backend settings-doc generate --output-format markdown
+    args:
+    - '--class'
+    - 'config.settings.site_specific.MetagridFrontendSettings'
+    - '--update'
+    - 'docs/docs/users/configurable_environment_variables.md'
+    - '--between'
+    - '<!-- start generated frontend settings markdown -->'
+    - '<!-- end generated frontend settings markdown -->'
+    - '--templates'
+    - 'docs/custom_templates'
+    - '--heading-offset'
+    - '3'
+
+# Front-end
+# ------------------------------------------------------------------------------
+# Runs the local node `lint` script to avoid dependencies being out of sync with `frontend/package.json`
+# https://github.com/pre-commit/pre-commit/issues/945
+- repo: local
+  hooks:
+  - id: eslint_and_tsc
+    name: eslint_and_tsc
+    language: system
+    files: .+(js|jsx|ts|tsx|json)$
+    entry: bash -c 'cd frontend && yarn precommit'

.vscode/settings.json

@@ -63,7 +63,7 @@
   // Jest
   // -----------------------------
   "jest.rootPath": "./frontend/src",
-  "jest.jestCommandLine": "yarn jest  --runInBand",
+  "jest.jestCommandLine": "yarn jest",
   "jest.monitorLongRun": "off",
   "jest.enable": true,
   "jest.runMode": {

backend/Dockerfile

@@ -22,8 +22,7 @@ RUN pip3 install --no-cache-dir -r /requirements/local.txt
 
 COPY . /app
 
-RUN python /app/manage.py collectstatic --noinput \
-    && mkdir -p /app/staticfiles/.well-known \
+RUN mkdir -p /app/staticfiles/.well-known \
     && cyclonedx-py requirements requirements/base.txt --output-format json --outfile /app/staticfiles/.well-known/bom
 
 WORKDIR /app

backend/config/settings/__init__.py

@@ -0,0 +1,24 @@
+from typing import Any, Iterable
+
+from .site_specific import MetagridBackendSettings, MetagridFrontendSettings
+from .static import DjangoStaticSettings
+
+
+def __dir__() -> Iterable[str]:
+    """The list of available options are retrieved from
+    the dict view of our DjangoSettings object.
+    """
+    return (
+        DjangoStaticSettings.model_fields.keys()
+        | MetagridBackendSettings.model_fields.keys()
+        | MetagridFrontendSettings.model_fields.keys()
+    )
+
+
+def __getattr__(name: str) -> Any:
+    """Turn the module access into a DjangoSettings access"""
+    _combined_settings = DjangoStaticSettings().model_dump()
+    _combined_settings |= MetagridBackendSettings().model_dump()
+    _combined_settings |= MetagridFrontendSettings().model_dump()
+
+    return _combined_settings[name]

backend/config/settings/base.py

@@ -344,8 +344,8 @@
     "urn:globus:auth:scope:transfer.api.globus.org:all",
 ]
 
-SOCIAL_AUTH_GLOBUS_KEY = env("GLOBUS_CLIENT_KEY", default="12345")
-SOCIAL_AUTH_GLOBUS_SECRET = env("GLOBUS_CLIENT_SECRET", default="12345")
+SOCIAL_AUTH_GLOBUS_KEY = env("GLOBUS_CLIENT_KEY", default="unset")
+SOCIAL_AUTH_GLOBUS_SECRET = env("GLOBUS_CLIENT_SECRET", default="unset")
 SOCIAL_AUTH_GLOBUS_AUTH_EXTRA_ARGUMENTS = {
     "requested_scopes": SOCIAL_AUTH_GLOBUS_SCOPE,
     "prompt": None,
@@ -392,10 +392,6 @@
     "REACT_APP_AUTHENTICATION_METHOD": env(
         "REACT_APP_AUTHENTICATION_METHOD", default="keycloak"
     ),
-    "REACT_APP_GLOBUS_REDIRECT": env(
-        "REACT_APP_GLOBUS_REDIRECT",
-        default="http://localhost:8080/cart/items",
-    ),
     "REACT_APP_GLOBUS_CLIENT_ID": env(
         "REACT_APP_GLOBUS_CLIENT_ID", default="frontend"
     ),
@@ -427,6 +423,9 @@
     ),
 }
 
+# Custom settings validation
+# -------------------------------------------------------------------------------
+
 if not isinstance(FRONTEND_SETTINGS["REACT_APP_GLOBUS_NODES"], list):
     raise environ.ImproperlyConfigured(
         f"REACT_APP_GLOBUS_NODES must be of type list, not "
@@ -442,3 +441,10 @@
         f"REACT_APP_AUTHENTICATION_METHOD must be one of keycloak or globus, "
         f"not {FRONTEND_SETTINGS['REACT_APP_AUTHENTICATION_METHOD']}"
     )
+
+if FRONTEND_SETTINGS["REACT_APP_AUTHENTICATION_METHOD"] == "globus" and (
+    SOCIAL_AUTH_GLOBUS_KEY == "unset" or SOCIAL_AUTH_GLOBUS_SECRET == "unset"
+):
+    raise environ.ImproperlyConfigured(
+        "When REACT_APP_AUTHENTICATION_METHOD is 'globus', both SOCIAL_AUTH_GLOBUS_KEY and SOCIAL_AUTH_GLOBUS_SECRET must be set via the environment variables GLOBUS_CLIENT_KEY and GLOBUS_CLIENT_SECRET respectively. You must obtain these credentials by registering a Portal at https://app.globus.org/settings/developers"
+    )