From 34db2d4b0f2fb9e757498950fa695956fe5f5159 Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 25 May 2023 06:55:23 +0000 Subject: [PATCH 1/5] new normsql anatomy --- .dae-devops/Makefile | 4 +- .dae-devops/docs/conventions.rst | 4 +- .dae-devops/docs/developing.rst | 5 +- .dae-devops/docs/devops.rst | 15 ++--- .dae-devops/docs/docs_structure.rst | 4 +- .dae-devops/docs/installing.rst | 7 +-- .dae-devops/docs/testing.rst | 12 ++-- .dae-devops/prepare_git_dependencies.sh | 4 +- .devcontainer/Dockerfile | 4 +- .devcontainer/devcontainer.json | 4 +- .github/CONTRIBUTING.rst | 4 +- .../actions/install_requirements/action.yml | 4 +- .github/dependabot.yml | 4 +- .github/pages/index.html | 4 +- .github/pages/make_switcher.py | 4 +- .github/workflows/code.yml | 4 +- .github/workflows/docs.yml | 4 +- .github/workflows/docs_clean.yml | 4 +- .github/workflows/linkcheck.yml | 4 +- .gitlab-ci.yml | 4 +- docs/_static/css/custom.css | 4 +- docs/conf.py | 4 +- pyproject.toml | 4 +- .../databases/database_definition.py | 12 ++-- .../databases/table_definitions.py | 0 src/dls_servbase_lib/databases/__init__.py | 0 src/dls_servbase_lib/databases/aiosqlite.py | 50 --------------- src/dls_servbase_lib/databases/databases.py | 46 -------------- src/dls_servbase_lib/datafaces/datafaces.py | 4 +- .../datafaces/{aiosqlite.py => normsql.py} | 16 +++-- tests/base_tester.py | 30 +++++++++ tests/configurations/servbase.yaml | 22 +++---- tests/test_database.py | 63 ++++++++++--------- 33 files changed, 142 insertions(+), 216 deletions(-) rename src/{dls_servbase_lib => dls_servbase_api}/databases/database_definition.py (80%) rename src/{dls_servbase_lib => dls_servbase_api}/databases/table_definitions.py (100%) delete mode 100644 src/dls_servbase_lib/databases/__init__.py delete mode 100644 src/dls_servbase_lib/databases/aiosqlite.py delete mode 100644 src/dls_servbase_lib/databases/databases.py rename src/dls_servbase_lib/datafaces/{aiosqlite.py => normsql.py} (93%) diff --git a/.dae-devops/Makefile b/.dae-devops/Makefile index 265e0ef..18deb55 100644 --- a/.dae-devops/Makefile +++ b/.dae-devops/Makefile @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase # --------------------------------------------------------------------- @@ -69,4 +69,4 @@ publish_docs: cp -r build/html/* $(DOCS_PUBLISH_ROOT) -# dae_devops_fingerprint dc970803af5fac409092e34299b9fa8a +# dae_devops_fingerprint 9d6b41a16b901f855bd09f8a5743bfa8 diff --git a/.dae-devops/docs/conventions.rst b/.dae-devops/docs/conventions.rst index e5e866c..c145e49 100644 --- a/.dae-devops/docs/conventions.rst +++ b/.dae-devops/docs/conventions.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-servbase Naming conventions @@ -31,4 +31,4 @@ repository lowercase, hyphens -.. # dae_devops_fingerprint de2f9c8adee74d13de393fe798054c9a +.. # dae_devops_fingerprint 94379720e1dca624dcebe759adc32cde diff --git a/.dae-devops/docs/developing.rst b/.dae-devops/docs/developing.rst index f661234..2937791 100644 --- a/.dae-devops/docs/developing.rst +++ b/.dae-devops/docs/developing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-servbase Developing @@ -22,7 +22,6 @@ Make sure to have at least python version 3.9 then:: Install the package in edit mode which will also install all its dependencies:: $ cd dls-servbase - $ export PIP_FIND_LINKS=/dls_sw/apps/bxflow/artifacts $ pip install -e .[dev] Now you may begin modifying the code. @@ -36,4 +35,4 @@ If you plan to modify the docs, you will need to:: -.. # dae_devops_fingerprint 85a9a86c29f573bd408647f7092b6fbe +.. # dae_devops_fingerprint eebb86b2433c3c3668ed0e68c2060ce3 diff --git a/.dae-devops/docs/devops.rst b/.dae-devops/docs/devops.rst index b79e3b1..02b940c 100644 --- a/.dae-devops/docs/devops.rst +++ b/.dae-devops/docs/devops.rst @@ -1,23 +1,24 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-servbase Devops ======================================================================= -There exists a a configuration file called ``.dae-devops/project.yaml``. +In the top level of the repository there exists a configuration file called ``.dae-devops/project.yaml``. This file defines the project information needed for CI/CD. It is parsed by the ``dae_devops.force`` command which creates these files: - pyproject.toml +- .githib/* - .gitlab-ci.yml - .dae-devops/Makefile - .dae-devops/docs/* Local CI/CD execution ------------------------------------------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ All the CI/CD ops which are run by the git server can be run at the command line. @@ -32,15 +33,15 @@ Validation of the code:: $ make -f .dae-devops/Makefile validate_pytest $ make -f .dae-devops/Makefile validate_docs -Packaging:: +Packaging (for the Diamond intranet):: $ make -f .dae-devops/Makefile package_pip -Publishing:: +Publishing (for the Diamond intranet):: $ make -f .dae-devops/Makefile publish_pip $ make -f .dae-devops/Makefile publish_docs +The Diamond intranet commands are not used for production. The production packaging and publishing are handled in the GitHub Actions workflows mechanism. - -.. # dae_devops_fingerprint d203976f0fb58a25033c89155e85466e +.. # dae_devops_fingerprint 56f856790e651a5410544bcfb52d77f9 diff --git a/.dae-devops/docs/docs_structure.rst b/.dae-devops/docs/docs_structure.rst index f87686e..94d05fd 100644 --- a/.dae-devops/docs/docs_structure.rst +++ b/.dae-devops/docs/docs_structure.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-servbase About the documentation @@ -21,4 +21,4 @@ improve most documentation - often immensely. `More information on this topic. `_ -.. # dae_devops_fingerprint 4c487ab610c69b04a69e018d6726347d +.. # dae_devops_fingerprint afcec7e660b5cb952ab0381111043f43 diff --git a/.dae-devops/docs/installing.rst b/.dae-devops/docs/installing.rst index 564f758..2efd660 100644 --- a/.dae-devops/docs/installing.rst +++ b/.dae-devops/docs/installing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-servbase Installing @@ -26,7 +26,6 @@ installation will not interfere with any existing Python software:: You can now use ``pip`` to install the library and its dependencies:: - $ export PIP_FIND_LINKS=/dls_sw/apps/bxflow/artifacts $ python3 -m pip install dls-servbase If you require a feature that is not currently released you can also install @@ -34,10 +33,10 @@ from git:: $ python3 -m pip install git+https://gitlab.diamond.ac.uk/kbp43231/dls-servbase.git -The library should now be installed and the commandline interface on your path. +The library should now be installed and the commandline should be available. You can check the version that has been installed by typing:: $ dls-servbase --version $ dls-servbase --version-json -.. # dae_devops_fingerprint 4595ce95067039bcca5e2ff516cd03af +.. # dae_devops_fingerprint d98511eda41e5ec9dfee88bd0263bb66 diff --git a/.dae-devops/docs/testing.rst b/.dae-devops/docs/testing.rst index 23e5d00..eb8760b 100644 --- a/.dae-devops/docs/testing.rst +++ b/.dae-devops/docs/testing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-servbase Testing @@ -11,17 +11,13 @@ If you want to run the tests, first get a copy of the code per the instructions Then you can run all tests by:: - $ pytest - -Or this, which is the command used by the CI runner. - - $ make -f .dae-devops/Makefile validate_pytest + $ tox -q -e pytest To run a single test you can do:: $ pytest tests/the_test_you_want.py -If you want to see more output of the test while it's running you can do: +If you want to see more output of the test while it's running you can do:: $ pytest -sv -ra --tb=line tests/the_test_you_want.py @@ -35,4 +31,4 @@ This allows peeking in there to see what's been written by the test. -.. # dae_devops_fingerprint 6b049b05d346f33c50ff7c08811e9754 +.. # dae_devops_fingerprint 813b0334290efd50c2d7ce4016da0dd9 diff --git a/.dae-devops/prepare_git_dependencies.sh b/.dae-devops/prepare_git_dependencies.sh index 12d6bf8..6df71b7 100644 --- a/.dae-devops/prepare_git_dependencies.sh +++ b/.dae-devops/prepare_git_dependencies.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase me=${BASH_SOURCE} @@ -13,4 +13,4 @@ function __install { } -# dae_devops_fingerprint 903ed54702867ede6f0174cc61d4b713 \ No newline at end of file +# dae_devops_fingerprint 05c763871274d8238b3646237d61135e \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d65ab49..4e67493 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase # This file is for use as a devcontainer and a runtime container @@ -40,4 +40,4 @@ ENV PATH=/venv/bin:$PATH ENTRYPOINT ["dls-servbase"] CMD ["--version"] -# dae_devops_fingerprint d98578ac4f9faec948660e9198138f35 +# dae_devops_fingerprint 0feaa75887d5dc7e333642766f3bf0e6 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8643fa7..f8defa2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ // ********** Please don't edit this file! -// ********** It has been generated automatically by dae_devops version 0.5.2. +// ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. // ********** For repository_name dls-servbase // For format details, see https://containers.dev/implementors/json_reference/ @@ -57,4 +57,4 @@ "postCreateCommand": "pip install -e .[dev]" } -// dae_devops_fingerprint a69fcd5d4332da3e6af3b97a7f171509 +// dae_devops_fingerprint 98e4a132e4fdcd0df6d661f6bc2b43be diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 8ff1b5e..e0f23a0 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.2. +.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. .. # ********** For repository_name dls-servbase Contributing to the project @@ -38,4 +38,4 @@ should follow. .. _Developer Guide: https://diamondlightsource.github.io/dls-servbase/main/developer/how-to/contribute.html -.. # dae_devops_fingerprint 65b30b94836cca6d96fc062a2f20730a +.. # dae_devops_fingerprint 5e16346b414b86d30bb6f91a53f2a74a diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index d4caf49..7503e21 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase name: Install requirements @@ -61,4 +61,4 @@ runs: shell: bash -# dae_devops_fingerprint c59b1a1b3a2300f4ab9f3fc9274b213c +# dae_devops_fingerprint 6e198d3569b69252542e666770294e24 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0355a5c..e44b654 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase # To get started with Dependabot version updates, you'll need to specify which @@ -19,4 +19,4 @@ updates: schedule: interval: "weekly" -# dae_devops_fingerprint 122e34b2a7528692f5975e2fb7cc167d +# dae_devops_fingerprint b3be937b70abef292f3c9cda6ae5b12d diff --git a/.github/pages/index.html b/.github/pages/index.html index 71587cf..a9bc5e9 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -1,5 +1,5 @@ - + @@ -14,4 +14,4 @@ - + diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index acf0d81..a21f442 100644 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase import json @@ -102,4 +102,4 @@ def main(args=None): if __name__ == "__main__": main() -# dae_devops_fingerprint fe647176edcb02077a904f8e42cf4367 +# dae_devops_fingerprint 63e494af602e6fae359363b564c6fe01 diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index c925bb2..b5271ec 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase name: Code CI @@ -212,4 +212,4 @@ jobs: with: password: ${{ secrets.PYPI_TOKEN }} -# dae_devops_fingerprint f06ae327848fe3cd45de89312a6cfadf +# dae_devops_fingerprint c83f7625277d38123457181b8c90e48f diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2c4126d..47bdfee 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase name: Docs CI @@ -56,4 +56,4 @@ jobs: publish_dir: .github/pages keep_files: true -# dae_devops_fingerprint 7eb227aeb2a5da22835482ebc7fe11dd +# dae_devops_fingerprint f3b488c471047c76aea8f825d07c18bd diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index d20021f..23edfba 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase name: Docs Cleanup CI @@ -46,4 +46,4 @@ jobs: git commit -am "Removing redundant docs version $DOCS_VERSION" git push -# dae_devops_fingerprint cd9ac693f5bf252fbaf5c9585cb1dd76 +# dae_devops_fingerprint 8c38ddcc9cef1d2542c8c10b7bf07ef9 diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 8d74ee8..8b3bf4a 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase name: Link Check @@ -27,4 +27,4 @@ jobs: - name: Check links run: tox -e docs build -- -b linkcheck -# dae_devops_fingerprint dddcd5c3991c4a5f1f8ae8a99b66fbcb +# dae_devops_fingerprint 867fe06b3073c6d7dbcb7419a7654068 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e5d3ce..f431cf9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase variables: @@ -87,4 +87,4 @@ package_pip: # # The validate_docs artifacts are in the build/html folder. # - make -f .dae-devops/Makefile publish_docs -# dae_devops_fingerprint e156b7559726d32f77aea7cfd628aee8 +# dae_devops_fingerprint 6db22c11e44b2e4faf915158345ffee7 diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index cad0176..5bc9040 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,5 +1,5 @@ /* ********** Please don't edit this file! */ -/* ********** It has been generated automatically by dae_devops version 0.5.2. */ +/* ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. */ /* ********** For repository_name dls-servbase */ /* The theme normally has this, but I think it's ok to use the full width of the window in all @media sizes. @@ -15,4 +15,4 @@ max-width: 100%; } -/* dae_devops_fingerprint 18ea0db9563c9bb76b88d1451e9cb254 */ +/* dae_devops_fingerprint b7a24fe59931733079a4121c7833c2d8 */ diff --git a/docs/conf.py b/docs/conf.py index 7692819..11bc6e1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase from pathlib import Path @@ -197,4 +197,4 @@ def setup(app): app.connect("source-read", ultimateReplace) -# dae_devops_fingerprint 9cd088d2151788486c59b6e240be074b +# dae_devops_fingerprint ba6ece62c7dc3689eb6fad112cadcd16 diff --git a/pyproject.toml b/pyproject.toml index e1a3376..01a4d77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.2. +# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. # ********** For repository_name dls-servbase [build-system] @@ -102,4 +102,4 @@ source = ["src", "**/site-packages/"] [tool.tox] legacy_tox_ini = "[tox]\nskipsdist=True\n\n[testenv:{pre-commit,mypy,pytest,docs}]\n# Don't create a virtualenv for the command, requires tox-direct plugin\ndirect = True\npassenv = *\nallowlist_externals = \n pytest \n pre-commit\n mypy\n sphinx-build\n sphinx-autobuild\ncommands =\n pytest: pytest {posargs}\n mypy: mypy src tests {posargs}\n pre-commit: pre-commit run --all-files {posargs}\n docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html\n" -# dae_devops_fingerprint b6059f181a9c0125cb482b270e753f4e +# dae_devops_fingerprint 44036d8720305615f23b52785a963a89 diff --git a/src/dls_servbase_lib/databases/database_definition.py b/src/dls_servbase_api/databases/database_definition.py similarity index 80% rename from src/dls_servbase_lib/databases/database_definition.py rename to src/dls_servbase_api/databases/database_definition.py index bc4a398..94c88d6 100644 --- a/src/dls_servbase_lib/databases/database_definition.py +++ b/src/dls_servbase_api/databases/database_definition.py @@ -3,7 +3,7 @@ from dls_servbase_api.databases.constants import CookieFieldnames, Tablenames # Base class for all aiosqlite database objects. -from dls_servbase_lib.databases.table_definitions import CookiesTableDefinition +from dls_servbase_api.databases.table_definitions import CookiesTableDefinition logger = logging.getLogger(__name__) @@ -24,13 +24,13 @@ def __init__(self): self.LATEST_REVISION = 2 # ---------------------------------------------------------------------------------------- - async def apply_revision(self, revision): + async def apply_revision(self, database, revision): if revision == 2: - await self.execute( + await database.execute( f"ALTER TABLE {Tablenames.COOKIES} ADD COLUMN {CookieFieldnames.NAME} TEXT", why=f"revision 2: add {Tablenames.COOKIES} {CookieFieldnames.NAME} column", ) - await self.execute( + await database.execute( "CREATE INDEX %s_%s ON %s(%s)" % ( Tablenames.COOKIES, @@ -41,10 +41,10 @@ async def apply_revision(self, revision): ) # ---------------------------------------------------------------------------------------- - async def add_table_definitions(self): + async def add_table_definitions(self, database): """ Make all the table definitions. """ # Table schemas in our database. - self.add_table_definition(CookiesTableDefinition()) + database.add_table_definition(CookiesTableDefinition()) diff --git a/src/dls_servbase_lib/databases/table_definitions.py b/src/dls_servbase_api/databases/table_definitions.py similarity index 100% rename from src/dls_servbase_lib/databases/table_definitions.py rename to src/dls_servbase_api/databases/table_definitions.py diff --git a/src/dls_servbase_lib/databases/__init__.py b/src/dls_servbase_lib/databases/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/dls_servbase_lib/databases/aiosqlite.py b/src/dls_servbase_lib/databases/aiosqlite.py deleted file mode 100644 index 0571d77..0000000 --- a/src/dls_servbase_lib/databases/aiosqlite.py +++ /dev/null @@ -1,50 +0,0 @@ -import logging - -# Base class for the aiosqlite database object. -from dls_normsql.aiosqlite import Aiosqlite as NormsqlAiosqlite - -# Base class for our database definition. -from dls_servbase_lib.databases.database_definition import DatabaseDefinition - -logger = logging.getLogger(__name__) - - -# ---------------------------------------------------------------------------------------- -class Aiosqlite(DatabaseDefinition, NormsqlAiosqlite): - """ - Class with coroutines for creating and querying a sqlite database. - We use dls-normsql to do the heavy lifting. - """ - - # ---------------------------------------------------------------------------------------- - def __init__(self, specification): - """ - Construct object. Do not connect to database. - """ - - # Constructor for the database definition. - DatabaseDefinition.__init__(self) - - # Constructor for the database implementation. - NormsqlAiosqlite.__init__(self, specification) - - # ---------------------------------------------------------------------------------------- - def reinstance(self): - """ - Clone database instance. - This is needed if a process inherits an instance, but needs its own connection. - """ - - return Aiosqlite(self.__filename) - - # ---------------------------------------------------------------------------------------- - async def add_table_definitions(self): - """ - Make all the table definitions. - """ - - # Add tables common in all implementations. - await NormsqlAiosqlite.add_table_definitions(self) - - # Add tables from our definition. - await DatabaseDefinition.add_table_definitions(self) diff --git a/src/dls_servbase_lib/databases/databases.py b/src/dls_servbase_lib/databases/databases.py deleted file mode 100644 index 0ed4f36..0000000 --- a/src/dls_servbase_lib/databases/databases.py +++ /dev/null @@ -1,46 +0,0 @@ -# Use standard logging in this module. -import logging - -# Class managing list of things. -from dls_utilpack.things import Things - -# Exceptions. -from dls_servbase_api.exceptions import NotFound - -logger = logging.getLogger(__name__) - - -class Databases(Things): - """ - List of available databases. - """ - - # ---------------------------------------------------------------------------------------- - def __init__(self, name=None): - Things.__init__(self, name) - - # ---------------------------------------------------------------------------------------- - def build_object(self, specification): - """""" - - database_class = self.lookup_class(specification["type"]) - - try: - database_object = database_class(specification) - except Exception as exception: - raise RuntimeError( - "unable to build database object for type %s" % (database_class) - ) from exception - - return database_object - - # ---------------------------------------------------------------------------------------- - def lookup_class(self, class_type): - """""" - - if class_type == "dls_servbase_lib.databases.aiosqlite": - from dls_servbase_lib.databases.aiosqlite import Aiosqlite - - return Aiosqlite - - raise NotFound("unable to get database class for type %s" % (class_type)) diff --git a/src/dls_servbase_lib/datafaces/datafaces.py b/src/dls_servbase_lib/datafaces/datafaces.py index 7b5b4d3..5c663ac 100644 --- a/src/dls_servbase_lib/datafaces/datafaces.py +++ b/src/dls_servbase_lib/datafaces/datafaces.py @@ -48,9 +48,9 @@ def lookup_class(self, class_type): return Aiohttp elif class_type == "dls_servbase_lib.datafaces.aiosqlite": - from dls_servbase_lib.datafaces.aiosqlite import Aiosqlite + from dls_servbase_lib.datafaces.normsql import Normsql - return Aiosqlite + return Normsql raise NotFound( "unable to get dls_servbase_dataface class for type %s" % (class_type) diff --git a/src/dls_servbase_lib/datafaces/aiosqlite.py b/src/dls_servbase_lib/datafaces/normsql.py similarity index 93% rename from src/dls_servbase_lib/datafaces/aiosqlite.py rename to src/dls_servbase_lib/datafaces/normsql.py index dfdca1a..12bf60f 100644 --- a/src/dls_servbase_lib/datafaces/aiosqlite.py +++ b/src/dls_servbase_lib/datafaces/normsql.py @@ -1,19 +1,20 @@ import logging +# Database manager. +from dls_normsql.databases import Databases + # Base class for generic things. from dls_utilpack.thing import Thing from dls_servbase_api.databases.constants import Tablenames - -# Database manager. -from dls_servbase_lib.databases.databases import Databases +from dls_servbase_api.databases.database_definition import DatabaseDefinition logger = logging.getLogger(__name__) -thing_type = "dls_servbase_lib.datafaces.aiosqlite" +thing_type = "dls_servbase_lib.datafaces.normsql" -class Aiosqlite(Thing): +class Normsql(Thing): """ """ # ---------------------------------------------------------------------------------------- @@ -37,7 +38,10 @@ async def disconnect(self): async def establish_database_connection(self): if self.__database is None: - self.__database = Databases().build_object(self.specification()["database"]) + self.__database = Databases().build_object( + self.specification()["database"], + DatabaseDefinition(), + ) await self.__database.connect() # ---------------------------------------------------------------------------------------- diff --git a/tests/base_tester.py b/tests/base_tester.py index 8bbdc02..8184704 100644 --- a/tests/base_tester.py +++ b/tests/base_tester.py @@ -33,3 +33,33 @@ def main(self, constants, output_directory): if failure_message is not None: pytest.fail(failure_message) + + +# ---------------------------------------------------------------------------------------- +class BaseTester2: + """ + Provide asyncio loop and error checking over *Tester classes. + """ + + def main(self, constants, specification, output_directory): + """ + This is the main program which calls the test using asyncio. + """ + + multiprocessing.current_process().name = "main" + + failure_message = None + try: + # Run main test in asyncio event loop. + asyncio.run( + self._main_coroutine(constants, specification, output_directory) + ) + + except Exception as exception: + logger.exception( + "unexpected exception in the test method", exc_info=exception + ) + failure_message = str(exception) + + if failure_message is not None: + pytest.fail(failure_message) diff --git a/tests/configurations/servbase.yaml b/tests/configurations/servbase.yaml index 790545e..d702ecf 100644 --- a/tests/configurations/servbase.yaml +++ b/tests/configurations/servbase.yaml @@ -1,13 +1,5 @@ type: dls_servbase_lib.dls_servbase_contexts.classic -visit: - beamline: b29 - year: 2022 - visit: cy29757-3 - directory: /dls/b29/data/2022/cy29757-3 - # Format to make actual data_filename using data_label as token. - data_path_pattern: "/dls/b29/data/2022/cy29757-3/Merlin/{data_label}_data.mib" - logging_settings: console: enabled: True @@ -31,11 +23,11 @@ logging_settings: # The external access bits. external_access_bits: dataface_port: &DATAFACE_PORT 27520 - dls_servbase_gui_server: &DLS_BILLY_GUI_SERVER http://*:22522 - dls_servbase_gui_client: &DLS_BILLY_GUI_CLIENT http://127.0.0.1:22522 + dls_servbase_gui_server: &DLS_SERVBASE_GUI_SERVER http://*:22522 + dls_servbase_gui_client: &DLS_SERVBASE_GUI_CLIENT http://127.0.0.1:22522 # The dls_servbase_dataface client/server composite. -dls_servbase_dataface_specification: &DLS_BILLY_DATAFACE_SPECIFICATION +dls_servbase_dataface_specification: &DLS_SERVBASE_DATAFACE_SPECIFICATION type: "dls_servbase_lib.datafaces.aiohttp" type_specific_tbd: # The remote dls_servbase_dataface server access. @@ -47,7 +39,7 @@ dls_servbase_dataface_specification: &DLS_BILLY_DATAFACE_SPECIFICATION actual_dataface_specification: type: "dls_servbase_lib.datafaces.aiosqlite" database: - type: "dls_servbase_lib.databases.aiosqlite" + type: "dls_normsql.aiosqlite" filename: "${output_directory}/dls_servbase_dataface.sqlite" log_level: "WARNING" context: @@ -59,12 +51,12 @@ dls_servbase_gui_specification: type_specific_tbd: # The remote dls_servbase_gui server access. aiohttp_specification: - server: *DLS_BILLY_GUI_SERVER - client: *DLS_BILLY_GUI_CLIENT + server: *DLS_SERVBASE_GUI_SERVER + client: *DLS_SERVBASE_GUI_CLIENT search_paths: ["examples/html"] cookie_specification: type: "dls_servbase_lib.cookies.dataface" type_specific_tbd: - dataface_specification: *DLS_BILLY_DATAFACE_SPECIFICATION + dataface_specification: *DLS_SERVBASE_DATAFACE_SPECIFICATION context: start_as: process diff --git a/tests/test_database.py b/tests/test_database.py index 56fe2c8..db1737d 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,11 +1,11 @@ -import asyncio import logging -import multiprocessing -import pytest +from dls_normsql.constants import ClassTypes +from dls_normsql.databases import Databases from dls_servbase_api.databases.constants import CookieFieldnames, Tablenames -from dls_servbase_lib.databases.databases import Databases +from dls_servbase_api.databases.database_definition import DatabaseDefinition +from tests.base_tester import BaseTester2 logger = logging.getLogger(__name__) @@ -20,7 +20,7 @@ def test(self, constants, logging_setup, output_directory): """ database_specification = { - "type": "dls_servbase_lib.databases.aiosqlite", + "type": ClassTypes.AIOSQLITE, "filename": "%s/dls_servbase_scheduler.sqlite" % (output_directory), } @@ -33,37 +33,35 @@ def test(self, constants, logging_setup, output_directory): # ---------------------------------------------------------------------------------------- -class _BaseTester: - """ - Provide asyncio loop and error checking over *Tester classes. - """ - - def main(self, constants, specification, output_directory): - """ - This is the main program which calls the test using asyncio. +class TestDatabaseMysql: + def test(self, constants, logging_setup, output_directory): """ + Tests the sqlite implementation of the Database interface. - multiprocessing.current_process().name = "main" - - failure_message = None - try: - # Run main test in asyncio event loop. - asyncio.run( - self._main_coroutine(constants, specification, output_directory) - ) + This does not use a service. + """ - except Exception as exception: - logger.exception( - "unexpected exception in the test method", exc_info=exception - ) - failure_message = str(exception) + database_specification = { + "type": ClassTypes.AIOMYSQL, + "type_specific_tbd": { + "database_name": "dls_servbase_pytest", + "host": "docker-mysql", + "port": 3306, + "username": "root", + "password": "root", + }, + } - if failure_message is not None: - pytest.fail(failure_message) + # Test direct SQL access to the database. + DatabaseTesterImage().main( + constants, + database_specification, + output_directory, + ) # ---------------------------------------------------------------------------------------- -class DatabaseTesterImage(_BaseTester): +class DatabaseTesterImage(BaseTester2): """ Test direct SQL access to the database. """ @@ -74,11 +72,14 @@ async def _main_coroutine( """ """ databases = Databases() - database = databases.build_object(database_specification) + database = databases.build_object( + database_specification, + DatabaseDefinition(), + ) try: # Connect to database. - await database.connect() + await database.connect(should_drop_database=True) # Write one record. await database.insert( From 78b3913726d45bc94b8a9823c674125b333f42c4 Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 25 May 2023 08:00:13 +0000 Subject: [PATCH 2/5] add tests for mysql --- src/dls_servbase_lib/datafaces/datafaces.py | 2 +- src/dls_servbase_lib/datafaces/normsql.py | 5 +- tests/configurations/mysql.yaml | 67 +++++++++++++++++++ .../{servbase.yaml => sqlite.yaml} | 2 +- tests/test_dataface.py | 17 ++++- tests/test_dataface_takeover.py | 56 ++++++++++++++-- tests/test_gui.py | 2 +- 7 files changed, 139 insertions(+), 12 deletions(-) create mode 100644 tests/configurations/mysql.yaml rename tests/configurations/{servbase.yaml => sqlite.yaml} (94%) diff --git a/src/dls_servbase_lib/datafaces/datafaces.py b/src/dls_servbase_lib/datafaces/datafaces.py index 5c663ac..0738eee 100644 --- a/src/dls_servbase_lib/datafaces/datafaces.py +++ b/src/dls_servbase_lib/datafaces/datafaces.py @@ -47,7 +47,7 @@ def lookup_class(self, class_type): return Aiohttp - elif class_type == "dls_servbase_lib.datafaces.aiosqlite": + elif class_type == "dls_servbase_lib.datafaces.normsql": from dls_servbase_lib.datafaces.normsql import Normsql return Normsql diff --git a/src/dls_servbase_lib/datafaces/normsql.py b/src/dls_servbase_lib/datafaces/normsql.py index 12bf60f..63746c4 100644 --- a/src/dls_servbase_lib/datafaces/normsql.py +++ b/src/dls_servbase_lib/datafaces/normsql.py @@ -21,6 +21,7 @@ class Normsql(Thing): def __init__(self, specification=None): Thing.__init__(self, thing_type, specification) + self.__should_drop_database = specification.get("should_drop_database") self.__database = None # ---------------------------------------------------------------------------------------- @@ -42,7 +43,9 @@ async def establish_database_connection(self): self.specification()["database"], DatabaseDefinition(), ) - await self.__database.connect() + await self.__database.connect( + should_drop_database=self.__should_drop_database + ) # ---------------------------------------------------------------------------------------- async def reinstance(self): diff --git a/tests/configurations/mysql.yaml b/tests/configurations/mysql.yaml new file mode 100644 index 0000000..4150a10 --- /dev/null +++ b/tests/configurations/mysql.yaml @@ -0,0 +1,67 @@ +type: dls_servbase_lib.dls_servbase_contexts.classic + +logging_settings: + console: + enabled: True + verbose: True + filters: + markers: + - "[RELCOOK]" + - "[COOKSEL]" + - "[COOKOFF]" + - "[GUITABS]" + - "[SETTINGVAL]" + logfile: + enabled: True + directory: ${output_directory}/logfile.log + graypy: + enabled: False + host: 172.23.7.128 + port: 12201 + protocol: UDP + +# The external access bits. +external_access_bits: + dataface_port: &DATAFACE_PORT 27520 + dls_servbase_gui_server: &DLS_SERVBASE_GUI_SERVER http://*:22522 + dls_servbase_gui_client: &DLS_SERVBASE_GUI_CLIENT http://127.0.0.1:22522 + +# The dls_servbase_dataface client/server composite. +dls_servbase_dataface_specification: &DLS_SERVBASE_DATAFACE_SPECIFICATION + type: "dls_servbase_lib.datafaces.aiohttp" + type_specific_tbd: + # The remote dls_servbase_dataface server access. + aiohttp_specification: + server_host: "*" + client_host: "127.0.0.1" + port: *DATAFACE_PORT + # The local implementation of the dls_servbase_dataface. + actual_dataface_specification: + type: "dls_servbase_lib.datafaces.normsql" + should_drop_database: True + database: + type: "dls_normsql.aiomysql" + type_specific_tbd: + database_name: "dls_servbase_pytest" + host: "docker-mysql" + port: 3306 + username: "root" + password: "root" + context: + start_as: process + +# The dls_servbase_gui specification. +dls_servbase_gui_specification: + type: "dls_servbase_lib.dls_servbase_guis.aiohttp" + type_specific_tbd: + # The remote dls_servbase_gui server access. + aiohttp_specification: + server: *DLS_SERVBASE_GUI_SERVER + client: *DLS_SERVBASE_GUI_CLIENT + search_paths: ["examples/html"] + cookie_specification: + type: "dls_servbase_lib.cookies.dataface" + type_specific_tbd: + dataface_specification: *DLS_SERVBASE_DATAFACE_SPECIFICATION + context: + start_as: process diff --git a/tests/configurations/servbase.yaml b/tests/configurations/sqlite.yaml similarity index 94% rename from tests/configurations/servbase.yaml rename to tests/configurations/sqlite.yaml index d702ecf..d8c3857 100644 --- a/tests/configurations/servbase.yaml +++ b/tests/configurations/sqlite.yaml @@ -37,7 +37,7 @@ dls_servbase_dataface_specification: &DLS_SERVBASE_DATAFACE_SPECIFICATION port: *DATAFACE_PORT # The local implementation of the dls_servbase_dataface. actual_dataface_specification: - type: "dls_servbase_lib.datafaces.aiosqlite" + type: "dls_servbase_lib.datafaces.normsql" database: type: "dls_normsql.aiosqlite" filename: "${output_directory}/dls_servbase_dataface.sqlite" diff --git a/tests/test_dataface.py b/tests/test_dataface.py index 3319e01..d20340a 100644 --- a/tests/test_dataface.py +++ b/tests/test_dataface.py @@ -12,7 +12,7 @@ # ---------------------------------------------------------------------------------------- -class TestDataface: +class TestDatafaceSqlite: """ Test that we can do a basic database operation through the service. """ @@ -20,7 +20,20 @@ class TestDataface: def test(self, constants, logging_setup, output_directory): """ """ - configuration_file = "tests/configurations/servbase.yaml" + configuration_file = "tests/configurations/sqlite.yaml" + DatafaceTester().main(constants, configuration_file, output_directory) + + +# ---------------------------------------------------------------------------------------- +class TestDatafaceMysql: + """ + Test that we can do a basic database operation through the service. + """ + + def test(self, constants, logging_setup, output_directory): + """ """ + + configuration_file = "tests/configurations/mysql.yaml" DatafaceTester().main(constants, configuration_file, output_directory) diff --git a/tests/test_dataface_takeover.py b/tests/test_dataface_takeover.py index eff4511..b105bd3 100644 --- a/tests/test_dataface_takeover.py +++ b/tests/test_dataface_takeover.py @@ -12,11 +12,24 @@ # ---------------------------------------------------------------------------------------- -class TestDatafaceTakeover: - def test_dataface_laptop(self, constants, logging_setup, output_directory): +class TestDatafaceTakeoverSqlite: + def test(self, constants, logging_setup, output_directory): """ """ - configuration_file = "tests/configurations/servbase.yaml" + configuration_file = "tests/configurations/sqlite.yaml" + DatafaceTakeoverTester().main(constants, configuration_file, output_directory) + + +# ---------------------------------------------------------------------------------------- +class TestDatafaceTakeoverMysql: + """ + Test that we can do a basic database operation through the service. + """ + + def test(self, constants, logging_setup, output_directory): + """ """ + + configuration_file = "tests/configurations/mysql.yaml" DatafaceTakeoverTester().main(constants, configuration_file, output_directory) @@ -55,14 +68,45 @@ async def _main_coroutine(self, constants, output_directory): } ], ) + all_sql = f"SELECT * FROM {Tablenames.COOKIES}" + records = await dataface.query(all_sql) - # Make a new dataface context with the same specification. + assert len(records) == 1 + assert records[0][CookieFieldnames.UUID] == "f0" + assert records[0][CookieFieldnames.CONTENTS] == "{'a': 'f000'}" + + # ---------------------------------------------------------------------------- + # Make a new dataface context with the same specification, except don't drop. + servbase_specification["type_specific_tbd"][ + "actual_dataface_specification" + ]["should_drop_database"] = False dls_servbase_server_context = ServerContext(servbase_specification) # Activate the new dataface which should send shutdown to the old process. async with dls_servbase_server_context: - all_sql = f"SELECT * FROM {Tablenames.COOKIES}" + # Check the final insert made it to the database. records = await dataface.query(all_sql) - assert len(records) == 1 assert records[0][CookieFieldnames.UUID] == "f0" assert records[0][CookieFieldnames.CONTENTS] == "{'a': 'f000'}" + + # Update the cookie record. + subs = [] + subs.append("f0") + await dataface.update( + Tablenames.COOKIES, + {CookieFieldnames.CONTENTS: "{'a': 'f001'}"}, + f"{CookieFieldnames.UUID} = ?", + subs=subs, + why="update database revision", + ) + + # ---------------------------------------------------------------------------- + # Make a new dataface context with the same specification, again without dropping. + dls_servbase_server_context = ServerContext(servbase_specification) + # Activate the new dataface which should send shutdown to the old process. + async with dls_servbase_server_context: + # Check the final update made it to the database. + records = await dataface.query(all_sql) + assert len(records) == 1 + assert records[0][CookieFieldnames.UUID] == "f0" + assert records[0][CookieFieldnames.CONTENTS] == "{'a': 'f001'}" diff --git a/tests/test_gui.py b/tests/test_gui.py index 268f537..2064851 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -30,7 +30,7 @@ class TestGui: def test(self, constants, logging_setup, output_directory): """ """ - configuration_file = "tests/configurations/servbase.yaml" + configuration_file = "tests/configurations/sqlite.yaml" GuiTester().main(constants, configuration_file, output_directory) From 0325753c57a5b8e2a3d90c749c963cb27a3d5ced Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 25 May 2023 18:38:46 +0100 Subject: [PATCH 3/5] comments and cosmetics --- src/dls_servbase_lib/datafaces/normsql.py | 4 ++++ tests/test_database.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/dls_servbase_lib/datafaces/normsql.py b/src/dls_servbase_lib/datafaces/normsql.py index 63746c4..0d2a08a 100644 --- a/src/dls_servbase_lib/datafaces/normsql.py +++ b/src/dls_servbase_lib/datafaces/normsql.py @@ -21,7 +21,9 @@ class Normsql(Thing): def __init__(self, specification=None): Thing.__init__(self, thing_type, specification) + # For testing, caller might want to drop the database on connection. self.__should_drop_database = specification.get("should_drop_database") + self.__database = None # ---------------------------------------------------------------------------------------- @@ -43,6 +45,8 @@ async def establish_database_connection(self): self.specification()["database"], DatabaseDefinition(), ) + + # For testing, caller might want to drop the database on connection. await self.__database.connect( should_drop_database=self.__should_drop_database ) diff --git a/tests/test_database.py b/tests/test_database.py index db1737d..30f708b 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -21,11 +21,11 @@ def test(self, constants, logging_setup, output_directory): database_specification = { "type": ClassTypes.AIOSQLITE, - "filename": "%s/dls_servbase_scheduler.sqlite" % (output_directory), + "filename": "%s/dls_servbase_pytest.sqlite" % (output_directory), } # Test direct SQL access to the database. - DatabaseTesterImage().main( + DatabaseTester().main( constants, database_specification, output_directory, @@ -53,7 +53,7 @@ def test(self, constants, logging_setup, output_directory): } # Test direct SQL access to the database. - DatabaseTesterImage().main( + DatabaseTester().main( constants, database_specification, output_directory, @@ -61,7 +61,7 @@ def test(self, constants, logging_setup, output_directory): # ---------------------------------------------------------------------------------------- -class DatabaseTesterImage(BaseTester2): +class DatabaseTester(BaseTester2): """ Test direct SQL access to the database. """ From d6f46ad9a64b2b4380538d397f2ca54336140255 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 08:11:05 +0100 Subject: [PATCH 4/5] environment MYSQL_HOST --- tests/configurations/mysql.yaml | 5 +++-- tests/test_database.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/configurations/mysql.yaml b/tests/configurations/mysql.yaml index 4150a10..1a1a9b4 100644 --- a/tests/configurations/mysql.yaml +++ b/tests/configurations/mysql.yaml @@ -43,8 +43,9 @@ dls_servbase_dataface_specification: &DLS_SERVBASE_DATAFACE_SPECIFICATION type: "dls_normsql.aiomysql" type_specific_tbd: database_name: "dls_servbase_pytest" - host: "docker-mysql" - port: 3306 + # These come straight from os.environ. + host: "$MYSQL_HOST" + port: "$MYSQL_PORT" username: "root" password: "root" context: diff --git a/tests/test_database.py b/tests/test_database.py index 30f708b..09b8797 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -45,8 +45,8 @@ def test(self, constants, logging_setup, output_directory): "type": ClassTypes.AIOMYSQL, "type_specific_tbd": { "database_name": "dls_servbase_pytest", - "host": "docker-mysql", - "port": 3306, + "host": "$MYSQL_HOST", + "port": "$MYSQL_PORT", "username": "root", "password": "root", }, From 830a30e1b0b112c1c72ef000947f6d84fe23c28e Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 26 May 2023 08:11:31 +0100 Subject: [PATCH 5/5] devops 5.3 --- .dae-devops/Makefile | 4 ++-- .dae-devops/docs/conventions.rst | 4 ++-- .dae-devops/docs/developing.rst | 4 ++-- .dae-devops/docs/devops.rst | 4 ++-- .dae-devops/docs/docs_structure.rst | 4 ++-- .dae-devops/docs/installing.rst | 4 ++-- .dae-devops/docs/testing.rst | 4 ++-- .dae-devops/prepare_git_dependencies.sh | 4 ++-- .devcontainer/Dockerfile | 4 ++-- .devcontainer/devcontainer.json | 4 ++-- .github/CONTRIBUTING.rst | 4 ++-- .github/actions/install_requirements/action.yml | 4 ++-- .github/dependabot.yml | 4 ++-- .github/pages/index.html | 4 ++-- .github/pages/make_switcher.py | 4 ++-- .github/workflows/code.yml | 10 +++++++--- .github/workflows/docs.yml | 4 ++-- .github/workflows/docs_clean.yml | 4 ++-- .github/workflows/linkcheck.yml | 4 ++-- .gitlab-ci.yml | 4 ++-- docs/_static/css/custom.css | 4 ++-- docs/conf.py | 4 ++-- pyproject.toml | 4 ++-- 23 files changed, 51 insertions(+), 47 deletions(-) diff --git a/.dae-devops/Makefile b/.dae-devops/Makefile index 18deb55..ce37ed8 100644 --- a/.dae-devops/Makefile +++ b/.dae-devops/Makefile @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase # --------------------------------------------------------------------- @@ -69,4 +69,4 @@ publish_docs: cp -r build/html/* $(DOCS_PUBLISH_ROOT) -# dae_devops_fingerprint 9d6b41a16b901f855bd09f8a5743bfa8 +# dae_devops_fingerprint 7a40fdc6afce4991715aeda7ae70c444 diff --git a/.dae-devops/docs/conventions.rst b/.dae-devops/docs/conventions.rst index c145e49..feb9d1b 100644 --- a/.dae-devops/docs/conventions.rst +++ b/.dae-devops/docs/conventions.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-servbase Naming conventions @@ -31,4 +31,4 @@ repository lowercase, hyphens -.. # dae_devops_fingerprint 94379720e1dca624dcebe759adc32cde +.. # dae_devops_fingerprint cbd0a78343b6577aa4ddb9d59a242b04 diff --git a/.dae-devops/docs/developing.rst b/.dae-devops/docs/developing.rst index 2937791..8068675 100644 --- a/.dae-devops/docs/developing.rst +++ b/.dae-devops/docs/developing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-servbase Developing @@ -35,4 +35,4 @@ If you plan to modify the docs, you will need to:: -.. # dae_devops_fingerprint eebb86b2433c3c3668ed0e68c2060ce3 +.. # dae_devops_fingerprint ac891cf352ef89b3b20d2c601dd9ddc4 diff --git a/.dae-devops/docs/devops.rst b/.dae-devops/docs/devops.rst index 02b940c..17d71fb 100644 --- a/.dae-devops/docs/devops.rst +++ b/.dae-devops/docs/devops.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-servbase Devops @@ -44,4 +44,4 @@ Publishing (for the Diamond intranet):: The Diamond intranet commands are not used for production. The production packaging and publishing are handled in the GitHub Actions workflows mechanism. -.. # dae_devops_fingerprint 56f856790e651a5410544bcfb52d77f9 +.. # dae_devops_fingerprint 84e8a32100211cbba64804a3e5c5b337 diff --git a/.dae-devops/docs/docs_structure.rst b/.dae-devops/docs/docs_structure.rst index 94d05fd..edcaea8 100644 --- a/.dae-devops/docs/docs_structure.rst +++ b/.dae-devops/docs/docs_structure.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-servbase About the documentation @@ -21,4 +21,4 @@ improve most documentation - often immensely. `More information on this topic. `_ -.. # dae_devops_fingerprint afcec7e660b5cb952ab0381111043f43 +.. # dae_devops_fingerprint 6ef08be920f12050c1c579cf6e9d4253 diff --git a/.dae-devops/docs/installing.rst b/.dae-devops/docs/installing.rst index 2efd660..c6af374 100644 --- a/.dae-devops/docs/installing.rst +++ b/.dae-devops/docs/installing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-servbase Installing @@ -39,4 +39,4 @@ You can check the version that has been installed by typing:: $ dls-servbase --version $ dls-servbase --version-json -.. # dae_devops_fingerprint d98511eda41e5ec9dfee88bd0263bb66 +.. # dae_devops_fingerprint 64d7ae0dde8939e479b9c793df928d15 diff --git a/.dae-devops/docs/testing.rst b/.dae-devops/docs/testing.rst index eb8760b..2c7843e 100644 --- a/.dae-devops/docs/testing.rst +++ b/.dae-devops/docs/testing.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-servbase Testing @@ -31,4 +31,4 @@ This allows peeking in there to see what's been written by the test. -.. # dae_devops_fingerprint 813b0334290efd50c2d7ce4016da0dd9 +.. # dae_devops_fingerprint 3bd80bbd88306473a1ee98b988b17986 diff --git a/.dae-devops/prepare_git_dependencies.sh b/.dae-devops/prepare_git_dependencies.sh index 6df71b7..09f5407 100644 --- a/.dae-devops/prepare_git_dependencies.sh +++ b/.dae-devops/prepare_git_dependencies.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase me=${BASH_SOURCE} @@ -13,4 +13,4 @@ function __install { } -# dae_devops_fingerprint 05c763871274d8238b3646237d61135e \ No newline at end of file +# dae_devops_fingerprint 663e11dbca4cdd5f84a218a64398a688 \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4e67493..e4d99b4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase # This file is for use as a devcontainer and a runtime container @@ -40,4 +40,4 @@ ENV PATH=/venv/bin:$PATH ENTRYPOINT ["dls-servbase"] CMD ["--version"] -# dae_devops_fingerprint 0feaa75887d5dc7e333642766f3bf0e6 +# dae_devops_fingerprint 80986671f8602f6e4666f78f98141826 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f8defa2..94fa52a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ // ********** Please don't edit this file! -// ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +// ********** It has been generated automatically by dae_devops version 0.5.3. // ********** For repository_name dls-servbase // For format details, see https://containers.dev/implementors/json_reference/ @@ -57,4 +57,4 @@ "postCreateCommand": "pip install -e .[dev]" } -// dae_devops_fingerprint 98e4a132e4fdcd0df6d661f6bc2b43be +// dae_devops_fingerprint 35358278c2556f05447815a0833301dc diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index e0f23a0..f846ea3 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -1,5 +1,5 @@ .. # ********** Please don't edit this file! -.. # ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +.. # ********** It has been generated automatically by dae_devops version 0.5.3. .. # ********** For repository_name dls-servbase Contributing to the project @@ -38,4 +38,4 @@ should follow. .. _Developer Guide: https://diamondlightsource.github.io/dls-servbase/main/developer/how-to/contribute.html -.. # dae_devops_fingerprint 5e16346b414b86d30bb6f91a53f2a74a +.. # dae_devops_fingerprint 0f4cbc1f77b787f03a9a4d268ed08055 diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 7503e21..2ae5439 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase name: Install requirements @@ -61,4 +61,4 @@ runs: shell: bash -# dae_devops_fingerprint 6e198d3569b69252542e666770294e24 +# dae_devops_fingerprint 492458ca501cde9a298a80a102ff22a4 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e44b654..c7de3cf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase # To get started with Dependabot version updates, you'll need to specify which @@ -19,4 +19,4 @@ updates: schedule: interval: "weekly" -# dae_devops_fingerprint b3be937b70abef292f3c9cda6ae5b12d +# dae_devops_fingerprint b5838c5e0d9b5041500c5cb701fb5e5b diff --git a/.github/pages/index.html b/.github/pages/index.html index a9bc5e9..9690395 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -1,5 +1,5 @@ - + @@ -14,4 +14,4 @@ - + diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index a21f442..e293a78 100644 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase import json @@ -102,4 +102,4 @@ def main(args=None): if __name__ == "__main__": main() -# dae_devops_fingerprint 63e494af602e6fae359363b564c6fe01 +# dae_devops_fingerprint 7a1b58b851c5198d7d380e4ab78d0b49 diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index b5271ec..c5aaf73 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase name: Code CI @@ -39,7 +39,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] # can add windows-latest, macos-latest - python: ["3.10", "3.11"] + python: ["3.10"] install: ["-e .[dev,docs]"] # Make one version be non-editable to test both paths of version code include: @@ -69,6 +69,10 @@ jobs: - name: List dependency tree run: pipdeptree + # TODO: Make startup of MySQL able to be configured. + - name: Start up the MySQL that comes with Unbuntu + run: sudo /etc/init.d/mysql start + - name: Run tests run: | sudo apt install environment-modules @@ -212,4 +216,4 @@ jobs: with: password: ${{ secrets.PYPI_TOKEN }} -# dae_devops_fingerprint c83f7625277d38123457181b8c90e48f +# dae_devops_fingerprint 480a147d8af93a934e6ad0fd272e60e5 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 47bdfee..19f4aeb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase name: Docs CI @@ -56,4 +56,4 @@ jobs: publish_dir: .github/pages keep_files: true -# dae_devops_fingerprint f3b488c471047c76aea8f825d07c18bd +# dae_devops_fingerprint fb2dde86086783119d0b0b805151bf09 diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index 23edfba..6bd3916 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase name: Docs Cleanup CI @@ -46,4 +46,4 @@ jobs: git commit -am "Removing redundant docs version $DOCS_VERSION" git push -# dae_devops_fingerprint 8c38ddcc9cef1d2542c8c10b7bf07ef9 +# dae_devops_fingerprint d46d59a6f5197bd11b1cece615feb4df diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 8b3bf4a..99e745c 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase name: Link Check @@ -27,4 +27,4 @@ jobs: - name: Check links run: tox -e docs build -- -b linkcheck -# dae_devops_fingerprint 867fe06b3073c6d7dbcb7419a7654068 +# dae_devops_fingerprint 0198bfb9fadda488379138f6b2e7f5e3 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f431cf9..a36d603 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase variables: @@ -87,4 +87,4 @@ package_pip: # # The validate_docs artifacts are in the build/html folder. # - make -f .dae-devops/Makefile publish_docs -# dae_devops_fingerprint 6db22c11e44b2e4faf915158345ffee7 +# dae_devops_fingerprint 814043c19b1379f7f1d7b188546a7734 diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 5bc9040..f1e3c38 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,5 +1,5 @@ /* ********** Please don't edit this file! */ -/* ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. */ +/* ********** It has been generated automatically by dae_devops version 0.5.3. */ /* ********** For repository_name dls-servbase */ /* The theme normally has this, but I think it's ok to use the full width of the window in all @media sizes. @@ -15,4 +15,4 @@ max-width: 100%; } -/* dae_devops_fingerprint b7a24fe59931733079a4121c7833c2d8 */ +/* dae_devops_fingerprint b94b4f0075614442e536fdd79bcf79f7 */ diff --git a/docs/conf.py b/docs/conf.py index 11bc6e1..5829875 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase from pathlib import Path @@ -197,4 +197,4 @@ def setup(app): app.connect("source-read", ultimateReplace) -# dae_devops_fingerprint ba6ece62c7dc3689eb6fad112cadcd16 +# dae_devops_fingerprint 529445ac120d879ddd71245754e51ecb diff --git a/pyproject.toml b/pyproject.toml index 01a4d77..3e4047e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ # ********** Please don't edit this file! -# ********** It has been generated automatically by dae_devops version 0.5.3.dev1+g36e9e1c.d20230523. +# ********** It has been generated automatically by dae_devops version 0.5.3. # ********** For repository_name dls-servbase [build-system] @@ -102,4 +102,4 @@ source = ["src", "**/site-packages/"] [tool.tox] legacy_tox_ini = "[tox]\nskipsdist=True\n\n[testenv:{pre-commit,mypy,pytest,docs}]\n# Don't create a virtualenv for the command, requires tox-direct plugin\ndirect = True\npassenv = *\nallowlist_externals = \n pytest \n pre-commit\n mypy\n sphinx-build\n sphinx-autobuild\ncommands =\n pytest: pytest {posargs}\n mypy: mypy src tests {posargs}\n pre-commit: pre-commit run --all-files {posargs}\n docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html\n" -# dae_devops_fingerprint 44036d8720305615f23b52785a963a89 +# dae_devops_fingerprint ca7c759c4aa01b455a13284486cf7eaa