Add improvements after working with FastApi projects

CHANGELOG.md

@@ -6,6 +6,9 @@ We follow [Semantic Versions](https://semver.org/).
 
 - Add `django.startapp` invocation.
 - Confirm support for python 3.12.
+- Add `secrets` invocations
+- Extend alembic invocations to be able to make db dumps
+- Add invocation for `celery` to run task
 
 ## 0.8.3
 

README.md

@@ -63,8 +63,13 @@ Collection of [invoke](https://www.pyinvoke.org/) commands used by Saritasa
     * [alembic.downgrade](#alembicdowngrade)
     * [alembic.check-for-migrations](#alembiccheck-for-migrations)
     * [alembic.check-for-adjust-messages](#alembiccheck-for-adjust-messages)
+    * [alembic.load-db-dump](#alembicload-db-dump)
+    * [alembic.backup-local-db](#alembicbackup-local-db)
+    * [alembic.backup-remote-db](#alembicbackup-remote-db)
+    * [alembic.load-remote-db](#alembicload-remote-db)
   * [celery](#celery)
     * [celery.run](#celeryrun)
+    * [celery.send-task](#celerysend-task)
   * [open-api](#open-api)
     * [open-api.validate-swagger](#open-apivalidate-swagger)
   * [db](#db)
@@ -374,7 +379,7 @@ Requires [django-extensions](https://django-extensions.readthedocs.io/en/latest/
 
 Settings:
 
-* `django_settings_path` default django settings (Default: `config.settings.local`)
+* `settings_path` default django settings (Default: `config.settings.local`)
 
 #### django.createsuperuser
 
@@ -448,7 +453,7 @@ Uses [backup_local_db](#dbbackup-local-db)
 
 Settings:
 
-* `django_settings_path` default django settings (Default: `config.settings.local`)
+* `settings_path` default django settings (Default: `config.settings.local`)
 
 #### django.backup-remote-db
 
@@ -458,7 +463,8 @@ Uses [create_dump](#db-k8screate-dump) and [get-dump](#db-k8sget-dump)
 
 Settings:
 
-* `django_settings_path` default django settings (Default: `config.settings.local`)
+* `settings_path` default django settings (Default: `config.settings.local`)
+* `remote_db_config_mapping` Mapping of db config (Default: `{"dbname": "RDS_DB_NAME", "host": "RDS_DB_HOST", "port": "RDS_DB_PORT", "username": "RDS_DB_USER", "password": "RDS_DB_PASSWORD"}`)
 
 #### django.load-remote-db
 
@@ -469,7 +475,7 @@ Uses [create_dump](#db-k8screate-dump) and [get-dump](#db-k8sget-dump) and
 
 Settings:
 
-* `django_settings_path` default django settings (Default: `config.settings.local`)
+* `settings_path` default django settings (Default: `config.settings.local`)
 
 #### django.startapp
 
@@ -536,6 +542,63 @@ Settings:
 * `migrations_folder` migrations files location (Default: `db/migrations/versions`)
 * `adjust_messages` list of alembic adjust messages (Default: `# ### commands auto generated by Alembic - please adjust! ###`, `# ### end Alembic commands ###`)
 
+#### alembic.load-db-dump
+
+Reset db and load db dump.
+
+Uses [downgrade](#alembicdowngrade) and [load-db-dump](#dbload-db-dump)
+
+Requires [python-decouple](https://github.com/HBNetwork/python-decouple)
+
+Installed with `[env_settings]`
+
+Settings:
+
+* `db_config_mapping` Mapping of db config (Default: `{ "dbname": "rds_db_name", "host": "rds_db_host", "port": "rds_db_port", "username": "rds_db_user", "password": "rds_db_password"}`)
+
+#### alembic.backup-local-db
+
+Back up local db.
+
+Uses [backup_local_db](#dbbackup-local-db)
+
+Requires [python-decouple](https://github.com/HBNetwork/python-decouple)
+
+Installed with `[env_settings]`
+
+Settings:
+
+* `db_config_mapping` Mapping of db config (Default: `{ "dbname": "rds_db_name", "host": "rds_db_host", "port": "rds_db_port", "username": "rds_db_user", "password": "rds_db_password"}`)
+
+#### alembic.backup-remote-db
+
+Make dump of remote db and download it.
+
+Uses [create_dump](#db-k8screate-dump) and [get-dump](#db-k8sget-dump)
+
+Requires [python-decouple](https://github.com/HBNetwork/python-decouple)
+
+Installed with `[env_settings]`
+
+Settings:
+
+* `db_config_mapping` Mapping of db config (Default: `{ "dbname": "rds_db_name", "host": "rds_db_host", "port": "rds_db_port", "username": "rds_db_user", "password": "rds_db_password"}`)
+
+#### alembic.load-remote-db
+
+Make dump of remote db and download it and apply to local db.
+
+Uses [create_dump](#db-k8screate-dump) and [get-dump](#db-k8sget-dump) and
+[load-db-dump](#alembicload-db-dump)
+
+Requires [python-decouple](https://github.com/HBNetwork/python-decouple)
+
+Installed with `[env_settings]`
+
+Settings:
+
+* `db_config_mapping` Mapping of db config (Default: `{ "dbname": "rds_db_name", "host": "rds_db_host", "port": "rds_db_port", "username": "rds_db_user", "password": "rds_db_password"}`)
+
 ### celery
 
 #### celery.run
@@ -544,9 +607,21 @@ Start celery worker.
 
 Settings:
 
-* `local_cmd` command for celery (Default: `celery --app config.celery:app worker --beat --scheduler=django --loglevel=info`)
+* `app` path to app  (Default: `config.celery.app`)
+* `scheduler` scheduler (Default: `django`)
+* `loglevel` log level for celery (Default: `info`)
+* `extra_params` extra params for worker (Default: `("--beat",)`)
+* `local_cmd` command for celery (Default: `celery --app {app} worker --scheduler={scheduler} --loglevel={info} {extra_params}`)
 * `service_name` name of celery service (Default: `celery`)
 
+#### celery.send-task
+
+Send task to celery worker.
+
+Settings:
+
+* `app` path to app  (Default: `config.celery.app`)
+
 ### open-api
 
 #### open-api.validate-swagger

saritasa_invocations/__init__.py

@@ -17,6 +17,7 @@
     pre_commit,
     pytest,
     python,
+    secrets,
     system,
 )
 from saritasa_invocations._config import (

saritasa_invocations/_config.py

@@ -120,10 +120,14 @@ class DjangoSettings:
 class CelerySettings:
     """Settings for celery module."""
 
+    app: str = "config.celery.app"
+    scheduler: str = "django"
     service_name: str = "celery"
+    loglevel: str = "info"
+    extra_params: tuple[str] = ("--beat",)
     local_cmd: str = (
-        "celery --app config.celery:app "
-        "worker --beat --scheduler=django --loglevel=info"
+        "celery --app {app} "
+        "worker --scheduler={scheduler} --loglevel={info} {extra_params}"
     )
 
 
@@ -150,6 +154,15 @@ class AlembicSettings:
         "# ### commands auto generated by Alembic - please adjust! ###",
         "# ### end Alembic commands ###",
     )
+    db_config_mapping: dict[str, str] = dataclasses.field(
+        default_factory=lambda: {
+            "dbname": "rds_db_name",
+            "host": "rds_db_host",
+            "port": "rds_db_port",
+            "username": "rds_db_user",
+            "password": "rds_db_password",
+        },
+    )
 
 
 @dataclasses.dataclass
@@ -249,6 +262,8 @@ class K8SSettings(metaclass=K8SSettingsMeta):
     default_entry: str | None = None
     python_shell: str | None = None
     health_check: str | None = None
+    secret_file_path_in_pod: str | None = None
+    temp_secret_file_path: str | None = None
     env_color: str | None = None
 
 
@@ -270,6 +285,8 @@ class K8SDefaultSettings:
     default_entry: str = "cnb/lifecycle/launcher bash"
     python_shell: str = "shell_plus"
     health_check: str = "health_check"
+    secret_file_path_in_pod: str | None = None
+    temp_secret_file_path: str = ".env.to_delete"
     env_color: str = "cyan"
 
 
@@ -290,6 +307,8 @@ class K8SGeneratedSettings:
     default_entry: str
     python_shell: str
     health_check: str
+    secret_file_path_in_pod: str
+    temp_secret_file_path: str
     env_color: str
 
     @classmethod

saritasa_invocations/alembic.py

@@ -2,7 +2,7 @@
 
 import invoke
 
-from . import _config, docker, printing, python
+from . import _config, db, db_k8s, docker, k8s, printing, python
 
 
 def wait_for_database(context: invoke.Context) -> None:
@@ -170,3 +170,100 @@ def _get_migration_files_paths(
         for path in pathlib.Path(migrations_folder).glob("*.py")
         if path.name not in ("__init__.py",)
     )
+
+
+@invoke.task
+def load_db_dump(
+    context: invoke.Context,
+    file: str = "",
+    env_file_path: str = "",
+    reset_db: bool = True,
+) -> None:
+    """Reset db and load db dump."""
+    if reset_db:
+        downgrade(context)
+    db.load_db_dump(
+        context,
+        file=file,
+        **_load_local_env_db_settings(context, file=env_file_path),
+    )
+
+
+@invoke.task
+def backup_local_db(
+    context: invoke.Context,
+    file: str = "",
+) -> None:
+    """Back up local db."""
+    db.backup_local_db(
+        context,
+        file=file,
+        **_load_local_env_db_settings(context),
+    )
+
+
+@invoke.task
+def backup_remote_db(
+    context: invoke.Context,
+    file: str = "",
+) -> str:
+    """Make dump of remote db and download it."""
+    settings = _load_remote_env_db_settings(context)
+    db_k8s.create_dump(context, file=file, **settings)
+    return db_k8s.get_dump(context, file=file)
+
+
+@invoke.task
+def load_remote_db(
+    context: invoke.Context,
+    file: str = "",
+) -> None:
+    """Make dump of remote db, download it and apply it."""
+    file = backup_remote_db(context, file=file)
+    load_db_dump(context, file=file)
+
+
+def _load_local_env_db_settings(
+    context: invoke.Context,
+    file: str = "",
+) -> dict[str, str]:
+    """Load local db settings from .env file.
+
+    Requires python-decouple:
+        https://github.com/HBNetwork/python-decouple
+
+    """
+    # decouple could not be installed during project init
+    # so we import decouple this way because it may not be installed
+    # at the project initialization stage
+    import decouple
+
+    secrets = decouple.Config(decouple.RepositoryEnv(file or ".env"))
+    config = _config.Config.from_context(context)
+    return {
+        arg: str(secrets(env_var))
+        for arg, env_var in config.alembic.db_config_mapping.items()
+    }
+
+
+def _load_remote_env_db_settings(
+    context: invoke.Context,
+) -> dict[str, str]:
+    """Load remote db settings from .env file.
+
+    Requires python-decouple:
+        https://github.com/HBNetwork/python-decouple
+
+    """
+    # decouple could not be installed during project init
+    # so we import decouple this way because it may not be installed
+    # at the project initialization stage
+    import decouple
+
+    with k8s.get_env_secrets(context) as file_path:
+        secrets = decouple.Config(decouple.RepositoryEnv(file_path))
+        config = _config.Config.from_context(context)
+        return {
+            arg: str(secrets(env_var))
+            for arg, env_var in config.alembic.db_config_mapping.items()
+        }

saritasa_invocations/celery.py

@@ -9,13 +9,30 @@ def run(
     detach: bool = True,
 ) -> None:
     """Start celery worker."""
-    config = _config.Config.from_context(context)
+    config = _config.Config.from_context(context).celery
     match python.get_python_env():
         case python.PythonEnv.LOCAL:
-            context.run(config.celery.local_cmd)
+            context.run(
+                config.local_cmd.format(
+                    app=config.app,
+                    scheduler=config.scheduler,
+                    loglevel=config.loglevel,
+                    extra_params=" ".join(config.loglevel),
+                ),
+            )
         case python.PythonEnv.DOCKER:
             docker.up_containers(
                 context,
-                (config.celery.service_name,),
+                (config.service_name,),
                 detach=detach,
             )
+
+
+@invoke.task
+def send_task(
+    context: invoke.Context,
+    task: str,
+) -> None:
+    """Send task to celery worker."""
+    config = _config.Config.from_context(context).celery
+    python.run(context, f"-m celery --app {config.app} call {task}")

saritasa_invocations/git.py

@@ -1,3 +1,5 @@
+import pathlib
+
 import invoke
 
 from . import _config, pre_commit, printing
@@ -29,3 +31,20 @@ def set_git_setting(
 ) -> None:
     """Set git setting in config."""
     context.run(f"git config --local --add {setting} {value}")
+
+
+@invoke.task
+def clone_repo(
+    context: invoke.Context,
+    repo_link: str,
+    repo_path: str | pathlib.Path,
+) -> None:
+    """Clone repo for work to folder."""
+    if not pathlib.Path(repo_path).exists():
+        printing.print_success(f"Cloning {repo_link} repository...")
+        context.run(f"git clone {repo_link} {repo_path}")
+        printing.print_success(f"Successfully cloned to '{repo_path}'!")
+    else:
+        printing.print_success(f"Pulling changes for {repo_link}...")
+        with context.cd(repo_path):
+            context.run("git pull")