From a67b192837214f334be0ec5bc30f66142a016767 Mon Sep 17 00:00:00 2001 From: Stanislav Khlud Date: Fri, 27 Oct 2023 17:26:53 +0700 Subject: [PATCH] Add improvements after working with FastApi projects --- CHANGELOG.md | 3 + README.md | 85 +++++++++++++++++++++++++-- saritasa_invocations/__init__.py | 1 + saritasa_invocations/_config.py | 23 +++++++- saritasa_invocations/alembic.py | 99 +++++++++++++++++++++++++++++++- saritasa_invocations/celery.py | 23 +++++++- saritasa_invocations/git.py | 19 ++++++ saritasa_invocations/k8s.py | 40 +++++++++++++ saritasa_invocations/secrets.py | 49 ++++++++++++++++ 9 files changed, 331 insertions(+), 11 deletions(-) create mode 100644 saritasa_invocations/secrets.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 983812a..46cb366 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ We follow [Semantic Versions](https://semver.org/). - Add `django.startapp` invocation. - Confirm support for python 3.12. +- Add `secrets` invocations +- Extend alembic invocations to be able to make db dumps +- Add invocation for `celery` to run task ## 0.8.3 diff --git a/README.md b/README.md index ff9e7ab..4cc08ae 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,13 @@ Collection of [invoke](https://www.pyinvoke.org/) commands used by Saritasa * [alembic.downgrade](#alembicdowngrade) * [alembic.check-for-migrations](#alembiccheck-for-migrations) * [alembic.check-for-adjust-messages](#alembiccheck-for-adjust-messages) + * [alembic.load-db-dump](#alembicload-db-dump) + * [alembic.backup-local-db](#alembicbackup-local-db) + * [alembic.backup-remote-db](#alembicbackup-remote-db) + * [alembic.load-remote-db](#alembicload-remote-db) * [celery](#celery) * [celery.run](#celeryrun) + * [celery.send-task](#celerysend-task) * [open-api](#open-api) * [open-api.validate-swagger](#open-apivalidate-swagger) * [db](#db) @@ -374,7 +379,7 @@ Requires [django-extensions](https://django-extensions.readthedocs.io/en/latest/ Settings: -* `django_settings_path` default django settings (Default: `config.settings.local`) +* `settings_path` default django settings (Default: `config.settings.local`) #### django.createsuperuser @@ -448,7 +453,7 @@ Uses [backup_local_db](#dbbackup-local-db) Settings: -* `django_settings_path` default django settings (Default: `config.settings.local`) +* `settings_path` default django settings (Default: `config.settings.local`) #### django.backup-remote-db @@ -458,7 +463,8 @@ Uses [create_dump](#db-k8screate-dump) and [get-dump](#db-k8sget-dump) Settings: -* `django_settings_path` default django settings (Default: `config.settings.local`) +* `settings_path` default django settings (Default: `config.settings.local`) +* `remote_db_config_mapping` Mapping of db config (Default: `{"dbname": "RDS_DB_NAME", "host": "RDS_DB_HOST", "port": "RDS_DB_PORT", "username": "RDS_DB_USER", "password": "RDS_DB_PASSWORD"}`) #### django.load-remote-db @@ -469,7 +475,7 @@ Uses [create_dump](#db-k8screate-dump) and [get-dump](#db-k8sget-dump) and Settings: -* `django_settings_path` default django settings (Default: `config.settings.local`) +* `settings_path` default django settings (Default: `config.settings.local`) #### django.startapp @@ -536,6 +542,63 @@ Settings: * `migrations_folder` migrations files location (Default: `db/migrations/versions`) * `adjust_messages` list of alembic adjust messages (Default: `# ### commands auto generated by Alembic - please adjust! ###`, `# ### end Alembic commands ###`) +#### alembic.load-db-dump + +Reset db and load db dump. + +Uses [downgrade](#alembicdowngrade) and [load-db-dump](#dbload-db-dump) + +Requires [python-decouple](https://github.com/HBNetwork/python-decouple) + +Installed with `[env_settings]` + +Settings: + +* `db_config_mapping` Mapping of db config (Default: `{ "dbname": "rds_db_name", "host": "rds_db_host", "port": "rds_db_port", "username": "rds_db_user", "password": "rds_db_password"}`) + +#### alembic.backup-local-db + +Back up local db. + +Uses [backup_local_db](#dbbackup-local-db) + +Requires [python-decouple](https://github.com/HBNetwork/python-decouple) + +Installed with `[env_settings]` + +Settings: + +* `db_config_mapping` Mapping of db config (Default: `{ "dbname": "rds_db_name", "host": "rds_db_host", "port": "rds_db_port", "username": "rds_db_user", "password": "rds_db_password"}`) + +#### alembic.backup-remote-db + +Make dump of remote db and download it. + +Uses [create_dump](#db-k8screate-dump) and [get-dump](#db-k8sget-dump) + +Requires [python-decouple](https://github.com/HBNetwork/python-decouple) + +Installed with `[env_settings]` + +Settings: + +* `db_config_mapping` Mapping of db config (Default: `{ "dbname": "rds_db_name", "host": "rds_db_host", "port": "rds_db_port", "username": "rds_db_user", "password": "rds_db_password"}`) + +#### alembic.load-remote-db + +Make dump of remote db and download it and apply to local db. + +Uses [create_dump](#db-k8screate-dump) and [get-dump](#db-k8sget-dump) and +[load-db-dump](#alembicload-db-dump) + +Requires [python-decouple](https://github.com/HBNetwork/python-decouple) + +Installed with `[env_settings]` + +Settings: + +* `db_config_mapping` Mapping of db config (Default: `{ "dbname": "rds_db_name", "host": "rds_db_host", "port": "rds_db_port", "username": "rds_db_user", "password": "rds_db_password"}`) + ### celery #### celery.run @@ -544,9 +607,21 @@ Start celery worker. Settings: -* `local_cmd` command for celery (Default: `celery --app config.celery:app worker --beat --scheduler=django --loglevel=info`) +* `app` path to app (Default: `config.celery.app`) +* `scheduler` scheduler (Default: `django`) +* `loglevel` log level for celery (Default: `info`) +* `extra_params` extra params for worker (Default: `("--beat",)`) +* `local_cmd` command for celery (Default: `celery --app {app} worker --scheduler={scheduler} --loglevel={info} {extra_params}`) * `service_name` name of celery service (Default: `celery`) +#### celery.send-task + +Send task to celery worker. + +Settings: + +* `app` path to app (Default: `config.celery.app`) + ### open-api #### open-api.validate-swagger diff --git a/saritasa_invocations/__init__.py b/saritasa_invocations/__init__.py index 8229d59..ffaa679 100644 --- a/saritasa_invocations/__init__.py +++ b/saritasa_invocations/__init__.py @@ -17,6 +17,7 @@ pre_commit, pytest, python, + secrets, system, ) from saritasa_invocations._config import ( diff --git a/saritasa_invocations/_config.py b/saritasa_invocations/_config.py index ec15d1b..d5764bd 100644 --- a/saritasa_invocations/_config.py +++ b/saritasa_invocations/_config.py @@ -120,10 +120,14 @@ class DjangoSettings: class CelerySettings: """Settings for celery module.""" + app: str = "config.celery.app" + scheduler: str = "django" service_name: str = "celery" + loglevel: str = "info" + extra_params: tuple[str] = ("--beat",) local_cmd: str = ( - "celery --app config.celery:app " - "worker --beat --scheduler=django --loglevel=info" + "celery --app {app} " + "worker --scheduler={scheduler} --loglevel={info} {extra_params}" ) @@ -150,6 +154,15 @@ class AlembicSettings: "# ### commands auto generated by Alembic - please adjust! ###", "# ### end Alembic commands ###", ) + db_config_mapping: dict[str, str] = dataclasses.field( + default_factory=lambda: { + "dbname": "rds_db_name", + "host": "rds_db_host", + "port": "rds_db_port", + "username": "rds_db_user", + "password": "rds_db_password", + }, + ) @dataclasses.dataclass @@ -249,6 +262,8 @@ class K8SSettings(metaclass=K8SSettingsMeta): default_entry: str | None = None python_shell: str | None = None health_check: str | None = None + secret_file_path_in_pod: str | None = None + temp_secret_file_path: str | None = None env_color: str | None = None @@ -270,6 +285,8 @@ class K8SDefaultSettings: default_entry: str = "cnb/lifecycle/launcher bash" python_shell: str = "shell_plus" health_check: str = "health_check" + secret_file_path_in_pod: str | None = None + temp_secret_file_path: str = ".env.to_delete" env_color: str = "cyan" @@ -290,6 +307,8 @@ class K8SGeneratedSettings: default_entry: str python_shell: str health_check: str + secret_file_path_in_pod: str + temp_secret_file_path: str env_color: str @classmethod diff --git a/saritasa_invocations/alembic.py b/saritasa_invocations/alembic.py index b294df7..f938d85 100644 --- a/saritasa_invocations/alembic.py +++ b/saritasa_invocations/alembic.py @@ -2,7 +2,7 @@ import invoke -from . import _config, docker, printing, python +from . import _config, db, db_k8s, docker, k8s, printing, python def wait_for_database(context: invoke.Context) -> None: @@ -170,3 +170,100 @@ def _get_migration_files_paths( for path in pathlib.Path(migrations_folder).glob("*.py") if path.name not in ("__init__.py",) ) + + +@invoke.task +def load_db_dump( + context: invoke.Context, + file: str = "", + env_file_path: str = "", + reset_db: bool = True, +) -> None: + """Reset db and load db dump.""" + if reset_db: + downgrade(context) + db.load_db_dump( + context, + file=file, + **_load_local_env_db_settings(context, file=env_file_path), + ) + + +@invoke.task +def backup_local_db( + context: invoke.Context, + file: str = "", +) -> None: + """Back up local db.""" + db.backup_local_db( + context, + file=file, + **_load_local_env_db_settings(context), + ) + + +@invoke.task +def backup_remote_db( + context: invoke.Context, + file: str = "", +) -> str: + """Make dump of remote db and download it.""" + settings = _load_remote_env_db_settings(context) + db_k8s.create_dump(context, file=file, **settings) + return db_k8s.get_dump(context, file=file) + + +@invoke.task +def load_remote_db( + context: invoke.Context, + file: str = "", +) -> None: + """Make dump of remote db, download it and apply it.""" + file = backup_remote_db(context, file=file) + load_db_dump(context, file=file) + + +def _load_local_env_db_settings( + context: invoke.Context, + file: str = "", +) -> dict[str, str]: + """Load local db settings from .env file. + + Requires python-decouple: + https://github.com/HBNetwork/python-decouple + + """ + # decouple could not be installed during project init + # so we import decouple this way because it may not be installed + # at the project initialization stage + import decouple + + secrets = decouple.Config(decouple.RepositoryEnv(file or ".env")) + config = _config.Config.from_context(context) + return { + arg: str(secrets(env_var)) + for arg, env_var in config.alembic.db_config_mapping.items() + } + + +def _load_remote_env_db_settings( + context: invoke.Context, +) -> dict[str, str]: + """Load remote db settings from .env file. + + Requires python-decouple: + https://github.com/HBNetwork/python-decouple + + """ + # decouple could not be installed during project init + # so we import decouple this way because it may not be installed + # at the project initialization stage + import decouple + + with k8s.get_env_secrets(context) as file_path: + secrets = decouple.Config(decouple.RepositoryEnv(file_path)) + config = _config.Config.from_context(context) + return { + arg: str(secrets(env_var)) + for arg, env_var in config.alembic.db_config_mapping.items() + } diff --git a/saritasa_invocations/celery.py b/saritasa_invocations/celery.py index 90664be..4912143 100644 --- a/saritasa_invocations/celery.py +++ b/saritasa_invocations/celery.py @@ -9,13 +9,30 @@ def run( detach: bool = True, ) -> None: """Start celery worker.""" - config = _config.Config.from_context(context) + config = _config.Config.from_context(context).celery match python.get_python_env(): case python.PythonEnv.LOCAL: - context.run(config.celery.local_cmd) + context.run( + config.local_cmd.format( + app=config.app, + scheduler=config.scheduler, + loglevel=config.loglevel, + extra_params=" ".join(config.loglevel), + ), + ) case python.PythonEnv.DOCKER: docker.up_containers( context, - (config.celery.service_name,), + (config.service_name,), detach=detach, ) + + +@invoke.task +def send_task( + context: invoke.Context, + task: str, +) -> None: + """Send task to celery worker.""" + config = _config.Config.from_context(context).celery + python.run(context, f"-m celery --app {config.app} call {task}") diff --git a/saritasa_invocations/git.py b/saritasa_invocations/git.py index 7d812fb..509178f 100644 --- a/saritasa_invocations/git.py +++ b/saritasa_invocations/git.py @@ -1,3 +1,5 @@ +import pathlib + import invoke from . import _config, pre_commit, printing @@ -29,3 +31,20 @@ def set_git_setting( ) -> None: """Set git setting in config.""" context.run(f"git config --local --add {setting} {value}") + + +@invoke.task +def clone_repo( + context: invoke.Context, + repo_link: str, + repo_path: str | pathlib.Path, +) -> None: + """Clone repo for work to folder.""" + if not pathlib.Path(repo_path).exists(): + printing.print_success(f"Cloning {repo_link} repository...") + context.run(f"git clone {repo_link} {repo_path}") + printing.print_success(f"Successfully cloned to '{repo_path}'!") + else: + printing.print_success(f"Pulling changes for {repo_link}...") + with context.cd(repo_path): + context.run("git pull") diff --git a/saritasa_invocations/k8s.py b/saritasa_invocations/k8s.py index 6c24bf6..cfd2b90 100644 --- a/saritasa_invocations/k8s.py +++ b/saritasa_invocations/k8s.py @@ -1,3 +1,8 @@ +import collections +import collections.abc +import contextlib +import typing + import invoke from . import _config, printing @@ -234,6 +239,41 @@ def download_file( ) +@contextlib.contextmanager +def download_file_and_remove_afterwards( + context: invoke.Context, + path_to_file_in_pod: str, + path_to_where_save_file: str, +) -> collections.abc.Generator[str, typing.Any, None]: + """Download file from k8s and delete it after work is done.""" + download_file( + context, + path_to_file_in_pod=path_to_file_in_pod, + path_to_where_save_file=path_to_where_save_file, + ) + try: + yield path_to_where_save_file + finally: + printing.print_success( + f"Deleting file({path_to_where_save_file}) after use", + ) + context.run(f"rm {path_to_where_save_file}") + + +@contextlib.contextmanager +def get_env_secrets( + context: invoke.Context, +) -> collections.abc.Generator[str, typing.Any, None]: + """Get secrets from k8s and save it to file.""" + config = get_current_env_config_from_context(context) + with download_file_and_remove_afterwards( + context, + path_to_file_in_pod=config.secret_file_path_in_pod, + path_to_where_save_file=config.temp_secret_file_path, + ) as file_path: + yield file_path + + def success( context: invoke.Context, message: str, diff --git a/saritasa_invocations/secrets.py b/saritasa_invocations/secrets.py new file mode 100644 index 0000000..2a0ade8 --- /dev/null +++ b/saritasa_invocations/secrets.py @@ -0,0 +1,49 @@ +import collections +import collections.abc +import re + +import invoke + +from . import k8s + + +@invoke.task +def setup_env_credentials( + context: invoke.Context, + credentials: collections.abc.Sequence[str], + env_file_path: str = "", +) -> None: + """Fill specified credentials. + + Requires python-decouple: + https://github.com/HBNetwork/python-decouple + + """ + # decouple could not be installed during project init + # so we import decouple this way because it may not be installed + # at the project initialization stage + + import decouple + + with k8s.get_env_secrets(context) as file_path: + secrets = decouple.Config(decouple.RepositoryEnv(file_path)) + cred_params = {cred: str(secrets(cred)) for cred in credentials} + env_secret_replacer( + env_file_path=env_file_path or ".env", + **cred_params, + ) + + +def env_secret_replacer(env_file_path: str, **credentials) -> None: + """Replace secret in env file.""" + with open(env_file_path, mode="r") as env_file: + env_data = env_file.read() + with open(env_file_path, mode="w") as env_file: + for cred, value in credentials.items(): + env_data = re.sub( + rf"{cred}=.*\n", + rf"{cred}={value}\n", + env_data, + count=1, + ) + env_file.write(env_data)