Merge pull request #9 from deeprave/docs-and-metadata-updates

Docs and metadata updates

.github/workflows/release-build-and-publish.yml

@@ -5,11 +5,11 @@ on:
     types: [created]
 
 jobs:
-  deploy:
+  checks:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.10", "3.11"]
         poetry-version: ["1.5.1"]
 
     steps:
@@ -31,6 +31,21 @@ jobs:
         run: |
           poetry run pytest
 
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup python and poetry
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: "3.10"
+          python-version: "1.5.1"
+
       - name: Set Poetry Config
         run: |
           poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}

README.md

@@ -5,8 +5,7 @@
 [![PyPI version](https://badge.fury.io/py/envex.svg)](https://badge.fury.io/py/envex)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
 
-Overview
---------
+## Overview
 
 This module provides a convenient interface for handling the environment, and therefore configuration of any application
 using 12factor.net principals removing many environment-specific variables and security sensitive information from
@@ -114,5 +113,32 @@ assert env.list('A_LIST_VALUE') == ['1', 'two', '3', 'four']
 assert env('A_LIST_VALUE', type=list) == ['1', 'two', '3', 'four']
 ```
 
-Environment variables are always stored as strings. This is enforced by the underlying os.environ, but also true of any
+Environment variables are always stored as strings.
+This is enforced by the underlying os.environ, but also true of any
 provided environment, which must use the `MutableMapping[str, str]` contract.
+
+## Vault
+
+In addition to handling of the os environment and .env files, Env supports fetching secrets from Hashicorp Vault using
+the kv.v2 engine.
+This provides a secure secrets store, without exposing them in plain text on the filesystem and in particular in
+published docker images.
+It also prevents storing secrets in the operating system’s environment, which can be inspected by external processes.
+Env can read secrets from the environment variables if you set Env(readenv=True).
+
+This document does not cover how to set up and configure a vault server, but you can find useful resources on the
+following websites:
+
+- [hashicorp.com](https://developer.hashicorp.com/vault) for the developer documentation and detailed information and
+  tutorials on setting up and hosting your own vault server, and
+- [vaultproject.io](https://www.vaultproject.io/) for information about HashiCorp's managed cloud offering.
+
+To access the vault server, you need a token with a role that has read permission for the path to the secrets.
+A read only profile is the recommended policy for tokens used at runtime by the application.
+
+This library provides a utility called `env2hvac` that can import (create or update) a typical .env file into vault. The
+utility uses a prefix that consists of an application name and an environment name, using the
+format <appname>/<envname>/<key> - for example, myapp/prod/DB_PASSWORD. The utility requires that the token has a role
+with create permission for the base secrets path on the vault server.
+The utility currently assumes that the kv secrets engine is mounted at secret/. The
+final path where the secrets are stored will be secret/data/<appname>/<envname>/<key>.

envex/dot_env.py

@@ -118,6 +118,8 @@ def process_line(_env_path: Path, _lineno: int, string: str):
                         )
         return _func, unquote(_key), unquote(_val)
 
+    files_not_found = []
+    files_found = False
     for env_path in _env_files(env_file, search_path, parents, errors):
         # insert PWD as container of env file
         env_path = Path(env_path).resolve()
@@ -133,9 +135,11 @@ def process_line(_env_path: Path, _lineno: int, string: str):
                         func, key, val = process_line(env_path, lineno, line)
                         if func is not None:
                             func(environ, key, val, overwrite=overwrite)
+                files_found = True
         except FileNotFoundError:
-            if errors:
-                raise
+            files_not_found.append(env_path)
+    if errors and not files_found and files_not_found:
+        raise FileNotFoundError(f"{env_file} as {[s.as_posix() for s in files_not_found]}")
     return environ
 
 

envex/env_wrapper.py

@@ -53,6 +53,7 @@ def __init__(
         @param verify: (optional) bool whether to verify server cert (default=True)
         @param cache_enabled: (optional) bool whether to cache secrets (default=True)
         @param base_path: (optional) str base path, or "environment" for secrets (default=None)
+        @param working_dirs: (optional) bool whether to include PWD/CWD (default=True)
         @param kwargs: (optional) environment variables to add/override
         """
         self._env = self.os_env() if environ is None else environ

scripts/env2hvac.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Import variables from a .env file to hashicorp vault.
+"""
+import logging
+import os
+
+import hvac
+
+import envex
+
+logging.captureWarnings(True)
+
+
+def main(
+    files: list[str],
+    url: str = None,
+    token: str = None,
+    cert: tuple = None,
+    verify: bool | str = True,
+    unseal: str = None,
+    namespace: str = None,
+    environ: str = None,
+):
+    client = hvac.Client(url=url, token=token, cert=cert, verify=verify)
+    try:
+        if unseal:
+            client.sys.submit_unseal_keys(keys=unseal.split(","))
+        if not client.is_authenticated():
+            # noinspection PyArgumentList
+            logging.fatal("Can't connect or authenticate with Vault", exitcode=1)
+            return
+    except hvac.v1.exceptions.VaultDown as e:
+        # sealed?
+        if client.seal_status["sealed"]:
+            # noinspection PyArgumentList
+            logging.fatal("Vault is currently sealed", exitcode=4)
+        # noinspection PyArgumentList
+        logging.fatal(f"Unknown exception connecting with Vault: {e}", exitcode=1)
+
+    def expand(p: str):
+        return os.path.expandvars(os.path.expanduser(p))
+
+    try:
+        path = f"{namespace}/{environ}" if environ else "{namespace}"
+        for filename in files:
+            filename = expand(filename)
+            try:
+                env = envex.Env(
+                    readenv=True,
+                    environ={},
+                    env_file=filename,
+                    update=False,
+                    errors=False,
+                    # pass these on in case we need them for completion
+                    url=url,
+                    token=token,
+                    cert=cert,
+                    verify=verify,
+                    base_path=path,
+                )
+                items = {k: v for k, v in env.items() if k not in ("CWD", "PWD")}
+                client.secrets.kv.v2.create_or_update_secret(
+                    path=path,
+                    secret=items,
+                    cas=None,
+                    mount_point="secret",
+                )
+                logging.info(f"Added or updated {len(items)} items from {filename} to '{path}'")
+                # comma_nl = ",\n    "
+                # print(f"[\n{comma_nl.join(items.keys())}\n]\n")
+            except IOError as e:
+                logging.error(f"{filename}: {e.__class__.__name__} - {e}")
+    finally:
+        # reseal the vault
+        if unseal:
+            client.sys.seal()
+
+
+if __name__ == "__main__":
+    import argparse
+
+    from scripts.lib.decr_action import Decrement
+    from scripts.lib.log import config as log_config
+    from scripts.lib.log import log_get_level, log_set_level
+
+    log_config()
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-n",
+        "--namespace",
+        type=str,
+        default="myapp",
+        help="Namespace or application name",
+    )
+    parser.add_argument(
+        "-e",
+        "--environ",
+        type=str,
+        default=None,
+        help="The code environment used to create or update variables",
+    )
+    parser.add_argument(
+        "-a",
+        "--address",
+        type=str,
+        default=None,
+        help="The address/url of the hashicorp vault server",
+    )
+    parser.add_argument(
+        "-t",
+        "--token",
+        type=str,
+        default=None,
+        help="The token used to authenticate to hashicorp vault",
+    )
+    parser.add_argument(
+        "-u",
+        "--unseal",
+        default="",
+        help="Unseal/reseal the vault with the provided comma-separated list of key",
+    )
+    parser.add_argument(
+        "-c",
+        "--cert",
+        type=str,
+        default=None,
+        help="Client cert (if any)",
+    )
+    parser.add_argument(
+        "-k",
+        "--key",
+        type=str,
+        default=None,
+        help="Client cert key (if any)",
+    )
+    parser.add_argument(
+        "-N",
+        "--noverify",
+        dest="verify",
+        action="store_false",
+        default=True,
+        help="Disable server certificate verification",
+    )
+    parser.add_argument(
+        "-C",
+        "--cacert",
+        dest="verify",
+        default=True,
+        help="Path to a custom CA certificate (do not use with -N)",
+    )
+    parser.add_argument(
+        "files",
+        nargs="+",
+        help="Filename(s) from which to read variables.",
+    )
+    default_level = log_get_level()
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=default_level,
+        dest="verbose",
+        help="Verbose output (specify multiple times for more verbosity)",
+    )
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        action=Decrement,
+        default=default_level,
+        dest="verbose",
+        help="Verbose output (specify multiple times for more verbosity)",
+    )
+    args = parser.parse_args()
+    certs = (args.cert, args.key) if args.cert and args.key else None
+
+    log_set_level(args.verbose)
+
+    main(
+        args.files,
+        url=args.address,
+        token=args.token,
+        cert=certs,
+        verify=args.verify,
+        unseal=args.unseal,
+        namespace=args.namespace,
+        environ=args.environ,
+    )

scripts/lib/decr_action.py

@@ -0,0 +1,20 @@
+import argparse
+from types import NoneType
+
+__all__ = ("Decrement",)
+
+
+# noinspection PyShadowingBuiltins
+class Decrement(argparse.Action):
+    def __init__(
+        self, option_strings, dest: str | NoneType, default: int = None, required: bool = False, help: str = None
+    ):
+        super().__init__(option_strings, dest, nargs=0, default=default, required=required, help=help)
+
+    # noinspection PyShadowingNames
+    def __call__(self, parser, namespace, values, option_string=None):
+        current_value = getattr(namespace, self.dest, self.default or 0)
+        try:
+            setattr(namespace, self.dest, current_value - 1)
+        except TypeError:
+            pass

scripts/lib/log.py

@@ -0,0 +1,65 @@
+import logging
+from types import NoneType
+
+__all__ = ("config", "log_set_level", "log_get_level")
+
+from typing import NoReturn
+
+
+def fatal(msg, *args, **kwargs) -> NoReturn:
+    """
+    Don't use this function, use critical() instead.
+    """
+    exitcode = kwargs.pop("exitcode", 1)
+    logging.critical(msg, *args, **kwargs)
+    exit(exitcode)
+
+
+logging.fatal = fatal
+
+
+def config(**kwargs):
+    kwargs.setdefault("level", logging.INFO),
+    kwargs.setdefault("format", "%(asctime)s %(message)s (%(levelname)s)"),
+    logging.basicConfig(**kwargs)
+
+
+__levelIndex = [
+    logging.FATAL,
+    logging.ERROR,
+    logging.WARNING,
+    logging.INFO,
+    logging.DEBUG,
+]
+
+
+__default_level = logging.WARNING
+__current_level = __default_level
+
+
+def log_set_level(level: int | NoneType = None):
+    """
+    Set the log level for the application.
+
+    :param level: The desired log level as an index.
+                  If not specified, the level is reset to the default.
+    :type level: int | NoneType
+    :return: The loggging level that was set.
+    :rtype: int
+    """
+    if level is None:
+        level = __default_level
+    else:
+        if level < 0:
+            level = 0
+        elif level >= len(__levelIndex):
+            level = len(__levelIndex) - 1
+        level = __levelIndex[level]
+    logging.getLogger().setLevel(level)
+    return level
+
+
+def log_get_level(level: int | NoneType = None):
+    if level is None:
+        level = __current_level
+    return __levelIndex.index(level)