fix: 47 refactor (#51)

.gitignore

@@ -164,6 +164,7 @@ junk.txt
 data/**
 data_prep/junk.py
 data_prep/docker_data/**
-data_prep/poetry.toml
-
+data_prep/docker_venv/**
+data_prep/data/**
+data_prep/.venv/**
 sync/src/junk.py

.vscode/launch.json

@@ -67,6 +67,17 @@
                 "PROD"
             ]
         },
+        {
+            "name": "oracle-extract-test",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/data_prep/main_ora_extract.py",
+            "console": "integratedTerminal",
+            "envFile": "${workspaceFolder}/.env",
+            "args": [
+                "TEST"
+            ]
+        },
         {
             "name": "oracle-ingest-test",
             "type": "python",

Dockerfile.ora_ingest

@@ -0,0 +1,62 @@
+# ----------------- STAGE 1 -----------------
+    FROM python:3.13-slim-bookworm as base
+    # using slim for development as the wheels are not always compat with
+    # alpine, and builds can be slow as a result
+    # FROM python:3.12-slim as base
+
+    ARG DEV=false
+    ENV VIRTUAL_ENV=/app/docker_venv \
+        PATH="/app/docker_venv/bin:$PATH"
+
+    # ----------------- STAGE 2 -----------------
+    FROM base as builder
+    WORKDIR /app
+
+    # Mostly config for poetry
+    #   - disable prompting
+    #   - disable virtualenv creation
+    #   - specify the path to the venv
+    ENV POETRY_NO_INTERACTION=1 \
+        POETRY_VIRTUALENVS_IN_PROJECT=0 \
+        POETRY_VIRTUALENVS_CREATE=0 \
+        POETRY_CACHE_DIR=/tmp/poetry_cache \
+        VIRTUAL_ENV=/app/docker_venv \
+        PATH="/app/docker_venv/bin:$PATH"
+
+    # install build tools, and dev libs required to compile postgres stuff
+    #   ideally these are not going to be used as for most of the libs
+    #   there are wheels available, however they are available in case we
+    #   need to compile something
+    RUN apt update && \
+        apt install -y --no-install-recommends gcc libpq-dev python3-dev && \
+        useradd -M nonroot
+
+    # Install Poetry to global python env
+    RUN pip install poetry==1.8.4
+
+    # Grab the app dependency declarations
+    COPY ./data_prep/pyproject.toml ./data_prep/poetry.lock /app/
+
+    # create the venv in $VIRTUAL_ENV so we can control where it is being created
+    #  unfortunately this is the only way that we can control the venv that
+    #  poetry will use, is to create it ourselves
+    RUN python -m venv $VIRTUAL_ENV; . $VIRTUAL_ENV/bin/activate
+
+    # have created and activated the venv, now install the dependencies into it
+    RUN poetry install --without dev --no-root -v && rm -rf $POETRY_CACHE_DIR
+
+    # ----------------- STAGE 3 -----------------
+    FROM base as runtime
+    WORKDIR /app
+
+    # install the postgres client, required by psycopg2
+    RUN apt update && apt install -y --no-install-recommends postgresql-client
+
+    # only copy over the venv from the builder stage
+    COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+    # RUN apk add bash
+    COPY data_prep/db_env_utils/  /app/db_env_utils/
+    COPY docker-compose.yml /app/
+
+    CMD ["python", "db_env_utils/main_ora_ingest.py", "TEST"]

Dockerfile.pg_ingest

@@ -0,0 +1,66 @@
+# ----------------- STAGE 1 -----------------
+# NEEDS the docker-compose file to retrieve the local database connection
+# information. To be able to include the docker-compose in the build this
+# dockerfile needs to be located in the parent directory of the docker-compose
+FROM python:3.13-slim-bookworm as base
+# using slim for development as the wheels are not always compat with
+# alpine, and builds can be slow as a result
+# FROM python:3.12-slim as base
+
+ARG DEV=false
+ENV VIRTUAL_ENV=/app/docker_venv \
+    PATH="/app/docker_venv/bin:$PATH"
+
+# ----------------- STAGE 2 -----------------
+FROM base as builder
+WORKDIR /app
+
+# Mostly config for poetry
+#   - disable prompting
+#   - disable virtualenv creation
+#   - specify the path to the venv
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=0 \
+    POETRY_VIRTUALENVS_CREATE=0 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache \
+    VIRTUAL_ENV=/app/docker_venv \
+    PATH="/app/docker_venv/bin:$PATH"
+
+# install build tools, and dev libs required to compile postgres stuff
+#   ideally these are not going to be used as for most of the libs
+#   there are wheels available, however they are available in case we
+#   need to compile something
+RUN apt update && \
+    apt install -y --no-install-recommends gcc libpq-dev python3-dev && \
+    useradd -M nonroot
+
+# Install Poetry to global python env
+RUN pip install poetry==1.8.4
+
+# Grab the app dependency declarations
+COPY ./data_prep/pyproject.toml data_prep/poetry.lock /app/
+
+# create the venv in $VIRTUAL_ENV so we can control where it is being created
+#  unfortunately this is the only way that we can control the venv that
+#  poetry will use, is to create it ourselves
+RUN python -m venv $VIRTUAL_ENV; . $VIRTUAL_ENV/bin/activate
+
+# have created and activated the venv, now install the dependencies into it
+RUN poetry install --without dev --no-root -v && rm -rf $POETRY_CACHE_DIR
+
+# ----------------- STAGE 3 -----------------
+FROM base as runtime
+WORKDIR /app
+
+
+# install the postgres client, required by psycopg2
+RUN apt update && apt install -y --no-install-recommends postgresql-client
+
+# only copy over the venv from the builder stage
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+# RUN apk add bash
+COPY data_prep/db_env_utils/  /app/db_env_utils/
+COPY docker-compose.yml /app/
+
+CMD ["python", "db_env_utils/main_spar_ingest.py", "TEST"]

data_prep/docker_parser.py → data_prep/db_env_utils/docker_parser.py

@@ -40,14 +40,81 @@ def __init__(self, compose_file_path: str | None = None) -> None:
             that the script was executed from.
         :type compose_file_path: str, path
         """
-        self.compose_file_path = compose_file_path
 
-        if self.compose_file_path is None:
-            self.compose_file_path = "./docker-compose.yml"
+        self.compose_file_path = self.get_composefile_path(compose_file_path)
 
         with pathlib.Path(self.compose_file_path).open("r") as fh:
             self.docker_comp = yaml.safe_load(fh)
 
+    def get_composefile_path(self, comp_file_path: str) -> str:
+        """
+        Return the path to the docker-compose file.
+
+        Expects the docker compose file to exist in either the current working
+        or back a single directory relative to this script.
+
+        :return: the path to the docker-compose file
+        :rtype: str
+        """
+        if not comp_file_path:
+            docker_comp_file_path = pathlib.Path(
+                "./docker-compose.yml"
+            ).resolve()
+        else:
+            docker_comp_file_path = pathlib.Path(comp_file_path)
+
+        if not docker_comp_file_path.exists():
+            LOGGER.debug(
+                "1. docker compose file does not exist: %s",
+                docker_comp_file_path,
+            )
+
+            # try to find the docker-compose file one file back from this files
+            # path
+            docker_comp_file_path = (
+                pathlib.Path(__file__)
+                .parent.joinpath("..", "docker-compose.yml")
+                .resolve()
+            )
+
+            if not docker_comp_file_path.exists():
+                LOGGER.debug(
+                    "2. docker compose file does not exist: %s",
+                    docker_comp_file_path,
+                )
+                # try going back one more directory
+                docker_comp_file_path = (
+                    pathlib.Path(__file__)
+                    .parent.joinpath("..", "..", "docker-compose.yml")
+                    .resolve()
+                )
+                if not docker_comp_file_path.exists():
+                    LOGGER.debug(
+                        "3. docker compose file does not exist: %s",
+                        docker_comp_file_path,
+                    )
+
+                    # try going back one more directory
+                    docker_comp_file_path = (
+                        pathlib.Path(__file__)
+                        .parent.joinpath(
+                            "..",
+                            "..",
+                            "..",
+                            "docker-compose.yml",
+                        )
+                        .resolve()
+                    )
+
+        if not docker_comp_file_path.exists():
+            raise FileNotFoundError(
+                "Could not find the docker-compose file in the current or parent directory, %s",
+                docker_comp_file_path,
+            )
+
+        LOGGER.info(f"Using docker-compose file: {docker_comp_file_path}")
+        return docker_comp_file_path
+
     def get_spar_conn_params(self) -> env_config.ConnectionParameters:
         """
         Return postgres connection parameters.

data_prep/main_common.py → data_prep/db_env_utils/main_common.py

@@ -36,8 +36,8 @@ def __init__(self, env_str: str, db: constants.DBType) -> None:
         self.curdir = pathlib.Path(__file__).parents[0]
         self.datadir = pathlib.Path(
             constants.DATA_DIR,
-            self.env_str,
-            db.name.lower(),
+            self.env_str.upper(),
+            db.name.upper(),
         )
         self.db_type = db
         self.kube_client = None

data_prep/oradb_demo/README.md

@@ -0,0 +1,2 @@
+Demo code to help get started with working with oracle in a container.
+