Merge pull request #2 from biocatchltd/develop

first version

.github/workflows/pull-requests.yml

@@ -0,0 +1,49 @@
+name: Blackbox Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  backend-lint-and-tests:
+    defaults:
+      run:
+        working-directory: backend
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Setup cache
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pypoetry
+        key: ${{ runner.os }}-pip
+    - name: Install poetry
+      run: pip install poetry
+    - name: Install dependencies
+      run: poetry install
+    - name: setup-docker
+      uses: docker-practice/[email protected]
+    - name: Linting backend
+      run: poetry run sh scripts/lint.sh
+    - name: Run unittests
+      run: poetry run sh scripts/unittest.sh
+    - name: Run blackbox tests
+      run: poetry run sh scripts/blackbox_test.sh
+  frontend-lint:
+    defaults:
+      run:
+        working-directory: frontend
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: '12'
+    - run: npm install
+    - run: npm run lint

.github/workflows/release-final.yml

@@ -0,0 +1,37 @@
+name: Release
+on:
+  push:
+    tags:
+      - '*.*.*'
+
+
+jobs:
+  verify_version:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Version verification
+      working-directory: backend
+      run: |
+        python -m pip install --upgrade pip
+        pip install --pre poetry
+        [ $(cut -d' ' -f2 <<< $(poetry version)) == ${GITHUB_REF#refs/tags/} ]
+
+  build_release:
+    needs: [verify_version]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build and push Docker images
+      uses: docker/[email protected]
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        repository: biocatchltd/hekshermgmt
+        tag_with_ref: true

.gitignore

@@ -1,2 +1,7 @@
 node_modules
 .DS_Store
+.vscode
+backend/.env
+*__pycache__*
+backend/.coverage
+backend/coverage.xml

Dockerfile

@@ -0,0 +1,125 @@
+FROM node:lts as build-stage
+WORKDIR /app
+COPY frontend/package*.json ./
+RUN npm install
+COPY frontend .
+RUN npm run build
+
+# production stage
+FROM python:3.8 as production-stage
+# Build nginx taken from https://github.com/nginxinc/docker-nginx/blob/master/mainline/debian/Dockerfile
+ENV NGINX_VERSION   1.19.6
+ENV NJS_VERSION     0.5.0
+ENV PKG_RELEASE     1~buster
+
+RUN set -x \
+# create nginx user/group first, to be consistent throughout docker variants
+    && addgroup --system --gid 102 nginx \
+    && adduser --system --disabled-login --ingroup nginx --no-create-home --home /nonexistent --gecos "nginx user" --shell /bin/false --uid 102 nginx \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates \
+    && \
+    NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
+    found=''; \
+    for server in \
+        ha.pool.sks-keyservers.net \
+        hkp://keyserver.ubuntu.com:80 \
+        hkp://p80.pool.sks-keyservers.net:80 \
+        pgp.mit.edu \
+    ; do \
+        echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
+        apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
+    done; \
+    test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
+    apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \
+    && dpkgArch="$(dpkg --print-architecture)" \
+    && nginxPackages=" \
+        nginx=${NGINX_VERSION}-${PKG_RELEASE} \
+    " \
+    && case "$dpkgArch" in \
+        amd64|i386|arm64) \
+# arches officialy built by upstream
+            echo "deb https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \
+            && apt-get update \
+            ;; \
+        *) \
+# we're on an architecture upstream doesn't officially build for
+# let's build binaries from the published source packages
+            echo "deb-src https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \
+            \
+# new directory for storing sources and .deb files
+            && tempDir="$(mktemp -d)" \
+            && chmod 777 "$tempDir" \
+# (777 to ensure APT's "_apt" user can access it too)
+            \
+# save list of currently-installed packages so build dependencies can be cleanly removed later
+            && savedAptMark="$(apt-mark showmanual)" \
+            \
+# build .deb files from upstream's source packages (which are verified by apt-get)
+            && apt-get update \
+            && apt-get build-dep -y $nginxPackages \
+            && ( \
+                cd "$tempDir" \
+                && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \
+                    apt-get source --compile $nginxPackages \
+            ) \
+# we don't remove APT lists here because they get re-downloaded and removed later
+            \
+# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies
+# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies)
+            && apt-mark showmanual | xargs apt-mark auto > /dev/null \
+            && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \
+            \
+# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be)
+            && ls -lAFh "$tempDir" \
+            && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \
+            && grep '^Package: ' "$tempDir/Packages" \
+            && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \
+# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes")
+#   Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
+#   ...
+#   E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages  Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
+            && apt-get -o Acquire::GzipIndexes=false update \
+            ;; \
+    esac \
+    \
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+                        $nginxPackages \
+                        gettext-base \
+                        curl \
+    && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \
+    \
+# if we have leftovers from building, let's purge them (including extra, unnecessary build deps)
+    && if [ -n "$tempDir" ]; then \
+        apt-get purge -y --auto-remove \
+        && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \
+    fi \
+# forward request and error logs to docker log collector
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log
+
+RUN pip install supervisor gunicorn
+COPY ./image/supervisord.ini /etc/supervisord.ini
+COPY ./image/nginx.conf /etc/nginx/conf.d/nginx.conf
+COPY ./image/gunicorn_conf.py /gunicorn_conf.py
+
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | POETRY_HOME=/opt/poetry python && \
+    cd /usr/local/bin && \
+    ln -s /opt/poetry/bin/poetry && \
+    poetry config virtualenvs.create false
+
+COPY ./backend /app
+RUN cd /app && poetry install --no-dev --no-root
+
+RUN cd /app \
+    && export APP_VERSION=$(poetry version | cut -d' ' -f2) \
+    && echo "__version__ = '$APP_VERSION'" > /app/hekshermgmt/_version.py
+
+
+COPY --from=build-stage /app/dist /usr/share/nginx/html
+
+ENV PYTHONOPTIMIZE=1
+ENV WEB_CONCURRENCY=1
+ENV PYTHONPATH="/app"
+EXPOSE 80
+CMD ["supervisord", "-c", "/etc/supervisord.ini"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Biocatch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,28 @@
+# HeksherMgmt
+HeksherMgmt is a complimentary service for managing your Heksher instance. It provides an UI for checking settings,
+adding and deleting rules, and possibly more later on.
+HeksherMgmt is split into frontend ([Vue.js](https://vuejs.org/) and [Vuetify](https://vuetifyjs.com/en/)) and backend ([FastAPI](https://fastapi.tiangolo.com/)).
+
+## How does it work
+HeksherMgmt has it's own backend as Heksher API should be used internally only, and we don't want to expose it's internal API.
+The internal API is less suited for UI also, so the backend serves as some sort of convinient authorization layer.
+HeksherMgmt is a standalone docker image, containing both the front and backend, as splitting those currently feels too synthetic.
+
+## Running it
+The backend requires an user header (currently supports only `x-forwarded-email`) to be passed from the reverse proxy.
+When running locally, the frontend automatically sends this header, which is of course unsecure in real life environment.
+
+## Deploying
+Our recommended deployment is using a sidecar http authentication solution such as [OAuth2-Proxy](http://oauth2-proxy.github.io/oauth2-proxy/).
+The sidecar handles authentication, and then passes the authenticated user as a header to Heksher's backend.
+
+## Environment Variables
+* `HEKSHERMGMT_HEKSHER_URL`: (required) URL to the Heksher service.
+* `HEKSHERMGMT_HEKSHER_HEADERS`: (optional) Headers to send to Heksher service (authorization, api keys, etc). Example - `apitoken:abcd authorization:abcd`
+* `SENTRY_DSN`: (optional) Send errors to the given Sentry DSN.
+* `HEKSHERMGMT_LOGSTASH_HOST`, `HEKSHERMGMT_LOGSTASH_PORT`, `HEKSHERMGMT_LOGSTASH_LEVEL`, `HEKSHERMGMT_LOGSTASH_TAGS`: Optional values
+  to allow sending logs to a logstash server.
+
+
+## License
+HeksherMgmt is registered under the MIT public license.

backend/hekshermgmt/_version.py

@@ -0,0 +1 @@
+__version__ = ""  # Auto generated by Dockerfile

backend/hekshermgmt/api/v1/__init__.py

@@ -0,0 +1,9 @@
+from fastapi import APIRouter
+
+from hekshermgmt.api.v1.rules import router as rules
+from hekshermgmt.api.v1.settings import router as settings
+from hekshermgmt.api.v1.utils import get_user_name
+
+router = APIRouter(prefix="/api/v1", dependencies=[get_user_name])
+router.include_router(settings)
+router.include_router(rules)

backend/hekshermgmt/api/v1/rules.py

@@ -0,0 +1,86 @@
+import datetime
+from logging import getLogger
+from typing import Any, Dict, Optional
+
+import httpx
+from fastapi import APIRouter
+from pydantic import BaseModel, Field, validator  # pytype: disable=import-error
+
+from hekshermgmt.api.v1.utils import application, httpx_error_to_response
+from hekshermgmt.app import HeksherManagement
+from hekshermgmt.context_vars import user
+
+router = APIRouter(prefix="/rule")
+
+logger = getLogger(__name__)
+
+
+class RuleAddOutput(BaseModel):
+    rule_id: int = Field(description="ID of the newly created rule.")
+
+
+class RuleAddInput(BaseModel):
+    setting: str = Field(description="the setting name the rule should apply to")
+    feature_values: Dict[str, str] = Field(
+        description="the exact-match conditions of the rule"
+    )
+    value: Any = Field(
+        description="the value of the setting in contexts that match the rule"
+    )
+    information: Optional[str] = Field(
+        description="information to store with the rule", max_length=100
+    )
+
+    @validator("feature_values")
+    @classmethod
+    def feature_values_not_empty(cls, v):
+        if not v:
+            raise ValueError("feature_values must not be empty")
+        return v
+
+
+@router.delete("/{rule_id}")
+async def delete_rule(rule_id: int, app: HeksherManagement = application):
+    """
+    Deletes a specific rule
+    """
+    try:
+        rule = await app.heksher_client.get_rule_data(rule_id)
+    except httpx.HTTPStatusError as error:
+        logger.warning("Error from Heksher API when deleting rule.", exc_info=error)
+        return httpx_error_to_response(error)
+    logger.info(
+        "Deleting rule.",
+        extra={
+            "rule_id": rule_id,
+            "setting_name": rule["setting"],
+            "value": rule["value"],
+        },
+    )
+    await app.heksher_client.delete_rule(rule_id)
+
+
+@router.post("", response_model=RuleAddOutput)
+async def add_rule(rule: RuleAddInput, app: HeksherManagement = application):
+    metadata = {
+        "added_by": user.get(),
+        "information": rule.information,
+        "date": datetime.datetime.now().isoformat(),
+    }
+    try:
+        rule_id = await app.heksher_client.add_rule(
+            rule.setting, rule.feature_values, rule.value, metadata
+        )
+    except httpx.HTTPStatusError as error:
+        logger.warning("Error from Heksher API when adding rule.", exc_info=error)
+        return httpx_error_to_response(error)
+    logger.info(
+        "Added rule.",
+        extra={
+            "rule_id": rule_id,
+            "rule_value": rule.value,
+            "feature_values": rule.feature_values,
+            "setting_name": rule.setting,
+        },
+    )
+    return RuleAddOutput(rule_id=rule_id)