From 3667bd18bde3b51b105a2ed9d421ae8333c7c326 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Wed, 18 Sep 2024 12:55:43 +0200 Subject: [PATCH 01/19] Add missing wget binary --- docker/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index da5c7383c..483f55282 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,6 +20,10 @@ COPY ./cedar-agent /tmp/cedar-agent WORKDIR /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release +# BUSYBOX IMAGE -------------------------------------- +# --------------------------------------------------- +FROM busybox:1.36.1-uclibc AS busybox-builder + # COMMON IMAGE -------------------------------------- # --------------------------------------------------- FROM python:3.10-slim-bookworm AS common @@ -33,6 +37,10 @@ COPY --from=build-stage /usr/local /usr/local RUN useradd -m -b / -s /bin/bash opal WORKDIR /opal +# copy wget binary +COPY --from=busybox-builder /bin/wget /bin/wget +RUN chmod +x /bin/wget + # copy wait-for script (create link at old path to maintain backward compatibility) COPY scripts/wait-for.sh . RUN chmod +x ./wait-for.sh From 000a901cccb504469204f193e9eeac08df0810d6 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Wed, 18 Sep 2024 13:09:25 +0200 Subject: [PATCH 02/19] Install wget package instead of copying static binary from busybox --- docker/Dockerfile | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 483f55282..58fb244f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,10 +20,6 @@ COPY ./cedar-agent /tmp/cedar-agent WORKDIR /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release -# BUSYBOX IMAGE -------------------------------------- -# --------------------------------------------------- -FROM busybox:1.36.1-uclibc AS busybox-builder - # COMMON IMAGE -------------------------------------- # --------------------------------------------------- FROM python:3.10-slim-bookworm AS common @@ -37,17 +33,13 @@ COPY --from=build-stage /usr/local /usr/local RUN useradd -m -b / -s /bin/bash opal WORKDIR /opal -# copy wget binary -COPY --from=busybox-builder /bin/wget /bin/wget -RUN chmod +x /bin/wget - # copy wait-for script (create link at old path to maintain backward compatibility) COPY scripts/wait-for.sh . RUN chmod +x ./wait-for.sh RUN ln -s /opal/wait-for.sh /usr/wait-for.sh -# netcat (nc) is used by the wait-for.sh script -RUN apt-get update && apt-get install -y netcat-traditional jq && apt-get clean +# netcat (nc) and wget are used by the wait-for.sh script +RUN apt-get update && apt-get install -y netcat-traditional jq wget && apt-get clean # copy startup script (create link at old path to maintain backward compatibility) COPY ./scripts/start.sh . From cecfafe57a4f3bfb0b6a1e156330c8c60b5a0d55 Mon Sep 17 00:00:00 2001 From: daveads Date: Thu, 19 Sep 2024 14:22:43 +0100 Subject: [PATCH 03/19] pulsar integration doc --- .../docs/tutorials/run_opal_with_pulsar.mdx | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 documentation/docs/tutorials/run_opal_with_pulsar.mdx diff --git a/documentation/docs/tutorials/run_opal_with_pulsar.mdx b/documentation/docs/tutorials/run_opal_with_pulsar.mdx new file mode 100644 index 000000000..8f069beb2 --- /dev/null +++ b/documentation/docs/tutorials/run_opal_with_pulsar.mdx @@ -0,0 +1,116 @@ +--- +sidebar_position: 12 +title: Run OPAL with Apache Pulsar +--- + +# Running OPAL-server with Apache Pulsar + +## Introduction + +OPAL-server supports multiple backbone pub/sub solutions for connecting distributed server instances. This guide explains how to set up and use Apache Pulsar as the backbone pub/sub (broadcast channel) for OPAL-server. + +## Apache Pulsar as the Backbone Pub/Sub + +### What is a backbone pub/sub? + +OPAL-server can scale out both in number of worker processes per server and across multiple servers. While OPAL provides a lightweight websocket pub/sub for OPAL-clients, multiple servers are linked together by a more robust messaging solution like Apache Pulsar, Kafka, Redis, or Postgres Listen/Notify. + +### Broadcaster Module + +Support for multiple backbone solutions is provided by the [Python Broadcaster package](https://pypi.org/project/broadcaster/). To use it with Apache Pulsar, install the `broadcaster[pulsar]` module: + +```bash +pip install broadcaster[pulsar] +``` + +## Setting Up OPAL-server with Apache Pulsar + +### Configuration + +To use Apache Pulsar as the backbone, set the `OPAL_BROADCAST_URI` environment variable: + +```bash +OPAL_BROADCAST_URI=pulsar://pulsar-host-name:6650 +``` + +The "pulsar://" prefix tells OPAL-server to use Apache Pulsar. + +### Pulsar Topic + +OPAL-server uses a single Pulsar topic named 'broadcast' for all communication. This topic is automatically created when the producer and consumer are initialized. + +## Docker Compose Example + +Here's an example `docker-compose.yml` configuration that includes Apache Pulsar: + +```yaml +version: '3' +services: + pulsar: + image: apachepulsar/pulsar:3.3.1 + command: bin/pulsar standalone + ports: + - 6650:6650 + - 8080:8080 + volumes: + - pulsardata:/pulsar/data + - pulsarconf:/pulsar/conf + + opal-server: + image: permitio/opal-server:latest + environment: + - OPAL_BROADCAST_URI=pulsar://pulsar:6650 + depends_on: + - pulsar + +volumes: + pulsardata: + pulsarconf: +``` + +Run this configuration with: + +```bash +docker-compose up --force-recreate +``` + +Allow a few seconds for Apache Pulsar and OPAL to start up before testing connectivity. + +## Triggering Events + +You can trigger events using the OPAL CLI: + +```bash +opal-client publish-data-update --src-url https://api.country.is/23.54.6.78 -t policy_data --dst-path /users/bob/location +``` + +You should see the effect in: +- OPAL-server logs: "Broadcasting incoming event" +- OPAL-client: Receiving and acting on the event +- Pulsar: Event data in the 'broadcast' topic + +## Supported Backends + +| Backend | Environment Variable | Docker Compose Service | +|----------|---------------------------------------------------------|------------------------| +| Kafka | `BROADCAST_URL=kafka://localhost:9092` | `docker-compose up kafka` | +| Redis | `BROADCAST_URL=redis://localhost:6379` | `docker-compose up redis` | +| Postgres | `BROADCAST_URL=postgres://localhost:5432/broadcaster` | `docker-compose up postgres` | +| Pulsar | `BROADCAST_URL=pulsar://localhost:6650` | `docker-compose up pulsar` | + +## Advanced: Publishing Events Directly to Pulsar + +You can trigger events by publishing messages directly to the 'broadcast' topic in Pulsar. Ensure the message format follows the OPAL-server schema for backbone events. + + +## Conclusion + +This guide covered setting up and using Apache Pulsar as the backbone pub/sub for OPAL-server. By following these instructions, you can effectively scale your OPAL deployment across multiple servers. + +## Further Resources + +- [OPAL Documentation](https://www.opal.ac/docs/) +- [Apache Pulsar Documentation](https://pulsar.apache.org/docs/en/standalone/) +- [Python Broadcaster Package](https://pypi.org/project/broadcaster/) + +For more information or support, please refer to the OPAL community forums or contact the maintainers. From d1fd0b8e6d34bee5220f087713de12a7fbc56e29 Mon Sep 17 00:00:00 2001 From: Gabriel Manor Date: Wed, 25 Sep 2024 15:18:24 +0300 Subject: [PATCH 04/19] Update run_opal_with_pulsar.mdx --- documentation/docs/tutorials/run_opal_with_pulsar.mdx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/documentation/docs/tutorials/run_opal_with_pulsar.mdx b/documentation/docs/tutorials/run_opal_with_pulsar.mdx index 8f069beb2..4433d7a65 100644 --- a/documentation/docs/tutorials/run_opal_with_pulsar.mdx +++ b/documentation/docs/tutorials/run_opal_with_pulsar.mdx @@ -17,10 +17,10 @@ OPAL-server can scale out both in number of worker processes per server and acro ### Broadcaster Module -Support for multiple backbone solutions is provided by the [Python Broadcaster package](https://pypi.org/project/broadcaster/). To use it with Apache Pulsar, install the `broadcaster[pulsar]` module: +Support for multiple backbone solutions is provided by the Permit's port of the [Python Broadcaster package](https://pypi.org/project/permit-broadcaster/). To use it with Apache Pulsar, install the `permit-broadcaster[pulsar]` module: ```bash -pip install broadcaster[pulsar] +pip install permit-broadcaster[pulsar] ``` ## Setting Up OPAL-server with Apache Pulsar @@ -102,7 +102,6 @@ You should see the effect in: You can trigger events by publishing messages directly to the 'broadcast' topic in Pulsar. Ensure the message format follows the OPAL-server schema for backbone events. - ## Conclusion This guide covered setting up and using Apache Pulsar as the backbone pub/sub for OPAL-server. By following these instructions, you can effectively scale your OPAL deployment across multiple servers. From 4f7789cb13c6281b473e07500e36b5733b051c74 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Thu, 26 Sep 2024 00:23:09 +0300 Subject: [PATCH 05/19] Changed authentication to be required (#669) --- packages/opal-common/opal_common/authentication/deps.py | 2 +- packages/opal-common/opal_common/authentication/signer.py | 2 +- .../opal-common/opal_common/authentication/verifier.py | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/deps.py b/packages/opal-common/opal_common/authentication/deps.py index 390e8ce2d..9ccf9156e 100644 --- a/packages/opal-common/opal_common/authentication/deps.py +++ b/packages/opal-common/opal_common/authentication/deps.py @@ -30,7 +30,7 @@ def verify_logged_in(verifier: JWTVerifier, token: Optional[str]) -> JWTClaims: try: if not verifier.enabled: logger.debug("JWT verification disabled, cannot verify requests!") - return {} + raise Unauthorized(description="JWT verification is disabled") if token is None: raise Unauthorized(description="access token was not provided") claims: JWTClaims = verifier.verify(token) diff --git a/packages/opal-common/opal_common/authentication/signer.py b/packages/opal-common/opal_common/authentication/signer.py index 89a27f53e..61a29d2bb 100644 --- a/packages/opal-common/opal_common/authentication/signer.py +++ b/packages/opal-common/opal_common/authentication/signer.py @@ -94,7 +94,7 @@ def _verify_crypto_keys(self): ) elif self._private_key is None and self._public_key is None: # valid situation, running in dev mode and api security is off - self._disable() + self._disable("No crypto keys provided") else: raise ValueError("Invalid JWT Signer input!") diff --git a/packages/opal-common/opal_common/authentication/verifier.py b/packages/opal-common/opal_common/authentication/verifier.py index 5243006f0..50008c043 100644 --- a/packages/opal-common/opal_common/authentication/verifier.py +++ b/packages/opal-common/opal_common/authentication/verifier.py @@ -63,9 +63,9 @@ def _verify_public_key(self): ) except jwt.exceptions.InvalidKeyError as e: logger.error(f"Invalid public key for jwt verification, error: {e}!") - self._disable() + self._disable(f"Invalid public key: {e}") else: - self._disable() + self._disable("No public key provided") def get_jwk(self) -> str: """returns the jwk json contents.""" @@ -74,7 +74,8 @@ def get_jwk(self) -> str: raise ValueError(f"invalid jwt algorithm: {self._algorithm}") return algorithm.to_jwk(self._public_key) - def _disable(self): + def _disable(self, reason: str): + logger.warning(f"Disabled JWT verification due to: {reason}") self._enabled = False @property From 3e809644702c5967936139fdfd8d70bc8548dec5 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Thu, 26 Sep 2024 12:40:54 +0300 Subject: [PATCH 06/19] Revert "Changed authentication to be required (#669)" (#670) This reverts commit 4f7789cb13c6281b473e07500e36b5733b051c74. --- packages/opal-common/opal_common/authentication/deps.py | 2 +- packages/opal-common/opal_common/authentication/signer.py | 2 +- .../opal-common/opal_common/authentication/verifier.py | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/deps.py b/packages/opal-common/opal_common/authentication/deps.py index 9ccf9156e..390e8ce2d 100644 --- a/packages/opal-common/opal_common/authentication/deps.py +++ b/packages/opal-common/opal_common/authentication/deps.py @@ -30,7 +30,7 @@ def verify_logged_in(verifier: JWTVerifier, token: Optional[str]) -> JWTClaims: try: if not verifier.enabled: logger.debug("JWT verification disabled, cannot verify requests!") - raise Unauthorized(description="JWT verification is disabled") + return {} if token is None: raise Unauthorized(description="access token was not provided") claims: JWTClaims = verifier.verify(token) diff --git a/packages/opal-common/opal_common/authentication/signer.py b/packages/opal-common/opal_common/authentication/signer.py index 61a29d2bb..89a27f53e 100644 --- a/packages/opal-common/opal_common/authentication/signer.py +++ b/packages/opal-common/opal_common/authentication/signer.py @@ -94,7 +94,7 @@ def _verify_crypto_keys(self): ) elif self._private_key is None and self._public_key is None: # valid situation, running in dev mode and api security is off - self._disable("No crypto keys provided") + self._disable() else: raise ValueError("Invalid JWT Signer input!") diff --git a/packages/opal-common/opal_common/authentication/verifier.py b/packages/opal-common/opal_common/authentication/verifier.py index 50008c043..5243006f0 100644 --- a/packages/opal-common/opal_common/authentication/verifier.py +++ b/packages/opal-common/opal_common/authentication/verifier.py @@ -63,9 +63,9 @@ def _verify_public_key(self): ) except jwt.exceptions.InvalidKeyError as e: logger.error(f"Invalid public key for jwt verification, error: {e}!") - self._disable(f"Invalid public key: {e}") + self._disable() else: - self._disable("No public key provided") + self._disable() def get_jwk(self) -> str: """returns the jwk json contents.""" @@ -74,8 +74,7 @@ def get_jwk(self) -> str: raise ValueError(f"invalid jwt algorithm: {self._algorithm}") return algorithm.to_jwk(self._public_key) - def _disable(self, reason: str): - logger.warning(f"Disabled JWT verification due to: {reason}") + def _disable(self): self._enabled = False @property From aef989dc89afae164bb336158ae550c8e07336bf Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Wed, 18 Sep 2024 12:55:43 +0200 Subject: [PATCH 07/19] Add missing wget binary --- docker/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index da5c7383c..483f55282 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,6 +20,10 @@ COPY ./cedar-agent /tmp/cedar-agent WORKDIR /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release +# BUSYBOX IMAGE -------------------------------------- +# --------------------------------------------------- +FROM busybox:1.36.1-uclibc AS busybox-builder + # COMMON IMAGE -------------------------------------- # --------------------------------------------------- FROM python:3.10-slim-bookworm AS common @@ -33,6 +37,10 @@ COPY --from=build-stage /usr/local /usr/local RUN useradd -m -b / -s /bin/bash opal WORKDIR /opal +# copy wget binary +COPY --from=busybox-builder /bin/wget /bin/wget +RUN chmod +x /bin/wget + # copy wait-for script (create link at old path to maintain backward compatibility) COPY scripts/wait-for.sh . RUN chmod +x ./wait-for.sh From 5a4318fb41b2302a1ab9ca78d245feedd15f6d60 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Wed, 18 Sep 2024 13:09:25 +0200 Subject: [PATCH 08/19] Install wget package instead of copying static binary from busybox --- docker/Dockerfile | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 483f55282..58fb244f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,10 +20,6 @@ COPY ./cedar-agent /tmp/cedar-agent WORKDIR /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release -# BUSYBOX IMAGE -------------------------------------- -# --------------------------------------------------- -FROM busybox:1.36.1-uclibc AS busybox-builder - # COMMON IMAGE -------------------------------------- # --------------------------------------------------- FROM python:3.10-slim-bookworm AS common @@ -37,17 +33,13 @@ COPY --from=build-stage /usr/local /usr/local RUN useradd -m -b / -s /bin/bash opal WORKDIR /opal -# copy wget binary -COPY --from=busybox-builder /bin/wget /bin/wget -RUN chmod +x /bin/wget - # copy wait-for script (create link at old path to maintain backward compatibility) COPY scripts/wait-for.sh . RUN chmod +x ./wait-for.sh RUN ln -s /opal/wait-for.sh /usr/wait-for.sh -# netcat (nc) is used by the wait-for.sh script -RUN apt-get update && apt-get install -y netcat-traditional jq && apt-get clean +# netcat (nc) and wget are used by the wait-for.sh script +RUN apt-get update && apt-get install -y netcat-traditional jq wget && apt-get clean # copy startup script (create link at old path to maintain backward compatibility) COPY ./scripts/start.sh . From b8cf78b62ca867a98233bb66b2e2496856b6f419 Mon Sep 17 00:00:00 2001 From: Oded Date: Sun, 29 Sep 2024 21:53:12 +0300 Subject: [PATCH 09/19] Add timeouts to avoid stuck actions --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 01b85cd5b..5d324cd39 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,6 +25,7 @@ jobs: --health-timeout 5s --health-retries 5 runs-on: ubuntu-latest + timeout-minutes: 60 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -53,6 +54,7 @@ jobs: test-docker: runs-on: ubuntu-latest + timeout-minutes: 60 steps: # BUILD PHASE - name: Checkout From 9aa097573227f53194066696daf57a93f723028e Mon Sep 17 00:00:00 2001 From: Ro'e Katz Date: Tue, 8 Oct 2024 20:03:05 +0300 Subject: [PATCH 10/19] Allow httpx>=0.27.0 --- packages/opal-common/requires.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/requires.txt b/packages/opal-common/requires.txt index dc8372bf6..57198ba7b 100644 --- a/packages/opal-common/requires.txt +++ b/packages/opal-common/requires.txt @@ -10,5 +10,5 @@ datadog>=0.44.0, <1 ddtrace>=2.8.1,<3 certifi>=2023.7.22 # not directly required, pinned by Snyk to avoid a vulnerability requests>=2.32.0 # not directly required, pinned by Snyk to avoid a vulnerability -httpx==0.27.0 +httpx>=0.27.0 urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability From 46ca93a2179250b7b81c4ebcb52acde62d6dccee Mon Sep 17 00:00:00 2001 From: gideonsmila Date: Wed, 9 Oct 2024 14:37:26 +0300 Subject: [PATCH 11/19] update docs --- documentation/docs/getting-started/configuration.mdx | 3 ++- .../running-opal/run-opal-server/security-parameters.mdx | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/documentation/docs/getting-started/configuration.mdx b/documentation/docs/getting-started/configuration.mdx index 5cd2ed4c9..8788f056a 100644 --- a/documentation/docs/getting-started/configuration.mdx +++ b/documentation/docs/getting-started/configuration.mdx @@ -24,7 +24,8 @@ Please use this table as a reference. | LOG_FILE_RETENTION | | | | LOG_FILE_COMPRESSION | | | | LOG_FILE_SERIALIZE | Serialize log messages in file into json format (useful for log aggregation platforms) | | -| LOG_FILE_LEVEL | | | +| LOG_FILE_LEVEL | +| LOG_DIAGNOSE | Include diagnosis in log messages | | | STATISTICS_ENABLED | Collect statistics about OPAL clients. | | | STATISTICS_ADD_CLIENT_CHANNEL | The topic to update about the new OPAL clients connection. | | | STATISTICS_REMOVE_CLIENT_CHANNEL | The topic to update about the OPAL clients disconnection. | | diff --git a/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx b/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx index 1862e7afd..af90d81a6 100644 --- a/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx +++ b/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx @@ -105,3 +105,5 @@ You must then configure the master token like so | Env Var Name | Function | | :--------------------- | :------------------------------------------------------------------- | | OPAL_AUTH_MASTER_TOKEN | the master token generated by the cli (or any other secret you pick) | + +Ensure LOG_DIAGNOSE is set to False to disable diagnostic logging that may expose sensitive information. \ No newline at end of file From 627b8c44e7054b8166627de88538c7a048dc0d8a Mon Sep 17 00:00:00 2001 From: gideonsmila Date: Sun, 13 Oct 2024 18:31:22 +0300 Subject: [PATCH 12/19] fix pre-commit --- documentation/docs/getting-started/configuration.mdx | 2 +- .../running-opal/run-opal-server/security-parameters.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/docs/getting-started/configuration.mdx b/documentation/docs/getting-started/configuration.mdx index 8788f056a..d3baa47b0 100644 --- a/documentation/docs/getting-started/configuration.mdx +++ b/documentation/docs/getting-started/configuration.mdx @@ -24,7 +24,7 @@ Please use this table as a reference. | LOG_FILE_RETENTION | | | | LOG_FILE_COMPRESSION | | | | LOG_FILE_SERIALIZE | Serialize log messages in file into json format (useful for log aggregation platforms) | | -| LOG_FILE_LEVEL | +| LOG_FILE_LEVEL | | LOG_DIAGNOSE | Include diagnosis in log messages | | | STATISTICS_ENABLED | Collect statistics about OPAL clients. | | | STATISTICS_ADD_CLIENT_CHANNEL | The topic to update about the new OPAL clients connection. | | diff --git a/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx b/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx index af90d81a6..8a0fba6ed 100644 --- a/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx +++ b/documentation/docs/getting-started/running-opal/run-opal-server/security-parameters.mdx @@ -106,4 +106,4 @@ You must then configure the master token like so | :--------------------- | :------------------------------------------------------------------- | | OPAL_AUTH_MASTER_TOKEN | the master token generated by the cli (or any other secret you pick) | -Ensure LOG_DIAGNOSE is set to False to disable diagnostic logging that may expose sensitive information. \ No newline at end of file +Ensure LOG_DIAGNOSE is set to False to disable diagnostic logging that may expose sensitive information. From f0c54f496cdcebea7df70cab30f97bb0291ed294 Mon Sep 17 00:00:00 2001 From: gideonsmila Date: Sun, 13 Oct 2024 18:40:07 +0300 Subject: [PATCH 13/19] update-pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 481d33c18..e57f8ef62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer From 1b2e9b602a2571604d3a9b1e491fc46ff9f395f0 Mon Sep 17 00:00:00 2001 From: Asaf Cohen Date: Tue, 5 Nov 2024 16:20:54 +0200 Subject: [PATCH 14/19] add ability to register to on ws connect and disconnect events of opal client --- packages/opal-client/opal_client/client.py | 10 ++++++++++ packages/opal-client/opal_client/data/updater.py | 10 ++++++++-- packages/opal-client/opal_client/policy/updater.py | 11 ++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/opal-client/opal_client/client.py b/packages/opal-client/opal_client/client.py index be8e5ca49..6a9958b98 100644 --- a/packages/opal-client/opal_client/client.py +++ b/packages/opal-client/opal_client/client.py @@ -13,6 +13,8 @@ import websockets from fastapi import FastAPI, status from fastapi.responses import JSONResponse +from fastapi_websocket_pubsub.pub_sub_client import PubSubOnConnectCallback +from fastapi_websocket_rpc.rpc_channel import OnDisconnectCallback from opal_client.callbacks.api import init_callbacks_api from opal_client.callbacks.register import CallbacksRegister from opal_client.config import PolicyStoreTypes, opal_client_config @@ -54,6 +56,10 @@ def __init__( store_backup_interval: Optional[int] = None, offline_mode_enabled: bool = False, shard_id: Optional[str] = None, + on_data_updater_connect: List[PubSubOnConnectCallback] = None, + on_data_updater_disconnect: List[OnDisconnectCallback] = None, + on_policy_updater_connect: List[PubSubOnConnectCallback] = None, + on_policy_updater_disconnect: List[OnDisconnectCallback] = None, ) -> None: """ Args: @@ -119,6 +125,8 @@ def __init__( policy_store=self.policy_store, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, + on_connect=on_policy_updater_connect, + on_disconnect=on_policy_updater_disconnect, ) else: self.policy_updater = None @@ -140,6 +148,8 @@ def __init__( callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, shard_id=self._shard_id, + on_connect=on_data_updater_connect, + on_disconnect=on_data_updater_disconnect, ) else: self.data_updater = None diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index e288b5963..860208652 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -9,7 +9,8 @@ import aiohttp from aiohttp.client import ClientError, ClientSession from fastapi_websocket_pubsub import PubSubClient -from fastapi_websocket_rpc.rpc_channel import RpcChannel +from fastapi_websocket_pubsub.pub_sub_client import PubSubOnConnectCallback +from fastapi_websocket_rpc.rpc_channel import OnDisconnectCallback, RpcChannel from opal_client.callbacks.register import CallbacksRegister from opal_client.callbacks.reporter import CallbacksReporter from opal_client.config import opal_client_config @@ -54,6 +55,8 @@ def __init__( callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, shard_id: Optional[str] = None, + on_connect: List[PubSubOnConnectCallback] = None, + on_disconnect: List[OnDisconnectCallback] = None, ): """Keeps policy-stores (e.g. OPA) up to date with relevant data Obtains data configuration on startup from OPAL-server Uses Pub/Sub to @@ -132,6 +135,8 @@ def __init__( self._updates_storing_queue = TakeANumberQueue(logger) self._tasks = TasksPool() self._polling_update_tasks = [] + self._on_connect_callbacks = on_connect or [] + self._on_disconnect_callbacks = on_disconnect or [] async def __aenter__(self): await self.start() @@ -278,7 +283,8 @@ async def _subscriber(self): self._data_topics, self._update_policy_data_callback, methods_class=TenantAwareRpcEventClientMethods, - on_connect=[self.on_connect], + on_connect=[self.on_connect, *self._on_connect_callbacks], + on_disconnect=[self.on_disconnect, *self._on_disconnect_callbacks], extra_headers=self._extra_headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py index 57d93099f..fe5640a63 100644 --- a/packages/opal-client/opal_client/policy/updater.py +++ b/packages/opal-client/opal_client/policy/updater.py @@ -3,7 +3,8 @@ import pydantic from fastapi_websocket_pubsub import PubSubClient -from fastapi_websocket_rpc.rpc_channel import RpcChannel +from fastapi_websocket_pubsub.pub_sub_client import PubSubOnConnectCallback +from fastapi_websocket_rpc.rpc_channel import OnDisconnectCallback, RpcChannel from opal_client.callbacks.register import CallbacksRegister from opal_client.callbacks.reporter import CallbacksReporter from opal_client.config import opal_client_config @@ -43,6 +44,8 @@ def __init__( data_fetcher: Optional[DataFetcher] = None, callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, + on_connect: List[PubSubOnConnectCallback] = None, + on_disconnect: List[OnDisconnectCallback] = None, ): """inits the policy updater. @@ -104,6 +107,8 @@ def __init__( ) self._policy_update_queue = asyncio.Queue() self._tasks = TasksPool() + self._on_connect_callbacks = on_connect or [] + self._on_disconnect_callbacks = on_disconnect or [] async def __aenter__(self): await self.start() @@ -243,8 +248,8 @@ async def _subscriber(self): self._client = PubSubClient( topics=self._topics, callback=self._update_policy_callback, - on_connect=[self._on_connect], - on_disconnect=[self._on_disconnect], + on_connect=[self._on_connect, *self._on_connect_callbacks], + on_disconnect=[self._on_disconnect, *self._on_disconnect_callbacks], extra_headers=self._extra_headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, From 778cc6faaa7bf68687e9d686d7a2810a14ef57c2 Mon Sep 17 00:00:00 2001 From: Asaf Cohen Date: Tue, 5 Nov 2024 16:28:50 +0200 Subject: [PATCH 15/19] fix _pipe_log_stream not running consistently for monitored subprocess --- packages/opal-client/opal_client/engine/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/opal-client/opal_client/engine/runner.py b/packages/opal-client/opal_client/engine/runner.py index 9cca62c28..762472232 100644 --- a/packages/opal-client/opal_client/engine/runner.py +++ b/packages/opal-client/opal_client/engine/runner.py @@ -136,8 +136,8 @@ async def _pipe_logs_stream(stream: asyncio.StreamReader): line = b"" - await asyncio.wait( - [ + await asyncio.gather( + *[ _pipe_logs_stream(self._process.stdout), _pipe_logs_stream(self._process.stderr), ] From 426ee50781c52c7b0c7934a3bf9f616e0b7867a8 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Wed, 18 Sep 2024 12:55:43 +0200 Subject: [PATCH 16/19] Add missing wget binary --- docker/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index da5c7383c..483f55282 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,6 +20,10 @@ COPY ./cedar-agent /tmp/cedar-agent WORKDIR /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release +# BUSYBOX IMAGE -------------------------------------- +# --------------------------------------------------- +FROM busybox:1.36.1-uclibc AS busybox-builder + # COMMON IMAGE -------------------------------------- # --------------------------------------------------- FROM python:3.10-slim-bookworm AS common @@ -33,6 +37,10 @@ COPY --from=build-stage /usr/local /usr/local RUN useradd -m -b / -s /bin/bash opal WORKDIR /opal +# copy wget binary +COPY --from=busybox-builder /bin/wget /bin/wget +RUN chmod +x /bin/wget + # copy wait-for script (create link at old path to maintain backward compatibility) COPY scripts/wait-for.sh . RUN chmod +x ./wait-for.sh From 0e288a4b50c92f8329e6b8271ec48c4b5dc8eafa Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Wed, 18 Sep 2024 13:09:25 +0200 Subject: [PATCH 17/19] Install wget package instead of copying static binary from busybox --- docker/Dockerfile | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 483f55282..58fb244f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,10 +20,6 @@ COPY ./cedar-agent /tmp/cedar-agent WORKDIR /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release -# BUSYBOX IMAGE -------------------------------------- -# --------------------------------------------------- -FROM busybox:1.36.1-uclibc AS busybox-builder - # COMMON IMAGE -------------------------------------- # --------------------------------------------------- FROM python:3.10-slim-bookworm AS common @@ -37,17 +33,13 @@ COPY --from=build-stage /usr/local /usr/local RUN useradd -m -b / -s /bin/bash opal WORKDIR /opal -# copy wget binary -COPY --from=busybox-builder /bin/wget /bin/wget -RUN chmod +x /bin/wget - # copy wait-for script (create link at old path to maintain backward compatibility) COPY scripts/wait-for.sh . RUN chmod +x ./wait-for.sh RUN ln -s /opal/wait-for.sh /usr/wait-for.sh -# netcat (nc) is used by the wait-for.sh script -RUN apt-get update && apt-get install -y netcat-traditional jq && apt-get clean +# netcat (nc) and wget are used by the wait-for.sh script +RUN apt-get update && apt-get install -y netcat-traditional jq wget && apt-get clean # copy startup script (create link at old path to maintain backward compatibility) COPY ./scripts/start.sh . From ccdd188e71a7023eb7c3937724ce323646cc477b Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Wed, 18 Sep 2024 12:55:43 +0200 Subject: [PATCH 18/19] Add missing wget binary --- docker/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 58fb244f4..3baca675b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,6 +20,10 @@ COPY ./cedar-agent /tmp/cedar-agent WORKDIR /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release +# BUSYBOX IMAGE -------------------------------------- +# --------------------------------------------------- +FROM busybox:1.36.1-uclibc AS busybox-builder + # COMMON IMAGE -------------------------------------- # --------------------------------------------------- FROM python:3.10-slim-bookworm AS common @@ -33,6 +37,10 @@ COPY --from=build-stage /usr/local /usr/local RUN useradd -m -b / -s /bin/bash opal WORKDIR /opal +# copy wget binary +COPY --from=busybox-builder /bin/wget /bin/wget +RUN chmod +x /bin/wget + # copy wait-for script (create link at old path to maintain backward compatibility) COPY scripts/wait-for.sh . RUN chmod +x ./wait-for.sh From 3aafa3f28c7d8b4e3d03020828cb826b3f6418cc Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Wed, 18 Sep 2024 13:09:25 +0200 Subject: [PATCH 19/19] Install wget package instead of copying static binary from busybox --- docker/Dockerfile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3baca675b..58fb244f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,10 +20,6 @@ COPY ./cedar-agent /tmp/cedar-agent WORKDIR /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release -# BUSYBOX IMAGE -------------------------------------- -# --------------------------------------------------- -FROM busybox:1.36.1-uclibc AS busybox-builder - # COMMON IMAGE -------------------------------------- # --------------------------------------------------- FROM python:3.10-slim-bookworm AS common @@ -37,10 +33,6 @@ COPY --from=build-stage /usr/local /usr/local RUN useradd -m -b / -s /bin/bash opal WORKDIR /opal -# copy wget binary -COPY --from=busybox-builder /bin/wget /bin/wget -RUN chmod +x /bin/wget - # copy wait-for script (create link at old path to maintain backward compatibility) COPY scripts/wait-for.sh . RUN chmod +x ./wait-for.sh