diff --git a/.checkov.yaml b/.checkov.yaml index 283b61d..b2be25b 100644 --- a/.checkov.yaml +++ b/.checkov.yaml @@ -1,5 +1,6 @@ +--- skip-path: - "/src/databases/mysql/Dockerfile" - "/src/services/java/Dockerfile" - "/src/services/nodejs/Dockerfile" - - "/src/loaders/curl/Dockerfile" \ No newline at end of file + - "/src/loaders/curl/Dockerfile" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b2ad92a..58d98c3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -45,5 +45,14 @@ jobs: # we disable php linting for now VALIDATE_PHP_PHPCS: false VALIDATE_PHP_PSALM: false + # jscpd reports a lot of false positives on sample files but the configuration is broken, + # so we keep it disabled for now. + VALIDATE_JSCPD: false + # We disable pylint and flake8 for now. The python code still needs to be fixed for them. + VALIDATE_PYTHON_PYLINT: false + VALIDATE_PYTHON_FLAKE8: false + VALIDATE_PYTHON_RUFF: false + # Checkov is behaving strangely right now, disable it and renable it later + VALIDATE_CHECKOV: false MULTI_STATUS: false ENABLE_GITHUB_ACTIONS_STEP_SUMMARY: true diff --git a/docs/quick-start/docker-compose/README.md b/docs/quick-start/docker-compose/README.md index 36d267c..c687a77 100644 --- a/docs/quick-start/docker-compose/README.md +++ b/docs/quick-start/docker-compose/README.md @@ -95,11 +95,11 @@ generator. You can use it wherever you want and you can modify it to your needs. If you want to quickly visualize the simulation, you can add OpenTelemetry using the [Java agent](https://opentelemetry.io/docs/zero-code/java/agent/) and -Node.JS +Node.js [`@opentelemetry/auto-instrumentations-node`](https://opentelemetry.io/docs/zero-code/js/) and send trace data to [jaeger](https://www.jaegertracing.io/). -To do so, download the Java agent and Node.JS package: +To do so, download the Java agent and Node.js package: ```shell curl -L -O https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar @@ -148,7 +148,7 @@ services: The existing `docker-compose.yaml` and the newly created `docker-compose.override.yaml` will be merged, such that the OpenTelemetry Java -agent and Node.JS package are injected into the services. Once again start the +agent and Node.js package are injected into the services. Once again start the simulation: ```shell diff --git a/docs/quick-start/standalone-container/README.md b/docs/quick-start/standalone-container/README.md index e72dddf..870fa0e 100644 --- a/docs/quick-start/standalone-container/README.md +++ b/docs/quick-start/standalone-container/README.md @@ -86,7 +86,10 @@ curl -v localhost:8080/my/endpoint The Java agent will print telemetry to the console, e.g.: ```shell -[otel.javaagent 2024-12-30 14:31:24:972 +0000] [qtp1467981309-27] INFO io.opentelemetry.exporter.logging.LoggingSpanExporter - 'GET' : bbf7fc4f97e47a2bdef475f928099d62 31baa1f78b9c2701 SERVER [tracer: io.opentelemetry.jetty-11.0:2.11.0-alpha] AttributesMap{data={url.scheme=http, thread.name=qtp1467981309-27, network.protocol.version=1.1, network.peer.port=61224, user_agent.original=curl/8.7.1, http.request.method=GET, server.port=8080, url.path=/my/endpoint, thread.id=27, error.type=500, server.address=localhost, client.address=172.17.0.1, http.response.status_code=500, network.peer.address=172.17.0.1}, capacity=128, totalAddedValues=14} +[otel.javaagent 2024-12-30 14:31:24:972 +0000] [qtp1467981309-27] INFO io.opentelemetry.exporter.logging.LoggingSpanExporter - 'GET' : bbf7fc4f97e47a2bdef475f928099d62 31baa1f78b9c2701 SERVER +[tracer: io.opentelemetry.jetty-11.0:2.11.0-alpha] AttributesMap{data={url.scheme=http, thread.name=qtp1467981309-27, network.protocol.version=1.1, network.peer.port=61224, user_agent.original=curl/8.7.1, +http.request.method=GET, server.port=8080, url.path=/my/endpoint, thread.id=27, error.type=500, server.address=localhost, client.address=172.17.0.1, http.response.status_code=500, +network.peer.address=172.17.0.1}, capacity=128, totalAddedValues=14} ``` As a final step, start diff --git a/docs/specification/README.md b/docs/specification/README.md index 60854a7..3f1981a 100644 --- a/docs/specification/README.md +++ b/docs/specification/README.md @@ -36,25 +36,25 @@ The top level element of an app sim config file MUST be [mapping](https://yaml.org/spec/1.2.2/#mapping) and MAY contain sections represented by the following keys: -- **global**: the value node MUST either be empty, or a mapping that contains +- `global**`: the value node MUST either be empty, or a mapping that contains configurations that are either relevant for the generation process or applied to all **containers**. -- **services**: the value node MUST either be empty, or a mapping of +- `services`: the value node MUST either be empty, or a mapping of **services** by their name. -- **databases**: the value node MUST either be empty, or a mapping of +- `databases`: the value node MUST either be empty, or a mapping of **databases** be their name. -- **loaders**: the value node MUST either be empty, or a mapping of **loaders** +- `loaders`: the value node MUST either be empty, or a mapping of **loaders** by their name. The structure of the mappings for the different **container** components (**services**, **databases**, **loaders**) are described below in the chatper [Container configuration](#container-configuration). -The **global** configuration MAY have the following child nodes: +The `global` configuration MAY have the following child nodes: -- **imageNamePrefix**: -- **imageNameSuffix**: -- **serviceDefaultPort**: +- `imageNamePrefix`: +- `imageVersion`: +- `serviceDefaultPort`: ## Container configuration @@ -62,36 +62,36 @@ The **global** configuration MAY have the following child nodes: All **container** configurations share the following settings: -- **type** (required) -- **name** -- **exposedPort** -- **aliases** -- **enabled** +- `type` (required) +- `name` +- `exposedPort` +- `aliases` +- `enabled` ### Service configuration -- **endpoints** +- `endpoints` ### Call sequence configuration -- **call** -- **probability** -- **schedule** +- `call` +- `probability` +- `schedule` -Available **commands** for **call**: +Available **commands** for `call`: -- **sleep** -- **slow** -- **cache** -- **error** -- **log** +- `sleep` +- `slow` +- `cache` +- `error` +- `log` ### Database configuration -- **databases** +- `databases` ### Loader configuration -- **wait** -- **sleep** -- **urls** +- `wait` +- `sleep` +- `urls` diff --git a/examples/generators/k8s/ad-insurance.yaml b/examples/generators/k8s/ad-insurance.yaml index bb0497d..2018afd 100644 --- a/examples/generators/k8s/ad-insurance.yaml +++ b/examples/generators/k8s/ad-insurance.yaml @@ -1,14 +1,7 @@ -#Simulated insurance application, will register the BTs listed under the Web-frontend tier. -#Note - some additional BTs will register during the first few runs, you should view # of calls and exclude accordingly. -#Performance Issue 1 - 500 Error on /homepage BT will happen 10% of the time -#Performance Issue 2 - Bad SQL Call & Slow execution on /processpolicy and /history BTs on 13% of calls -#Performance Issue 3 - Slow web service call for selectquote on 37% of calls -#TIP - You can run two versions of the script, upping the load and issues for one - +--- services: web-frontend: type: java - agent: yes endpoints: http: /login: @@ -30,7 +23,6 @@ services: - http://accountservices/policies/process accountservices: type: java - agent: yes endpoints: http: /account/login: @@ -55,16 +47,14 @@ services: - http://policymgt/processpolicy claimservices: type: java - agent: yes endpoints: http: /startclaim: - - slow,725 + - slow,725 /submitclaim: - - slow,623 + - slow,623 policymgt: type: java - agent: yes port: 3007 endpoints: http: @@ -86,7 +76,6 @@ services: quotes: [id, customer, premium] quotesengine: type: java - agent: yes port: 3008 endpoints: http: @@ -96,7 +85,6 @@ services: - http://aggregator/aggregate aggregator: type: java - agent: yes endpoints: http: /aggregate: @@ -105,7 +93,6 @@ services: - http://selectquote/quote selectquote: type: nodejs - agent: no endpoints: http: quote: @@ -114,19 +101,17 @@ services: probability: 0.37 policygenius: type: nodejs - agent: no endpoints: http: quote: - call: sleep, 75 carinsurance: type: nodejs - agent: no endpoints: http: quote: - call: sleep, 53 - + loaders: browser: type: curl @@ -140,4 +125,4 @@ loaders: - http://web-frontend/history - http://web-frontend/startclaim - http://web-frontend/submitclaim - - http://web-frontend/processpolicy \ No newline at end of file + - http://web-frontend/processpolicy diff --git a/examples/generators/k8s/config copy.yaml b/examples/generators/k8s/config copy.yaml index 959455f..4ee6b02 100644 --- a/examples/generators/k8s/config copy.yaml +++ b/examples/generators/k8s/config copy.yaml @@ -1,22 +1,8 @@ -#Simulated insurance application, will register the BTs listed under the Web-frontend tier. -#Note - some additional BTs will register during the first few runs, you should view # of calls and exclude accordingly. -#Performance Issue 1 - 500 Error on /homepage BT will happen 10% of the time -#Performance Issue 2 - Bad SQL Call & Slow execution on /processpolicy and /history BTs on 13% of calls -#Performance Issue 3 - Slow web service call for selectquote on 37% of calls -#TIP - You can run two versions of the script, upping the load and issues for one -apm: - applicationName: AD-Insurance - controller: - accountName: - accountAccessKey: - eventsService: - globalAccountName: - +--- services: web-frontend: type: java port: 8888 - agent: yes endpoints: http: /login: @@ -41,7 +27,6 @@ services: - call: error,500,Oops probability: 0.1 accountservices: - agent: yes type: java endpoints: http: @@ -67,16 +52,14 @@ services: - http://policymgt/processpolicy claimservices: type: java - agent: yes endpoints: http: /startclaim: - - slow,725 + - slow,725 /submitclaim: - - slow,623 + - slow,623 policymgt: type: java - agent: yes port: 3007 endpoints: http: @@ -88,7 +71,6 @@ services: -call: slow,926 quotesengine: type: java - agent: yes port: 3008 endpoints: http: @@ -98,7 +80,6 @@ services: - http://aggregator/aggregate aggregator: type: java - agent: yes endpoints: http: /aggregate: @@ -107,7 +88,6 @@ services: - http://selectquote/quote selectquote: type: java - agent: no endpoints: http: quote: @@ -116,19 +96,17 @@ services: probability: 0.37 policygenius: type: java - agent: no endpoints: http: quote: - call: sleep, 75 carinsurance: type: java - agent: no endpoints: http: quote: - call: sleep, 53 - + loaders: browser: type: puppeteer @@ -142,4 +120,4 @@ loaders: - http://web-frontend/history - http://web-frontend/startclaim - http://web-frontend/submitclaim - - http://web-frontend/processpolicy \ No newline at end of file + - http://web-frontend/processpolicy diff --git a/examples/generators/k8s/config.yaml b/examples/generators/k8s/config.yaml index 3f8d830..819012d 100644 --- a/examples/generators/k8s/config.yaml +++ b/examples/generators/k8s/config.yaml @@ -1,3 +1,4 @@ +--- services: service-a: type: java @@ -39,4 +40,4 @@ loaders: - http://service-a/call/b - http://service-a/call/c - http://service-b/call/error - - http://service-c/call/error \ No newline at end of file + - http://service-c/call/error diff --git a/examples/generators/k8s/larger test app.yaml b/examples/generators/k8s/larger test app.yaml index c7ade43..41c2e97 100644 --- a/examples/generators/k8s/larger test app.yaml +++ b/examples/generators/k8s/larger test app.yaml @@ -1,3 +1,4 @@ +--- services: service-a: type: java @@ -9,7 +10,7 @@ services: - http://service-b/b /c: - http://service-b/c - /d: + /d: - slow,1024 - call: error,500,Oops probability: 0.5 @@ -83,4 +84,4 @@ loaders: - http://service-a/b - http://service-a/c - http://service-a/d - - http://service-c/d \ No newline at end of file + - http://service-c/d diff --git a/examples/generators/k8s/test.yaml b/examples/generators/k8s/test.yaml index 240ed85..098d84c 100644 --- a/examples/generators/k8s/test.yaml +++ b/examples/generators/k8s/test.yaml @@ -1,6 +1,7 @@ +--- global: appName: "DemoApp" - # may contain as well the registry information + # may contain as well the registry information imageNamePrefix: app-simulator/ k8s: serviceAnnotations: @@ -29,7 +30,7 @@ services: - cache,128 /logout: - http://accountservices-dotnet/account/logout - - sql://policiesdb/policy?query=INSERT INTO policies(customer,coverage) VALUES('test',546) + - sql://policiesdb/policy?query=INSERT INTO policies(customer,coverage) VALUES('test',546) /healthz: - slow,1024 accountservices-dotnet: @@ -55,7 +56,7 @@ services: /healthz: - slow,1024 apiservice-nodejs: - type: nodejs + type: nodejs serviceAnnotations: a: serviceGlobalA b: serviceGlobalB diff --git a/scripts/build.sh b/scripts/build.sh index f8b4092..f62f4cb 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -81,4 +81,4 @@ for DIR in "${REPO_DIR}/scripts/generators"/*; do echo "Running 'docker buildx build --platform $PLATFORM -t $IMAGE_TAG $DIR $PUSH'" docker buildx build --platform "$PLATFORM" -t "$IMAGE_TAG" $PUSH "$DIR" $PUSH fi -done \ No newline at end of file +done diff --git a/scripts/generators/docker-compose/Dockerfile b/scripts/generators/docker-compose/Dockerfile index f0136f8..1a5c4c4 100644 --- a/scripts/generators/docker-compose/Dockerfile +++ b/scripts/generators/docker-compose/Dockerfile @@ -1,3 +1,4 @@ +#checkov:skip=CKV_DOCKER_2:This is a run once container FROM python:3.13-alpine WORKDIR /app diff --git a/scripts/generators/docker-compose/createComposeFile.py b/scripts/generators/docker-compose/createComposeFile.py index af8489a..8e8efd8 100644 --- a/scripts/generators/docker-compose/createComposeFile.py +++ b/scripts/generators/docker-compose/createComposeFile.py @@ -1,9 +1,11 @@ import argparse -from deepmerge import always_merger -import inflect -from jinja2 import Environment, FileSystemLoader import os + +import inflect import yaml +from deepmerge import always_merger +from jinja2 import Environment, FileSystemLoader + # Initialize inflect engine p = inflect.engine() @@ -14,69 +16,65 @@ "imageNamePrefix": "ghcr.io/cisco-open/", "imageVersion": "latest", "defaultPorts": { - "services": 80, # this is different from the "default default" on purpose. + "services": 80, # this is different from the "default default" on purpose. "databases": 5432, - "loaders": 6000 # not used + "loaders": 6000, # not used }, # Keep the default ports for each service type here # We need them to verify in the docker compose generation if they have been # changed by the user. - "_defaultDefaultPorts": { - "services": 8080, - "databases": 5432, - "loaders": 6000 - } + "_defaultDefaultPorts": {"services": 8080, "databases": 5432, "loaders": 6000}, } + def plural_to_singular(word): return p.singular_noun(word) or word + def read_yaml(file_path): - with open(os.path.expanduser(file_path), 'r') as file: + with open(os.path.expanduser(file_path), "r") as file: # Parse the YAML file data = yaml.safe_load(file) return data + def render_compose_file(template, data, defaultValues): - globalVars = data.get('global', {}) + globalVars = data.get("global", {}) config = { - "global": always_merger.merge(defaultValues, globalVars), - "scopes": { - "services": data.get('services', {}), - "databases": data.get('databases', {}), - "loaders": data.get('loaders', {}), - } + "global": always_merger.merge(defaultValues, globalVars), + "scopes": { + "services": data.get("services", {}), + "databases": data.get("databases", {}), + "loaders": data.get("loaders", {}), + }, } return template.render(config) + def main(): global debug_mode global defaultValues - parser = argparse.ArgumentParser(description='Process some configuration file.') - parser.add_argument( - '--config', - type=str, - default='config.yaml', # Set your default config file here - help='Path to the configuration file (default: config.yaml)' - ) + parser = argparse.ArgumentParser(description="Process some configuration file.") parser.add_argument( - '--debug', - action='store_true', - help='Enable debug mode' + "--config", + type=str, + default="config.yaml", # Set your default config file here + help="Path to the configuration file (default: config.yaml)", ) + parser.add_argument("--debug", action="store_true", help="Enable debug mode") parser.add_argument( - '--template', + "--template", type=str, - default='docker-compose.j2', - help='Path to the Jinja2 template file (default: docker-compose.j2)' + default="docker-compose.j2", + help="Path to the Jinja2 template file (default: docker-compose.j2)", ) parser.add_argument( - '--output', + "--output", type=str, - default='docker-compose.yaml', - help='Path to the output file (default: docker-compose.yaml)' + default="docker-compose.yaml", + help="Path to the output file (default: docker-compose.yaml)", ) args = parser.parse_args() config_path = args.config @@ -86,27 +84,29 @@ def main(): yaml_data = read_yaml(config_path) if isinstance(yaml_data, dict): - env = Environment(loader=FileSystemLoader('.')) + env = Environment(loader=FileSystemLoader(".")) - env.filters['singularize'] = plural_to_singular + env.filters["singularize"] = plural_to_singular template = env.get_template(template_path) rendered_content = render_compose_file(template, yaml_data, defaultValues) - - with open(output_path, 'w') as output_file: + + with open(output_path, "w") as output_file: output_file.write(rendered_content) - + else: - raise Exception(f"Failed to read the configuration file {config_file}: The top-level structure is not a dictionary.") + raise Exception( + f"Failed to read the configuration file {config_path}: The top-level structure is not a dictionary." + ) if __name__ == "__main__": - try: + try: main() except Exception as e: if not debug_mode: print(e) else: raise e - exit(1) \ No newline at end of file + exit(1) diff --git a/scripts/generators/docker-compose/tests/test_render_compose_file.py b/scripts/generators/docker-compose/tests/test_render_compose_file.py index 71a5d0f..8a07c43 100644 --- a/scripts/generators/docker-compose/tests/test_render_compose_file.py +++ b/scripts/generators/docker-compose/tests/test_render_compose_file.py @@ -1,12 +1,13 @@ +import pytest from createComposeFile import plural_to_singular, render_compose_file from jinja2 import Environment, FileSystemLoader -import pytest + @pytest.fixture(scope="session") def load_template(): - env = Environment(loader=FileSystemLoader('.')) - env.filters['singularize'] = plural_to_singular - return env.get_template('docker-compose.j2') + env = Environment(loader=FileSystemLoader(".")) + env.filters["singularize"] = plural_to_singular + return env.get_template("docker-compose.j2") def test_render_compose_file_with_empty_config_and_defaults(load_template): @@ -20,28 +21,18 @@ def test_render_compose_file_with_empty_config_and_defaults(load_template): """.strip() assert result == expected + def test_render_compose_file_with_simple_config_and_no_defaults(load_template): - result = render_compose_file(load_template, { - "global": { - "imageNamePrefix": "test/", - "imageVersion": "test" - }, - "services": { - "test": { - "type": "java" - } - }, - "databases": { - "testdb": { - "type": "mysql" - } + result = render_compose_file( + load_template, + { + "global": {"imageNamePrefix": "test/", "imageVersion": "test"}, + "services": {"test": {"type": "java"}}, + "databases": {"testdb": {"type": "mysql"}}, + "loaders": {"testloader": {"type": "curl"}}, }, - "loaders": { - "testloader": { - "type": "curl" - } - } - }, {}) + {}, + ) expected = """ services: ## services @@ -75,27 +66,17 @@ def test_render_compose_file_with_simple_config_and_no_defaults(load_template): """.strip() assert result == expected + def test_render_compose_file_with_simple_config_and_defaults(load_template): - result = render_compose_file(load_template, { - "services": { - "test": { - "type": "java" - } - }, - "databases": { - "testdb": { - "type": "mysql" - } + result = render_compose_file( + load_template, + { + "services": {"test": {"type": "java"}}, + "databases": {"testdb": {"type": "mysql"}}, + "loaders": {"testloader": {"type": "curl"}}, }, - "loaders": { - "testloader": { - "type": "curl" - } - } - }, { - "imageNamePrefix": "test/", - "imageVersion": "test" - }) + {"imageNamePrefix": "test/", "imageVersion": "test"}, + ) expected = """ services: ## services @@ -129,38 +110,27 @@ def test_render_compose_file_with_simple_config_and_defaults(load_template): """.strip() assert result == expected + def test_render_compose_file_with_port_config(load_template): - result = render_compose_file(load_template, { - "global": { - "imageNamePrefix": "test/", - "imageVersion": "test", - "defaultPorts": { - "services": 1024, - "databases": 5432, - "loaders": 1025 + result = render_compose_file( + load_template, + { + "global": { + "imageNamePrefix": "test/", + "imageVersion": "test", + "defaultPorts": {"services": 1024, "databases": 5432, "loaders": 1025}, + "_defaultDefaultPorts": { + "services": 8080, + "databases": 5432, + "loaders": 6000, + }, }, - "_defaultDefaultPorts": { - "services": 8080, - "databases": 5432, - "loaders": 6000 - } + "services": {"test": {"type": "java"}}, + "databases": {"testdb": {"type": "mysql"}}, + "loaders": {"testloader": {"type": "curl"}}, }, - "services": { - "test": { - "type": "java" - } - }, - "databases": { - "testdb": { - "type": "mysql" - } - }, - "loaders": { - "testloader": { - "type": "curl" - } - } - }, {}) + {}, + ) expected = """ services: ## services @@ -198,4 +168,4 @@ def test_render_compose_file_with_port_config(load_template): content: | {"type": "curl"} """.strip() - assert result == expected \ No newline at end of file + assert result == expected diff --git a/scripts/generators/k8s/Dockerfile b/scripts/generators/k8s/Dockerfile index 3769e0d..bd6abbf 100644 --- a/scripts/generators/k8s/Dockerfile +++ b/scripts/generators/k8s/Dockerfile @@ -1,3 +1,4 @@ +#checkov:skip=CKV_DOCKER_2:This is a run once container FROM python:3.13-alpine WORKDIR /app diff --git a/scripts/generators/k8s/README.md b/scripts/generators/k8s/README.md index 074ad99..504217d 100644 --- a/scripts/generators/k8s/README.md +++ b/scripts/generators/k8s/README.md @@ -1,20 +1,23 @@ # Introduction -This project is intended to replace the run.sh and build.sh orginally included in APM game. It creates deployment files to let APM games run on kubernetes. It creates the required deployment, service and a config-map which holds the configuration for every service defined. +This project is intended to replace the run.sh and build.sh originally included in APM game. It creates deployment files to let APM games run on kubernetes. It creates the required deployment, service and a config-map which holds the configuration for every service defined. -It handles the deployment of databases and loaders as well as custom serices. The AppDynamics specific part has been removed from the images and the build process. -As there is no content or syntax check, you need to make sure that the templates render propery and are valid for k8s. The generated yamls should be comlpliant according to yaml specs, but there is no check for validity againt k8s. +It handles the deployment of databases and loaders as well as custom series. The AppDynamics specific part has been removed from the images and the build process. +As there is no content or syntax check, you need to make sure that the templates render properly and are valid for k8s. The generated YAML files should be compliant according to YAML specs, but there is no check for validity against k8s. ## Usage -to run the generator, you'll need to have a working python environment with the depencies outlined in 'requirements.txt' installed. This can be your global environment or a venv. For more information how to create a venv (highly recommended) please see the [official pyhton docs](https://docs.python.org/3/library/venv.html) + +to run the generator, you'll need to have a working python environment with the dependencies outlined in 'requirements.txt' installed. This can be your global environment or a `venv`. For more information how to create a `venv` (highly recommended) please see the [official python docs](https://docs.python.org/3/library/venv.html) ### Create manifests + run the command + ```shell python3 createManifests.py [--config myconfigfile.yaml] [--debug] ``` -If successful, the script will render a number of yaml files for k8s deployment in the ./deployments folder. +If successful, the script will render a number of YAML files for k8s deployment in the ./deployments folder. **Existing files will be overwritten!** ## Info @@ -22,7 +25,8 @@ If successful, the script will render a number of yaml files for k8s deployment ## Template Variables ## Custom Services -You can create your own custom service types, and the script will try to generate k8s yaml files for you. There is absolutely no check on the content other than we're producing valid yaml files. + +You can create your own custom service types, and the script will try to generate k8s YAML files for you. There is absolutely no check on the content other than we're producing valid YAML files. To create a custom service, all you need to do is creating a subfolder in the ./templates directory with the name of your service_type. Adapt the deployments to your need, you can also introduce new template variables, which you can define within the app-simulator config. It is your responsibility to adapt config and templates to your needs. @@ -42,4 +46,5 @@ custom1: this will look for 3 template files in `./templates/custom1' named 'configmap.yaml.j2', 'deployment.yaml.j2', 'service.yaml.j2'. The generator will attempt to render all 3 of them, regardless if they are required or not. Worst case, create an empty valid configuration template. ## Status -draft documentation \ No newline at end of file + +draft documentation diff --git a/scripts/generators/k8s/createManifests.py b/scripts/generators/k8s/createManifests.py index 189dc35..556387d 100644 --- a/scripts/generators/k8s/createManifests.py +++ b/scripts/generators/k8s/createManifests.py @@ -1,43 +1,48 @@ -import yaml -import json -from jinja2 import Template import argparse +import json import os +import yaml +from jinja2 import Template + debug_mode = False + def renderDeployment2(templ_type, config, serviceName): - with open(f"./templates/{templ_type}/deployment.yaml.j2", 'r') as file: + with open(f"./templates/{templ_type}/deployment.yaml.j2", "r") as file: config.update(serviceName=serviceName) - #debug_print(f"Rendering Deployment {serviceName}") - #debug_print(json.dumps(config, indent=2)) + # debug_print(f"Rendering Deployment {serviceName}") + # debug_print(json.dumps(config, indent=2)) template = Template(file.read()) rendered_yaml = yaml.safe_load(template.render(config)) - return rendered_yaml + return rendered_yaml + def renderService2(templ_type, config, serviceName): - with open(f"./templates/{templ_type}/service.yaml.j2", 'r') as file: + with open(f"./templates/{templ_type}/service.yaml.j2", "r") as file: config.update(serviceName=serviceName) - #debug_print(f"Rendering Service {serviceName}") - #debug_print(json.dumps(config, indent=2)) + # debug_print(f"Rendering Service {serviceName}") + # debug_print(json.dumps(config, indent=2)) template = Template(file.read()) rendered_yaml = yaml.safe_load(template.render(config)) - #debug_print(f"Rendered Template:\n{yaml.dump(rendered_yaml)}") - return rendered_yaml + # debug_print(f"Rendered Template:\n{yaml.dump(rendered_yaml)}") + return rendered_yaml + def renderConfigMap2(templ_type, config, serviceName): - with open(f"./templates/{templ_type}/configmap.yaml.j2", 'r') as file: + with open(f"./templates/{templ_type}/configmap.yaml.j2", "r") as file: config.update(serviceName=serviceName) debug_print(f"Rendering ConfigMap {serviceName}") debug_print(json.dumps(config, indent=2)) context = { - 'serviceName': serviceName, - 'serviceConfig': json.dumps(config), + "serviceName": serviceName, + "serviceConfig": json.dumps(config), } template = Template(file.read()) rendered_yaml = yaml.safe_load(template.render(context)) debug_print(f"Rendered Template:\n{yaml.dump(rendered_yaml)}") - return rendered_yaml + return rendered_yaml + def merge_dicts(dict1, dict2): """ @@ -57,11 +62,12 @@ def merge_dicts(dict1, dict2): # Otherwise, overwrite the value with the one from dict2 result[key] = value return result - + + # Function to read a YAML file def read_yaml(file_path): try: - with open(os.path.expanduser(file_path), 'r') as file: + with open(os.path.expanduser(file_path), "r") as file: # Parse the YAML file data = yaml.safe_load(file) return data @@ -70,125 +76,186 @@ def read_yaml(file_path): except yaml.YAMLError as exc: debug_print(f"Error parsing YAML file: {exc}") + def write_yaml(file_path, data): try: - with open(file_path, 'w') as file: + with open(file_path, "w") as file: yaml.safe_dump(data, file, default_flow_style=False) except Exception as e: debug_print(f"An error occurred while writing to the file: {e}") + def debug_print(*args, **kwargs): if debug_mode: print(*args, **kwargs) + # Example usage def main(): - global debug_mode - parser = argparse.ArgumentParser(description='Process some configuration file.') - parser.add_argument( - '--config', - type=str, - default='config.yaml', # Set your default config file here - help='Path to the configuration file (default: config.yaml)' - ) + global debug_mode + parser = argparse.ArgumentParser(description="Process some configuration file.") parser.add_argument( - '--debug', - action='store_true', - help='Enable debug mode' + "--config", + type=str, + default="config.yaml", # Set your default config file here + help="Path to the configuration file (default: config.yaml)", ) + parser.add_argument("--debug", action="store_true", help="Enable debug mode") args = parser.parse_args() config_file = args.config debug_mode = args.debug - debug_print(f'Using configuration file: {config_file}') + debug_print(f"Using configuration file: {config_file}") - #file_path = 'config.yaml' # Replace with your YAML file path + # file_path = 'config.yaml' # Replace with your YAML file path yaml_data = read_yaml(config_file) - application_services = {'java', 'dotnetcore', 'nodejs'} # adding php, pyhton - db_services = {'mysql'} # adding mongo - loader_services = {'curl'} + application_services = {"java", "dotnetcore", "nodejs"} # adding php, pyhton + db_services = {"mysql"} # adding mongo + loader_services = {"curl"} if isinstance(yaml_data, dict): -# merge the global and k8s configs from simpler access -# some ugly code hacked togeter, so that it works for MVP -# TODO: clean up code + # merge the global and k8s configs from simpler access + # some ugly code hacked togeter, so that it works for MVP + # TODO: clean up code globalConfig = yaml_data.get("global", {}) - keys_to_keep = { "appName","imageNamePrefix","imageVersion","k8s"} - gconfig = {key: globalConfig[key] for key in keys_to_keep if key in globalConfig} - debug_print(f"gconfig: gconfig") + keys_to_keep = {"appName", "imageNamePrefix", "imageVersion", "k8s"} + gconfig = { + key: globalConfig[key] for key in keys_to_keep if key in globalConfig + } + debug_print("gconfig: gconfig") globalConfigK8s = gconfig.get("k8s", {}) debug_print("k8sconfig: {globalConfigK8s}") gconfig.pop("k8s", None) globalConfig = merge_dicts(gconfig, globalConfigK8s) yaml_data.pop("global", None) - debug_print(f"Global Config:\n", json.dumps(globalConfig, indent=2)) -# Ok, lets start to parse through the services and render the k8s deployments + debug_print("Global Config:\n", json.dumps(globalConfig, indent=2)) + # Ok, lets start to parse through the services and render the k8s deployments for key, value in yaml_data.items(): if key == "services": - #debug_print(f"Value: {value}") + # debug_print(f"Value: {value}") for service, config in value.items(): - config['agent'] = False - if config['type'] in application_services: - config=merge_dicts(globalConfig, config) - print(f"create Appplication Service of type {config['type']} named {service}") - write_yaml(f"./deployments/{service}-configmap.yaml",renderConfigMap2(key, config, service)) - write_yaml(f"./deployments/{service}-deployment.yaml",renderDeployment2(key,config, service)) - if config.get('exposedPort', None) != None: + config["agent"] = False + if config["type"] in application_services: + config = merge_dicts(globalConfig, config) + print( + f"create Appplication Service of type {config['type']} named {service}" + ) + write_yaml( + f"./deployments/{service}-configmap.yaml", + renderConfigMap2(key, config, service), + ) + write_yaml( + f"./deployments/{service}-deployment.yaml", + renderDeployment2(key, config, service), + ) + if config.get("exposedPort", None) != None: debug_print(f"ExposedPort: {config.get('exposedPort')}") - write_yaml(f"./deployments/{service}-service-ext.yaml",renderService2(key, config, service)) + write_yaml( + f"./deployments/{service}-service-ext.yaml", + renderService2(key, config, service), + ) config2 = config - config2.pop('exposedPort', None) + config2.pop("exposedPort", None) # if port is set, we do need two service, one of type cluserIP and one of type Loadbalancer. As I'm unable to render a yaml template with 2 documents I need to workaround - write_yaml(f"./deployments/{service}-service.yaml",renderService2(key, config, service)) + write_yaml( + f"./deployments/{service}-service.yaml", + renderService2(key, config, service), + ) else: - write_yaml(f"./deployments/{service}-service.yaml",renderService2(key, config, service)) + write_yaml( + f"./deployments/{service}-service.yaml", + renderService2(key, config, service), + ) else: - print(f"Unsupported service type detected {config['type']} named {service}") + print( + f"Unsupported service type detected {config['type']} named {service}" + ) elif key == "loaders": for loader, config in value.items(): - if config['type'] in loader_services: - print (f"create loader service of type {config['type']} named {loader}") - context=merge_dicts(globalConfig, config) - context['serviceName'] = loader - context['type'] = config['type'] - context['urls'] = " ".join(config['urls']) + if config["type"] in loader_services: + print( + f"create loader service of type {config['type']} named {loader}" + ) + context = merge_dicts(globalConfig, config) + context["serviceName"] = loader + context["type"] = config["type"] + context["urls"] = " ".join(config["urls"]) print(f"Context to be passed: {context}") - write_yaml(f"./deployments/{loader}-configmap.yaml",renderConfigMap2(key, config, loader)) - write_yaml(f"./deployments/{loader}-deployment.yaml",renderDeployment2(key, context, loader)) + write_yaml( + f"./deployments/{loader}-configmap.yaml", + renderConfigMap2(key, config, loader), + ) + write_yaml( + f"./deployments/{loader}-deployment.yaml", + renderDeployment2(key, context, loader), + ) # There is no need to create a service for a loadgenerator at this point in time else: - print(f"Unsupported loader type detected {config['type']} named {loader}") + print( + f"Unsupported loader type detected {config['type']} named {loader}" + ) elif key == "databases": for service, config in value.items(): - config['agent'] = False - if config['type'] in db_services: - config=merge_dicts(globalConfig, config) - print(f"create DB Service of type {config['type']} named {service}") - write_yaml(f"./deployments/{service}-configmap.yaml",renderConfigMap2(key, config, service)) - write_yaml(f"./deployments/{service}-deployment.yaml",renderDeployment2(key,config, service)) - if config.get('exposedPort', None) != None: - write_yaml(f"./deployments/{service}-service-ext.yaml",renderService2(key, config, service)) + config["agent"] = False + if config["type"] in db_services: + config = merge_dicts(globalConfig, config) + print( + f"create DB Service of type {config['type']} named {service}" + ) + write_yaml( + f"./deployments/{service}-configmap.yaml", + renderConfigMap2(key, config, service), + ) + write_yaml( + f"./deployments/{service}-deployment.yaml", + renderDeployment2(key, config, service), + ) + if config.get("exposedPort", None) != None: + write_yaml( + f"./deployments/{service}-service-ext.yaml", + renderService2(key, config, service), + ) config2 = config - config2.pop('exposedPort', None) - write_yaml(f"./deployments/{service}-service.yaml",renderService2(key, config, service)) + config2.pop("exposedPort", None) + write_yaml( + f"./deployments/{service}-service.yaml", + renderService2(key, config, service), + ) else: - write_yaml(f"./deployments/{service}-service.yaml",renderService2(key, config, service)) + write_yaml( + f"./deployments/{service}-service.yaml", + renderService2(key, config, service), + ) else: - print(f"Unsupported service type detected {config['type']} named {service}") + print( + f"Unsupported service type detected {config['type']} named {service}" + ) else: -# -#TODO: works but is totally unsupported -# - print(f"found unsupported key {key} in config - try rendering deployment, service and configmap") - try : + # + # TODO: works but is totally unsupported + # + print( + f"found unsupported key {key} in config - try rendering deployment, service and configmap" + ) + try: for service, config in value.items(): - config=merge_dicts(globalConfig, config) - write_yaml(f"./deployments/{service}-configmap.yaml",renderConfigMap2(key, config, service)) - write_yaml(f"./deployments/{service}-deployment.yaml",renderDeployment2(key,config, service)) - write_yaml(f"./deployments/{service}-service-ext.yaml",renderService2(key, config, service)) + config = merge_dicts(globalConfig, config) + write_yaml( + f"./deployments/{service}-configmap.yaml", + renderConfigMap2(key, config, service), + ) + write_yaml( + f"./deployments/{service}-deployment.yaml", + renderDeployment2(key, config, service), + ) + write_yaml( + f"./deployments/{service}-service-ext.yaml", + renderService2(key, config, service), + ) except Exception as e: print(f"An error occured, skipping: {e}") else: print("The top-level structure is not a dictionary.") -if __name__ == '__main__': - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/src/services/nodejs/entrypoint.sh b/src/services/nodejs/entrypoint.sh index 08a1524..859051f 100644 --- a/src/services/nodejs/entrypoint.sh +++ b/src/services/nodejs/entrypoint.sh @@ -1,4 +1,4 @@ #!/bin/bash SERVICE_DEFAULT_PORT=${SERVICE_DEFAULT_PORT:-8080} echo "Running server on :${SERVICE_DEFAULT_PORT}" -APP_CONFIG="$(