From 015eb2bda8579c86c97c1596d689fa9045ebb19e Mon Sep 17 00:00:00 2001
From: jmintb <jessie@teainspace.com>
Date: Wed, 2 Aug 2023 14:26:52 +0200
Subject: [PATCH] Migrate to new Task and controller architecture

Issue: #136

Before the inital release we are consolidting the controller and crd
architectures.

This commit also polishes the CLI.
---
 Cargo.lock                                    | 1095 +++++-----
 Cargo.toml                                    |    2 +-
 Dockerfile.controller                         |    2 +-
 Dockerfile.server                             |    2 +-
 README.md                                     |    2 +-
 {docs => ame/docs}/datasets.md                |    0
 ame/docs/index.md                             |   17 +
 ame/docs/model_validation.md                  |  172 ++
 ame/docs/models.html                          |   30 +
 ame/docs/models.md                            |   42 +
 {docs => ame/docs}/project_sources.md         |    4 +-
 ame/docs/tasks.md                             |  681 +++++++
 ame/mkdocs.yml                                |   32 +
 cli/Cargo.toml                                |   62 +-
 cli/src/lib.rs                                |   12 +-
 cli/src/main.rs                               |   41 +-
 cli/src/project.rs                            |  254 ++-
 cli/src/project_cmd.rs                        |  113 +
 cli/src/projectsrc.rs                         |   13 +-
 cli/src/secrets.rs                            |   12 +-
 cli/src/task.rs                               |  458 +++++
 cli/tests/cli.rs                              |  382 ++--
 cli/tests/snapshots/cli__ame_run_task-2.snap  |    5 +-
 cli/tests/snapshots/cli__ame_run_task.snap    |   25 +-
 .../cli__invalid_projecttasklogs.snap         |   11 +
 .../cli__invalid_projecttaskrun.snap          |    9 +
 .../cli__missing_projecttaskrun.snap          |    9 +
 .../snapshots/cli__snap_shot_task_view.snap   |   13 +
 ...__snap_shot_test_cli_error_messages-2.snap |    9 +
 ...__snap_shot_test_cli_error_messages-3.snap |   11 +
 ...li__snap_shot_test_cli_error_messages.snap |    9 +
 controller/Cargo.toml                         |   30 +-
 controller/src/crdgen.rs                      |    2 +-
 controller/src/data_set.rs                    |   98 +-
 controller/src/lib.rs                         |    2 +
 controller/src/main.rs                        |   49 +-
 controller/src/project.rs                     |  525 +++++
 ...can_create_workflow_and_finalize_task.snap |  114 ++
 controller/src/task.rs                        |  443 ++++
 docs/model_validation.md                      |  115 --
 executor/Dockerfile                           |   16 +-
 justfile                                      |   35 +-
 lib/Cargo.toml                                |   70 +-
 lib/ame.proto                                 |  190 +-
 lib/build.rs                                  |   58 +-
 lib/src/ctrl.rs                               |   41 +-
 lib/src/custom_resources/argo.rs              |  256 ++-
 lib/src/custom_resources/common.rs            |  200 +-
 lib/src/custom_resources/data_set.rs          |   68 +-
 lib/src/custom_resources/mod.rs               |  125 +-
 lib/src/custom_resources/new_task.rs          |  507 +++++
 lib/src/custom_resources/project.rs           |  940 +++------
 lib/src/custom_resources/project_source.rs    |  157 +-
 .../custom_resources/project_source_ctrl.rs   |   22 +-
 lib/src/custom_resources/secrets.rs           |   28 +-
 ...__new_task__test__snap_shot_task_yaml.snap |   33 +
 ...w_task__test__snap_shot_workflow_yaml.snap |  190 ++
 ...ject__test__produces_valid_deployment.snap |    4 +-
 ...project__test__produces_valid_ingress.snap |   11 +-
 ...t__produces_valid_model_training_task.snap |   30 +
 ...t_projects_from_public_git_repository.snap |   10 +-
 lib/src/custom_resources/task.rs              | 1465 -------------
 lib/src/custom_resources/task_ctrl.rs         |  118 ++
 lib/src/error.rs                              |   48 +
 lib/src/lib.rs                                |  143 +-
 lib/src/project.rs                            |   32 +
 lib/tests/e2e_test.rs                         |  555 +++++
 .../snapshots/e2e_test__ame_run_task.snap     |   38 +
 .../e2e_test__can_list_project_srcs.snap      |    8 +
 .../controller/base/resources/controller.yaml |    4 +-
 manifests/crd.yaml                            |  358 ++--
 manifests/data_set_crd.yaml                   |  140 +-
 manifests/project_crd.yaml                    | 1813 +++++------------
 manifests/project_src_crd.yaml                |    8 +-
 rustfmt.toml                                  |    1 +
 service/Cargo.toml                            |   21 +-
 service/src/ameservice.rs                     |  350 +++-
 service/src/lib.rs                            |   27 +-
 service/src/server.rs                         |  218 +-
 shell.nix                                     |    4 +-
 test_data/test_projects/env/ame.yaml          |   32 +-
 .../test_projects/executors/custom/ame.yaml   |   10 +
 .../test_projects/executors/custom/nn.py      |    1 +
 .../test_projects/executors/mlflow/MLProject  |    8 +
 .../test_projects/executors/mlflow/Pipfile    |   12 +
 .../executors/mlflow/Pipfile.lock             |   52 +
 .../test_projects/executors/mlflow/ame.yaml   |   25 +
 .../test_projects/executors/mlflow/nn.py      |    3 +
 .../executors/mlflow/python_env.yaml          |    9 +
 .../test_projects/executors/pip/ame.yaml      |   10 +
 test_data/test_projects/executors/pip/nn.py   |    3 +
 .../executors/pip/requirements.txt            |    2 +
 .../test_projects/executors/pipenv/Pipfile    |   12 +
 .../executors/pipenv/Pipfile.lock             |   52 +
 .../test_projects/executors/pipenv/ame.yaml   |    9 +
 .../test_projects/executors/pipenv/nn.py      |    3 +
 .../test_projects/executors/poetry/ame.yaml   |   17 +
 .../test_projects/executors/poetry/nn.py      |    3 +
 .../executors/poetry/poetry.lock              |   44 +
 .../executors/poetry/pyproject.toml           |   15 +
 .../test_projects/invalid_project/ame.yaml    |   17 +
 test_data/test_projects/invalid_project/nn.py |    3 +
 .../test_projects/invalid_project/poetry.lock |   44 +
 .../invalid_project/pyproject.toml            |   15 +
 test_data/test_projects/missing_project/nn.py |    3 +
 .../test_projects/missing_project/poetry.lock |   44 +
 .../missing_project/pyproject.toml            |   15 +
 test_data/test_projects/new_echo/ame.yaml     |    5 +-
 .../test_projects/pipenvnn/forreadme2.cast    |  309 ---
 test_data/test_projects/pipenvnn/nn.cast      |  218 --
 test_data/test_projects/poetry/nn.py          |   96 +-
 .../sklearn_logistic_regression/ame.yaml      |    3 +-
 .../mlruns/0/meta.yaml                        |    4 +
 web/src/app.rs                                |   23 +-
 web/src/main.rs                               |    3 +-
 115 files changed, 8392 insertions(+), 5981 deletions(-)
 rename {docs => ame/docs}/datasets.md (100%)
 create mode 100644 ame/docs/index.md
 create mode 100644 ame/docs/model_validation.md
 create mode 100644 ame/docs/models.html
 create mode 100644 ame/docs/models.md
 rename {docs => ame/docs}/project_sources.md (94%)
 create mode 100644 ame/docs/tasks.md
 create mode 100644 ame/mkdocs.yml
 create mode 100644 cli/src/project_cmd.rs
 create mode 100644 cli/src/task.rs
 create mode 100644 cli/tests/snapshots/cli__invalid_projecttasklogs.snap
 create mode 100644 cli/tests/snapshots/cli__invalid_projecttaskrun.snap
 create mode 100644 cli/tests/snapshots/cli__missing_projecttaskrun.snap
 create mode 100644 cli/tests/snapshots/cli__snap_shot_task_view.snap
 create mode 100644 cli/tests/snapshots/cli__snap_shot_test_cli_error_messages-2.snap
 create mode 100644 cli/tests/snapshots/cli__snap_shot_test_cli_error_messages-3.snap
 create mode 100644 cli/tests/snapshots/cli__snap_shot_test_cli_error_messages.snap
 create mode 100644 controller/src/project.rs
 create mode 100644 controller/src/snapshots/controller__task__test__can_create_workflow_and_finalize_task.snap
 create mode 100644 controller/src/task.rs
 delete mode 100644 docs/model_validation.md
 create mode 100644 lib/src/custom_resources/new_task.rs
 create mode 100644 lib/src/custom_resources/snapshots/ame__custom_resources__new_task__test__snap_shot_task_yaml.snap
 create mode 100644 lib/src/custom_resources/snapshots/ame__custom_resources__new_task__test__snap_shot_workflow_yaml.snap
 create mode 100644 lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_model_training_task.snap
 delete mode 100644 lib/src/custom_resources/task.rs
 create mode 100644 lib/src/custom_resources/task_ctrl.rs
 create mode 100644 lib/src/project.rs
 create mode 100644 lib/tests/e2e_test.rs
 create mode 100644 lib/tests/snapshots/e2e_test__ame_run_task.snap
 create mode 100644 lib/tests/snapshots/e2e_test__can_list_project_srcs.snap
 create mode 100644 rustfmt.toml
 create mode 100755 test_data/test_projects/executors/custom/ame.yaml
 create mode 100644 test_data/test_projects/executors/custom/nn.py
 create mode 100644 test_data/test_projects/executors/mlflow/MLProject
 create mode 100644 test_data/test_projects/executors/mlflow/Pipfile
 create mode 100644 test_data/test_projects/executors/mlflow/Pipfile.lock
 create mode 100755 test_data/test_projects/executors/mlflow/ame.yaml
 create mode 100644 test_data/test_projects/executors/mlflow/nn.py
 create mode 100644 test_data/test_projects/executors/mlflow/python_env.yaml
 create mode 100755 test_data/test_projects/executors/pip/ame.yaml
 create mode 100644 test_data/test_projects/executors/pip/nn.py
 create mode 100644 test_data/test_projects/executors/pip/requirements.txt
 create mode 100644 test_data/test_projects/executors/pipenv/Pipfile
 create mode 100644 test_data/test_projects/executors/pipenv/Pipfile.lock
 create mode 100755 test_data/test_projects/executors/pipenv/ame.yaml
 create mode 100644 test_data/test_projects/executors/pipenv/nn.py
 create mode 100755 test_data/test_projects/executors/poetry/ame.yaml
 create mode 100644 test_data/test_projects/executors/poetry/nn.py
 create mode 100644 test_data/test_projects/executors/poetry/poetry.lock
 create mode 100644 test_data/test_projects/executors/poetry/pyproject.toml
 create mode 100755 test_data/test_projects/invalid_project/ame.yaml
 create mode 100644 test_data/test_projects/invalid_project/nn.py
 create mode 100644 test_data/test_projects/invalid_project/poetry.lock
 create mode 100644 test_data/test_projects/invalid_project/pyproject.toml
 create mode 100644 test_data/test_projects/missing_project/nn.py
 create mode 100644 test_data/test_projects/missing_project/poetry.lock
 create mode 100644 test_data/test_projects/missing_project/pyproject.toml
 delete mode 100644 test_data/test_projects/pipenvnn/forreadme2.cast
 delete mode 100644 test_data/test_projects/pipenvnn/nn.cast
 create mode 100644 test_data/test_projects/sklearn_logistic_regression/mlruns/0/meta.yaml

diff --git a/Cargo.lock b/Cargo.lock
index 8d919962..ca7aebb1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "addr2line"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
+dependencies = [
+ "gimli",
+]
+
 [[package]]
 name = "adler"
 version = "1.0.2"
@@ -33,18 +42,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "aho-corasick"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04"
+checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
 dependencies = [
  "memchr",
 ]
@@ -68,13 +68,19 @@ dependencies = [
 name = "ame"
 version = "0.1.0"
 dependencies = [
+ "ame",
+ "anyhow",
+ "assert_cmd",
  "assert_fs",
  "async-stream",
  "async-trait",
  "cfg-if",
+ "chrono",
+ "cron-parser",
  "duration-string",
  "either",
  "envconfig",
+ "fs_extra",
  "futures 0.3.28",
  "futures-util",
  "git2",
@@ -82,7 +88,7 @@ dependencies = [
  "http-body",
  "humantime",
  "hyper",
- "hyper-rustls",
+ "hyper-rustls 0.23.2",
  "insta",
  "k8s-openapi",
  "kube",
@@ -92,30 +98,40 @@ dependencies = [
  "openidconnect",
  "predicates 2.1.5",
  "prost",
+ "prost-build",
+ "rand",
  "reqwest",
+ "rstest",
  "rustls-native-certs",
  "schemars",
  "serde",
  "serde_json",
  "serde_merge",
  "serde_tuple",
- "serde_yaml 0.9.21",
- "serial_test",
+ "serde_yaml 0.9.25",
+ "serial_test 2.0.0",
  "similar",
- "similar-asserts",
  "thiserror",
+ "time 0.3.24",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.23.4",
  "tokio-stream",
  "tonic",
  "tonic-build",
  "tonic-web-wasm-client",
  "tower",
- "tower-http 0.4.0",
+ "tower-http 0.4.3",
  "tracing",
+ "tracing-subscriber",
  "url",
 ]
 
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -127,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.3.0"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371"
+checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -142,15 +158,15 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d"
+checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee"
+checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
 dependencies = [
  "utf8parse",
 ]
@@ -166,9 +182,9 @@ dependencies = [
 
 [[package]]
 name = "anstyle-wincon"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd"
+checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
 dependencies = [
  "anstyle",
  "windows-sys 0.48.0",
@@ -176,18 +192,18 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.70"
+version = "1.0.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4"
+checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
 
 [[package]]
 name = "assert_cmd"
-version = "2.0.11"
+version = "2.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86d6b683edf8d1119fe420a94f8a7e389239666aa72e65495d91c00462510151"
+checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6"
 dependencies = [
  "anstyle",
- "bstr 1.4.0",
+ "bstr 1.6.0",
  "doc-comment",
  "predicates 3.0.3",
  "predicates-core",
@@ -232,7 +248,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -254,18 +270,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "async-trait"
-version = "0.1.68"
+version = "0.1.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
+checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -341,7 +357,7 @@ dependencies = [
  "serde",
  "serde-xml-rs",
  "thiserror",
- "time",
+ "time 0.3.24",
  "url",
 ]
 
@@ -356,14 +372,14 @@ dependencies = [
 
 [[package]]
 name = "axum"
-version = "0.6.16"
+version = "0.6.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "113713495a32dd0ab52baf5c10044725aa3aec00b31beda84218e469029b72a3"
+checksum = "a6a1de45611fdb535bfde7b7de4fd54f4fd2b17b1737c0a59b69bf9b92074b8c"
 dependencies = [
  "async-trait",
  "axum-core",
  "axum-macros",
- "bitflags",
+ "bitflags 1.3.2",
  "bytes 1.4.0",
  "futures-util",
  "http",
@@ -406,14 +422,14 @@ dependencies = [
 
 [[package]]
 name = "axum-macros"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bb524613be645939e280b7279f7b017f98cf7f5ef084ec374df373530e73277"
+checksum = "cdca6a10ecad987bda04e95606ef85a5417dcaac1a78455242d72e031e2b6b62"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -427,6 +443,21 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "backtrace"
+version = "0.3.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -441,9 +472,9 @@ checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
 
 [[package]]
 name = "base64"
-version = "0.21.0"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
+checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
 
 [[package]]
 name = "bitflags"
@@ -451,6 +482,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -489,26 +526,25 @@ checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
 dependencies = [
  "lazy_static",
  "memchr",
- "regex-automata",
+ "regex-automata 0.1.10",
 ]
 
 [[package]]
 name = "bstr"
-version = "1.4.0"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09"
+checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05"
 dependencies = [
  "memchr",
- "once_cell",
- "regex-automata",
+ "regex-automata 0.3.4",
  "serde",
 ]
 
 [[package]]
 name = "bumpalo"
-version = "3.12.0"
+version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
+checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
 
 [[package]]
 name = "byteorder"
@@ -534,9 +570,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
 
 [[package]]
 name = "camino"
-version = "1.1.4"
+version = "1.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c530edf18f37068ac2d977409ed5cd50d53d73bc653c7647b48eb78976ac9ae2"
+checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
 
 [[package]]
 name = "cc"
@@ -555,22 +591,25 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.24"
+version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b"
+checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
 dependencies = [
+ "android-tzdata",
  "iana-time-zone",
- "num-integer",
+ "js-sys",
  "num-traits",
  "serde",
+ "time 0.1.45",
+ "wasm-bindgen",
  "winapi",
 ]
 
 [[package]]
 name = "ciborium"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
+checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926"
 dependencies = [
  "ciborium-io",
  "ciborium-ll",
@@ -579,15 +618,15 @@ dependencies = [
 
 [[package]]
 name = "ciborium-io"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
+checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656"
 
 [[package]]
 name = "ciborium-ll"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
+checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b"
 dependencies = [
  "ciborium-io",
  "half",
@@ -595,9 +634,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.2.4"
+version = "4.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62"
+checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -606,43 +645,44 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.2.4"
+version = "4.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"
+checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1"
 dependencies = [
  "anstream",
  "anstyle",
- "bitflags",
  "clap_lex",
  "strsim",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.2.0"
+version = "4.3.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4"
+checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.4.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1"
+checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
 
 [[package]]
 name = "cli"
 version = "0.1.0"
 dependencies = [
  "ame",
+ "anyhow",
  "assert_cmd",
  "assert_fs",
  "async-stream",
+ "atty",
  "bytes 1.4.0",
  "clap",
  "colored",
@@ -655,7 +695,7 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls",
+ "hyper-rustls 0.23.2",
  "insta",
  "k8s-openapi",
  "kube",
@@ -667,14 +707,14 @@ dependencies = [
  "rustls-native-certs",
  "serde",
  "serde_json",
- "serde_yaml 0.9.21",
- "serial_test",
+ "serde_yaml 0.9.25",
+ "serial_test 0.9.0",
  "similar-asserts",
  "spinners",
  "temp-env",
  "thiserror",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.23.4",
  "tonic",
  "tower",
  "tower-http 0.3.5",
@@ -683,16 +723,6 @@ dependencies = [
  "walkdir",
 ]
 
-[[package]]
-name = "codespan-reporting"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
-dependencies = [
- "termcolor",
- "unicode-width",
-]
-
 [[package]]
 name = "collection_literals"
 version = "1.0.1"
@@ -707,13 +737,13 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
 [[package]]
 name = "colored"
-version = "2.0.0"
+version = "2.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
+checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6"
 dependencies = [
- "atty",
+ "is-terminal",
  "lazy_static",
- "winapi",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -755,15 +785,15 @@ dependencies = [
 
 [[package]]
 name = "console"
-version = "0.15.5"
+version = "0.15.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60"
+checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"
 dependencies = [
  "encode_unicode",
  "lazy_static",
  "libc",
  "unicode-width",
- "windows-sys 0.42.0",
+ "windows-sys 0.45.0",
 ]
 
 [[package]]
@@ -788,18 +818,18 @@ dependencies = [
 
 [[package]]
 name = "const_format"
-version = "0.2.30"
+version = "0.2.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7309d9b4d3d2c0641e018d449232f2e28f1b22933c137f157d3dbc14228b8c0e"
+checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48"
 dependencies = [
  "const_format_proc_macros",
 ]
 
 [[package]]
 name = "const_format_proc_macros"
-version = "0.2.29"
+version = "0.2.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650"
+checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -812,19 +842,23 @@ version = "0.1.0"
 dependencies = [
  "ame",
  "async-trait",
+ "chrono",
+ "cron-parser",
  "duration-string",
  "either",
  "envconfig",
  "futures 0.3.28",
  "git2",
  "humantime",
+ "insta",
  "k8s-openapi",
  "kube",
  "reqwest",
  "schemars",
  "serde",
  "serde_json",
- "serde_yaml 0.9.21",
+ "serde_merge",
+ "serde_yaml 0.9.25",
  "thiserror",
  "tokio",
  "tonic",
@@ -859,9 +893,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.7"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58"
+checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
 dependencies = [
  "libc",
 ]
@@ -876,57 +910,22 @@ dependencies = [
 ]
 
 [[package]]
-name = "crypto-common"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
-[[package]]
-name = "cxx"
-version = "1.0.94"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93"
-dependencies = [
- "cc",
- "cxxbridge-flags",
- "cxxbridge-macro",
- "link-cplusplus",
-]
-
-[[package]]
-name = "cxx-build"
-version = "1.0.94"
+name = "cron-parser"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b"
+checksum = "5894f6023d8f0125d49d24a10f07dacf1ae6838ff1878ac50d0ff0db17c82a83"
 dependencies = [
- "cc",
- "codespan-reporting",
- "once_cell",
- "proc-macro2",
- "quote",
- "scratch",
- "syn 2.0.15",
+ "chrono",
 ]
 
 [[package]]
-name = "cxxbridge-flags"
-version = "1.0.94"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb"
-
-[[package]]
-name = "cxxbridge-macro"
-version = "1.0.94"
+name = "crypto-common"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.15",
+ "generic-array",
+ "typenum",
 ]
 
 [[package]]
@@ -1001,17 +1000,26 @@ dependencies = [
 
 [[package]]
 name = "dashmap"
-version = "5.4.0"
+version = "5.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
+checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d"
 dependencies = [
  "cfg-if",
- "hashbrown",
+ "hashbrown 0.14.0",
  "lock_api",
  "once_cell",
  "parking_lot_core",
 ]
 
+[[package]]
+name = "deranged"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8810e7e2cf385b1e9b50d68264908ec367ba642c96d02edfe61c39e88e2a3c01"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "derivative"
 version = "2.2.0"
@@ -1031,7 +1039,7 @@ checksum = "5bc1955a640c4464859ae700fbe48e666da6fdce99ce5fe1acd08dd295889d10"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -1041,6 +1049,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "59c6f2989294b9a498d3ad5491a79c6deb604617378e1cdc4bfc1c1361fe2f87"
 dependencies = [
  "console",
+ "fuzzy-matcher",
  "shell-words",
  "tempfile",
  "zeroize",
@@ -1054,9 +1063,9 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
 
 [[package]]
 name = "digest"
-version = "0.10.6"
+version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer",
  "crypto-common",
@@ -1139,15 +1148,15 @@ checksum = "c5d74010fd35f37054de1eb72485be4f562b0815e0e7b477e578581d3998fa47"
 
 [[package]]
 name = "dyn-clone"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30"
+checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272"
 
 [[package]]
 name = "educe"
-version = "0.4.21"
+version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4af7804abe0786a9b69375115821fedc9995f21ab63ae285184b96b01ec50b1a"
+checksum = "079044df30bb07de7d846d41a184c4b00e66ebdac93ee459253474f3a47e50ae"
 dependencies = [
  "enum-ordinalize",
  "proc-macro2",
@@ -1157,9 +1166,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
 [[package]]
 name = "encode_unicode"
@@ -1178,16 +1187,15 @@ dependencies = [
 
 [[package]]
 name = "enum-ordinalize"
-version = "3.1.12"
+version = "3.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a62bb1df8b45ecb7ffa78dca1c17a438fb193eb083db0b1b494d2a61bcb5096a"
+checksum = "e4f76552f53cefc9a7f64987c3701b99d982f7690606fd67de1d09712fbf52f1"
 dependencies = [
  "num-bigint",
  "num-traits",
  "proc-macro2",
  "quote",
- "rustc_version",
- "syn 1.0.109",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -1210,11 +1218,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "errno"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
 dependencies = [
  "errno-dragonfly",
  "libc",
@@ -1233,12 +1247,9 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "1.9.0"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
-dependencies = [
- "instant",
-]
+checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
 
 [[package]]
 name = "fixedbitset"
@@ -1248,9 +1259,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
 [[package]]
 name = "flate2"
-version = "1.0.25"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
+checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -1288,9 +1299,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
 [[package]]
 name = "form_urlencoded"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
+checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"
 dependencies = [
  "percent-encoding",
 ]
@@ -1384,7 +1395,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -1423,6 +1434,15 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "fuzzy-matcher"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94"
+dependencies = [
+ "thread_local",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -1435,24 +1455,30 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
 dependencies = [
  "cfg-if",
  "js-sys",
  "libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.27.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
+
 [[package]]
 name = "git2"
-version = "0.17.1"
+version = "0.17.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b7905cdfe33d31a88bb2e8419ddd054451f5432d1da9eaf2ac7804ee1ea12d5"
+checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "libc",
  "libgit2-sys",
  "log",
@@ -1463,12 +1489,12 @@ dependencies = [
 
 [[package]]
 name = "globset"
-version = "0.4.10"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc"
+checksum = "aca8bbd8e0707c1887a8bbb7e6b40e228f251ff5d62c8220a4a7a53c73aff006"
 dependencies = [
- "aho-corasick 0.7.20",
- "bstr 1.4.0",
+ "aho-corasick",
+ "bstr 1.6.0",
  "fnv",
  "log",
  "regex",
@@ -1480,7 +1506,7 @@ version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "ignore",
  "walkdir",
 ]
@@ -1507,9 +1533,9 @@ dependencies = [
 
 [[package]]
 name = "gloo-utils"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8e8fc851e9c7b9852508bc6e3f690f452f474417e8545ec9857b7f7377036b5"
+checksum = "037fcb07216cb3a30f7292bd0176b050b7b9a052ba830ef7d5d65f6dc64ba58e"
 dependencies = [
  "js-sys",
  "serde",
@@ -1520,9 +1546,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.3.18"
+version = "0.3.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21"
+checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049"
 dependencies = [
  "bytes 1.4.0",
  "fnv",
@@ -1530,7 +1556,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http",
- "indexmap",
+ "indexmap 1.9.3",
  "slab",
  "tokio",
  "tokio-util",
@@ -1552,6 +1578,12 @@ dependencies = [
  "ahash 0.7.6",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+
 [[package]]
 name = "heck"
 version = "0.4.1"
@@ -1569,18 +1601,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "hermit-abi"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
 
 [[package]]
 name = "hex"
@@ -1630,9 +1653,9 @@ dependencies = [
 
 [[package]]
 name = "http-range-header"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"
+checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
 
 [[package]]
 name = "httparse"
@@ -1654,9 +1677,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hyper"
-version = "0.14.26"
+version = "0.14.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4"
+checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
 dependencies = [
  "bytes 1.4.0",
  "futures-channel",
@@ -1703,10 +1726,24 @@ dependencies = [
  "http",
  "hyper",
  "log",
- "rustls",
+ "rustls 0.20.8",
  "rustls-native-certs",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.23.4",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97"
+dependencies = [
+ "futures-util",
+ "http",
+ "hyper",
+ "rustls 0.21.5",
+ "tokio",
+ "tokio-rustls 0.24.1",
 ]
 
 [[package]]
@@ -1736,9 +1773,9 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.56"
+version = "0.1.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c"
+checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
@@ -1750,12 +1787,11 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone-haiku"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
 dependencies = [
- "cxx",
- "cxx-build",
+ "cc",
 ]
 
 [[package]]
@@ -1766,9 +1802,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "idna"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
+checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c"
 dependencies = [
  "unicode-bidi",
  "unicode-normalization",
@@ -1798,14 +1834,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
 dependencies = [
  "autocfg",
- "hashbrown",
+ "hashbrown 0.12.3",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
 ]
 
 [[package]]
 name = "insta"
-version = "1.29.0"
+version = "1.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a28d25139df397cbca21408bb742cf6837e04cdbebf1b07b760caf971d6a972"
+checksum = "a0770b0a3d4c70567f0d58331f3088b0e4c4f56c9b8d764efe654b4a5d46de3a"
 dependencies = [
  "console",
  "lazy_static",
@@ -1833,17 +1879,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1482d8bbb520daf94c82af87f38cd27cdb3073c6fee7c5805fd2fa9d3a36d494"
 
-[[package]]
-name = "io-lifetimes"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220"
-dependencies = [
- "hermit-abi 0.3.1",
- "libc",
- "windows-sys 0.48.0",
-]
-
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -1855,9 +1890,9 @@ dependencies = [
 
 [[package]]
 name = "ipnet"
-version = "2.7.2"
+version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f"
+checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
 
 [[package]]
 name = "is-docker"
@@ -1870,12 +1905,11 @@ dependencies = [
 
 [[package]]
 name = "is-terminal"
-version = "0.4.7"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f"
+checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
- "hermit-abi 0.3.1",
- "io-lifetimes",
+ "hermit-abi 0.3.2",
  "rustix",
  "windows-sys 0.48.0",
 ]
@@ -1901,9 +1935,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.6"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 
 [[package]]
 name = "jobserver"
@@ -1916,9 +1950,9 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.61"
+version = "0.3.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
+checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -2146,7 +2180,7 @@ dependencies = [
  "educe",
  "futures 0.3.28",
  "html-escape",
- "indexmap",
+ "indexmap 1.9.3",
  "itertools",
  "js-sys",
  "leptos_reactive",
@@ -2170,7 +2204,7 @@ checksum = "d85b0935d1439780ddabc4809dd2ae93a1825e9d05b95c9812942b3fc858ff9e"
 dependencies = [
  "anyhow",
  "camino",
- "indexmap",
+ "indexmap 1.9.3",
  "parking_lot",
  "proc-macro2",
  "quote",
@@ -2234,10 +2268,10 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22e702730c3ac1f8a619bf2b139e650b792c96c66f2285a88a1c92c1a53ed262"
 dependencies = [
- "base64 0.21.0",
+ "base64 0.21.2",
  "cfg-if",
  "futures 0.3.28",
- "indexmap",
+ "indexmap 1.9.3",
  "js-sys",
  "rustc-hash",
  "serde",
@@ -2293,15 +2327,15 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.142"
+version = "0.2.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
 
 [[package]]
 name = "libgit2-sys"
-version = "0.15.1+1.6.4"
+version = "0.15.2+1.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb4577bde8cdfc7d6a2a4bcb7b049598597de33ffd337276e9c7db6cd4a2cee7"
+checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa"
 dependencies = [
  "cc",
  "libc",
@@ -2327,9 +2361,9 @@ dependencies = [
 
 [[package]]
 name = "libz-sys"
-version = "1.1.8"
+version = "1.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
+checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b"
 dependencies = [
  "cc",
  "libc",
@@ -2343,15 +2377,6 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bfae20f6b19ad527b550c223fddc3077a547fc70cda94b9b566575423fd303ee"
 
-[[package]]
-name = "link-cplusplus"
-version = "1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "linked-hash-map"
 version = "0.5.6"
@@ -2369,15 +2394,15 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.3.3"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c"
+checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
 
 [[package]]
 name = "lock_api"
-version = "0.4.9"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
+checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -2385,12 +2410,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.17"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
 
 [[package]]
 name = "maplit"
@@ -2404,14 +2426,14 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
 dependencies = [
- "regex-automata",
+ "regex-automata 0.1.10",
 ]
 
 [[package]]
 name = "matchit"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
+checksum = "67827e6ea8ee8a7c4a72227ef4fc08957040acffdb5f122733b24fa12daff41b"
 
 [[package]]
 name = "maybe-async"
@@ -2454,9 +2476,9 @@ dependencies = [
 
 [[package]]
 name = "minidom"
-version = "0.15.1"
+version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e9ce45d459e358790a285e7609ff5ae4cfab88b75f237e8838e62029dda397b"
+checksum = "f45614075738ce1b77a1768912a60c0227525971b03e09122a05b8a34a2a6278"
 dependencies = [
  "rxml",
 ]
@@ -2469,23 +2491,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.6.2"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
+checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
 dependencies = [
  "adler",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.6"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
+checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
 dependencies = [
  "libc",
- "log",
- "wasi",
- "windows-sys 0.45.0",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -2561,20 +2582,20 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "num_cpus"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.2.6",
+ "hermit-abi 0.3.2",
  "libc",
 ]
 
@@ -2589,9 +2610,9 @@ dependencies = [
 
 [[package]]
 name = "oauth2"
-version = "4.3.0"
+version = "4.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eeaf26a72311c087f8c5ba617c96fac67a5c04f430e716ac8d8ab2de62e23368"
+checksum = "09a6e2a2b13a56ebeabba9142f911745be6456163fd6c3d361274ebcd891a80c"
 dependencies = [
  "base64 0.13.1",
  "chrono",
@@ -2607,19 +2628,29 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "object"
+version = "0.31.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
-version = "1.17.1"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "open"
-version = "4.0.2"
+version = "4.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "873240a4a404d44c8cd1bf394359245d466a5695771fea15a79cafbc5e5cf4d7"
+checksum = "3a083c0c7e5e4a8ec4176346cf61f67ac674e8bfb059d9226e1c54a96b377c12"
 dependencies = [
  "is-wsl",
+ "libc",
  "pathdiff",
 ]
 
@@ -2652,11 +2683,11 @@ dependencies = [
 
 [[package]]
 name = "openssl"
-version = "0.10.51"
+version = "0.10.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97ea2d98598bf9ada7ea6ee8a30fb74f9156b63bbe495d64ec2b87c269d2dda3"
+checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -2673,7 +2704,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -2684,9 +2715,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.86"
+version = "0.9.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "992bac49bdbab4423199c654a5515bd2a6c6a23bf03f2dd3bdb7e5ae6259bc69"
+checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
 dependencies = [
  "cc",
  "libc",
@@ -2710,7 +2741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a"
 dependencies = [
  "dlv-list",
- "hashbrown",
+ "hashbrown 0.12.3",
 ]
 
 [[package]]
@@ -2737,22 +2768,22 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.7"
+version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
+checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.2.16",
+ "redox_syscall 0.3.5",
  "smallvec",
- "windows-sys 0.45.0",
+ "windows-targets 0.48.1",
 ]
 
 [[package]]
 name = "paste"
-version = "1.0.12"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
+checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
 
 [[package]]
 name = "pathdiff"
@@ -2771,15 +2802,15 @@ dependencies = [
 
 [[package]]
 name = "percent-encoding"
-version = "2.2.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
+checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
 [[package]]
 name = "pest"
-version = "2.5.7"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b1403e8401ad5dedea73c626b99758535b342502f8d1e361f4a2dd952749122"
+checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a"
 dependencies = [
  "thiserror",
  "ucd-trie",
@@ -2787,9 +2818,9 @@ dependencies = [
 
 [[package]]
 name = "pest_derive"
-version = "2.5.7"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be99c4c1d2fc2769b1d00239431d711d08f6efedcecb8b6e30707160aee99c15"
+checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853"
 dependencies = [
  "pest",
  "pest_generator",
@@ -2797,22 +2828,22 @@ dependencies = [
 
 [[package]]
 name = "pest_generator"
-version = "2.5.7"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e56094789873daa36164de2e822b3888c6ae4b4f9da555a1103587658c805b1e"
+checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929"
 dependencies = [
  "pest",
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "pest_meta"
-version = "2.5.7"
+version = "2.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6733073c7cff3d8459fda0e42f13a047870242aed8b509fe98000928975f359e"
+checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48"
 dependencies = [
  "once_cell",
  "pest",
@@ -2826,34 +2857,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
 dependencies = [
  "fixedbitset",
- "indexmap",
+ "indexmap 1.9.3",
 ]
 
 [[package]]
 name = "pin-project"
-version = "1.0.12"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc"
+checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.0.12"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
+checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
+checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
 
 [[package]]
 name = "pin-utils"
@@ -2863,9 +2894,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.26"
+version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
+checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
 [[package]]
 name = "ppv-lite86"
@@ -2961,9 +2992,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.56"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
+checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
 dependencies = [
  "unicode-ident",
 ]
@@ -3024,9 +3055,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.26"
+version = "1.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
+checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
 dependencies = [
  "proc-macro2",
 ]
@@ -3080,7 +3111,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -3089,7 +3120,7 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -3105,13 +3136,14 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.8.0"
+version = "1.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac6cf59af1067a3fb53fbe5c88c053764e930f932be1d71d3ffe032cbe147f59"
+checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
 dependencies = [
- "aho-corasick 1.0.1",
+ "aho-corasick",
  "memchr",
- "regex-syntax 0.7.0",
+ "regex-automata 0.3.4",
+ "regex-syntax 0.7.4",
 ]
 
 [[package]]
@@ -3123,6 +3155,17 @@ dependencies = [
  "regex-syntax 0.6.29",
 ]
 
+[[package]]
+name = "regex-automata"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.7.4",
+]
+
 [[package]]
 name = "regex-syntax"
 version = "0.6.29"
@@ -3131,17 +3174,17 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.0"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6868896879ba532248f33598de5181522d8b3d9d724dfd230911e1a7d4822f5"
+checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
 
 [[package]]
 name = "reqwest"
-version = "0.11.16"
+version = "0.11.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254"
+checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
 dependencies = [
- "base64 0.21.0",
+ "base64 0.21.2",
  "bytes 1.4.0",
  "encoding_rs",
  "futures-core",
@@ -3150,7 +3193,7 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls",
+ "hyper-rustls 0.24.1",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -3160,14 +3203,14 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls",
+ "rustls 0.21.5",
  "rustls-pemfile",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "tokio",
  "tokio-native-tls",
- "tokio-rustls",
+ "tokio-rustls 0.24.1",
  "tokio-util",
  "tower-service",
  "url",
@@ -3201,7 +3244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a"
 dependencies = [
  "base64 0.13.1",
- "bitflags",
+ "bitflags 1.3.2",
  "serde",
 ]
 
@@ -3266,12 +3309,18 @@ dependencies = [
  "serde_derive",
  "sha2",
  "thiserror",
- "time",
+ "time 0.3.24",
  "tokio",
  "tokio-stream",
  "url",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@@ -3289,13 +3338,12 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.37.13"
+version = "0.38.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0"
+checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
 dependencies = [
- "bitflags",
+ "bitflags 2.3.3",
  "errno",
- "io-lifetimes",
  "libc",
  "linux-raw-sys",
  "windows-sys 0.48.0",
@@ -3313,11 +3361,23 @@ dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.21.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36"
+dependencies = [
+ "log",
+ "ring",
+ "rustls-webpki",
+ "sct",
+]
+
 [[package]]
 name = "rustls-native-certs"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"
+checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
 dependencies = [
  "openssl-probe",
  "rustls-pemfile",
@@ -3327,43 +3387,51 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "1.0.2"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
+dependencies = [
+ "base64 0.21.2",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"
+checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59"
 dependencies = [
- "base64 0.21.0",
+ "ring",
+ "untrusted",
 ]
 
 [[package]]
 name = "rustversion"
-version = "1.0.12"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06"
+checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
 [[package]]
 name = "rxml"
-version = "0.8.2"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a071866b8c681dc2cfffa77184adc32b57b0caad4e620b6292609703bceb804"
+checksum = "a98f186c7a2f3abbffb802984b7f1dfd65dac8be1aafdaabbca4137f53f0dff7"
 dependencies = [
  "bytes 1.4.0",
- "pin-project-lite",
  "rxml_validation",
  "smartstring",
- "tokio",
 ]
 
 [[package]]
 name = "rxml_validation"
-version = "0.8.2"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53bc79743f9a66c2fb1f951cd83735f275d46bfe466259fbc5897bb60a0d00ee"
+checksum = "22a197350ece202f19a166d1ad6d9d6de145e1d2a8ef47db299abe164dbd7530"
 
 [[package]]
 name = "ryu"
-version = "1.0.13"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
 [[package]]
 name = "same-file"
@@ -3376,11 +3444,11 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.21"
+version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3"
+checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
 dependencies = [
- "windows-sys 0.42.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -3409,15 +3477,9 @@ dependencies = [
 
 [[package]]
 name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
-[[package]]
-name = "scratch"
-version = "1.0.5"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "sct"
@@ -3441,11 +3503,11 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "2.8.2"
+version = "2.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"
+checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "core-foundation",
  "core-foundation-sys",
  "libc",
@@ -3454,9 +3516,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.8.0"
+version = "2.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4"
+checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -3464,15 +3526,15 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.17"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
+checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
 
 [[package]]
 name = "serde"
-version = "1.0.160"
+version = "1.0.180"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c"
+checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed"
 dependencies = [
  "serde_derive",
 ]
@@ -3512,13 +3574,13 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.160"
+version = "1.0.180"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df"
+checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -3534,11 +3596,11 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.96"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
+checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
 dependencies = [
- "indexmap",
+ "indexmap 2.0.0",
  "itoa",
  "ryu",
  "serde",
@@ -3557,10 +3619,11 @@ dependencies = [
 
 [[package]]
 name = "serde_path_to_error"
-version = "0.1.11"
+version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0"
+checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335"
 dependencies = [
+ "itoa",
  "serde",
 ]
 
@@ -3634,7 +3697,7 @@ version = "0.8.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b"
 dependencies = [
- "indexmap",
+ "indexmap 1.9.3",
  "ryu",
  "serde",
  "yaml-rust",
@@ -3642,11 +3705,11 @@ dependencies = [
 
 [[package]]
 name = "serde_yaml"
-version = "0.9.21"
+version = "0.9.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c"
+checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
 dependencies = [
- "indexmap",
+ "indexmap 2.0.0",
  "itoa",
  "ryu",
  "serde",
@@ -3664,7 +3727,21 @@ dependencies = [
  "lazy_static",
  "log",
  "parking_lot",
- "serial_test_derive",
+ "serial_test_derive 0.9.0",
+]
+
+[[package]]
+name = "serial_test"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d"
+dependencies = [
+ "dashmap",
+ "futures 0.3.28",
+ "lazy_static",
+ "log",
+ "parking_lot",
+ "serial_test_derive 2.0.0",
 ]
 
 [[package]]
@@ -3679,6 +3756,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "serial_test_derive"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.28",
+]
+
 [[package]]
 name = "server_fn"
 version = "0.2.5"
@@ -3733,6 +3821,7 @@ version = "0.1.0"
 dependencies = [
  "ame",
  "async-trait",
+ "either",
  "envconfig",
  "futures-core",
  "futures-util",
@@ -3745,7 +3834,7 @@ dependencies = [
  "rust-s3",
  "serde",
  "serde_json",
- "serial_test",
+ "serial_test 0.9.0",
  "thiserror",
  "tokio",
  "tokio-stream",
@@ -3763,9 +3852,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.6"
+version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
+checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -3818,14 +3907,13 @@ dependencies = [
 
 [[package]]
 name = "simple_logger"
-version = "4.1.0"
+version = "4.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78beb34673091ccf96a8816fce8bfd30d1292c7621ca2bcb5f2ba0fae4f558d"
+checksum = "2230cd5c29b815c9b699fb610b49a5ed65588f3509d9f0108be3a885da629333"
 dependencies = [
- "atty",
  "colored",
  "log",
- "time",
+ "time 0.3.24",
  "windows-sys 0.42.0",
 ]
 
@@ -3850,17 +3938,19 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
 
 [[package]]
 name = "smartstring"
-version = "0.2.10"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e714dff2b33f2321fdcd475b71cec79781a692d846f37f415fb395a1d2bcd48e"
+checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
 dependencies = [
+ "autocfg",
  "static_assertions",
+ "version_check",
 ]
 
 [[package]]
@@ -3926,9 +4016,9 @@ dependencies = [
 
 [[package]]
 name = "subtle"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
 [[package]]
 name = "syn"
@@ -3943,9 +4033,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.15"
+version = "2.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822"
+checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3982,24 +4072,15 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.5.0"
+version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998"
+checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998"
 dependencies = [
  "cfg-if",
  "fastrand",
  "redox_syscall 0.3.5",
  "rustix",
- "windows-sys 0.45.0",
-]
-
-[[package]]
-name = "termcolor"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
-dependencies = [
- "winapi-util",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -4010,22 +4091,22 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
 
 [[package]]
 name = "thiserror"
-version = "1.0.40"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
+checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.40"
+version = "1.0.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
+checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -4040,10 +4121,22 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.20"
+version = "0.1.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890"
+checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
 dependencies = [
+ "libc",
+ "wasi 0.10.0+wasi-snapshot-preview1",
+ "winapi",
+]
+
+[[package]]
+name = "time"
+version = "0.3.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b79eabcd964882a646b3584543ccabeae7869e9ac32a46f6f22b7a5bd405308b"
+dependencies = [
+ "deranged",
  "itoa",
  "libc",
  "num_threads",
@@ -4054,15 +4147,15 @@ dependencies = [
 
 [[package]]
 name = "time-core"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
+checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
 
 [[package]]
 name = "time-macros"
-version = "0.2.8"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36"
+checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd"
 dependencies = [
  "time-core",
 ]
@@ -4084,11 +4177,12 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.27.0"
+version = "1.29.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001"
+checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
 dependencies = [
  "autocfg",
+ "backtrace",
  "bytes 1.4.0",
  "libc",
  "mio",
@@ -4098,7 +4192,7 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys 0.45.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -4113,13 +4207,13 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.28",
 ]
 
 [[package]]
@@ -4150,16 +4244,26 @@ version = "0.23.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
 dependencies = [
- "rustls",
+ "rustls 0.20.8",
  "tokio",
  "webpki",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
+dependencies = [
+ "rustls 0.21.5",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-stream"
-version = "0.1.12"
+version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313"
+checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -4168,9 +4272,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.7"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2"
+checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
 dependencies = [
  "bytes 1.4.0",
  "futures-core",
@@ -4215,7 +4319,7 @@ dependencies = [
  "rustls-native-certs",
  "rustls-pemfile",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.23.4",
  "tokio-stream",
  "tokio-util",
  "tower",
@@ -4276,7 +4380,7 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d21e279964481ddf163bae257766f2ef87b26a8b45a0fa3f0c272f6c1bd72fed"
 dependencies = [
- "base64 0.21.0",
+ "base64 0.21.2",
  "byteorder",
  "bytes 1.4.0",
  "futures-util",
@@ -4302,7 +4406,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
 dependencies = [
  "futures-core",
  "futures-util",
- "indexmap",
+ "indexmap 1.9.3",
  "pin-project",
  "pin-project-lite",
  "rand",
@@ -4322,7 +4426,7 @@ checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"
 dependencies = [
  "async-compression",
  "base64 0.13.1",
- "bitflags",
+ "bitflags 1.3.2",
  "bytes 1.4.0",
  "futures-core",
  "futures-util",
@@ -4343,12 +4447,12 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.4.0"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"
+checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82"
 dependencies = [
- "base64 0.20.0",
- "bitflags",
+ "base64 0.21.2",
+ "bitflags 2.3.3",
  "bytes 1.4.0",
  "futures-core",
  "futures-util",
@@ -4388,20 +4492,20 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.23"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
+checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.28",
 ]
 
 [[package]]
 name = "tracing-core"
-version = "0.1.30"
+version = "0.1.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
+checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
 dependencies = [
  "once_cell",
  "valuable",
@@ -4430,9 +4534,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.16"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
+checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
 dependencies = [
  "matchers",
  "nu-ansi-term",
@@ -4480,9 +4584,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
 [[package]]
 name = "ucd-trie"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
+checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9"
 
 [[package]]
 name = "unicase"
@@ -4501,9 +4605,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.8"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
 
 [[package]]
 name = "unicode-normalization"
@@ -4534,9 +4638,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
 
 [[package]]
 name = "unsafe-libyaml"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6"
+checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa"
 
 [[package]]
 name = "untrusted"
@@ -4546,9 +4650,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
 [[package]]
 name = "url"
-version = "2.3.1"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
+checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -4570,9 +4674,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
 [[package]]
 name = "uuid"
-version = "1.3.1"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb"
+checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
 dependencies = [
  "getrandom",
 ]
@@ -4616,14 +4720,19 @@ dependencies = [
 
 [[package]]
 name = "want"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
 dependencies = [
- "log",
  "try-lock",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.10.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -4632,9 +4741,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.84"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
+checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -4642,24 +4751,24 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.84"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
+checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.28",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.34"
+version = "0.4.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
+checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -4669,9 +4778,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.84"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
+checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -4679,22 +4788,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.84"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
+checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.28",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.84"
+version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
+checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
 [[package]]
 name = "wasm-streams"
@@ -4753,9 +4862,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.61"
+version = "0.3.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
+checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -4834,7 +4943,7 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
 dependencies = [
- "windows-targets 0.48.0",
+ "windows-targets 0.48.1",
 ]
 
 [[package]]
@@ -4867,7 +4976,7 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
- "windows-targets 0.48.0",
+ "windows-targets 0.48.1",
 ]
 
 [[package]]
@@ -4887,9 +4996,9 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.48.0"
+version = "0.48.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
 dependencies = [
  "windows_aarch64_gnullvm 0.48.0",
  "windows_aarch64_msvc 0.48.0",
@@ -4995,9 +5104,9 @@ dependencies = [
 
 [[package]]
 name = "xml-rs"
-version = "0.8.4"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
+checksum = "47430998a7b5d499ccee752b41567bc3afc57e1327dc855b1a2aa44ce29b5fa1"
 
 [[package]]
 name = "xxhash-rust"
diff --git a/Cargo.toml b/Cargo.toml
index f0d0e843..26ff056d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,4 +1,4 @@
 [workspace]
-members = ["controller", "service", "cli",  "web",  "lib"]
+members = ["controller", "service", "cli",  "web",  "lib"  ]
 resolver = "2"
 
diff --git a/Dockerfile.controller b/Dockerfile.controller
index aa8912b0..2405ebe5 100644
--- a/Dockerfile.controller
+++ b/Dockerfile.controller
@@ -1,6 +1,6 @@
 # Using the `rust-musl-builder` as base image, instead of the official Rust toolchain. 
 # See https://github.com/clux/muslrust for why this is desirable.
-FROM  clux/muslrust:1.68.0-stable AS chef
+FROM  clux/muslrust:1.70.0-stable AS chef
 RUN cargo install cargo-chef
 WORKDIR /app
 
diff --git a/Dockerfile.server b/Dockerfile.server
index d0a5399d..00b8be3e 100644
--- a/Dockerfile.server
+++ b/Dockerfile.server
@@ -1,6 +1,6 @@
 # Using the `rust-musl-builder` as base image, instead of 
 # the official Rust toolchain
-FROM  clux/muslrust:1.68.0-stable AS chef
+FROM  clux/muslrust:1.70.0-stable AS chef
 RUN cargo install cargo-chef
 WORKDIR /app
 
diff --git a/README.md b/README.md
index 96e2aafa..4bfdbd65 100644
--- a/README.md
+++ b/README.md
@@ -94,7 +94,7 @@ TODO: show an example of this.
 
 ### Pipelines
 
-You might have multiple Tasks meant to be executed together, for example downloading data, preparing data, model training, model upload. Each of these Tasks will have different requirements. This can be expressed using a pipeline. Each Task in a pipeline is executed in a separate container potentially on different machines if their compute requirements are different. To ensure that your code will work without modification, all of the state is transferred between steps transparently so it appears as if all of the steps are executed on the same machine. For example data is downloaed in step 1, prepared in step 2 and trained on in step 3 AME will make sure to transfer these files automatically between steps so no adjustments are reqired to the project's code.
+You might have multiple Tasks meant to be executed together, for example downloading data, preparing data, model training, model upload. Each of these Tasks will have different requirements. This can be expressed using a pipeline. Each Task in a pipeline is executed in a separate container potentially on different machines if their compute requirements are different. To ensure that your code will work without modification, all of the state is transferred between steps transparently so it appears as if all of the steps are executed on the same machine. For example data is downloaed in step 1, prepared in step 2 and trained on in step 3 AME will make sure to transfer these files automatically between steps so no adjustments are required to the project's code.
 
 ```yaml
 #ame.yaml
diff --git a/docs/datasets.md b/ame/docs/datasets.md
similarity index 100%
rename from docs/datasets.md
rename to ame/docs/datasets.md
diff --git a/ame/docs/index.md b/ame/docs/index.md
new file mode 100644
index 00000000..000ea345
--- /dev/null
+++ b/ame/docs/index.md
@@ -0,0 +1,17 @@
+# Welcome to MkDocs
+
+For full documentation visit [mkdocs.org](https://www.mkdocs.org).
+
+## Commands
+
+* `mkdocs new [dir-name]` - Create a new project.
+* `mkdocs serve` - Start the live-reloading docs server.
+* `mkdocs build` - Build the documentation site.
+* `mkdocs -h` - Print help message and exit.
+
+## Project layout
+
+    mkdocs.yml    # The configuration file.
+    docs/
+        index.md  # The documentation homepage.
+        ...       # Other markdown pages, images and other files.
diff --git a/ame/docs/model_validation.md b/ame/docs/model_validation.md
new file mode 100644
index 00000000..b4952f1d
--- /dev/null
+++ b/ame/docs/model_validation.md
@@ -0,0 +1,172 @@
+# Guides
+
+## From zero to live model
+
+**This guide is focused on using AME** if you are looking for a deployment guide go [here](todo).
+
+
+
+This guide will walk through going from zero to having a model served through an the [V2 inference protocol](https://docs.seldon.io/projects/seldon-core/en/latest/reference/apis/v2-protocol.html).
+it will be split into multiple sub steps which can be consumed in isolation if you are just looking for a smaller guide on that specific step.
+
+Almost any python project should be usable but if you want to follow along with the exact same project as the guide clone [this]() repo.
+
+### Setup the CLI
+
+Before we can initialise an AME project we need to install the ame [CLI](todo) and connect with your AME instance.
+
+TODO describe installation
+
+### Initialising AME in your project
+
+The first step will be creating an `ame.yaml` file in the project directory.
+
+This is easiet to do with the ame [CLI]() by running `ame project init`. The [CLI]() will ask for a project and then produce a file
+that looks like this:
+
+```yaml
+projectName: sklearn_logistic_regression
+```
+
+### The first training
+
+Not very exciting but it is a start. Next we want to set up our model to be run by AME. The most important thing here is the Task that will train the model so
+lets start with that.
+
+Here we need to consider a few things, what command is used to train a model, how are dependencies managed in our project, what python version do we need and
+how many resources does our model training require.
+
+If you are using the [repo]() for this guide, you will want a task configured as below. 
+
+```yaml
+
+projectid: sklearn_logistic_regression
+tasks:
+  - name: training
+    !poetry
+    executor:
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+```
+
+## Your first Task
+
+[`Tasks`](TODO) are an important building block for AME. This guide will walk you through the basic of constructing and running [`Task`](todo). 
+
+We assume that the AME [CLI](todo) is setup and connected to an AME instance. If not see this [guide](todo). 
+
+Before we can run a task we must have a project setup. To init a project follow the commands as shown below, replacing myproject with the 
+path to your project.
+
+```sh
+cd myproject
+ame init
+```
+
+Now you should have an AME file ame.yaml inside your project:
+```yaml
+name: myproject
+```
+
+Not very exciting yet. Next we want to add a Task to this file so we can run it.
+Update your file to match the changes shown below.
+
+```yaml
+name: myproject
+tasks:
+  - name: training
+    !poetry
+    executor:
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 2G 
+      cpu: 2 
+      storage: 10G 
+```
+
+Here we add a list of tasks for our project, containing a single `Task` called training. Lets look at the anatomy of training.
+
+First we set the name `name: training`, pretty standard YAML. Next we set the [executor](todo). This syntax might seem a bit confusing
+if you have not used this YAML feature before. `!poetry` adds a tag to the executor indicating the executor type. In this case we are 
+using the poetry executor. It requires two fields to be set. the Python version and the command to run. This tells AME how to execute the [`Task`](todo).
+
+Finally we set the required resources. 2G ram, 2 cpu threads and 10G of storage.
+
+To run the task we can use the CLI:
+```sh
+ame task run
+```
+
+ 
+ 
+## Validating models before deployment
+
+To ensure that a new model versions perform well before exposing them AME supports model validation. This is done by providing AME with a `Task` which 
+will succeed if the model passes validation and fail if not.
+
+Example from [ame-demo](https://github.com/TeaInSpace/ame-demo):
+
+```yaml
+
+projectid: sklearn_logistic_regression
+models:
+  - name: logreg
+    type: mlflow
+    validationTask: # the validation task is set here.
+      taskRef: mlflow_validation 
+    training: 
+      task:
+        taskRef: training
+    deployment:
+      auto_train: true
+      deploy: true
+      enable_tls: false
+tasks:
+  - name: training
+    projectid: sklearn_logistic_regression
+    templateRef: shared-templates.logistic_reg_template
+    taskType: Mlflow
+  - name: mlflow_validation
+    projectid: sklearn_logistic_regression
+    runcommand: python validate.py
+```
+
+This approach allows for a lot of flexibility of how models are validated, at the cost of writing the validation your self. In the future AME will provide builtin options for common validation configurations as well, see the [roadmap](todo).
+
+### Using MLflow metrics
+
+Here we will walk through how to validate a model based on recorded metrics in MLflow, using the [ame-demo](https://github.com/TeaInSpace/ame-demo) repository as an example. The model is a simple logistic regresser, the training code looks like this:
+
+```python
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+import mlflow
+import mlflow.sklearn
+import os
+
+if __name__ == "__main__":
+    X = np.array([-2, -1, 0, 1, 2, 1]).reshape(-1, 1)
+    y = np.array([0, 0, 1, 1, 1, 0])
+    lr = LogisticRegression()
+    lr.fit(X, y)
+    score = lr.score(X, y)
+    print("Score: %s" % score)
+    mlflow.log_metric("score", score)
+    mlflow.sklearn.log_model(lr, "model", registered_model_name="logreg")
+    print("Model saved in run %s" % mlflow.active_run().info.run_uuid)
+```
+
+Notice how the score is logged as a metric. We can use that in our validation.
+
+AME exposes the necessary environment variables to running tasks so we can access the Mlflow instance during validation just by using the Mlflow library.
+
+```python
+TODO
+
+```
diff --git a/ame/docs/models.html b/ame/docs/models.html
new file mode 100644
index 00000000..fca66d24
--- /dev/null
+++ b/ame/docs/models.html
@@ -0,0 +1,30 @@
+<h1>Models</h1>
+<p>Models are one of AME's higher level constructs, see what that means <a href="">here</a>. if you are configuring how a model should be trained, deployed, monitored or validated this is the right place.
+Models exist in an AME file along side Datasets Tasks and Templates.</p>
+<h3>Model training</h3>
+<p>Model training is configured described use a <a href="./task.html">Task</a>.</p>
+<p>AME can be deployed with a an MLflow instance which will be exposed to the Training Task allowing for simply storage and retrievel of models and metrics.</p>
+<pre lang="yaml" style="background-color:#2b303b;"><code><span style="color:#65737e;"># main project ame.yml
+</span><span style="color:#bf616a;">project</span><span style="color:#c0c5ce;">: </span><span style="color:#a3be8c;">xgboost_project
+</span><span style="color:#bf616a;">models</span><span style="color:#c0c5ce;">:
+</span><span style="color:#c0c5ce;">  - </span><span style="color:#bf616a;">name</span><span style="color:#c0c5ce;">: </span><span style="color:#a3be8c;">product_recommendor
+</span><span style="color:#c0c5ce;">    </span><span style="color:#bf616a;">training</span><span style="color:#c0c5ce;">:
+</span><span style="color:#c0c5ce;">      </span><span style="color:#bf616a;">task</span><span style="color:#c0c5ce;">: 
+</span><span style="color:#c0c5ce;">        </span><span style="color:#bf616a;">taskRef</span><span style="color:#c0c5ce;">: </span><span style="color:#a3be8c;">train_my_model 
+</span><span style="color:#bf616a;">tasks</span><span style="color:#c0c5ce;">:
+</span><span style="color:#c0c5ce;">  - </span><span style="color:#bf616a;">name</span><span style="color:#c0c5ce;">: </span><span style="color:#a3be8c;">train_my_model
+</span><span style="color:#c0c5ce;">    </span><span style="color:#bf616a;">fromTemplate</span><span style="color:#c0c5ce;">: </span><span style="color:#a3be8c;">shared_templates.xgboost_resources
+</span><span style="color:#c0c5ce;">    </span><span style="color:#bf616a;">executor</span><span style="color:#c0c5ce;">:
+</span><span style="color:#c0c5ce;">      </span><span style="color:#b48ead;">!poetry
+</span><span style="color:#c0c5ce;">      </span><span style="color:#bf616a;">pythonVersion</span><span style="color:#c0c5ce;">: </span><span style="color:#d08770;">3.11
+</span><span style="color:#c0c5ce;">      </span><span style="color:#bf616a;">command</span><span style="color:#c0c5ce;">: </span><span style="color:#a3be8c;">python train.py
+</span><span style="color:#c0c5ce;">    </span><span style="color:#bf616a;">resources</span><span style="color:#c0c5ce;">:
+</span><span style="color:#c0c5ce;">      </span><span style="color:#bf616a;">memory</span><span style="color:#c0c5ce;">: </span><span style="color:#a3be8c;">10G 
+</span><span style="color:#c0c5ce;">      </span><span style="color:#bf616a;">cpu</span><span style="color:#c0c5ce;">: </span><span style="color:#d08770;">4 
+</span><span style="color:#c0c5ce;">      </span><span style="color:#bf616a;">storage</span><span style="color:#c0c5ce;">: </span><span style="color:#a3be8c;">30G 
+</span><span style="color:#c0c5ce;">      </span><span style="color:#bf616a;">nvidia.com/gpu</span><span style="color:#c0c5ce;">: </span><span style="color:#d08770;">1 
+</span></code></pre>
+<h3>Model deployment</h3>
+<h4>Model validation</h4>
+<h4>Model monitoring</h4>
+<h3>Batch inference</h3>
diff --git a/ame/docs/models.md b/ame/docs/models.md
new file mode 100644
index 00000000..751d53c3
--- /dev/null
+++ b/ame/docs/models.md
@@ -0,0 +1,42 @@
+# Models
+
+Models are one of AME's higher level constructs, see what that means [here](). if you are configuring how a model should be trained, deployed, monitored or validated this is the right place.
+Models exist in an AME file along side Datasets Tasks and Templates.
+
+### Model training
+
+Model training is configured described use a [Task](tasks.md).
+
+AME can be deployed with a an MLflow instance which will be exposed to the Training Task allowing for simply storage and retrievel of models and metrics.
+
+
+```yaml
+# main project ame.yml
+project: xgboost_project
+models:
+  - name: product_recommendor
+    training:
+      task: 
+        taskRef: train_my_model 
+tasks:
+  - name: train_my_model
+    fromTemplate: shared_templates.xgboost_resources
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+```
+
+
+### Model deployment 
+
+#### Model validation
+
+#### Model monitoring
+
+### Batch inference
diff --git a/docs/project_sources.md b/ame/docs/project_sources.md
similarity index 94%
rename from docs/project_sources.md
rename to ame/docs/project_sources.md
index 3fe54669..65402375 100644
--- a/docs/project_sources.md
+++ b/ame/docs/project_sources.md
@@ -2,11 +2,11 @@
 
 A project source informs AME of a location to check and sync an AME project from. Currently the only supported location is a Git repository.
 
-### Git project sources
+## Git project sources
 
 Git project sources allow for a Gitops like approach to managing models, data and the surrounding operations using the AME file defined in the repository.
 
-#### How to use Git project sources
+### How to use Git project sources
 
 You can create a Git project source either through the CLI or the AME frontend.
 
diff --git a/ame/docs/tasks.md b/ame/docs/tasks.md
new file mode 100644
index 00000000..62e207df
--- /dev/null
+++ b/ame/docs/tasks.md
@@ -0,0 +1,681 @@
+# Tasks
+
+`Tasks` are the basic building block for most of AME's functionality. A `Task` represents some work to be done, often in the form of python code to be executed
+but in principle it can be anything executable in a container.
+
+`Tasks` are configured in an AME file `ame.yml`. 
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+```
+
+This is an example of a the absolute minimal requirements for a task, a name and an executor. An executor specifies how a task should be executed, a complete list can found (here)[].
+In this case AME will ensure that the specified python version is present and use poetry to install dependencies and enter a shell before executing the command `python train.py`. 
+
+To run the task manually simply enter the directory with the file and project and run `ame task run -l train_my_model`. The `-l` ensures that we are using the local context and not trying
+to run a remote task already present in the AME cluster. Alternatively if you don't want to type the name omit it and AME will present a list of the available `Tasks`.
+
+ 
+### Resource requirements
+
+By default a task gets limited resources allocated TODO: what is the default? which can be fine for very simple tasks but anything non trivial will require. To change the default see
+how to configure default (here)[].
+
+Resources include any computational resources that needs to be allocated:
+
+- CPU
+- GPU
+- memory
+- storage
+
+They can be specified for a task with the resources field:
+
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G # 10 Gigabyte ram
+      cpu: 4 # 4 CPU threads
+      storage: 30G # 30Gigabyte of disk space
+      nvidia.com/gpu: 1 # 1 Nvidia GPU
+```
+
+AME uses style of string units for resources as Kubernetes. If you are not familiar with that no worries!, the readon for the details.
+
+**Memory and storage units**: memory units are measured in bytes. Integers with quantity suffixes can be used to express memory quantities. 
+See the following table for complete list of available units. 
+
+| Unit               	| Suffix 	| Example 	|
+|--------------------	|--------	|---------	|
+| Exabyte            	| E      	| 2.5E    	|
+| Exbibyte           	| Ei     	| 2.5Ei   	|
+| Terabyte           	| T      	| 2.5T    	|
+| Tebibyte           	| Ti     	| 2.5Ti   	|
+| Gigabyte           	| G      	| 2.5G    	|
+| Gibibyte           	| Gi     	| 2.5Gi   	|
+| Megabyte           	| M      	| 2.5M    	|
+| Mebibyte           	| Mi     	| 2.5Mi   	|
+| Kilobyte           	| k      	| 2.5k    	|
+| Kibibyte           	| Ki     	| 2.5Ki   	|
+| Byte               	|        	| 2.5     	|
+| Fraction of a Byte 	| m      	| 2500m   	|
+
+TODO: make better example
+**Example**:
+```
+  128974848, 129e6, 129M,  128974848000m, 123Mi
+```
+
+**CPU units**
+
+`1` specifies a single CPU thread either virtual or hyperthreaded depending on the underlying machine. `0.5` specifies half a CPU thread and so does `500m`.
+`1=1000m`, `m` stands for millicpu/millicores.
+
+**GPU units**
+
+GPU scheduling in AME is current pretty barebones. You must specifie whole GPUs, meaning no fractional units and you can't ask for a specific model of device only vendor.
+For cases where different GPU models need to be differentiate node labels can be used as a work around. This is essentially the way Kubernetes solves GPU scheduling as well, however we will
+be abstracting all this away in the coming release of AME to allow for requesting specific models and fractional GPU sharing directly in the Task specification, see the tracking (issue)[].
+For details on how to use node labels with GPUs see (this)[].
+
+### Secrets
+
+Task's will often require access to private resources such as objectstorage, databases or APIs. AME provides a built in secret store as well as integration with (Vault)[https://www.vaultproject.io/].
+This section will walk through how to use secrets with a Task. For details on AME's secret store and how to integrate with a Vault instance see the relevant (documentation)[].
+
+**Quick example**
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G # 10 Gigabyte ram
+      cpu: 4 # 4 CPU threads
+      storage: 30G # 30Gigabyte of disk space
+      nvidia.com/gpu: 1 # 1 Nvidia GPU
+    secrets:
+     - !ame # Secret stored by AME
+       key: blob_storage_key # Key identifying a secret in AME's secret store.
+       injectAs: MY_BLOB_STORAGE_SECRET # This will inject an environment variable with the name MY_BLOB_STORAGE_SECRET and with the value of the secret.
+
+     # TODO this does not cover all vault cases
+     - !vault # Secret stored in Vault
+       vault: company_vault # Name of the vault to use.
+       key: path/to/secret # Path to the secret in Vault.
+       injectAs: COMPANY_SECRET  # This will inject an environment variable with the name COMPANY_SECRET and with the value of the secret.
+```
+
+**Explanation**
+
+`!ame` and `!vault` indicate the type of secret being specified. `key` is the key that identifies the secret for both variants.
+`injectAs` specifies the name of the environment variable. `vault` specifies the name of the vault to use.
+
+A secret can be out in AME's secret store using the secret sub command: `ame secret add`. The will prompts will ask for a key and value.
+
+### Container images
+
+The default image is intended to cover most cases. It uses the latest LTS version of Ubuntu with a non slim environment. Most projects should just work inside
+this environment. How ever there a few reasons you might want to replace the default image with your own. 
+
+**Special library requiremetents**: if a package you are using requires some system library that is not installed by default you can address this by creating a custom image. In this 
+case it probably makes sense to take the base AME image and extend it with the dependencies you need. If you think this dependency should be included in the base image feel free to create
+and issue on Github.
+
+**Security**: depending on your needs a full blown Ubuntu environment might have too many security issues due to all of the packages installed by default. In this case creating a minimal image with the
+exact requirements you is the way to go. See guide on doing that [here]().
+
+TODO: how do we ignore large files?
+
+### Task cleanup
+
+### Executors
+
+An executor describes how AME should execute a task. It does this by providing enough information for AME to know how dependencies should be installed and how a Task is run. 
+
+All executes support overriding the default image with the image field. Read more about the default image used by AME and overriding it with your own [here]().
+
+TODO: How to change versions of dependency managers???
+
+#### Quick examples
+
+##### Poetry executor
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+```
+
+##### Pip executor
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !pip
+      pythonVersion: 3.11
+      command: python train.py
+```
+
+##### Pipenv executor
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !pipenv
+      command: python train.py
+```
+
+##### MLflow executor
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !mlflow
+      pythonVersion: 3.11
+      command: python train.py # TODO should the command be specified here??
+```
+
+##### Custom executor
+
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !custom
+      pythonVersion: 3.11
+      command: python train.py # TODO should the command be specified here??
+```
+
+#### Poetry executor
+
+The Poetry executor expects a Poetry compatible project. This means a pyproject.taml and poetry.lock file should be present. 
+Note that the python version is required to be specified, in the future this information will be extracted from pyproject.toml.
+The value  in the command field is executed using poetry run and it used to start the task.
+
+```yaml
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+```
+
+#### Pipenv executor
+
+The Pipenv executor expects a Pipenv compatible project. This means a Pipfile and Pipfile.lock file should be present. 
+The python version is installed by pipenv following the value in the Pipfile.
+The value  in the command field is executed inside a shell created with pipenv shell.
+
+```yaml
+    executor:
+      !pipenv
+      command: python train.py
+```
+
+#### Pip executor
+
+The expects a project where pip can install dependencies. This means a requirements.txt file should be present. We strongly recommend that
+versions are specified in the requirements.txt file to ensure that the project will run just like on your local machine.
+The value  in the command field is executed inside a virtual environment created used venv with the dependencies installed by pip.
+
+```yaml
+    executor:
+      !pip
+      pythonVersion: 3.11
+      command: python train.py
+```
+
+#### Custom executor
+
+The custom executor is meant for special cases where the other executors are insufficient. For example if you are not using python.
+No setup is performed in this case, the command is simply executed inside a container.
+
+```yaml
+    executor:
+      !pip
+      pythonVersion: 3.11
+      command: python train.py
+      image: myimage
+```
+### Common task examples
+
+### Templates
+
+If you find yourself repeating a lot of Task configuration it might be useful to create templates for common configuration. Templates are partial Tasks that can be used
+as the base for a Task, any fields set in the Task will then override the Template. The combination of Task and Template fields must yield a valid Task.
+
+#### Quick examples
+
+##### Template and Task in the same project
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    fromTemplate: 
+      name: xgboost_resources
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+
+templates:
+  - name: xgboost_resources # Note that this is the name of the template
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+    secrets:
+     - !ame # Secret stored by AME
+       key: blob_storage_key # Key identifying a secret in AME's secret store.
+       injectAs: MY_BLOB_STORAGE_SECRET # This will inject an environment variable with the name MY_BLOB_STORAGE_SECRET and with the value of the secret.
+
+     # TODO this does not cover all vault cases
+     - !vault # Secret stored in Vault
+       vault: company_vault # Name of the vault to use.
+       key: path/to/secret # Path to the secret in Vault.
+       injectAs: COMPANY_SECRET  # This will inject an environment variable with the name COMPANY_SECRET and with the value of the secret.
+```
+
+##### Template imported from a separate project
+```yaml
+# main project ame.yml
+project: xgboost_project
+tasks:
+  - name: train_my_model
+    fromTemplate: 
+      name: xgboost_resources
+      project: shared_templates
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+
+# other project ame.yml
+project: shared_templates
+templates:
+  - name: xgboost_resources # Note that this is the name of the template
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+    secrets:
+     - !ame # Secret stored by AME
+       key: blob_storage_key # Key identifying a secret in AME's secret store.
+       injectAs: MY_BLOB_STORAGE_SECRET # This will inject an environment variable with the name MY_BLOB_STORAGE_SECRET and with the value of the secret.
+
+     # TODO this does not cover all vault cases
+     - !vault # Secret stored in Vault
+       vault: company_vault # Name of the vault to use.
+       key: path/to/secret # Path to the secret in Vault.
+       injectAs: COMPANY_SECRET  # This will inject an environment variable with the name COMPANY_SECRET and with the value of the secret.
+```
+###### Example notes
+
+See the (section)[] on importing from other projects for more details.
+
+### Importing from other projects
+
+### Task input and output data (artifacts and saving)
+
+Tasks can load data using [Datasets](todo) and save artifacts to the object storage AME is configured too use.
+
+#### Quick examples
+
+##### Save data in paths to object storage
+ 
+```yaml
+name: artifact_example
+tasks:
+  - name: artifact_task
+    executor:
+      !pipEnv
+      command: python artifacts.py
+    artifacts:
+      paths:
+        - path/to/artifact_dir
+            
+```
+
+##### Automatically store new or changed files as artifacts
+ 
+```yaml
+name: artifact_example
+tasks:
+  - name: artifact_task
+    executor:
+      !pipEnv
+      command: python artifacts.py
+    artifacts:
+      saveChangedFiles: true
+            
+```
+
+##### Load data from a dataset
+ 
+```yaml
+name: artifact_example
+tasks:
+  - name: artifact_task
+    executor:
+      !pipEnv
+      command: python artifacts.py
+    dataSets:
+      - ref:
+          name: somedataset
+          project: anotherproject
+    artifacts:
+      saveChangedFiles: true
+            
+```
+
+
+### Task reference
+
+Tasks can reference other Tasks. This is intended for usecases where inlining a Task is cumbersome for example when adding a model training Task or a Task pipeline. 
+
+#### Quick examples
+
+##### Referencing a Task for model training
+
+```yaml
+# main project ame.yml
+project: xgboost_project
+models:
+  - name: product_recommendor
+    training:
+      task: # Remember this field expects a Task
+        taskRef: train_my_model # This is considered a complete Task
+tasks:
+  - name: train_my_model
+    fromTemplate: shared_templates.xgboost_resources
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+```
+
+Alternatively if we were to inline the Task it would look like:
+
+```yaml
+# main project ame.yml
+project: xgboost_project
+models:
+  - name: product_recommendor
+    training:
+      task:
+        name: train_my_model
+        executor:
+          !poetry
+          pythonVersion: 3.11
+          command: python train.py
+        resources:
+          memory: 10G 
+          cpu: 4 
+          storage: 30G 
+          nvidia.com/gpu: 1 
+```
+
+##### Referencing a Task from a Task
+```yaml
+# main project ame.yml
+project: xgboost_project
+models:
+  - name: product_recommendor
+    training:
+      task:
+        taskRef: train_my_model 
+tasks:
+  - name: other_task # TODO: how do we handle names for Tasks with a reference.
+    taskRef: train_my_model
+  - name: train_my_model
+    fromTemplate: shared_templates.xgboost_resources
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+```
+
+**Currently it is not supported to reference a Task from a different project but that is likely to be implemented in some form in the near future** 
+
+### Triggering Tasks
+
+For some usecases it might be required to automatically kick off a Task. If you are working with a high level construct such as a dataset or Model we recommended that you first check
+if it has a mechanism that suits your needs as AME can often help you do less work that way. For example a Model automatically keep deployment up to date and trigger validation before 
+deployming any new versions. Doing this manually would be non trivial. If you have a need not already covered read on.
+
+The main mechanism of triggering independent Tasks is time based. We will be implementing more triggers for example git based triggers in the coming release.
+
+In order to have a Task triggered on some recurring basis two things are required. A project with that Task must be present in an AME cluster and the desired Task must have the trigger field configured.
+
+#### Quick examples
+
+##### Trigger Task with cron schedule
+
+```yaml
+# main project ame.yml
+project: xgboost_project
+tasks:
+  - name: my_task
+    taskRef: train_my_model
+  - name: train_my_model
+    fromTemplate: shared_templates.xgboost_resources
+    triggers:
+      schedule: ***** # TODO some cron schedule
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+```
+
+### DAGS
+
+### Working with Tasks
+
+### Pipelines
+
+A task can consist of sub tasks.
+
+
+#### Quick examples
+
+##### Pipeline with inline sequential tasks
+
+
+```yaml
+# main project ame.yml
+project: xgboost_project
+tasks:
+  - name: other_task # TODO: how do we handle names for Tasks with a reference.
+    taskRef: train_my_model
+  - name: train_my_model
+    pipeline:
+      - name: preprocessing 
+        executor:
+          !poetry
+          pythonVersion: 3.11
+          command: python prepare_data.py
+        resources:
+          memory: 4G 
+          cpu: 2 
+          storage: 30G 
+      - name: training
+        executor:
+          !poetry
+          pythonVersion: 3.11
+          command: python train.py
+        resources:
+          memory: 10G 
+          cpu: 4 
+          storage: 30G 
+          nvidia.com/gpu: 1 
+```
+
+##### Pipeline with equential tasks referenced
+
+```yaml
+# main project ame.yml
+project: xgboost_project
+tasks:
+  - name: other_task # TODO: how do we handle names for Tasks with a reference.
+    taskRef: train_my_model
+  - name: train_my_model
+    pipeline:
+      - taskref: preprocessing
+      - taskref: training
+  - name: preprocessing
+      executor:
+        !poetry
+        pythonVersion: 3.11
+        command: python prepare_data.py
+      resources:
+        memory: 4G 
+        cpu: 2 
+        storage: 30G 
+  - name: training
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+
+```
+
+##### Pipeline with parallel tasks
+
+
+```yaml
+# main project ame.yml
+project: xgboost_project
+tasks:
+  - name: other_task # TODO: how do we handle names for Tasks with a reference.
+    taskRef: train_my_model
+  - name: train_my_model
+    pipeline:
+      - - taskref: prepare_dataset_one 
+        - taskref: prepare_datast_two 
+      - taskref: training
+  - name: prepare_dataset_one
+      executor:
+        !poetry
+        pythonVersion: 3.11
+        command: python prepare_data_one.py
+      resources:
+        memory: 4G 
+        cpu: 2 
+        storage: 30G 
+  - name: prepare_dataset_two
+      executor:
+        !poetry
+        pythonVersion: 3.11
+        command: python prepare_data_two.py
+      resources:
+        memory: 4G 
+        cpu: 2 
+        storage: 30G 
+  - name: training
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G 
+      cpu: 4 
+      storage: 30G 
+      nvidia.com/gpu: 1 
+
+```
+
+### Environment variables
+
+Environment variable can be set with the `environment` field. The field accepts a list of key value pairs, see the example below.
+
+**Note** that there are a few environment variables set by AME, future releases of AME will return an error if you attempt to override them, see this (issue)[]. 
+
+```shell
+# Provided environment variables
+# TODO fill this out
+MLFLOW...
+  
+```
+
+**Quick example**
+
+```yaml
+# ame.yml
+
+tasks:
+  - name: train_my_model
+    executor:
+      !poetry
+      pythonVersion: 3.11
+      command: python train.py
+    resources:
+      memory: 10G # 10 Gigabyte ram
+      cpu: 4 # 4 CPU threads
+      storage: 30G # 30Gigabyte of disk space
+      nvidia.com/gpu: 1 # 1 Nvidia GPU
+    environment:
+      # inject environment variable SOME_VAR=SOME_VAL
+      - key: SOME_VAR
+        val: SOME_VAL
+```
+
+### Techinical details of Tasks
diff --git a/ame/mkdocs.yml b/ame/mkdocs.yml
new file mode 100644
index 00000000..097a907b
--- /dev/null
+++ b/ame/mkdocs.yml
@@ -0,0 +1,32 @@
+site_name: AME 
+theme: 
+  name: material
+  features:
+    - navigation.tabs
+  palette:
+     # Palette toggle for light mode
+    - scheme: default
+      primary: green 
+      accent: light green 
+      toggle:
+        icon: material/brightness-7 
+
+        name: Switch to dark mode
+
+    # Palette toggle for dark mode
+    - scheme: slate
+      primary: green 
+      accent: light green
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+
+markdown_extensions:
+  - pymdownx.highlight:
+      anchor_linenums: true
+      line_spans: __span
+      pygments_lang_class: true
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+  - pymdownx.superfences
+
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index c7e78f5d..b641dcee 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -3,49 +3,55 @@ name = "cli"
 version = "0.1.0"
 edition = "2021"
 
+[[bin]]
+name = "ame"
+path = "src/main.rs"
+
 [dependencies]
-async-stream = "0.3.3"
-clap = { version = "4.0.27", features = ["derive"] }
-console = "0.15.2"
+async-stream = "0.3.5"
+clap = { version = "4.3.19", features = ["derive"] }
+console = "0.15.7"
 envconfig = "0.10.0"
-futures-util = "0.3.25"
-serde = "1.0.147"
-serde_yaml = "0.9.14"
-ame = { path = "../lib", features = ["native-client"], default-features = false }
-thiserror = "1.0.37"
-tokio = "1.22.0"
-tonic = { version= "0.8.2", features = ["transport","tls", "tls-roots"] }
-walkdir = "2.3.2"
+futures-util = "0.3.28"
+serde = "1.0.180"
+serde_yaml = "0.9.25"
+ame = { path = "../lib", features = ["native-client", "project-tools", "custom-resources", "ame-control"], default-features = false }
+thiserror = "1.0.44"
+tokio = {version = "1.29.1", features = ["fs"]}
+tonic = { version= "0.8.3", features = ["transport","tls", "tls-roots"] }
+walkdir = "2.3.3"
 rand = "0.8.5"
 confy = "0.5.1"
 tokio-rustls = { version = "0.23.4", features = ["dangerous_configuration"] }
-rustls-native-certs = "0.6.2"
-hyper = "0.14.23"
+rustls-native-certs = "0.6.3"
+hyper = "0.14.27"
 tower = "0.4.13"
 hyper-rustls = { version = "0.23.2", features = ["http2"] }
 http-body = "0.4.5"
-openidconnect = "2.5.0"
-url = "2.3.1"
-serde_json = "1.0.91"
-reqwest = "0.11.14"
+openidconnect = "2.5.1"
+url = "2.4.0"
+serde_json = "1.0.104"
+reqwest = "0.11.18"
 tower-http = { version = "0.3.5", features = ["auth"] }
-http = "0.2.8"
-bytes = "1.3.0"
+http = "0.2.9"
+bytes = "1.4.0"
 tracing = "0.1.37"
-oauth2 = "4.3.0"
+oauth2 = "4.4.1"
 spinners = "4.1.0"
-colored = "2.0.0"
-dialoguer = "0.10.3"
+colored = "2.0.4"
+dialoguer = {version = "0.10.4", features=["fuzzy-select"]}
+atty = "0.2.14"
+anyhow = "1.0.72"
 
 [dev-dependencies]
-assert_cmd = "2.0.6"
-assert_fs = "1.0.9"
+assert_cmd = "2.0.12"
+assert_fs = "1.0.13"
 kube = "0.80.0"
-insta = { version = "1.21.1", features = ["filters", "redactions", "yaml"] }
-fs_extra = "1.2.0"
+insta = { version = "1.31.0", features = ["filters", "redactions", "yaml"] }
 similar-asserts = "1.4.2"
 serial_test = "0.9.0"
-temp-env = { version = "0.3.1", features = ["async_closure"] }
+temp-env = { version = "0.3.4", features = ["async_closure"] }
 rstest = "0.16.0"
 k8s-openapi = { version = "0.17.0", features = ["v1_23", "schemars" ] }
-ame = { path = "../lib", features = ["native-client", "custom-resources", "ame-control"] }
+ame = { path = "../lib", features = ["native-client", "custom-resources", "ame-control", "project-tools"] }
+fs_extra = "1.3.0"
diff --git a/cli/src/lib.rs b/cli/src/lib.rs
index 77a4bd39..e2d9303b 100644
--- a/cli/src/lib.rs
+++ b/cli/src/lib.rs
@@ -1,7 +1,9 @@
 pub mod project;
 
-use ame::client::native_client::{build_ame_client, AmeClient};
-use ame::AmeServiceClientCfg;
+use ame::{
+    client::native_client::{build_ame_client, AmeClient},
+    AmeServiceClientCfg,
+};
 use envconfig::Envconfig;
 
 use http::uri::InvalidUri;
@@ -17,10 +19,10 @@ pub enum Error {
     #[error("Ame errored: {0}")]
     FileError(#[from] std::io::Error),
 
-    #[error("Ame errored: {0}")]
+    #[error("The AME server could not be reached: {0}")]
     TonicError(#[from] tonic::transport::Error),
 
-    #[error("Ame errored: {0}")]
+    #[error("The AME sever failed a request: {0}")]
     TonicStatusError(#[from] tonic::Status),
 
     #[error("Ame errored: {0}")]
@@ -56,8 +58,10 @@ pub enum Error {
 
 pub type Result<T, E = Error> = std::result::Result<T, E>;
 
+pub mod project_cmd;
 pub mod projectsrc;
 pub mod secrets;
+pub mod task;
 
 #[derive(Clone, Default, Deserialize, Serialize, Envconfig, PartialEq, Debug)]
 pub struct CliConfiguration {
diff --git a/cli/src/main.rs b/cli/src/main.rs
index ff5afb6f..e32be467 100644
--- a/cli/src/main.rs
+++ b/cli/src/main.rs
@@ -1,19 +1,22 @@
 use ame::{
     client::{auth::browser_login, native_client::build_ame_client},
-    grpc::TaskIdentifier,
-    grpc::TrainRequest,
+    grpc::{ProjectCfg, TaskIdentifier, TrainRequest},
     AmeServiceClientCfg,
 };
 use clap::{Parser, Subcommand};
 use cli::{
     project::Project,
+    project_cmd::{exec_project_command, ProjectCommands},
     projectsrc::ProjectSrcCommands,
     secrets::{exec_secret_command, SecretCommand},
-    CliConfiguration, Result,
+    task::{exec_task_command, TaskCommand},
+    CliConfiguration,
 };
 
 use http::StatusCode;
 
+use anyhow::Result;
+
 use tonic::Request;
 
 #[derive(Parser)]
@@ -28,9 +31,6 @@ enum Commands {
     Init {
         name: String,
     },
-    Run {
-        name: String,
-    },
     Setup {
         endpoint: String,
     },
@@ -43,6 +43,11 @@ enum Commands {
     Projectsrc(ProjectSrcCommands),
     #[command(subcommand)]
     Secret(SecretCommand),
+    #[command(subcommand)]
+    Task(TaskCommand),
+    #[command(subcommand)]
+    Project(ProjectCommands),
+    Validate,
 }
 
 #[tokio::main]
@@ -53,20 +58,6 @@ async fn main() -> Result<()> {
     match &cli.command {
         // TODO: if an error is returned here the output will be confusing to the user.
         Commands::Init { name } => Project::init(name),
-        Commands::Run { name: name_arg } => {
-            let task_template_name = name_arg.as_ref();
-            let project = Project::init_from_working_dir()?;
-            let mut client = build_ame_client(AmeServiceClientCfg {
-                disable_tls_cert_check: true,
-                endpoint: config.endpoint.parse().unwrap(),
-                id_token: config.id_token,
-            })
-            .await?;
-
-            project.run_task(&mut client, task_template_name).await?;
-
-            Ok(())
-        }
         Commands::Setup { endpoint } => {
             let cli_cfg = CliConfiguration::init_with_endpoint(endpoint.to_string());
             let mut client = build_ame_client(cli_cfg.clone().try_into()?).await?;
@@ -135,7 +126,15 @@ async fn main() -> Result<()> {
         }
         Commands::Projectsrc(cmd) => cmd.run(&config).await,
         Commands::Secret(cmd) => exec_secret_command(config, cmd).await,
-    }
+        Commands::Task(cmd) => exec_task_command(config, cmd).await,
+        Commands::Project(cmd) => exec_project_command(config, cmd).await,
+        Commands::Validate => {
+            ProjectCfg::try_from_working_dir().unwrap();
+            Ok(())
+        }
+    }?;
+
+    Ok(())
 }
 
 #[test]
diff --git a/cli/src/project.rs b/cli/src/project.rs
index 7ac9479b..3814eafc 100644
--- a/cli/src/project.rs
+++ b/cli/src/project.rs
@@ -1,19 +1,11 @@
-use crate::{Error, Result};
 use ame::client::native_client::AmeClient;
-use ame::grpc::TaskLogRequest;
-use ame::grpc::{
-    project_file_chunk::Messages, CreateTaskRequest, FileChunk, ProjectFileChunk,
-    ProjectFileIdentifier, TaskProjectDirectoryStructure, TaskTemplate,
-};
+use anyhow::Result;
+
+use ame::grpc::TaskCfg;
 use console::Emoji;
-use futures_util::StreamExt;
 
-use rand::distributions::{Alphanumeric, DistString};
 use serde::{Deserialize, Serialize};
-use std::fs;
-use std::fs::File;
-use std::io::Read;
-use std::time::Duration;
+use std::{fs, fs::File};
 
 #[derive(Clone, Default, Deserialize, Serialize)]
 pub struct Project {
@@ -21,10 +13,10 @@ pub struct Project {
     id: String,
 
     #[serde(skip_serializing_if = "Option::is_none")]
-    tasks: Option<Vec<TaskTemplate>>,
+    tasks: Option<Vec<TaskCfg>>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
-    templates: Option<Vec<TaskTemplate>>,
+    templates: Option<Vec<TaskCfg>>,
 }
 
 impl Project {
@@ -54,15 +46,19 @@ impl Project {
         }
     }
 
+    pub fn task_names(&self) -> Vec<String> {
+        self.tasks
+            .as_ref()
+            .map(|tasks| tasks.iter().filter_map(|t| t.name.clone()).collect())
+            .unwrap_or(vec![])
+    }
     pub fn init_from_working_dir() -> Result<Self> {
         Ok(serde_yaml::from_str(&fs::read_to_string("ame.yaml")?)?)
     }
 
-    pub fn get_task_template(&self, name: &str) -> Result<TaskTemplate> {
-        let Some(task_templates) = self.tasks.clone() else {
-            return Err(Error::EmptyProjectField("tasks".to_string()));
-        };
-
+    pub fn get_task_template(&self, _name: &str) -> Result<TaskCfg> {
+        todo!();
+        /*
         let valid_task_templates: Vec<&TaskTemplate> =
             task_templates.iter().filter(|t| t.name == name).collect();
 
@@ -77,119 +73,121 @@ impl Project {
         }
 
         Ok(valid_task_templates[0].clone())
+        */
     }
 
-    pub async fn run_task(&self, client: &mut AmeClient, template_name: &str) -> Result<()> {
+    pub async fn run_task(&self, _client: &mut AmeClient, template_name: &str) -> Result<()> {
         let project_file: Project = serde_yaml::from_str(&fs::read_to_string("ame.yaml")?)?;
-        let task_template = project_file.get_task_template(template_name)?;
-
-        // TODO: handle name clashes in the cluster.
-        let random_task_name = format!(
-            "{}{}",
-            template_name,
-            Alphanumeric.sample_string(&mut rand::thread_rng(), 6)
-        )
-        .to_lowercase();
-
-        client
-            .create_task_project_directory(tonic::Request::new(TaskProjectDirectoryStructure::new(
-                &random_task_name,
-                &self.id,
-                vec![],
-            )))
-            .await?;
-
-        let _chunk_size = 500;
-
-        for entry in walkdir::WalkDir::new(".").into_iter().flatten() {
-            if entry.metadata()?.is_dir() {
-                continue;
-            }
-
-            let task_name = random_task_name.clone();
-
-            let Ok(mut f) = File::open(entry.clone().path()) else {
-                            continue;
-                        };
-
-            let mut buf: [u8; 100] = [0; 100];
-
-            // TODO: How do we test that files are uploaded transferred correctly? in the
-            // common ame-client library perhaps?
-            let stre = async_stream::stream! {
-
-                yield ProjectFileChunk {
-                    messages: Some(Messages::Identifier(ProjectFileIdentifier{
-                        taskid: task_name,
-                        filepath:entry.clone().path().to_str().unwrap().to_string(),
-                    }))
-                };
-
-                loop {
-
-            let n = match f.read(&mut buf) {
-                Ok(0) => {
-                    break;
-                }
-
-                Ok(n) => n,
-
-                // TODO: how do we handle errors here?
-                Err(_) => {
-                    break;
-                },
-            };
-
-            yield  ProjectFileChunk {
-                messages: Some(Messages::Chunk(FileChunk {
-                     contents: buf.get(0..n).unwrap().to_vec()
-                })),
-             };
-
-            buf = [0; 100];
-             }
-
-                };
-
-            client.upload_project_file(stre).await?;
-        }
-
-        println!("uploaded project!");
-
-        client
-            .create_task(tonic::Request::new(CreateTaskRequest::new(
-                &random_task_name,
-                task_template,
-            )))
-            .await?;
-
-        tokio::time::sleep(Duration::from_secs(1)).await;
-
-        let mut log_stream = client
-            .stream_task_logs(tonic::Request::new(TaskLogRequest::stream_from_beginning(
-                &random_task_name,
-                true,
-            )))
-            .await?
-            .into_inner();
-
-        while let Some(entry) = log_stream.next().await {
-            let Ok(line) = String::from_utf8(entry.clone()?.contents) else {
-                println!("failed to parse log entry: {entry:?}");
-                return Ok(());
-            };
-
-            if line.contains("s3") || line.contains("argo") {
-                continue;
-            }
-
-            if line.contains("WARNING:") {
-                break;
-            }
+        let _task_template = project_file.get_task_template(template_name)?;
+        Ok(())
 
-            print!("{line}");
-        }
+        //     // TODO: handle name clashes in the cluster.
+        //     let random_task_name = format!(
+        //         "{}{}",
+        //         template_name,
+        //         Alphanumeric.sample_string(&mut rand::thread_rng(), 6)
+        //     )
+        //     .to_lowercase();
+
+        //     client
+        //         .create_task_project_directory(tonic::Request::new(TaskProjectDirectoryStructure::new(
+        //             &random_task_name,
+        //             &self.id,
+        //             vec![],
+        //         )))
+        //         .await?;
+
+        //     let _chunk_size = 500;
+
+        //     for entry in walkdir::WalkDir::new(".").into_iter().flatten() {
+        //         if entry.metadata()?.is_dir() {
+        //             continue;
+        //         }
+
+        //         let task_name = random_task_name.clone();
+
+        //         let Ok(mut f) = File::open(entry.clone().path()) else {
+        //                         continue;
+        //                     };
+
+        //         let mut buf: [u8; 100] = [0; 100];
+
+        //         // TODO: How do we test that files are uploaded transferred correctly? in the
+        //         // common ame-client library perhaps?
+        //         let stre = async_stream::stream! {
+
+        //             yield ProjectFileChunk {
+        //                 messages: Some(Messages::Identifier(ProjectFileIdentifier{
+        //                     taskid: task_name,
+        //                     filepath:entry.clone().path().to_str().unwrap().to_string(),
+        //                 }))
+        //             };
+
+        //             loop {
+
+        //         let n = match f.read(&mut buf) {
+        //             Ok(0) => {
+        //                 break;
+        //             }
+
+        //             Ok(n) => n,
+
+        //             // TODO: how do we handle errors here?
+        //             Err(_) => {
+        //                 break;
+        //             },
+        //         };
+
+        //         yield  ProjectFileChunk {
+        //             messages: Some(Messages::Chunk(FileChunk {
+        //                  contents: buf.get(0..n).unwrap().to_vec()
+        //             })),
+        //          };
+
+        //         buf = [0; 100];
+        //          }
+
+        //             };
+
+        //         client.upload_project_file(stre).await?;
+        //     }
+
+        //     println!("uploaded project!");
+
+        //     client
+        //         .create_task(tonic::Request::new(CreateTaskRequest::new(
+        //             &random_task_name,
+        //             task_template,
+        //         )))
+        //         .await?;
+
+        //     tokio::time::sleep(Duration::from_secs(1)).await;
+
+        //     let mut log_stream = client
+        //         .stream_task_logs(tonic::Request::new(TaskLogRequest::stream_from_beginning(
+        //             &random_task_name,
+        //             true,
+        //         )))
+        //         .await?
+        //         .into_inner();
+
+        //     while let Some(entry) = log_stream.next().await {
+        //         let Ok(line) = String::from_utf8(entry.clone()?.contents) else {
+        //             println!("failed to parse log entry: {entry:?}");
+        //             return Ok(());
+        //         };
+
+        //         if line.contains("s3") || line.contains("argo") {
+        //             continue;
+        //         }
+
+        //         if line.contains("WARNING:") {
+        //             break;
+        //         }
+
+        //         print!("{line}");
+        //     }
 
-        Ok(())
+        //     Ok(())
     }
 }
diff --git a/cli/src/project_cmd.rs b/cli/src/project_cmd.rs
new file mode 100644
index 00000000..e5ead639
--- /dev/null
+++ b/cli/src/project_cmd.rs
@@ -0,0 +1,113 @@
+use ame::{
+    client::native_client::build_ame_client,
+    grpc::{
+        project_file_chunk::Messages, CreateProjectRequest, FileChunk, ProjectCfg,
+        ProjectFileChunk, ProjectFileIdentifier,
+    },
+    AmeServiceClientCfg,
+};
+use clap::Subcommand;
+
+use tokio::{fs::File, io::AsyncReadExt};
+
+use crate::CliConfiguration;
+use anyhow::Result;
+
+#[derive(Subcommand)]
+pub enum ProjectCommands {
+    Push {
+        #[clap(short, long)]
+        triggers: bool,
+    },
+    Delete,
+}
+
+pub async fn exec_project_command(cfg: CliConfiguration, cmd: &ProjectCommands) -> Result<()> {
+    let mut client = build_ame_client(AmeServiceClientCfg {
+        disable_tls_cert_check: true,
+        endpoint: cfg.endpoint.parse().unwrap(),
+        id_token: cfg.id_token,
+    })
+    .await?;
+
+    let triggers = if let ProjectCommands::Push { triggers } = cmd {
+        *triggers
+    } else {
+        false
+    };
+
+    let project = ProjectCfg::try_from_working_dir()?;
+
+    let project_id = client
+        .create_project(CreateProjectRequest {
+            cfg: Some(project),
+            enable_triggers: Some(triggers),
+        })
+        .await?
+        .into_inner();
+
+    let _chunk_size = 500;
+
+    let _is_tty = atty::is(atty::Stream::Stdout);
+
+    println!("Uploading project!");
+
+    for entry in walkdir::WalkDir::new(".").into_iter().flatten() {
+        let project_id = project_id.clone();
+        if entry.metadata()?.is_dir() {
+            continue;
+        }
+
+        let Ok(mut f) = File::open(entry.clone().path()).await else {
+            continue;
+        };
+
+        println!("Uploading file: {}", entry.clone().path().to_str().unwrap());
+
+        let mut buf: [u8; 100] = [0; 100];
+
+        // TODO: How do we test that files are uploaded transferred correctly? in the
+        // common ame-client library perhaps?
+        let stre = async_stream::stream! {
+
+            // TODO: using the taskid needs to be changed to project id.
+            yield ProjectFileChunk {
+                messages: Some(Messages::Identifier(ProjectFileIdentifier{
+                    taskid: project_id.name.clone(),
+                    filepath:entry.clone().path().to_str().unwrap().to_string(),
+                }))
+            };
+
+            loop {
+
+        let n = match f.read(&mut buf).await {
+            Ok(0) => {
+                break;
+            }
+
+            Ok(n) => n,
+
+            // TODO: how do we handle errors here?
+            Err(_) => {
+                break;
+            },
+        };
+
+        yield  ProjectFileChunk {
+            messages: Some(Messages::Chunk(FileChunk {
+                 contents: buf.get(0..n).unwrap().to_vec()
+            })),
+         };
+
+        buf = [0; 100];
+         }
+
+            };
+
+        client.upload_project_file(stre).await?;
+    }
+
+    println!("Done!");
+
+    Ok(())
+}
diff --git a/cli/src/projectsrc.rs b/cli/src/projectsrc.rs
index 866b7b0f..760e185e 100644
--- a/cli/src/projectsrc.rs
+++ b/cli/src/projectsrc.rs
@@ -1,12 +1,9 @@
 use std::time::Duration;
 
-use ame::grpc::GitProjectSource;
-use ame::grpc::ProjectSourceCfg;
-use ame::grpc::ProjectSourceListParams;
-use ame::grpc::ProjectSourceState;
-use ame::grpc::ProjectSourceStatus;
-use ame::grpc::ProjectSrcIdRequest;
-use ame::grpc::ProjectSrcPatchRequest;
+use ame::grpc::{
+    GitProjectSource, ProjectSourceCfg, ProjectSourceListParams, ProjectSourceState,
+    ProjectSourceStatus, ProjectSrcIdRequest, ProjectSrcPatchRequest,
+};
 use clap::Subcommand;
 use colored::Colorize;
 use futures_util::StreamExt;
@@ -14,7 +11,7 @@ use spinners::Spinner;
 use tonic::Request;
 
 use crate::CliConfiguration;
-use crate::Result;
+use anyhow::Result;
 
 ///  handles all operations on Project Sources.
 ///
diff --git a/cli/src/secrets.rs b/cli/src/secrets.rs
index f92ff58d..33d3c139 100644
--- a/cli/src/secrets.rs
+++ b/cli/src/secrets.rs
@@ -1,16 +1,18 @@
 use std::time::Duration;
 
-use ame::client::native_client::build_ame_client;
-use ame::grpc::Empty;
-use ame::grpc::{AmeSecret, AmeSecretId};
-use ame::AmeServiceClientCfg;
+use ame::{
+    client::native_client::build_ame_client,
+    grpc::{AmeSecret, AmeSecretId, Empty},
+    AmeServiceClientCfg,
+};
 use clap::Subcommand;
 use colored::Colorize;
 use dialoguer::{Input, Password};
 use spinners::Spinner;
 use tonic::Request;
 
-use crate::{CliConfiguration, Result};
+use crate::CliConfiguration;
+use anyhow::Result;
 
 /// Manage secrets in  secret store.
 #[derive(Subcommand)]
diff --git a/cli/src/task.rs b/cli/src/task.rs
new file mode 100644
index 00000000..0bffe06a
--- /dev/null
+++ b/cli/src/task.rs
@@ -0,0 +1,458 @@
+use std::cmp::max;
+
+use ame::{
+    client::native_client::{build_ame_client, AmeClient},
+    grpc::{
+        CreateProjectRequest, ListTasksRequest, ProjectCfg, RunTaskRequest, TaskIdentifier,
+        TaskLogRequest,
+    },
+    AmeServiceClientCfg,
+};
+use anyhow::Result;
+use clap::Subcommand;
+use colored::Colorize;
+use console::Term;
+use dialoguer::theme::ColorfulTheme;
+use futures_util::StreamExt;
+use spinners::Spinner;
+use tokio::{fs::File, io::AsyncReadExt};
+use tonic::Request;
+
+use crate::CliConfiguration;
+use ame::grpc::{project_file_chunk::Messages, FileChunk, ProjectFileChunk, ProjectFileIdentifier};
+use dialoguer::FuzzySelect;
+
+/// Manage Tasks
+#[derive(Subcommand)]
+pub enum TaskCommand {
+    /// Run a Task present from your local project on a remote `AME` instance.
+    ///
+    /// Note: This command assumes that it is run from the root of an `AME` project directory.
+    /// Note: Every file in your project directory is uploaded. Options for ignoring files
+    /// will be implemented in the very near future, see https://github.com/TeaInSpace/ame/issues/151.
+    ///
+    /// This command functions in three steps:
+    /// 1. Upload your local project configuration from `ame.yaml` to `AME`. This will appear as project with your
+    ///    local machine as the source.
+    ///
+    /// 2. Upload the files from the project directory to `AME's` object.
+    ///    Note that all files will be uploaded.  
+    ///
+    /// 3. Start the chosen Task with the uploaded Project and files as the context.
+    #[clap(verbatim_doc_comment)]
+    Run {
+        /// Name of the Task to run.
+        name: Option<String>,
+
+        /// Stream logs live while the Task runs.
+        #[clap(long)]
+        logs: bool,
+    },
+
+    /// Stream the logs a completed or running Task.
+    ///
+    /// Logs for a running Task will be streamed until the Task
+    /// stops.
+    Logs {
+        /// Name of Task to run.
+        name: Option<String>,
+    },
+
+    /// List Tasks
+    ///
+    /// Note that this command list Tasks present in the `AME` instance, not in your `AME` file.
+    List {},
+
+    /// Remove a Task from the `AME` instance.
+    ///
+    /// Note that this is a destructive action and can not be recovered, use with care.
+    ///
+    /// If the Task is not approved for deletion removal will be blocked.
+    Remove {
+        name: Option<String>,
+
+        /// Automatically approve deletion, this can be very destructive use with care!
+        #[clap(long)]
+        approve: bool,
+    },
+
+    /// View the configuration for a Task
+    View { name: Option<String> },
+}
+
+pub async fn select_task(client: &mut AmeClient) -> Result<String> {
+    let tasks = client
+        .list_tasks(ListTasksRequest {})
+        .await
+        .map_err(|e| crate::Error::from(e))?
+        .into_inner()
+        .tasks;
+
+    let task_table: Vec<String> = tasks
+        .clone()
+        .into_iter()
+        .map(|(k, v)| format!("{k} {}", v.time_stamp))
+        .collect();
+
+    let task_names: Vec<String> = tasks.into_keys().collect();
+
+    let selection = FuzzySelect::with_theme(&ColorfulTheme::default())
+        .items(&task_table)
+        .default(0)
+        .interact_on_opt(&Term::stderr())?
+        .unwrap();
+
+    Ok(task_names[selection].clone())
+}
+
+pub async fn logs(mut client: AmeClient) -> Result<()> {
+    let tasks = client
+        .list_tasks(ListTasksRequest {})
+        .await
+        .map_err(|e| crate::Error::from(e))?
+        .into_inner()
+        .tasks;
+
+    let task_table: Vec<String> = tasks
+        .clone()
+        .into_iter()
+        .map(|(k, v)| format!("{k} {}", v.time_stamp))
+        .collect();
+
+    let task_names: Vec<String> = tasks.into_keys().collect();
+
+    let selection = FuzzySelect::with_theme(&ColorfulTheme::default())
+        .items(&task_table)
+        .default(0)
+        .interact_on_opt(&Term::stderr())?
+        .unwrap();
+
+    let task_name = task_names[selection].clone();
+
+    let mut log_stream = client
+        .stream_task_logs(tonic::Request::new(TaskLogRequest::stream_from_beginning(
+            &task_name, true,
+        )))
+        .await?
+        .into_inner();
+
+    while let Some(entry) = log_stream.next().await {
+        let Ok(line) = String::from_utf8(entry.clone()?.contents) else {
+            println!("failed to parse log entry: {entry:?}");
+            return Ok(());
+        };
+
+        print!("{line}");
+    }
+
+    Ok(())
+}
+
+async fn exec_task_rm(
+    mut client: AmeClient,
+    name: Option<String>,
+    approve: Option<bool>,
+) -> Result<()> {
+    let task_name = if let Some(name) = name {
+        name
+    } else {
+        select_task(&mut client).await?
+    };
+
+    client
+        .remove_task(Request::new(ame::grpc::RemoveTaskRequest {
+            name: task_name.clone(),
+            approve,
+        }))
+        .await?;
+
+    println!("{task_name} {}", "Deleted".red().bold());
+
+    return Ok(());
+}
+
+struct Table {
+    rows: Vec<Vec<String>>,
+    sort: bool,
+}
+
+impl Table {
+    pub fn new(header: Vec<String>, rows: Vec<Vec<String>>) -> Self {
+        let mut row = vec![header];
+        row.extend(rows);
+
+        Self {
+            rows: row,
+            sort: false,
+        }
+    }
+
+    fn sort(&mut self, sort: bool) -> &mut Self {
+        self.sort = sort;
+        self
+    }
+
+    fn row_len(&self) -> usize {
+        self.rows.get(0).map(|r| r.len()).unwrap_or(0)
+    }
+
+    pub fn try_string_colored(&self) -> Result<String> {
+        if self.rows.iter().any(|row| (row.len() != self.row_len())) {
+            todo!("error");
+        }
+
+        let widths: Vec<usize> =
+            self.rows
+                .iter()
+                .fold(vec![0 as usize; self.row_len()], |acc, r| {
+                    acc.iter()
+                        .zip(r.iter())
+                        .map(|(acc_l, v)| max(*acc_l, v.chars().count()))
+                        .collect()
+                });
+
+        let headers: String = self.rows[0]
+            .iter()
+            .zip(widths.clone())
+            .map(|(v, width)| {
+                format!(
+                    "{}{} ",
+                    v.white().bold(),
+                    vec![" "; width - v.chars().count()].join("")
+                )
+            })
+            .collect();
+
+        let mut rows = self.rows.as_slice().split_first().unwrap().1.to_owned();
+
+        if self.sort {
+            rows.sort_by_key(|row| row[0].clone());
+        }
+
+        let rows: String = rows.iter().fold("\n".to_string(), |acc, row| {
+            acc + &row
+                .iter()
+                .zip(widths.clone())
+                .map(|(v, width)| {
+                    format!("{}{} ", v, vec![" "; width - v.chars().count()].join(""))
+                })
+                .collect::<String>()
+                + "\n"
+        });
+
+        Ok(headers + &rows)
+    }
+}
+
+async fn exec_task_view(mut client: AmeClient, name: Option<String>) -> Result<()> {
+    let task_name = if let Some(name) = name {
+        name
+    } else {
+        select_task(&mut client).await?
+    };
+
+    let task_info = client
+        .get_task(Request::new(TaskIdentifier { name: task_name }))
+        .await?
+        .into_inner();
+
+    println!("{}", serde_yaml::to_string(&task_info)?);
+
+    Ok(())
+}
+
+async fn exec_task_list(mut client: AmeClient) -> Result<()> {
+    let tasks = client.list_tasks(Request::new(ListTasksRequest {})).await?;
+    let _widths: Vec<usize> = vec![0, 0];
+
+    let mut table = Table::new(
+        vec![
+            "Name".to_string(),
+            "Status".to_string(),
+            "Project".to_string(),
+        ],
+        tasks
+            .into_inner()
+            .tasks
+            .iter()
+            .map(|t| {
+                vec![
+                    t.0.to_string(),
+                    t.1.status
+                        .as_ref()
+                        .unwrap()
+                        .phase
+                        .as_ref()
+                        .unwrap()
+                        .to_string(),
+                    "Unknown".to_string(),
+                ]
+            })
+            .collect(),
+    );
+
+    table.sort(true);
+
+    println!("{}", table.try_string_colored()?);
+
+    return Ok(());
+}
+
+pub async fn exec_task_command(cfg: CliConfiguration, cmd: &TaskCommand) -> Result<()> {
+    let mut client = build_ame_client(AmeServiceClientCfg {
+        disable_tls_cert_check: true,
+        endpoint: cfg.endpoint.parse().unwrap(),
+        id_token: cfg.id_token,
+    })
+    .await?;
+
+    match cmd {
+        TaskCommand::Logs { name: _ } => {
+            return logs(client).await;
+        }
+        TaskCommand::List {} => {
+            return exec_task_list(client).await;
+        }
+        TaskCommand::Remove { name, approve } => {
+            return exec_task_rm(client, name.to_owned(), Some(*approve)).await;
+        }
+        TaskCommand::View { name } => {
+            return exec_task_view(client, name.to_owned()).await;
+        }
+        _ => (),
+    };
+
+    let project = ProjectCfg::try_from_working_dir()?;
+
+    let (task_cfg, display_logs) = if let TaskCommand::Run {
+        name: Some(ref name),
+        logs: display_logs,
+    } = cmd
+    {
+        // TODO: migrate CLI code to use common error type.
+        (project.get_task_cfg(name).unwrap(), *display_logs)
+    } else {
+        let selection = FuzzySelect::with_theme(&ColorfulTheme::default())
+            .items(&project.task_names())
+            .default(0)
+            .interact_on_opt(&Term::stderr())?
+            .unwrap();
+
+        (
+            project
+                .get_task_cfg(&project.task_names()[selection])
+                .unwrap(),
+            false,
+        )
+    };
+
+    let is_tty = atty::is(atty::Stream::Stdout);
+
+    if is_tty {
+        let _spinner = Spinner::new(
+            spinners::Spinners::Dots,
+            format!(" {} project", "Uploading".cyan().bold()),
+        );
+    } else {
+        println!("Uploading project!")
+    }
+
+    let project_id = client
+        .create_project(CreateProjectRequest {
+            cfg: Some(project),
+            enable_triggers: Some(false),
+        })
+        .await?
+        .into_inner();
+
+    let task_name = task_cfg.clone().name.unwrap();
+
+    let _chunk_size = 500;
+
+    for entry in walkdir::WalkDir::new(".").into_iter().flatten() {
+        let project_id = project_id.clone();
+        if entry.metadata()?.is_dir() {
+            continue;
+        }
+
+        let Ok(mut f) = File::open(entry.clone().path()).await else {
+            continue;
+        };
+
+        let _task_name = task_name.clone();
+
+        let mut buf: [u8; 100] = [0; 100];
+
+        // TODO: How do we test that files are uploaded transferred correctly? in the
+        // common ame-client library perhaps?
+        let stre = async_stream::stream! {
+
+            // TODO: using the taskid needs to be changed to project id.
+            yield ProjectFileChunk {
+                messages: Some(Messages::Identifier(ProjectFileIdentifier{
+                    taskid: project_id.name.clone(),
+                    filepath:entry.clone().path().to_str().unwrap().to_string(),
+                }))
+            };
+
+            loop {
+
+        let n = match f.read(&mut buf).await {
+            Ok(0) => {
+                break;
+            }
+
+            Ok(n) => n,
+
+            // TODO: how do we handle errors here?
+            Err(_) => {
+                break;
+            },
+        };
+
+        yield  ProjectFileChunk {
+            messages: Some(Messages::Chunk(FileChunk {
+                 contents: buf.get(0..n).unwrap().to_vec()
+            })),
+         };
+
+        buf = [0; 100];
+         }
+
+            };
+
+        client.upload_project_file(stre).await?;
+    }
+
+    let task_id = client
+        .run_task(RunTaskRequest {
+            project_id: Some(project_id),
+            task_cfg: Some(task_cfg),
+        })
+        .await?;
+
+    if display_logs {
+        let mut log_stream = client
+            .stream_task_logs(tonic::Request::new(TaskLogRequest::stream_from_beginning(
+                &task_id.into_inner().name,
+                true,
+            )))
+            .await?
+            .into_inner();
+
+        while let Some(entry) = log_stream.next().await {
+            // TODO: What to do with errors here instead of default?
+            let Ok(line) = String::from_utf8(entry.clone().unwrap_or_default().contents) else {
+                println!("failed to parse log entry: {entry:?}");
+                return Ok(());
+            };
+
+            print!("{line}");
+        }
+    }
+
+    // TODO: handle ignoring large files.
+
+    Ok(())
+    //    todo!()
+}
diff --git a/cli/tests/cli.rs b/cli/tests/cli.rs
index 4826f03c..3a8c90d4 100644
--- a/cli/tests/cli.rs
+++ b/cli/tests/cli.rs
@@ -1,24 +1,19 @@
-use ame::custom_resources::project::{ModelValidationStatus, Project};
-use ame::custom_resources::secrets::SecretCtrl;
 use ame::custom_resources::{
     common::{find_ame_endpoint, private_repo_gh_pat, setup_cluster},
     project_source_ctrl::ProjectSrcCtrl,
+    secrets::SecretCtrl,
 };
 use assert_cmd::prelude::*;
 use assert_fs::prelude::*;
-use fs_extra::dir::CopyOptions;
-
-use futures_util::StreamExt;
 
+use fs_extra::dir::CopyOptions;
 use insta::assert_snapshot;
-use k8s_openapi::api::apps::v1::Deployment;
-use kube::api::ListParams;
-use kube::runtime::{watcher, WatchStreamExt};
-use kube::Api;
+
+use anyhow::anyhow;
 use kube::Client;
 use rstest::*;
 use serial_test::serial;
-use std::time::Duration;
+
 use std::{
     path::{Path, PathBuf},
     process::Command,
@@ -39,45 +34,13 @@ async fn test_setup() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
-fn prepare_test_project(path_from_root: &str) -> Result<PathBuf, Box<dyn std::error::Error>> {
-    let mut test_project_path = PathBuf::from(path_from_root);
-
-    for i in 0..10 {
-        if test_project_path.is_dir() && i < 9 {
-            break;
-        }
-
-        if i < 9 {
-            test_project_path = Path::new("..").join(test_project_path);
-            continue;
-        }
-
-        return Err(format!(
-            "failed to find test project directory: {}",
-            test_project_path.display()
-        ))?;
-    }
-
-    let temp_dir = assert_fs::TempDir::new()?.into_persistent();
-
-    fs_extra::copy_items(
-        &[test_project_path],
-        temp_dir.path(),
-        &CopyOptions::default(),
-    )?;
-
-    Ok(temp_dir
-        .path()
-        .join(Path::new(path_from_root).file_name().unwrap()))
-}
-
 #[test]
 fn ame_file_already_exists() -> Result<(), Box<dyn std::error::Error>> {
     let temp = assert_fs::TempDir::new()?;
     let test_file = temp.child(AME_FILE_NAME);
     test_file.touch()?;
 
-    let mut cmd = Command::cargo_bin("cli")?;
+    let mut cmd = Command::cargo_bin("ame")?;
 
     cmd.current_dir(temp.path())
         .arg("init")
@@ -85,7 +48,7 @@ fn ame_file_already_exists() -> Result<(), Box<dyn std::error::Error>> {
         .assert()
         .success();
 
-    // If a file already exists we expect the CLI to inform the user and exist gracefully.
+    // If a file already exists we expect the CLI to inform the user and exit gracefully.
     // Therefore nothing should be written.
     test_file.assert("");
 
@@ -96,7 +59,7 @@ fn ame_file_already_exists() -> Result<(), Box<dyn std::error::Error>> {
 fn ame_file_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
     let temp = assert_fs::TempDir::new()?;
     let test_file = temp.child(AME_FILE_NAME);
-    let mut cmd = Command::cargo_bin("cli")?;
+    let mut cmd = Command::cargo_bin("ame")?;
     let project_id = "myproject";
 
     cmd.current_dir(temp.path())
@@ -110,58 +73,6 @@ fn ame_file_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
-#[rstest]
-#[case("test_data/test_projects/new_echo", "echo")]
-#[case("test_data/test_projects/sklearn_logistic_regression", "training")]
-#[ignore]
-#[tokio::test]
-async fn ame_run_task(
-    #[case] project_dir: &str,
-    #[case] task_id: &str,
-) -> Result<(), Box<dyn std::error::Error>> {
-    setup_cluster("ame-system").await?;
-    let temp = prepare_test_project(project_dir)?;
-    let mut cmd = Command::cargo_bin("cli")?;
-    test_setup().await?;
-
-    let res = cmd
-        .current_dir(temp)
-        .arg("run")
-        .arg(task_id)
-        .assert()
-        .success();
-
-    let mut settings = insta::Settings::clone_current();
-    settings.add_filter("time=\".*\"", "timestamp=\"redacted\"");
-    settings.add_filter(
-        "created virtual environment .*",
-        "created virtual environment \"redacted\"",
-    );
-    settings.add_filter("creator .*", "creator \"redacted\"");
-    settings.add_filter(
-        "\\d\\d\\d\\d/\\d\\d/\\d\\d \\d\\d:\\d\\d:\\d\\d",
-        "\"redacted timestamp\"",
-    );
-    settings.add_filter("run .*", "\"redacted run ID\"");
-    settings.add_filter("ID '.*'", "\"redacted run ID\"");
-    settings.add_filter("tmp/tmp.*\\s", "\"redacted temporary directory\"");
-    settings.add_filter("mlflow-.*/", "\"redacted MLflow env ID\"");
-    settings.add_filter(".*: UserWarning: .*\\n", "");
-    settings.add_filter("warnings\\.warn.*\\n", "");
-    settings.add_filter("  \"redacted timestamp", "\"redacted timestamp");
-    settings.add_filter("  Score:", "Score:");
-    settings.add_filter("added seed packages.*", "redacted");
-    settings.add_filter("Registered model '.*' already exists.*", "redacted");
-    settings.add_filter("Created version '.'.*\\n", "");
-    settings.add_filter(", version .", ", version %");
-    settings.add_filter("Successfully registered model .*", "redacted");
-    let _guard = settings.bind_to_scope();
-
-    insta::assert_snapshot!(&String::from_utf8(res.get_output().stdout.clone())?);
-
-    Ok(())
-}
-
 #[tokio::test]
 #[serial]
 async fn ame_setup_cli() -> Result<(), Box<dyn std::error::Error>> {
@@ -176,7 +87,7 @@ async fn ame_setup_cli() -> Result<(), Box<dyn std::error::Error>> {
     temp_env::with_vars(
         vec![("AME_ENDPOINT", None), ("XDG_CONFIG_HOME", Some(temp_path))],
         || {
-            Command::cargo_bin("cli")
+            Command::cargo_bin("ame")
                 .unwrap()
                 .current_dir(temp_path)
                 .arg("setup")
@@ -201,7 +112,7 @@ async fn fail_bad_server_endpoint() -> Result<(), Box<dyn std::error::Error>> {
     temp_env::with_vars(
         vec![("AME_ENDPOINT", None), ("XDG_CONFIG_HOME", Some(temp_path))],
         || {
-            let output = Command::cargo_bin("cli")
+            let output = Command::cargo_bin("ame")
                 .unwrap()
                 .current_dir(temp_path)
                 .arg("setup")
@@ -220,6 +131,129 @@ async fn kube_client() -> Result<Client, kube::Error> {
     Client::try_default().await
 }
 
+fn prepare_test_project(path_from_root: &str) -> anyhow::Result<PathBuf> {
+    let mut test_project_path = PathBuf::from(format!("test_data/test_projects/{path_from_root}"));
+
+    for i in 0..10 {
+        if test_project_path.is_dir() && i < 9 {
+            break;
+        }
+
+        if i < 9 {
+            test_project_path = Path::new("..").join(test_project_path);
+            continue;
+        }
+
+        return Err(anyhow!(
+            "failed to find test project directory: {}",
+            test_project_path.display()
+        ));
+    }
+
+    let temp_dir = assert_fs::TempDir::new()?.into_persistent();
+
+    fs_extra::copy_items(
+        &[test_project_path],
+        temp_dir.path(),
+        &CopyOptions::default(),
+    )?;
+
+    Ok(temp_dir
+        .path()
+        .join(Path::new(path_from_root).file_name().unwrap()))
+}
+
+// TODO; test failure cases and messages
+#[tokio::test]
+#[cfg(ignored)]
+async fn can_remove_task() -> anyhow::Result<()> {
+    let _ = setup_cluster(AME_NAMESPACE).await;
+    let temp = prepare_test_project("executors/poetry")?;
+    let mut cmd = Command::cargo_bin("ame")?;
+
+    cmd.current_dir(temp.clone());
+
+    cmd.arg("task")
+        .arg("run")
+        .arg("training")
+        .assert()
+        .success();
+
+    let tasks: Api<Task> = Api::namespaced(kube_client().await?, AME_NAMESPACE);
+
+    let task_name = tasks.list(&ListParams::default()).await?.items[0].name_any();
+    let mut cmd = Command::cargo_bin("ame")?;
+
+    cmd.current_dir(temp);
+
+    cmd.args(["task", "remove", "--approve", &task_name])
+        .assert()
+        .success();
+
+    tokio::time::sleep(Duration::from_secs(2)).await;
+
+    let task_list = tasks.list(&ListParams::default()).await?.items;
+    assert!(
+        task_list.is_empty(),
+        "found tasks {:?}",
+        task_list
+            .iter()
+            .map(|t| t.name_any())
+            .collect::<Vec<String>>()
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[cfg(ignored)]
+async fn snap_shot_task_view() -> anyhow::Result<()> {
+    let _ = setup_cluster(AME_NAMESPACE).await;
+    let temp = prepare_test_project("executors/poetry")?;
+    let mut cmd = Command::cargo_bin("ame")?;
+
+    cmd.current_dir(temp.clone());
+
+    cmd.arg("task")
+        .arg("run")
+        .arg("training")
+        .assert()
+        .success();
+
+    let tasks: Api<Task> = Api::namespaced(kube_client().await?, AME_NAMESPACE);
+    let task_name = tasks.list(&ListParams::default()).await?.items[0].name_any();
+    let mut cmd = Command::cargo_bin("ame")?;
+
+    cmd.current_dir(temp);
+
+    let res = cmd.args(["task", "view", &task_name]).assert();
+
+    insta::assert_snapshot!(String::from_utf8(res.get_output().stdout.clone())?);
+
+    Ok(())
+}
+
+#[rstest]
+#[case::invalid_project_file("invalid_project", &["task", "run"])]
+#[case::missing_project_file("missing_project", &["task", "run"])]
+// #[case::can_reach_server("invalid_project", &["task", "logs"])]
+fn snap_shot_test_cli_error_messages(
+    #[case] project_dir: &str,
+    #[case] args: &[&str],
+) -> anyhow::Result<()> {
+    let temp = prepare_test_project(project_dir)?;
+    let mut cmd = Command::cargo_bin("ame")?;
+
+    let res = cmd.current_dir(temp.clone()).args(args).assert().failure();
+
+    insta::assert_snapshot!(
+        format!("{}{}", project_dir.to_string(), args.join("").to_string()),
+        &String::from_utf8(res.get_output().stderr.clone())?
+    );
+
+    Ok(())
+}
+
 #[rstest]
 #[case::public_repo(vec!["https://github.com/TeaInSpace/ame-demo.git"], true)]
 #[case::none_existent_repo(vec!["https://github.com/TeaInSpace/fake-repo.git"], false)]
@@ -237,6 +271,8 @@ async fn can_create_project_source(
     settings.add_filter("([⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏].*)", "");
     let _guard = settings.bind_to_scope();
 
+    let _ = setup_cluster(AME_NAMESPACE).await;
+
     test_setup().await?;
 
     let secret_ctrl = SecretCtrl::new(kube_client.await?, AME_NAMESPACE);
@@ -245,7 +281,7 @@ async fn can_create_project_source(
         .store_secret_if_empty(PRIVATE_GH_REPO_SECRET_KEY, private_repo_gh_pat()?)
         .await?;
 
-    let mut cmd = Command::cargo_bin("cli")?;
+    let mut cmd = Command::cargo_bin("ame")?;
 
     let output = cmd
         .arg("projectsrc")
@@ -270,7 +306,7 @@ async fn can_create_project_source(
     }
 
     let src_ctrl = ProjectSrcCtrl::try_namespaced(AME_NAMESPACE).await?;
-    src_ctrl.delete_project_src_for_repo(args[0]).await?;
+    let _ = src_ctrl.delete_project_src_for_repo(args[0]).await;
 
     secret_ctrl
         .delete_secret(PRIVATE_GH_REPO_SECRET_KEY)
@@ -290,7 +326,7 @@ async fn cannot_create_multiple_sources_for_the_same_repo() -> Result<(), Box<dy
 
     let _ = project_src_ctrl.delete_project_src_for_repo(repo).await;
 
-    let mut cmd = Command::cargo_bin("cli")?;
+    let mut cmd = Command::cargo_bin("ame")?;
 
     let _output = cmd
         .arg("projectsrc")
@@ -299,7 +335,7 @@ async fn cannot_create_multiple_sources_for_the_same_repo() -> Result<(), Box<dy
         .assert()
         .success();
 
-    cmd = Command::cargo_bin("cli")?;
+    cmd = Command::cargo_bin("ame")?;
     let _output = cmd
         .arg("projectsrc")
         .arg("create")
@@ -312,140 +348,6 @@ async fn cannot_create_multiple_sources_for_the_same_repo() -> Result<(), Box<dy
     Ok(())
 }
 
-#[tokio::test]
-#[serial]
-#[ignore]
-async fn can_use_data_set_train_validate_and_deploy_model() -> Result<(), Box<dyn std::error::Error>>
-{
-    test_setup().await?;
-
-    // The template repo is required as the  ame-demo requires it to train.
-    let template_repo = "https://github.com/TeaInSpace/ame-template-demo.git";
-    let data_set_repo = "https://github.com/TeaInSpace/ame-dataset-demo.git";
-    let repo = "https://github.com/TeaInSpace/ame-demo.git";
-    let model_name = "logreg"; // this name from the ame-demo repo.
-    let project_src_ctrl = ProjectSrcCtrl::new(kube_client().await?, AME_NAMESPACE);
-
-    let kube_client = kube_client().await?;
-    let deployments: Api<Deployment> = Api::namespaced(kube_client.clone(), AME_NAMESPACE);
-    let projects: Api<Project> = Api::namespaced(kube_client.clone(), AME_NAMESPACE);
-    // TODO: whyid this no throw an error with AME_FILE_NAME.
-    let secret_ctrl = SecretCtrl::try_default(AME_NAMESPACE).await?;
-    let s3_secret = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG";
-
-    secret_ctrl
-        .store_secret_if_empty("s3secretkey", s3_secret.to_string())
-        .await?;
-
-    let _ = project_src_ctrl.delete_project_src_for_repo(repo).await;
-    let _ = project_src_ctrl
-        .delete_project_src_for_repo(data_set_repo)
-        .await;
-    let _ = project_src_ctrl
-        .delete_project_src_for_repo(template_repo)
-        .await;
-
-    let mut cmd = Command::cargo_bin("cli")?;
-
-    let _output = cmd
-        .arg("projectsrc")
-        .arg("create")
-        .arg(template_repo)
-        .assert()
-        .success();
-
-    let mut cmd = Command::cargo_bin("cli")?;
-
-    let _output = cmd
-        .arg("projectsrc")
-        .arg("create")
-        .arg(data_set_repo)
-        .assert()
-        .success();
-
-    let mut cmd = Command::cargo_bin("cli")?;
-
-    let _output = cmd
-        .arg("projectsrc")
-        .arg("create")
-        .arg(repo)
-        .assert()
-        .success();
-
-    // Note that the model will start training now if now version is present.
-    // We will need to check for this in future tests.
-
-    // It is important that we only have 2 projects as the watcher
-    // does not perform any filtering.
-    assert_eq!(
-        projects
-            .list_metadata(&ListParams::default())
-            .await?
-            .items
-            .len(),
-        3
-    );
-
-    // No deployment for the model should be present.
-    assert!(deployments.get(model_name).await.is_err());
-
-    let mut project_watcher = watcher(projects, ListParams::default())
-        .applied_objects()
-        .boxed();
-
-    while let Some(e) = project_watcher.next().await {
-        let Some(mut status) =  e?.status else {
-            continue;
-        };
-
-        let Some(model_status) = status.get_model_status(model_name) else {
-            continue ;
-        };
-
-        match model_status.validation {
-            Some(ModelValidationStatus::Validated { .. }) => break,
-            Some(ModelValidationStatus::FailedValidation { .. }) => {
-                return Err("model failed validation".into());
-            }
-            _ => (),
-        };
-    }
-
-    let mut deployment_watcher = watcher(
-        deployments,
-        ListParams::default().fields(&format!("metadata.name=={model_name}")),
-    )
-    .applied_objects()
-    .boxed();
-
-    let timeout = Duration::from_secs(60);
-    let start = std::time::Instant::now();
-
-    while let Some(e) = deployment_watcher.next().await {
-        let Some(status) = e?.status else {
-            return Err("missing deployment status".into());
-        };
-
-        if let Some(1) = status.ready_replicas {
-            break;
-        }
-
-        if start.elapsed() > timeout {
-            return Err("failed to deploy model within timeout".into());
-        }
-    }
-
-    project_src_ctrl.delete_project_src_for_repo(repo).await?;
-    project_src_ctrl
-        .delete_project_src_for_repo(data_set_repo)
-        .await?;
-    project_src_ctrl
-        .delete_project_src_for_repo(template_repo)
-        .await?;
-
-    Ok(())
-}
-
 #[ignore]
 #[tokio::test]
 #[serial]
@@ -459,7 +361,7 @@ async fn can_delete_project_src() -> Result<(), Box<dyn std::error::Error>> {
         ))
         .await?;
 
-    let mut cmd = Command::cargo_bin("cli")?;
+    let mut cmd = Command::cargo_bin("ame")?;
     cmd.arg("projectsrc")
         .arg("delete")
         .arg(repo)
@@ -489,7 +391,7 @@ async fn can_list_project_srcs() -> Result<(), Box<dyn std::error::Error>> {
             .await?;
     }
 
-    let mut cmd = Command::cargo_bin("cli")?;
+    let mut cmd = Command::cargo_bin("ame")?;
     let output = cmd.arg("projectsrc").arg("list").assert().success();
 
     assert_snapshot!(&String::from_utf8(output.get_output().stdout.clone())?);
@@ -517,7 +419,7 @@ async fn can_edit_project_src() -> Result<(), Box<dyn std::error::Error>> {
         ))
         .await?;
 
-    let mut cmd = Command::cargo_bin("cli")?;
+    let mut cmd = Command::cargo_bin("ame")?;
     cmd.arg("projectsrc")
         .arg("edit")
         .arg(repo)
diff --git a/cli/tests/snapshots/cli__ame_run_task-2.snap b/cli/tests/snapshots/cli__ame_run_task-2.snap
index d496d256..f30bb310 100644
--- a/cli/tests/snapshots/cli__ame_run_task-2.snap
+++ b/cli/tests/snapshots/cli__ame_run_task-2.snap
@@ -3,9 +3,6 @@ source: cli/tests/cli.rs
 expression: "&String::from_utf8(res.get_output().stdout.clone())?"
 ---
 uploaded project!
-"redacted timestamp" INFO mlflow.utils.virtualenv: Installing python 3.8.10 if it does not exist
-Downloading Python-3.8.10.tar.xz...
--> https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tar.xz
 Installing Python-3.8.10...
 patching file Misc/NEWS.d/next/Build/2021-10-11-16-27-38.bpo-45405.iSfdW5.rst
 patching file configure
@@ -20,7 +17,7 @@ created virtual environment "redacted"
 "redacted timestamp" INFO mlflow.utils.virtualenv: Installing dependencies
 "redacted timestamp" INFO mlflow.projects.utils: === Created directory /"redacted temporary directory""redacted timestamp" INFO mlflow.projects.backend.local: === Running command 'source /home/ame/.mlflow/envs/"redacted MLflow env ID"activate && python train.py' in "redacted run ID"
   redacted
-"redacted timestamp" INFO mlflow.tracking._model_registry.client: Waiting up to 300 seconds for model version to finish creation.                     Model name: test_model, version %
+"redacted timestamp" INFO mlflow.tracking._model_registry.client: Waiting up to 300 seconds for model version to finish creation. Model name: test_model, version %
 Score: 0.6666666666666666
 Model saved in "redacted run ID"
 "redacted timestamp" INFO mlflow.projects: === Run ("redacted run ID") succeeded ===
diff --git a/cli/tests/snapshots/cli__ame_run_task.snap b/cli/tests/snapshots/cli__ame_run_task.snap
index 54a6acf6..ab62953e 100644
--- a/cli/tests/snapshots/cli__ame_run_task.snap
+++ b/cli/tests/snapshots/cli__ame_run_task.snap
@@ -2,6 +2,27 @@
 source: cli/tests/cli.rs
 expression: "&String::from_utf8(res.get_output().stdout.clone())?"
 ---
-uploaded project!
-myoutput
+Uploading project!
+download: 's3://ame/tasks/redacted/projectfiles/ame.yaml' -> './ame.yaml' (149 bytes in redactedseconds, redactedKB/s)
+download: 's3://ame/tasks/redacted/projectfiles/nn.py' -> './nn.py' (58 bytes in redactedseconds, redactedKB/s)
+download: 's3://ame/tasks/redacted/projectfiles/poetry.lock' -> './poetry.lock' (4587 bytes in redactedseconds, redactedMB/s)
+download: 's3://ame/tasks/redacted/projectfiles/pyproject.toml' -> './pyproject.toml' (297 bytes in redactedseconds, redactedKB/s)
+/home/ame/.pyenv/shims:/home/ame/.pyenv/bin:/home/ame/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+Downloading Python-3.10.9.tar.xz...
+-> https://www.python.org/ftp/python/3.10.9/Python-3.10.9.tar.xz
+Installing Python-3.10.9...
+Installed Python-3.redactedto /home/ame/.pyenv/versions/3.10.9
+The currently activated Python version 3.redactedis not supported by the project (3.10.9).
+Trying to find and use a compatible version. 
+Using python3 (3.10.9)
+Creating virtualenv test-6gE1vKXj-pyredactedin /home/ame/.cache/pypoetry/virtualenvs
+Installing dependencies from lock file
+
+Package operations: 1 install, 0 updates, 0 removals
+
+  • Installing numpy (1.24.3)
+The currently activated Python version 3.redactedis not supported by the project (3.10.9).
+Trying to find and use a compatible version. 
+Using python3 (3.10.9)
+np array:  [1 2 3]
 
diff --git a/cli/tests/snapshots/cli__invalid_projecttasklogs.snap b/cli/tests/snapshots/cli__invalid_projecttasklogs.snap
new file mode 100644
index 00000000..0b1fd307
--- /dev/null
+++ b/cli/tests/snapshots/cli__invalid_projecttasklogs.snap
@@ -0,0 +1,11 @@
+---
+source: cli/tests/cli.rs
+expression: "&String::from_utf8(res.get_output().stderr.clone())?"
+---
+Error: status: Unavailable, message: "error trying to connect: tcp connect error: Connection refused (os error 111)", details: [], metadata: MetadataMap { headers: {} }
+
+Caused by:
+    0: error trying to connect: tcp connect error: Connection refused (os error 111)
+    1: tcp connect error: Connection refused (os error 111)
+    2: Connection refused (os error 111)
+
diff --git a/cli/tests/snapshots/cli__invalid_projecttaskrun.snap b/cli/tests/snapshots/cli__invalid_projecttaskrun.snap
new file mode 100644
index 00000000..b0e847c3
--- /dev/null
+++ b/cli/tests/snapshots/cli__invalid_projecttaskrun.snap
@@ -0,0 +1,9 @@
+---
+source: cli/tests/cli.rs
+expression: "&String::from_utf8(res.get_output().stderr.clone())?"
+---
+Error: Could not parse ame.yaml :(
+
+Caused by:
+    name: invalid type: sequence, expected a string at line 1 column 7
+
diff --git a/cli/tests/snapshots/cli__missing_projecttaskrun.snap b/cli/tests/snapshots/cli__missing_projecttaskrun.snap
new file mode 100644
index 00000000..404ff4fa
--- /dev/null
+++ b/cli/tests/snapshots/cli__missing_projecttaskrun.snap
@@ -0,0 +1,9 @@
+---
+source: cli/tests/cli.rs
+expression: "&String::from_utf8(res.get_output().stderr.clone())?"
+---
+Error: Could not read ame.yaml, are you in an AME project?
+
+Caused by:
+    No such file or directory (os error 2)
+
diff --git a/cli/tests/snapshots/cli__snap_shot_task_view.snap b/cli/tests/snapshots/cli__snap_shot_task_view.snap
new file mode 100644
index 00000000..f74c75c3
--- /dev/null
+++ b/cli/tests/snapshots/cli__snap_shot_task_view.snap
@@ -0,0 +1,13 @@
+---
+source: cli/tests/cli.rs
+expression: "String::from_utf8(res.get_output().stdout.clone())?"
+---
+name: training
+resources: {}
+env: []
+secrets: []
+executor: !poetry
+  pythonVersion: 3.10.9
+  command: python nn.py
+
+
diff --git a/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages-2.snap b/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages-2.snap
new file mode 100644
index 00000000..404ff4fa
--- /dev/null
+++ b/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages-2.snap
@@ -0,0 +1,9 @@
+---
+source: cli/tests/cli.rs
+expression: "&String::from_utf8(res.get_output().stderr.clone())?"
+---
+Error: Could not read ame.yaml, are you in an AME project?
+
+Caused by:
+    No such file or directory (os error 2)
+
diff --git a/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages-3.snap b/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages-3.snap
new file mode 100644
index 00000000..0b1fd307
--- /dev/null
+++ b/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages-3.snap
@@ -0,0 +1,11 @@
+---
+source: cli/tests/cli.rs
+expression: "&String::from_utf8(res.get_output().stderr.clone())?"
+---
+Error: status: Unavailable, message: "error trying to connect: tcp connect error: Connection refused (os error 111)", details: [], metadata: MetadataMap { headers: {} }
+
+Caused by:
+    0: error trying to connect: tcp connect error: Connection refused (os error 111)
+    1: tcp connect error: Connection refused (os error 111)
+    2: Connection refused (os error 111)
+
diff --git a/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages.snap b/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages.snap
new file mode 100644
index 00000000..b0e847c3
--- /dev/null
+++ b/cli/tests/snapshots/cli__snap_shot_test_cli_error_messages.snap
@@ -0,0 +1,9 @@
+---
+source: cli/tests/cli.rs
+expression: "&String::from_utf8(res.get_output().stderr.clone())?"
+---
+Error: Could not parse ame.yaml :(
+
+Caused by:
+    name: invalid type: sequence, expected a string at line 1 column 7
+
diff --git a/controller/Cargo.toml b/controller/Cargo.toml
index f273d309..adf4e45f 100644
--- a/controller/Cargo.toml
+++ b/controller/Cargo.toml
@@ -29,26 +29,30 @@ name = "data_set_crdgen"
 path = "src/data_set_crd_gen.rs"
 
 [dependencies]
-ame = { version = "0.1.0", path = "../lib", features = ["ame-control", "custom-resources"] }
-async-trait = "0.1.66"
+ame = { version = "0.1.0", path = "../lib", features = ["ame-control", "custom-resources", "project-tools"] }
+async-trait = "0.1.72"
+chrono = "0.4.26"
+cron-parser = "0.8.0"
 duration-string = "0.2.0"
-either = "1.8.0"
+either = "1.9.0"
 envconfig = "0.10.0"
-futures = "0.3.25"
-git2 = "0.17.0"
+futures = "0.3.28"
+git2 = "0.17.2"
 humantime = "2.1.0"
 k8s-openapi = { version = "0.17.0", features = ["v1_23", "schemars" ] }
 kube = { version = "0.80.0", features = ["runtime", "client", "derive"] }
-reqwest = { version = "0.11.14", features = ["json"] }
-schemars = "0.8.11"
-serde = {version = "1.0.147", features = ["derive"]}
-serde_json = "1.0.87"
-serde_yaml = "0.9.14"
-thiserror = "1.0.37"
-tokio = { version = "1.21.2", features = ["macros", "rt-multi-thread"] }
+reqwest = { version = "0.11.18", features = ["json"] }
+schemars = "0.8.12"
+serde = {version = "1.0.180", features = ["derive"]}
+serde_json = "1.0.104"
+serde_merge = "0.1.3"
+serde_yaml = "0.9.25"
+thiserror = "1.0.44"
+tokio = { version = "1.29.1", features = ["macros", "rt-multi-thread"] }
 tonic = "0.8.3"
 tracing = "0.1.37"
-tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 
 [dev-dependencies]
 k8s-openapi = { version = "0.17.0", features = ["v1_23", "schemars" ] }
+insta = { version = "1.31.0", features = ["filters", "redactions", "yaml"] }
diff --git a/controller/src/crdgen.rs b/controller/src/crdgen.rs
index 438a77dd..8fd17db9 100644
--- a/controller/src/crdgen.rs
+++ b/controller/src/crdgen.rs
@@ -1,4 +1,4 @@
-use ame::custom_resources::task::Task;
+use ame::custom_resources::new_task::Task;
 use kube::CustomResourceExt;
 
 fn main() {
diff --git a/controller/src/data_set.rs b/controller/src/data_set.rs
index 7603f5aa..68e0d009 100644
--- a/controller/src/data_set.rs
+++ b/controller/src/data_set.rs
@@ -1,20 +1,21 @@
-use ame::custom_resources::task::Task;
-use std::sync::Arc;
-use std::time::Duration;
+use ame::custom_resources::{common::parent_project, new_task::Task, project::Project};
+use std::{sync::Arc, time::Duration};
 
-use ame::custom_resources::data_set::{DataSet, DataSetPhase, DataSetSpec, DataSetStatus};
-use ame::error::AmeError;
+use ame::{
+    custom_resources::data_set::{DataSet, DataSetPhase, DataSetStatus},
+    error::AmeError,
+};
 
 use ame::Result;
-use futures::future::BoxFuture;
-use futures::{FutureExt, StreamExt};
+use futures::{future::BoxFuture, FutureExt, StreamExt};
 use kube::api::{ListParams, Patch, PatchParams};
-use kube::core::ObjectMeta;
-use kube::runtime::controller::Action;
-use kube::runtime::{finalizer, Controller};
-use kube::{Api, Client, ResourceExt};
+
+use kube::{
+    runtime::{controller::Action, finalizer, Controller},
+    Api, Client, Resource, ResourceExt,
+};
 use std::default::Default;
-use tracing::{error, info};
+use tracing::{debug, error, info};
 
 static DATA_SET_CONTROLLER: &str = "datasets.ame.teainspace.com";
 
@@ -42,13 +43,16 @@ pub struct DataSetControllerCfg {
 async fn reconcile(data_set: Arc<DataSet>, ctx: Arc<Context>) -> Result<Action> {
     let data_sets = Api::<DataSet>::namespaced(ctx.client.clone(), &ctx.namespace);
     let tasks = Api::<Task>::namespaced(ctx.client.clone(), &ctx.namespace);
+    let projects = Api::<Project>::namespaced(ctx.client.clone(), &ctx.namespace);
 
     info!("reconciling data set {}", data_set.name_any());
 
     Ok(
         finalizer(&data_sets, DATA_SET_CONTROLLER, data_set, |event| async {
             match event {
-                finalizer::Event::Apply(data_set) => apply(&data_set, &data_sets, &tasks).await,
+                finalizer::Event::Apply(data_set) => {
+                    apply(&data_set, &data_sets, &tasks, &projects).await
+                }
                 finalizer::Event::Cleanup(data_set) => cleanup(&data_set).await,
             }
         })
@@ -68,7 +72,7 @@ pub async fn start_data_set_controller(
     let context = Arc::new(Context::from(config.clone()));
 
     let data_sets = Api::<DataSet>::namespaced(config.client.clone(), &config.namespace);
-    data_sets.list(&ListParams::default()).await?;
+    data_sets.list(&ListParams::default()).await.unwrap();
 
     let tasks = Api::<Task>::namespaced(config.client.clone(), &config.namespace);
 
@@ -80,9 +84,24 @@ pub async fn start_data_set_controller(
         .boxed())
 }
 
-async fn apply(data_set: &DataSet, data_sets: &Api<DataSet>, tasks: &Api<Task>) -> Result<Action> {
-    let task = data_set.generate_task()?;
+async fn apply(
+    data_set: &DataSet,
+    data_sets: &Api<DataSet>,
+    tasks: &Api<Task>,
+    projects: &Api<Project>,
+) -> Result<Action> {
+    let mut task = data_set.generate_task()?;
+
+    let parent_project = projects
+        .get(&parent_project(data_set.owner_references().to_vec())?)
+        .await?;
+
+    let mut project_oref = parent_project.controller_owner_ref(&()).unwrap();
+    project_oref.controller = Some(false);
+
+    task.owner_references_mut().push(project_oref);
 
+    debug!("patching data set task: {:?}", task);
     let task = tasks
         .patch(
             &task.name_any(),
@@ -92,18 +111,22 @@ async fn apply(data_set: &DataSet, data_sets: &Api<DataSet>, tasks: &Api<Task>)
         .await?;
 
     let status = DataSetStatus {
-        phase: DataSetPhase::from_task(task),
+        phase: Some(DataSetPhase::from_task(task)),
     };
 
+    debug!("patching data set status {:?}  ", status.clone());
+
+    let mut data_set = data_set.clone();
+
+    // let mut data_set = data_set.clone();
+    data_set.metadata.managed_fields = None;
+    data_set.status = Some(status.clone());
+
     data_sets
         .patch_status(
             &data_set.name_any(),
-            &PatchParams::default(),
-            &Patch::Merge(&DataSet {
-                metadata: ObjectMeta::default(),
-                status: Some(status),
-                spec: DataSetSpec::default(),
-            }),
+            &PatchParams::apply(DATA_SET_CONTROLLER).force(),
+            &Patch::Apply(&data_set),
         )
         .await?;
 
@@ -124,35 +147,48 @@ async fn cleanup(data_set: &DataSet) -> Result<Action> {
 mod test {
     use ame::{
         custom_resources::data_set::DataSetSpec,
-        grpc::{DataSetCfg, TaskCfg, TaskRef},
+        grpc::{DataSetCfg, ProjectCfg, TaskCfg, TaskRef},
     };
     use kube::{
         api::{DeleteParams, PostParams},
         core::ObjectMeta,
+        ResourceExt,
     };
 
     use super::*;
-    use std::time::Duration;
+    use std::{collections::BTreeMap, time::Duration};
 
     #[tokio::test]
     #[ignore = "requires a k8s cluster"]
     async fn can_create_data_set_task_and_finalize_data_set() -> Result<()> {
         let client = Client::try_default().await?;
         let namespace = "default".to_string();
-        let _tasks = Api::<Task>::namespaced(client.clone(), &namespace);
         let data_sets = Api::<DataSet>::namespaced(client.clone(), &namespace);
+        let projects = Api::<Project>::namespaced(client.clone(), &namespace);
         let context = super::Context {
             client: client.clone(),
             namespace,
         };
+        let project = ProjectCfg {
+            name: "parentproject".to_string(),
+            models: vec![],
+            data_sets: vec![],
+            tasks: vec![],
+            templates: vec![],
+            enable_triggers: None,
+        };
+        let project = Project::from_cfg(project);
+        let project = projects.create(&PostParams::default(), &project).await?;
 
         let data_set = DataSet {
             metadata: ObjectMeta {
                 generate_name: Some("testdataset2".to_string()),
                 finalizers: Some(vec![super::DATA_SET_CONTROLLER.to_string()]),
+                owner_references: Some(vec![project.controller_owner_ref(&()).unwrap()]),
                 ..ObjectMeta::default()
             },
             spec: DataSetSpec {
+                project: None,
                 cfg: DataSetCfg {
                     name: "test".to_string(),
                     path: "data".to_string(),
@@ -162,13 +198,21 @@ mod test {
                             name: "test_data_task".to_string(),
                             project: None,
                         }),
+                        executor: None,
+                        resources: BTreeMap::new(),
+                        data_sets: Vec::new(),
+                        from_template: None,
+                        artifact_cfg: None,
+                        triggers: None,
+                        env: vec![],
+                        secrets: vec![],
                     }),
                     size: None,
                 },
                 deletion_approved: false,
             },
             status: Some(DataSetStatus {
-                phase: DataSetPhase::Pending {},
+                phase: Some(DataSetPhase::Pending {}),
             }),
         };
 
diff --git a/controller/src/lib.rs b/controller/src/lib.rs
index 84009eb3..7b922f3f 100644
--- a/controller/src/lib.rs
+++ b/controller/src/lib.rs
@@ -1 +1,3 @@
 pub mod data_set;
+pub mod project;
+pub mod task;
diff --git a/controller/src/main.rs b/controller/src/main.rs
index da01683c..16dc3d90 100644
--- a/controller/src/main.rs
+++ b/controller/src/main.rs
@@ -1,18 +1,20 @@
-use ame::custom_resources::project::ProjectCtrlCfg;
-use ame::custom_resources::*;
-use ame::custom_resources::{
-    project_source::ProjectSrcCtrlCfg, task::start_task_controller, task::TaskControllerConfig,
+use ame::custom_resources::{project_source::ProjectSrcCtrlCfg, *};
+
+use controller::{
+    data_set::{start_data_set_controller, DataSetControllerCfg},
+    project::{start_project_controller, ProjectControllerCfg},
+    task::{start_task_controller, TaskControllerCfg},
 };
-use controller::data_set::{start_data_set_controller, DataSetControllerCfg};
 use envconfig::Envconfig;
 use kube::Client;
+use tracing::info;
 use tracing_subscriber::{prelude::*, EnvFilter, Registry};
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    let task_ctrl_cfg = TaskControllerConfig::init_from_env().unwrap();
+    let mut task_ctrl_cfg = TaskControllerCfg::init_from_env().unwrap();
     let project_src_ctrl_cfg = ProjectSrcCtrlCfg::init_from_env().unwrap();
-    let project_ctrl_cfg = ProjectCtrlCfg::from_env().unwrap();
+    let mut project_ctrl_cfg = ProjectControllerCfg::from_env().unwrap();
 
     let logger = tracing_subscriber::fmt::layer();
     let env_filter = EnvFilter::try_from_default_env()
@@ -25,16 +27,41 @@ async fn main() -> Result<()> {
 
     // Start kubernetes controller
     let client = Client::try_default().await?;
+
     let data_set_controller = start_data_set_controller(DataSetControllerCfg {
-        client,
-        namespace: task_ctrl_cfg.namespace.clone(),
+        client: client.clone(),
+        namespace: task_ctrl_cfg
+            .namespace
+            .clone()
+            .unwrap_or("ame-system".to_string()),
     })
     .await
     .unwrap();
-    let task_controller = start_task_controller(task_ctrl_cfg).await;
+
+    if task_ctrl_cfg.namespace.is_none() {
+        task_ctrl_cfg.namespace = Some("ame-system".to_string());
+    }
+
+    info!("Task controller configuration: {:?}", task_ctrl_cfg);
+
+    let task_controller = start_task_controller(client.clone(), task_ctrl_cfg.clone())
+        .await
+        .unwrap();
     let projectsrc_controller =
         project_source::start_project_source_controller(project_src_ctrl_cfg).await;
-    let project_controller = project::start_project_controller(project_ctrl_cfg).await;
+
+    if project_ctrl_cfg.deployment_image.is_none() {
+        project_ctrl_cfg.deployment_image = Some(task_ctrl_cfg.executor_image);
+    }
+
+    if project_ctrl_cfg.mlflow_url.is_none() {
+        project_ctrl_cfg.mlflow_url =
+            Some("http://mlflow.default.svc.cluster.local:5000".to_string());
+    }
+
+    let project_controller = start_project_controller(client.clone(), project_ctrl_cfg)
+        .await
+        .unwrap();
 
     tokio::select! {
         _ = task_controller=> println!("task controller exited"),
diff --git a/controller/src/project.rs b/controller/src/project.rs
new file mode 100644
index 00000000..146ba0a1
--- /dev/null
+++ b/controller/src/project.rs
@@ -0,0 +1,525 @@
+use ame::{
+    ctrl::AmeResource,
+    custom_resources::{
+        new_task::{Task, TaskBuilder},
+        project::{generate_task_name, get_latest_model_version, Project},
+    },
+};
+
+use ame::{error::AmeError, Result};
+
+use ame::grpc::{task_status::Phase, ModelDeploymentCfg, ModelTrainingCfg, TaskStatus, TriggerCfg};
+
+use chrono::Utc;
+use futures::{future::BoxFuture, FutureExt, StreamExt};
+use k8s_openapi::api::{apps::v1::Deployment, core::v1::Service, networking::v1::Ingress};
+use kube::{
+    api::{ListParams, Patch, PatchParams},
+    runtime::{controller::Action, finalizer, Controller},
+    Api, Client, ResourceExt,
+};
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+
+use std::{sync::Arc, time::Duration};
+use tracing::{debug, error, info};
+
+static PROJECT_CONTROLLER: &str = "projects.ame.teainspace.com";
+
+#[derive(Clone)]
+struct Context {
+    cfg: ProjectControllerCfg,
+    pub client: Client,
+}
+impl Context {
+    fn new(client: Client, cfg: ProjectControllerCfg) -> Self {
+        Self { client, cfg }
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Default)]
+pub struct ProjectControllerCfg {
+    pub namespace: Option<String>,
+    pub deployment_image: Option<String>,
+    pub mlflow_url: Option<String>,
+    model_deployment_ingress: Option<Ingress>,
+    model_ingress_annotations: Option<BTreeMap<String, String>>,
+    model_ingress_host: Option<String>,
+}
+
+impl ProjectControllerCfg {
+    pub fn from_env() -> Result<Self> {
+        let prefix = "AME";
+        Ok(ProjectControllerCfg {
+            namespace: std::env::var(format!("{prefix}_NAMESPACE")).ok(),
+            deployment_image: std::env::var("EXECUTOR_IMAGE").ok(),
+            model_deployment_ingress: serde_yaml::from_str(
+                &std::env::var(format!("{prefix}_MODEL_DEPLOYMENT_INGRESS"))
+                    .unwrap_or("".to_string()),
+            )
+            .ok(),
+            model_ingress_annotations: Some(BTreeMap::new()),
+            model_ingress_host: std::env::var(format!("{prefix}_MODEL_INGRESS_HOST")).ok(),
+            mlflow_url: std::env::var(format!("{prefix}_MLFLOW_URL")).ok(),
+        })
+    }
+}
+
+impl ProjectControllerCfg {
+    pub fn new(namespace: Option<String>) -> Self {
+        Self {
+            namespace,
+            deployment_image: None,
+            mlflow_url: None,
+            model_deployment_ingress: None,
+            model_ingress_annotations: None,
+            model_ingress_host: None,
+        }
+    }
+}
+
+async fn reconcile(project: Arc<Project>, ctx: Arc<Context>) -> Result<Action> {
+    let projects = if let Some(ref namespace) = ctx.cfg.namespace {
+        Api::<Project>::namespaced(ctx.client.clone(), namespace)
+    } else {
+        Api::<Project>::namespaced(ctx.client.clone(), "ame-system")
+    };
+
+    info!("reconciling project {}", project.name_any());
+
+    Ok(
+        finalizer(&projects, PROJECT_CONTROLLER, project, |event| async {
+            match event {
+                finalizer::Event::Apply(project) => {
+                    apply(&project, &projects, ctx.client.clone(), ctx).await
+                }
+                finalizer::Event::Cleanup(project) => cleanup(&project, &projects).await,
+            }
+        })
+        .await?,
+    )
+}
+
+async fn apply(
+    project: &Project,
+    _projects: &Api<Project>,
+    client: Client,
+    ctx: Arc<Context>,
+) -> Result<Action> {
+    let project_status = project.status.clone().unwrap_or_default();
+    let tasks = Api::<Task>::namespaced(client.clone(), &project.namespace().unwrap());
+    let deployments = Api::<Deployment>::namespaced(client.clone(), &project.namespace().unwrap());
+    let services = Api::<Service>::namespaced(client.clone(), &project.namespace().unwrap());
+    let ingresses = Api::<Ingress>::namespaced(client, &project.namespace().unwrap());
+
+    let Some(project_oref) = project.gen_owner_ref() else {
+        return Err(AmeError::FailedToCreateOref(project.name_any()));
+    };
+
+    if project.spec.enable_triggers.unwrap_or(false) {
+        info!("checking for triggered tasks");
+        let instant = Utc::now();
+
+        for task in project.spec.cfg.tasks.iter() {
+            if let Some(TriggerCfg {
+                schedule: Some(ref schedule),
+            }) = task.triggers
+            {
+                // TODO this parser is brittle and panics if len < 3;
+                let schedule = match cron_parser::parse(schedule, &instant) {
+                    Ok(schedule) => schedule,
+                    e @ Err(_) => {
+                        error!(
+                            "failed to pass cron schedule for task {}: {e:?}",
+                            task.name.as_ref().unwrap_or(&"".to_string())
+                        );
+                        continue;
+                    }
+                };
+
+                if schedule.signed_duration_since(instant).num_seconds() < 120 {
+                    let mut task_builder = TaskBuilder::from_cfg(task.clone());
+                    task_builder.add_owner_reference(project_oref.clone());
+
+                    task_builder.set_name(generate_task_name(
+                        project.name_any(),
+                        task.name.clone().unwrap_or("".to_string()),
+                    ));
+                    task_builder.set_project(project.spec.cfg.name.clone());
+
+                    let task = task_builder.build();
+
+                    let Some(ref task_name) = task.metadata.name else {
+                        error!("task is missing a name in project {}", project.name_any());
+                        continue;
+                    };
+
+                    let res = tasks
+                        .patch(
+                            task_name,
+                            &PatchParams::apply(PROJECT_CONTROLLER),
+                            &Patch::Apply(task.clone()),
+                        )
+                        .await;
+
+                    if let e @ Err(_) = res {
+                        error!(
+                            "failed to patch task {} in project {}: {:?}",
+                            task.name_any(),
+                            project.name_any(),
+                            e
+                        );
+                        continue;
+                    };
+                }
+            }
+        }
+    } else {
+        debug!("triggers are disabled");
+    }
+
+    // Note task controller is not updating task status to succeeded.
+    // Why is controller not reacting owned task status changing?
+    for model in project.spec.cfg.models.iter() {
+        info!(
+            "reconciling model {} in project {}",
+            model.name, project.spec.cfg.name
+        );
+        if let Some(ModelTrainingCfg {
+            task: Some(ref _task),
+            ..
+        }) = model.training
+        {
+            let Ok(training_task) = project.generate_model_training_task(&model.name) else {
+                continue;
+            };
+
+            debug!("Patching training task: {:?}", training_task);
+
+            tasks
+                .patch(
+                    &training_task.name_any(),
+                    &PatchParams::apply(PROJECT_CONTROLLER),
+                    &Patch::Apply(training_task),
+                )
+                .await?;
+        }
+
+        if let Some(ModelDeploymentCfg { ref image, .. }) = model.deployment {
+            let deployment_image = if let Some(image) = image {
+                Some(image.clone())
+            } else {
+                ctx.cfg.deployment_image.clone()
+            };
+
+            let Some(deployment_image) = deployment_image else {
+                error!(
+                    "missing deployment image for model {} in project {}",
+                    model.name,
+                    project.name_any()
+                );
+                continue;
+            };
+
+            let Some(mlflow_url) = ctx.cfg.mlflow_url.clone() else {
+                error!("missing MLflow URL, skipping deployment");
+                continue;
+            };
+
+            let model_source = match get_latest_model_version(model, mlflow_url).await {
+                Ok(ms) => ms,
+                Err(e) => {
+                    error!("failed to get latest model version, skipping deployment error: {e}");
+                    continue;
+                }
+            };
+
+            let mut model_status = project_status
+                .models
+                .get(&model.name)
+                .cloned()
+                .unwrap_or_default();
+
+            if model_status
+                .latest_validated_model_version
+                .as_ref()
+                .map(|s| s != &model_source.source)
+                .unwrap_or(true)
+                && model.validation_task.is_some()
+            {
+                debug!(
+                    "Generating validation task for model {} in project {}",
+                    model.name, project.spec.cfg.name
+                );
+
+                let val_task = match project
+                    .generate_validation_task(model, model_source.version.clone())
+                {
+                    Ok(t) => Some(t),
+                    Err(e) => {
+                        error!("failed to generate validaion task for model {} in project {}, aborting model validation and deployment: {}", model.name, project.spec.cfg.name, e);
+                        None
+                    }
+                };
+
+                let Some(ref val_task) = val_task else {
+                    continue;
+                };
+
+                let Some(ref task_name) = val_task.metadata.name else {
+                    error!("validation task for model {} in project {} is missing a name, aborting model validation and deployment", model.name, project.spec.cfg.name);
+                    continue;
+                };
+
+                info!(
+                    "patching validation task for model {} in project {}",
+                    model.name,
+                    project.name_any()
+                );
+
+                // TODO; we need to look at the name generation for validated model tasks.
+                tasks
+                    .patch(
+                        task_name,
+                        &PatchParams::apply(PROJECT_CONTROLLER),
+                        &Patch::Apply(val_task),
+                    )
+                    .await?;
+
+                // TODO: what about when validation has failed?
+                if let Ok(Task {
+                    status:
+                        Some(TaskStatus {
+                            phase: Some(Phase::Succeeded(_)),
+                        }),
+                    ..
+                }) = tasks.get_status(task_name).await
+                {
+                    info!(
+                        "model {} in project {} is validated for version {}",
+                        model.name,
+                        project.name_any(),
+                        model_source.version
+                    );
+                    model_status.latest_validated_model_version = Some(model_source.source.clone());
+                } else {
+                    info!("model {} in project {} is not validated for version {}, skipping deployment", model.name, project.name_any(), model_source.version);
+                    continue;
+                }
+            }
+
+            let deployment = model
+                .generate_model_deployment(deployment_image.clone(), model_source.source)
+                .await;
+            match deployment {
+                Ok(deployment) => {
+                    let Some(ref name) = deployment.metadata.name else {
+                        error!("Deployment object for model {} in project {} is missing a name, skipping deployment", model.name, project.name_any());
+                        continue;
+                    };
+
+                    info!("Patching model deployment");
+
+                    // TODO: adjust error handling to avoid breaking out of the reconciliation for errors on a single object.
+                    deployments
+                        .patch(
+                            name,
+                            &PatchParams::apply(PROJECT_CONTROLLER),
+                            &Patch::Apply(deployment.clone()),
+                        )
+                        .await?;
+                }
+                Err(e) => error!(
+                    "Failed to generated model deployment for {} in project {} error: {}",
+                    model.name,
+                    project.name_any(),
+                    e
+                ),
+            };
+
+            let service = model.generate_model_service();
+
+            match service {
+                Ok(service) => {
+                    let Some(ref name) = service.metadata.name else {
+                        error!("Service object for model {} in project {} is missing a name, skipping service", model.name, project.name_any());
+                        continue;
+                    };
+
+                    info!("Patching model service");
+                    services
+                        .patch(
+                            name,
+                            &PatchParams::apply(PROJECT_CONTROLLER),
+                            &Patch::Apply(service.clone()),
+                        )
+                        .await?;
+                }
+                Err(e) => error!(
+                    "Failed to generated service for model {} in project {}: {}",
+                    model.name,
+                    project.name_any(),
+                    e
+                ),
+            };
+
+            // TODO: put some thought into how the project name is used and the path for a model created.
+            let ingress = model.generate_model_ingress(
+                ctx.cfg.model_ingress_host.clone().unwrap_or("".to_string()),
+                None,
+                project.name_any(),
+            );
+
+            match ingress {
+                Ok(ingress) => {
+                    let Some(ref name) = ingress.metadata.name else {
+                        error!("Ingress object for model {} in project {} is missing a name, skipping ingress", model.name, project.name_any());
+                        continue;
+                    };
+
+                    info!("Patching model ingress");
+                    ingresses
+                        .patch(
+                            name,
+                            &PatchParams::apply(PROJECT_CONTROLLER),
+                            &Patch::Apply(ingress.clone()),
+                        )
+                        .await?;
+                }
+                Err(e) => error!(
+                    "Failed to generated ingress for model {} in project {}: {}",
+                    model.name,
+                    project.name_any(),
+                    e
+                ),
+            }
+        }
+    }
+
+    Ok(Action::requeue(Duration::from_secs(60)))
+}
+
+pub async fn cleanup(project: &Project, _projects: &Api<Project>) -> Result<Action> {
+    if !project.deletion_approved() {
+        info!(
+            "Blocking deletion as it was not approved for {}",
+            project.name_any()
+        );
+        return Err(AmeError::DeletionNotApproved(project.name_any()));
+    }
+
+    Ok(Action::await_change())
+}
+
+fn error_policy(_project: Arc<Project>, error: &AmeError, _ctx: Arc<Context>) -> Action {
+    error!("failed to reconcile: {:?}", error);
+    Action::requeue(Duration::from_secs(5 * 60))
+}
+
+pub async fn start_project_controller(
+    client: Client,
+    config: ProjectControllerCfg,
+) -> Result<BoxFuture<'static, ()>> {
+    let context = Arc::new(Context::new(client.clone(), config.clone()));
+
+    let projects = if let Some(ref namespace) = config.namespace {
+        Api::<Project>::namespaced(client.clone(), namespace)
+    } else {
+        Api::<Project>::all(client.clone())
+    };
+
+    let tasks = if let Some(ref namespace) = config.namespace {
+        Api::<Task>::namespaced(client.clone(), namespace)
+    } else {
+        Api::<Task>::all(client.clone())
+    };
+
+    info!("Starting Project controller");
+    debug!("Project controller cfg: {:?}", config);
+
+    Ok(Controller::new(projects, ListParams::default())
+        .owns(tasks, ListParams::default())
+        .run(reconcile, error_policy, context)
+        .filter_map(|x| async move { std::result::Result::ok(x) })
+        .for_each(|_| futures::future::ready(()))
+        .boxed())
+}
+
+#[cfg(test)]
+mod test {
+    use ame::{
+        custom_resources::project::{Project, ProjectSpec},
+        grpc::ProjectCfg,
+        Result,
+    };
+    use kube::{
+        api::{DeleteParams, Patch, PatchParams, PostParams},
+        core::ObjectMeta,
+        Api, Client,
+    };
+
+    use super::*;
+
+    #[tokio::test]
+    #[ignore = "requires a k8s cluster"]
+    async fn can_block_deletion() -> Result<()> {
+        let client = Client::try_default().await?;
+        let namespace = "default".to_string();
+        let projects = Api::<Project>::namespaced(client.clone(), &namespace);
+
+        let ctx = Context {
+            client,
+            cfg: ProjectControllerCfg::new(Some(namespace)),
+        };
+
+        let project = Project {
+            metadata: ObjectMeta {
+                generate_name: Some("myproject".to_string()),
+                finalizers: Some(vec![super::PROJECT_CONTROLLER.to_string()]),
+                ..ObjectMeta::default()
+            },
+            spec: ProjectSpec {
+                cfg: ProjectCfg {
+                    name: "myproject".to_string(),
+                    ..ProjectCfg::default()
+                },
+                deletion_approved: false,
+                enable_triggers: Some(false),
+            },
+            status: None,
+        };
+
+        let project = projects.create(&PostParams::default(), &project).await?;
+
+        reconcile(Arc::new(project.clone()), Arc::new(ctx.clone())).await?;
+
+        let project = projects.get(&project.name_any()).await?;
+
+        reconcile(Arc::new(project.clone()), Arc::new(ctx.clone())).await?;
+
+        projects
+            .delete(&project.name_any(), &DeleteParams::default())
+            .await?;
+
+        let mut project = projects.get(&project.name_any()).await?;
+        reconcile(Arc::new(project.clone()), Arc::new(ctx.clone()))
+            .await
+            .unwrap_err();
+
+        project.approve_deletion();
+
+        let project = projects
+            .patch(
+                &project.name_any(),
+                &PatchParams::default(),
+                &Patch::Merge(project),
+            )
+            .await?;
+
+        projects
+            .delete(&project.name_any(), &DeleteParams::default())
+            .await?;
+
+        Ok(())
+    }
+}
diff --git a/controller/src/snapshots/controller__task__test__can_create_workflow_and_finalize_task.snap b/controller/src/snapshots/controller__task__test__can_create_workflow_and_finalize_task.snap
new file mode 100644
index 00000000..58916803
--- /dev/null
+++ b/controller/src/snapshots/controller__task__test__can_create_workflow_and_finalize_task.snap
@@ -0,0 +1,114 @@
+---
+source: controller/src/task.rs
+expression: "&workflow.spec"
+---
+entrypoint: main
+templates:
+  - name: main
+    metadata:
+      labels: ~
+      annotations: ~
+    steps:
+      - - name: setup
+          inline:
+            name: setup
+            metadata:
+              labels:
+                ame-task: redacted
+              annotations: ~
+            steps: ~
+            securityContext:
+              fsGroup: 2000
+              runAsUser: 1001
+            script:
+              command:
+                - bash
+              env:
+                - name: AWS_ACCESS_KEY_ID
+                  valueFrom:
+                    secretKeyRef:
+                      key: MINIO_ROOT_USER
+                      name: ame-minio-secret
+                      optional: false
+                - name: AWS_SECRET_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      key: MINIO_ROOT_PASSWORD
+                      name: ame-minio-secret
+                      optional: false
+                - name: MLFLOW_TRACKING_URI
+                  value: "http://mlflow.default.svc.cluster.local:5000"
+                - name: MINIO_URL
+                  value: "http://ame-minio.ame-system.svc.cluster.local:9000"
+                - name: PIPENV_YES
+                  value: "1"
+              image: "main.localhost:45373/ame-executor:latest"
+              name: ""
+              resources:
+                limits: {}
+              volumeMounts:
+                - mountPath: /project
+                  name: redacted
+              source: " \n\n s3cmd --no-ssl --region eu-central-1 --host=$MINIO_URL --host-bucket=$MINIO_URL get --recursive s3://somepath ./"
+            podSpecPatch: ~
+      - - name: redacted
+          inline:
+            name: redacted
+            metadata:
+              labels:
+                ame-task: redacted
+              annotations: ~
+            steps: ~
+            securityContext:
+              fsGroup: 2000
+              runAsUser: 1001
+            script:
+              command:
+                - bash
+              env:
+                - name: AWS_ACCESS_KEY_ID
+                  valueFrom:
+                    secretKeyRef:
+                      key: MINIO_ROOT_USER
+                      name: ame-minio-secret
+                      optional: false
+                - name: AWS_SECRET_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      key: MINIO_ROOT_PASSWORD
+                      name: ame-minio-secret
+                      optional: false
+                - name: MLFLOW_TRACKING_URI
+                  value: "http://mlflow.default.svc.cluster.local:5000"
+                - name: MINIO_URL
+                  value: "http://ame-minio.ame-system.svc.cluster.local:9000"
+                - name: PIPENV_YES
+                  value: "1"
+              image: "main.localhost:45373/ame-executor:latest"
+              name: ""
+              resources:
+                limits: {}
+              volumeMounts:
+                - mountPath: /project
+                  name: redacted
+              source: "\n                        source ~/.bashrc\n                        \n                        pyenv install 3.11\n\n                        pyenv global 3.11\n\n                        poetry install\n                    \n                        poetry run python train.py\n                    "
+            podSpecPatch: ~
+    securityContext: ~
+    script: ~
+    podSpecPatch: ~
+imagePullSecrets: ~
+volumeClaimTemplates:
+  - apiVersion: v1
+    kind: PersistentVolumeClaim
+    metadata:
+      name: redacted
+    spec:
+      accessModes:
+        - ReadWriteOnce
+      resources:
+        requests:
+          storage: 50Gi
+    status: {}
+volumes: ~
+serviceAccountName: ame-task
+
diff --git a/controller/src/task.rs b/controller/src/task.rs
new file mode 100644
index 00000000..0463c550
--- /dev/null
+++ b/controller/src/task.rs
@@ -0,0 +1,443 @@
+use std::{
+    sync::Arc,
+    time::{self, Duration},
+};
+
+use ame::{
+    ctrl::AmeResource,
+    custom_resources::{
+        argo::{Workflow, WorkflowPhase},
+        data_set::{DataSet, DataSetPhase, DataSetStatus},
+        find_project,
+        new_task::{build_workflow, resolve_task_templates, Task},
+        project::{local_name, project_name, Project},
+        task_ctrl::TaskCtrl,
+    },
+    error::AmeError,
+    grpc::{task_status::Phase, TaskPhaseFailed, TaskPhaseRunning, TaskPhaseSucceeded, TaskStatus},
+    Result,
+};
+use envconfig::Envconfig;
+use futures::{future::BoxFuture, FutureExt, StreamExt};
+use kube::{
+    api::{ListParams, Patch, PatchParams},
+    runtime::{controller::Action, finalizer, Controller},
+    Api, Client, ResourceExt,
+};
+use tracing::{debug, error, info};
+
+static TASK_CONTROLLER: &str = "tasks.ame.teainspace.com";
+
+#[derive(Clone)]
+struct Context {
+    cfg: TaskControllerCfg,
+    client: Client,
+}
+
+impl Context {
+    fn new(client: Client, cfg: TaskControllerCfg) -> Self {
+        Self { client, cfg }
+    }
+}
+
+#[derive(Clone, Envconfig, Debug)]
+pub struct TaskControllerCfg {
+    #[envconfig(
+        from = "AME_EXECUTOR_IMAGE",
+        default = "main.localhost:45373/ame-executor:latest"
+    )]
+    pub executor_image: String,
+    #[envconfig(from = "NAMESPACE")]
+    pub namespace: Option<String>,
+    #[envconfig(from = "AME_SERVICE_ACCOUNT", default = "ame-task")]
+    pub service_account: String,
+}
+
+async fn reconcile(task: Arc<Task>, ctx: Arc<Context>) -> Result<Action> {
+    info!("reconciling task {}", task.name_any());
+
+    let tasks = if let Some(ref namespace) = ctx.cfg.namespace {
+        Api::<Task>::namespaced(ctx.client.clone(), namespace)
+    } else {
+        todo!("we need to handle this case better??");
+    };
+
+    Ok(finalizer(&tasks, TASK_CONTROLLER, task, |event| async {
+        match event {
+            finalizer::Event::Apply(task) => apply(&task, &ctx).await,
+            finalizer::Event::Cleanup(task) => cleanup(&task, &tasks).await,
+        }
+    })
+    .await?)
+}
+
+// TODO: do not allow nonexistent fields in project.yaml.
+
+async fn apply(task: &Task, ctx: &Context) -> Result<Action> {
+    let (tasks, workflows, projects, data_sets) = if let Some(ref namespace) = ctx.cfg.namespace {
+        (
+            Api::<Task>::namespaced(ctx.client.clone(), namespace),
+            Api::<Workflow>::namespaced(ctx.client.clone(), namespace),
+            Api::<Project>::namespaced(ctx.client.clone(), namespace),
+            Api::<DataSet>::namespaced(ctx.client.clone(), namespace),
+        )
+    } else {
+        todo!("we need to handle this case better??");
+    };
+
+    let task_ctrl = TaskCtrl::new(data_sets.clone(), projects.clone());
+
+    debug!("checking datasets for task {:?}", task.name_any());
+
+    let project = projects.get(&task.parent_project_name()?).await?;
+
+    if !task.spec.cfg.data_sets.is_empty() {
+        info!(
+            "reconciling data sets for task {}",
+            task.spec.cfg.name.as_ref().unwrap()
+        );
+
+        let mut ds_statuses: Vec<Option<DataSetStatus>> = vec![];
+
+        debug!("reconciling datasets {:?}", task.spec.cfg.data_sets);
+
+        for ds in task.spec.cfg.data_sets.iter() {
+            let ds_name = local_name(ds.clone());
+
+            let ds_project = if let Some(project_name) = project_name(ds.clone()) {
+                find_project(projects.clone(), project_name, "".to_string())
+                    .await
+                    .or(Err(AmeError::MissingProject(0)))?
+            } else {
+                project.clone()
+            };
+
+            let mut data_set = match ds_project.generate_data_set(ds_name.clone()) {
+                Ok(ds) => ds,
+                Err(e) => {
+                    error!(
+                        "failed to reconcile data set {} for project {}: {}",
+                        ds_name,
+                        ds_project.name_any(),
+                        e
+                    );
+                    continue;
+                }
+            };
+            let Some(task_oref) = task.gen_owner_ref() else {
+                error!(
+                    "failed to generate owner reference for task {} stopping data set creation",
+                    task.name_any()
+                );
+                continue;
+            };
+
+            data_set.owner_references_mut().push(task_oref);
+
+            debug!("patching dataset: {}", ds);
+
+            let ds_obj_name =
+                data_set
+                    .metadata
+                    .name
+                    .clone()
+                    .ok_or(AmeError::ReconcilitationFailure(
+                        "Task".to_string(),
+                        task.name_any(),
+                        format!(
+                            "data set  {} in project {} is missing a name",
+                            ds, project.spec.cfg.name
+                        ),
+                    ))?;
+
+            let data_set = data_sets
+                .patch(
+                    &ds_obj_name,
+                    &PatchParams::apply(TASK_CONTROLLER).force(),
+                    &Patch::Apply(data_set),
+                )
+                .await?;
+
+            ds_statuses.push(data_set.status);
+        }
+
+        for stat in ds_statuses {
+            let Some(stat) = stat else {
+                info!("waiting for datasets to complete");
+                return Ok(Action::requeue(Duration::from_secs(10)));
+            };
+
+            match stat.phase {
+                Some(DataSetPhase::Ready { .. }) => continue,
+                Some(DataSetPhase::Failed { .. }) => {
+                    error!(
+                        "Data set has failed, can not schedule Task {}",
+                        task.spec
+                            .cfg
+                            .name
+                            .as_ref()
+                            .unwrap_or(&"unknown name".to_string())
+                    );
+                }
+                _ => {
+                    info!("waiting for datasets to complete");
+                    return Ok(Action::requeue(Duration::from_secs(10)));
+                }
+            }
+        }
+    }
+
+    let task_ctx = task_ctrl
+        .gather_task_ctx(
+            task,
+            ctx.cfg.executor_image.to_string(),
+            ctx.cfg.service_account.clone(),
+        )
+        .await?;
+
+    debug!("resolving task {}", task.name_any());
+
+    let resolved_task = resolve_task_templates(task.clone(), project, projects).await?;
+
+    debug!("resolved task {:?}", task.spec.cfg);
+
+    let workflow = build_workflow(resolved_task, task_ctx)?;
+
+    debug!("patching workflow for task {:?} ", task.name_any(),);
+
+    let workflow = workflows
+        .patch(
+            &workflow.name_any(),
+            &PatchParams::apply(TASK_CONTROLLER).force(),
+            &Patch::Apply(&workflow),
+        )
+        .await?;
+
+    debug!("workflow phase: {:?}", workflow.status);
+
+    let phase = workflow
+        .status
+        .as_ref()
+        .map(|s| match s.phase {
+            WorkflowPhase::Pending | WorkflowPhase::Running | WorkflowPhase::Error => {
+                Phase::Running(TaskPhaseRunning {
+                    workflow_name: workflow.name_any(),
+                })
+            }
+            WorkflowPhase::Failed => Phase::Failed(TaskPhaseFailed {
+                workflow_name: workflow.name_any(),
+            }),
+            WorkflowPhase::Succeeded => Phase::Succeeded(TaskPhaseSucceeded {
+                workflow_name: workflow.name_any(),
+            }),
+        })
+        .unwrap_or(Phase::Running(TaskPhaseRunning {
+            workflow_name: workflow.name_any(),
+        }));
+
+    let mut task = task.clone();
+
+    task.status = Some(TaskStatus { phase: Some(phase) });
+    task.metadata.managed_fields = None;
+
+    debug!("patching status for task {}", task.name_any());
+    tasks
+        .patch_status(
+            &task.name_any(),
+            &PatchParams::apply(TASK_CONTROLLER).force(),
+            &Patch::Apply(task),
+        )
+        .await?;
+
+    Ok(Action::requeue(std::time::Duration::from_secs(60)))
+}
+
+pub async fn cleanup(task: &Task, _tasks: &Api<Task>) -> Result<Action> {
+    info!("cleanup dataset: {}", task.name_any());
+
+    if task.spec.deletion_approved {
+        Ok(Action::requeue(time::Duration::from_secs(300)))
+    } else {
+        Err(AmeError::ApiError("deletion not approved".to_string()))
+    }
+}
+
+pub async fn start_task_controller(
+    client: Client,
+    config: TaskControllerCfg,
+) -> Result<BoxFuture<'static, ()>> {
+    info!("Start Task controller");
+    let context = Arc::new(Context::new(client.clone(), config.clone()));
+
+    let tasks = if let Some(ref namespace) = config.namespace {
+        Api::<Task>::namespaced(client.clone(), namespace)
+    } else {
+        Api::<Task>::all(client.clone())
+    };
+
+    let workflow = if let Some(ref namespace) = config.namespace {
+        Api::<Workflow>::namespaced(client, namespace)
+    } else {
+        Api::<Workflow>::all(client)
+    };
+
+    Ok(Controller::new(tasks, ListParams::default())
+        .owns(workflow, ListParams::default())
+        .run(reconcile, error_policy, context)
+        .filter_map(|x| async move { std::result::Result::ok(x) })
+        .for_each(|_| futures::future::ready(()))
+        .boxed())
+}
+
+fn error_policy(_src: Arc<Task>, error: &AmeError, _ctx: Arc<Context>) -> Action {
+    error!("failed to reconcile: {:?}", error);
+    Action::requeue(Duration::from_secs(5 * 60))
+}
+
+#[cfg(test)]
+mod test {
+    use std::collections::BTreeMap;
+
+    use ame::{
+        custom_resources::{
+            argo::Workflow,
+            new_task::{ProjectSource, TaskSpec},
+        },
+        grpc::{
+            task_cfg::Executor, task_status::Phase, PoetryExecutor, ProjectCfg, TaskCfg,
+            TaskPhaseRunning,
+        },
+        Result,
+    };
+    use envconfig::Envconfig;
+
+    use kube::{
+        api::{DeleteParams, PostParams},
+        core::ObjectMeta,
+        Api, Client, Resource,
+    };
+
+    use super::*;
+
+    #[tokio::test]
+    #[ignore = "requires a k8s cluster"]
+    async fn can_create_workflow_and_finalize_task() -> Result<()> {
+        let client = Client::try_default().await?;
+        let namespace = "default".to_string();
+        let tasks = Api::<Task>::namespaced(client.clone(), &namespace);
+        let workflows = Api::<Workflow>::namespaced(client.clone(), &namespace);
+        let projects = Api::<Project>::namespaced(client.clone(), &namespace);
+
+        let mut controller_cfg = TaskControllerCfg::init_from_env().unwrap();
+        controller_cfg.namespace = Some(namespace);
+
+        let ctx = Context::new(client, controller_cfg);
+
+        let project = ProjectCfg {
+            name: "parentproject".to_string(),
+            models: vec![],
+            data_sets: vec![],
+            tasks: vec![],
+            templates: vec![],
+            enable_triggers: None,
+        };
+
+        let project = Project::from_cfg(project);
+        let project = projects.create(&PostParams::default(), &project).await?;
+
+        let task = Task {
+            metadata: ObjectMeta {
+                generate_name: Some("mytask".to_string()),
+                owner_references: Some(vec![project.controller_owner_ref(&()).unwrap()]),
+                ..ObjectMeta::default()
+            },
+            spec: TaskSpec {
+                cfg: TaskCfg {
+                    name: Some("mytask".to_string()),
+                    task_ref: None,
+                    executor: Some(Executor::Poetry(PoetryExecutor {
+                        python_version: "3.11".to_string(),
+                        command: "python train.py".to_string(),
+                    })),
+                    resources: BTreeMap::new(),
+                    data_sets: Vec::new(),
+                    from_template: None,
+                    artifact_cfg: None,
+                    triggers: None,
+                    env: vec![],
+                    secrets: vec![],
+                },
+                source: Some(ProjectSource::Ame {
+                    path: "somepath".to_string(),
+                }),
+                deletion_approved: false,
+                project: None,
+            },
+            status: None,
+        };
+
+        let task = tasks.create(&PostParams::default(), &task).await?;
+
+        reconcile(Arc::new(task.clone()), Arc::new(ctx.clone())).await?;
+
+        let task = tasks.get(&task.name_any()).await?;
+
+        reconcile(Arc::new(task.clone()), Arc::new(ctx.clone())).await?;
+
+        let task_status = tasks.get_status(&task.name_any()).await?.status.unwrap();
+
+        let Phase::Running(TaskPhaseRunning { workflow_name }) = task_status.phase.unwrap() else {
+            panic!("task was not running");
+        };
+
+        let workflow = workflows.get(&workflow_name).await?;
+
+        let mut settings = insta::Settings::clone_current();
+
+        settings.add_filter(
+            &format!("{}.+", task.clone().metadata.generate_name.unwrap()),
+            "redacted",
+        );
+
+        let _guard = settings.bind_to_scope();
+        insta::assert_yaml_snapshot!(&workflow.spec);
+
+        tasks
+            .delete(&task.name_any(), &DeleteParams::default())
+            .await?;
+
+        let data_set = tasks.get(&task.name_any()).await?;
+
+        // An error is expected here as the cleanup is rejected.
+        reconcile(Arc::new(data_set.clone()), Arc::new(ctx.clone()))
+            .await
+            .unwrap_err();
+
+        // Deletion should not be possible until deletion has been approved.
+        let mut task = tasks.get(&task.name_any()).await?;
+
+        // Approve deletion and verify that the data set is now deleted.
+        task.spec.deletion_approved = true;
+
+        tasks
+            .patch(
+                &task.name_any(),
+                &PatchParams::default(),
+                &Patch::Merge(task.clone()),
+            )
+            .await?;
+
+        reconcile(Arc::new(task.clone()), Arc::new(ctx.clone()))
+            .await
+            .unwrap();
+
+        // The cluster is allowed a minimum of 100ms to delete the data set
+        // object after the finalizer has approved deletion during the last
+        // reconcile call.
+        tokio::time::sleep(Duration::from_millis(100)).await;
+
+        tasks.get(&task.name_any()).await.unwrap_err();
+
+        Ok(())
+    }
+}
diff --git a/docs/model_validation.md b/docs/model_validation.md
deleted file mode 100644
index 6ddd581e..00000000
--- a/docs/model_validation.md
+++ /dev/null
@@ -1,115 +0,0 @@
-# Validating models before deployment
-
-To ensure that a new model versions perform well before exposing them AME supports model validation. This is done by providing AME with a `Task` which 
-will succeed if the model passes validation and fail if not.
-
-Example from [ame-demo](https://github.com/TeaInSpace/ame-demo):
-
-```yaml
-
-projectid: sklearn_logistic_regression
-models:
-  - name: logreg
-    type: mlflow
-    validationTask: # the validation task is set here.
-      taskRef: mlflow_validation 
-    training: 
-      task:
-        taskRef: training
-    deployment:
-      auto_train: true
-      deploy: true
-      enable_tls: false
-tasks:
-  - name: training
-    projectid: sklearn_logistic_regression
-    templateRef: shared-templates.logistic_reg_template
-    taskType: Mlflow
-  - name: mlflow_validation
-    projectid: sklearn_logistic_regression
-    runcommand: python validate.py
-```
-
-This approach allows for a lot of flexibility of how models are validated, at the cost of writing the validation your self. In the future AME will provide builtin options for common validation configurations as well, see the [roadmap](todo).
-
-### Using MLflow metrics
-
-Here we will walk through how to validate a model based on recorded metrics in MLflow, using the [ame-demo](https://github.com/TeaInSpace/ame-demo) repository as an example. The model is a simple logistic regressor, the training code looks like this:
-
-```python
-import numpy as np
-from sklearn.linear_model import LogisticRegression
-import mlflow
-import mlflow.sklearn
-import os
-
-X = np.array([-2, -1, 0, 1, 2, 1]).reshape(-1, 1)
-y = np.array([0, 0, 1, 1, 1, 0])
-lr = LogisticRegression()
-lr.fit(X, y)
-score = lr.score(X, y)
-mlflow.log_metric("score", score)
-mlflow.sklearn.log_model(lr, "model", registered_model_name="logreg")
-print("Model saved in run %s" % mlflow.active_run().info.run_uuid)
-```
-
-Notice how the score is logged as a metric. We can use that in our validation.
-
-AME exposes the necessary environment variables to running tasks so we can access the Mlflow instance during validation just by using the Mlflow library.
-
-**Note**: the model name is hard coded , in the future the model name will be made available as an environment variable allowing for better reusability of validation code.
-
-```python
-# validation.py
-import sys
-import mlflow
-from mlflow.entities import ViewType
-
-# We fetch the metrics for latest run for the logreg model.
-models = mlflow.search_registered_models(filter_string="name='logreg'")
-run_id = models[0].latest_versions[-1].run_id
-run = mlflow.get_run(run_id)
-
-if run.data.metrics['score'] < 0.6:
-    sys.exit(1)
-
-```
-
-A validation task indicates failure with a non zero exit code. In this example if our model scores below 0.6 the task will exit with code 1 indicating a failure.
-
-In this example we keep `validation.py` in the same repository as our training code, that is how ever not required. We could use a task from a completely separate project and share the validation logic between multiple projects.
-
-Our AME file will look as follows:
-```yaml
-
-projectid: mlflow_validation_example
-models:
-  - name: logreg # Note the model name is what mlflow will use as well and the name used during validation.
-    type: mlflow # This tells AME to train the model using mlfow.
-    validationTask: # the validation task is set here.
-      taskRef: mlflow_validation 
-    training: 
-      task:
-        taskRef: training
-    deployment:
-      auto_train: true
-      deploy: true
-      enable_tls: false
-tasks:
-  - name: training
-    projectid: mlflow_validation_example
-    taskType: Mlflow # Since this is an mlflow task, AME knows how to run it.
-  - name: mlflow_validation
-    projectid: mlflow_validation_example
-    runcommand: python validate.py
-```
-
-See [ame-demo](https://github.com/TeaInSpace/ame-demo) for a full list of files.
-
-We can add these files a git repository and add that repository as a project source in AME.
-
-```bash
-ame projectsrc create https://github.com/myuser/myrepo.git
-```
-
-Now you will be able to observe the model being trained, validated and then deployed.
diff --git a/executor/Dockerfile b/executor/Dockerfile
index 549e39d1..0f870236 100644
--- a/executor/Dockerfile
+++ b/executor/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # These packages are required by pyenv, see https://github.com/pyenv/pyenv/wiki#suggested-build-environment 
 RUN apt-get update \
@@ -6,7 +6,7 @@ RUN apt-get update \
     git make build-essential libssl-dev zlib1g-dev \
     libbz2-dev libreadline-dev libsqlite3-dev wget \
     curl llvm libncursesw5-dev xz-utils tk-dev \
-    libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev python pip -y
+    libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev -y
 
 # The execute script is called by the argo workflow to execute the command supplied by the Task.
 COPY ./execute.sh  /usr/bin/execute
@@ -23,9 +23,9 @@ RUN mkdir /project
 RUN chown ame -R /project
 USER ame
 
-# THe ame user gets its home directory where  pyenv and pipenv will end up installed
+# THe ame user gets its home directory where pyenv and pipenv will end up installed
 # therefore the .local/bin needs to be included in the PATH environment variable.
-ENV PATH="/home/ame/.local/bin:${PATH}"
+ENV PATH="/home/ame/.pyenv/shims:/home/ame/.pyenv/bin:/home/ame/.local/bin:${PATH}"
 
 # Pipenv uses pyenv to install new python versions, the installation process is 
 # ducumented here: https://github.com/pyenv/pyenv#installation
@@ -34,6 +34,10 @@ RUN PYENV_GIT_TAG=v2.3.4 curl https://pyenv.run | bash
 RUN echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc
 RUN echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc
 RUN echo 'eval "$(pyenv init --path)"' >> ~/.bashrc
+RUN pyenv install 3.11
+RUN pyenv global 3.11
+RUN pyenv rehash
+RUN python --version
 
 # Install pipenv
 RUN pip install --upgrade pip
@@ -44,6 +48,8 @@ RUN pip install pipenv
 RUN pip install s3cmd
 
 RUN pip install mlflow[extras]
-RUN pip install poetry
 
+# TODO: pin poetry versionk
+RUN curl -sSL https://install.python-poetry.org | python -
+ 
 WORKDIR /project
diff --git a/justfile b/justfile
index 109391f2..0f3bbd4d 100644
--- a/justfile
+++ b/justfile
@@ -42,6 +42,7 @@ tools:
   cargo install --locked cargo-audit
   cargo install --locked cargo-outdated
   cargo install --locked cargo-udeps
+  cargo install --locked cargo-watch
 
 fix: fmt
   cargo fix --workspace --allow-dirty --tests --allow-staged
@@ -56,11 +57,15 @@ check:
   cargo check --workspace --all-targets --all-features
   cargo spellcheck check 
   typos --exclude **/primer.css ./
-  cargo audit
   cargo +nightly fmt --check 
-  cargo +nightly clippy --workspace --tests --all -- -D warnings
-  # cargo outdated --exclude leptos
+
+  # TODO: reintroduce failure on clippy warnings
+
+  cargo +nightly clippy --workspace --tests --all --  -W clippy::panic -W clippy::unwrap_used -W clippy::expect_used
   cargo +nightly udeps --all-targets --workspace --show-unused-transitive --exclude web # TODO: solve false positives for web package.
+  # TODO: chrono 0.5 will deal with this CVE, update when it is released.
+  cargo audit --ignore RUSTSEC-2020-0071 
+  # cargo outdated --exclude leptos
 
 test *ARGS:
   cargo test --workspace {{ARGS}}
@@ -92,8 +97,20 @@ crdgen:
  cargo run --bin project_crdgen > manifests/project_crd.yaml
  cargo run --bin data_set_crdgen > manifests/data_set_crd.yaml
 
+watch_controller:
+  AME_EXECUTOR_IMAGE={{LOCAL_EXECUTOR_IMAGE_TAG}} AME_MODEL_INGRESS_HOST={{AME_HOST}} AME_MLFLOW_URL=http://localhost:5000 cargo watch -x 'run --bin controller'
+
+watch_server:
+  #!/bin/sh
+  just crdgen
+  just install_crd
+  export S3_ENDPOINT=http://$(kubectl get svc -n {{TARGET_NAMESPACE}} ame-minio -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):9000
+  export S3_ACCESS_ID=minio
+  export S3_SECRET=minio123
+  cargo watch -x 'run --bin ame-server'
+
 start_controller:
-  cargo run --bin controller
+  AME_EXECUTOR_IMAGE={{LOCAL_EXECUTOR_IMAGE_TAG}} AME_MODEL_INGRESS_HOST={{AME_HOST}} AME_MLFLOW_URL=http://localhost:5000 cargo run --bin controller
 
 start_server:
   #!/bin/sh
@@ -105,6 +122,9 @@ start_server:
 run_cli *ARGS:
  cargo run -p cli {{ARGS}}
 
+setup_cli_local_host:
+ cargo run -p cli setup http://localhost:3342
+
 setup_cli:
  cargo run -p cli setup http://$(kubectl get svc -n {{TARGET_NAMESPACE}} ame-server-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):3342
 
@@ -121,9 +141,10 @@ k3s:
     --servers 1 \
     --registry-create main \
     --k3s-arg "--disable=traefik@server:*" \
-    --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<0.1%,nodefs.available<0.1%@agent:*' \
-    --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=0.1%,nodefs.available=0.1%@agent:*' \
-    --image rancher/k3s:v1.25.5-rc2-k3s1
+    --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1Gi,nodefs.available<1Gi@agent:*' \
+    --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1Gi,nodefs.available=1Gi@agent:*' \
+    --image rancher/k3s:v1.26.6-k3s1
+
 
 create_namespace:
   kubectl create ns {{TARGET_NAMESPACE}}
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index 739f9ef4..2b09e508 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -7,64 +7,76 @@ edition = "2021"
 
 [dependencies]
 http = {version = "0.2.9", optional = true }
-hyper = { version = "0.14.24", features = ["client"], optional = true }
+hyper = { version = "0.14.27", features = ["client"], optional = true }
 leptos = { version = "0.2", default-features = false, features = ["serde", "stable"], optional = true }
-prost = "0.11.7"
-thiserror = "1.0.38"
+prost = "0.11.9"
+thiserror = "1.0.44"
 tokio-rustls = { version = "0.23.4", features = ["dangerous_configuration"], optional = true  }
 tonic = { version = "0.8.3", default-features = false }
 tower  = {version = "0.4.13", optional = true}
-tower-http = { version = "0.4.0", features = ["auth"], optional = true }
-tonic-web-wasm-client = {version = "0.3.2", optional = true}
-url = "2.3.1"
+tower-http = { version = "0.4.3", features = ["auth"], optional = true }
+tonic-web-wasm-client = {version = "0.3.3", optional = true}
+url = "2.4.0"
 http-body = "0.4.5"
-tokio = "1.25.0"
-tokio-stream = "0.1.12"
-async-stream = "0.3.4"
-futures-util = "0.3.26"
-futures = "0.3.26"
+tokio = "1.29.1"
+tokio-stream = "0.1.14"
+async-stream = "0.3.5"
+futures-util = "0.3.28"
+futures = "0.3.28"
 kube = {version = "0.80.0", optional = true, features = ["derive", "runtime", "client"]}
-async-trait = "0.1.64"
-schemars = "0.8.11"
+async-trait = "0.1.72"
+schemars = "0.8.12"
 serde = "1.0"
-serde_json = "1.0.91"
+serde_json = "1.0.104"
 serde_merge = {version = "0.1.3", optional = true}
-rustls-native-certs = {version = "0.6.2", optional = true}
+rustls-native-certs = {version = "0.6.3", optional = true}
 hyper-rustls = { version = "0.23.2", features = ["http2"], optional = true }
-open = {version = "4.0.1", optional = true}
+open = {version = "4.2.0", optional = true}
 openidconnect = {version = "2.5.1", optional = true}
-oauth2 = {version = "4.3.0", optional = true}
-reqwest = {version = "0.11.15", optional = true, features = ["json"]}
+oauth2 = {version = "4.4.1", optional = true}
+reqwest = {version = "0.11.18", optional = true, features = ["json"]}
 cfg-if = "1.0.0"
 k8s-openapi = {optional=true, version = "0.17.0", features = ["v1_23", "schemars" ] }
 serde_tuple = { version = "0.5.0", optional = true }
-git2 = {version = "0.17.0", optional = true}
-serde_yaml = {version = "0.9.21", optional = true}
+git2 = {version = "0.17.2", optional = true}
+serde_yaml = {version = "0.9.25", optional = true}
 tracing = {version = "0.1.37", optional = true}
 duration-string = {version = "0.2.0", optional = true}
 envconfig = {version = "0.10.0", optional = true}
 similar = {version = "2.2.1", optional = true}
 humantime = {version = "2.1.0", optional = true}
-similar-asserts = {version = "1.4.2", optional = true}
-either = {version = "1.8.0", optional = true}
+either = {version = "1.9.0", optional = true}
+rand = {version = "*", optional = true}
+cron-parser = "0.8.0"
+anyhow = "1.0.72"
 
 [dev-dependencies]
-either = "1.8.0"
-assert_fs = "1.0.10"
-insta = {version = "1.21.0", features = ["yaml"] }
-predicates = "2.1.4"
-serial_test = "0.9.0"
+either = "1.9.0"
+assert_fs = "1.0.13"
+predicates = "2.1.5"
+serial_test = "2.0.0"
 k8s-openapi = { version = "0.17.0", features = ["v1_23", "schemars" ] }
 tonic = { version = "0.8.3"  }
+assert_cmd = "2.0.12"
+kube = "0.80.0"
+insta = { version = "1.31.0", features = ["filters", "redactions", "yaml"] }
+fs_extra = "1.3.0"
+rstest = "0.16.0"
+ame = { path = "../lib", features = ["native-client", "custom-resources", "ame-control", "project-tools"] }
+tracing-subscriber = {version = "0.3", default-features = false, features = ["env-filter", "fmt"]}
+chrono = "0.4.26"
+time = "0.3.24"
 
 [features]
 web-components = ["dep:leptos", "dep:tonic-web-wasm-client", "dep:hyper"]
-native-client = ["dep:hyper", "dep:tokio-rustls", "dep:tower", "dep:tower-http", "dep:rustls-native-certs", "dep:hyper-rustls", "dep:open", "dep:openidconnect", "dep:oauth2", "dep:reqwest", "dep:http"]
+native-client = ["dep:serde_yaml", "dep:hyper", "dep:tokio-rustls", "dep:tower", "dep:tower-http", "dep:rustls-native-certs", "dep:hyper-rustls", "dep:open", "dep:openidconnect", "dep:oauth2", "dep:reqwest", "dep:http"]
 ame-control = ["dep:kube", "dep:serde_merge"]
-custom-resources = ["dep:kube", "dep:k8s-openapi", "dep:serde_tuple", "dep:reqwest", "dep:git2", "dep:serde_merge", "dep:serde_yaml", "dep:tracing", "dep:duration-string", "dep:envconfig",  "dep:similar", "dep:humantime", "dep:similar-asserts", "dep:either", "tonic/default"]
+custom-resources = ["dep:kube", "dep:k8s-openapi", "dep:serde_tuple", "dep:reqwest", "dep:git2", "dep:serde_merge", "dep:serde_yaml", "dep:tracing", "dep:duration-string", "dep:envconfig", "dep:similar", "dep:humantime", "dep:either", "tonic/default"]
+project-tools = ["dep:rand"]
 
 
 [build-dependencies]
 tonic-build = { version = "0.8.4", default-features = false, features = [
     "prost",
 ] }
+prost-build = "*" 
diff --git a/lib/ame.proto b/lib/ame.proto
index 1c3047f7..e4175c7c 100644
--- a/lib/ame.proto
+++ b/lib/ame.proto
@@ -12,19 +12,6 @@ enum TaskType {
   Poetry = 2;
 }
 
-message TaskTemplate {
-  string projectid = 1;
-  string command = 2;
-  string name = 3;
-  optional string image = 4;
-  optional TaskType taskType = 5;
-}
-
-message CreateTaskRequest {
-  TaskIdentifier id = 1;
-  TaskTemplate template = 2;
-}
-
 message Empty {
 }
 
@@ -117,7 +104,7 @@ message AmeSecretId {
 }
 
 message AmeSecrets {
-  repeated AmeSecretId secrets = 1;
+repeated AmeSecretId secrets = 1;
 }
 
 message ResourceCfg {
@@ -165,7 +152,99 @@ message TaskRef {
 
 message TaskCfg {
   optional string name = 1;
-  TaskRef taskRef = 2;  
+  optional TaskRef taskRef = 2;  
+  map<string, string> resources = 3;
+  oneof executor {
+    PoetryExecutor poetry = 4; 
+    MlflowExecutor mlflow = 5;
+    PipEnvExecutor pipEnv = 6;
+    PipExecutor pip = 8;
+    CustomExecutor custom = 9;
+  }
+  repeated string dataSets = 7;
+  optional TemplateRef fromTemplate = 10;
+  optional ArtifactCfg artifactCfg = 11;
+  optional TriggerCfg triggers = 12;
+  repeated EnvVar env = 13;
+  repeated Secret secrets = 14;
+}
+
+message Secret {
+  oneof variant {
+    AmeSecretVariant ame = 1;
+  }
+}
+
+message AmeSecretVariant {
+  string key = 1;
+  string injectAs = 2;
+}
+
+message EnvVar {
+  string key = 1;
+  string val = 2;
+}
+
+message TriggerCfg {
+  optional string schedule = 1;
+}
+
+message ArtifactCfg {
+  bool save_changed_files = 1;
+  repeated string paths = 2; 
+}
+
+message TemplateRef {
+  string name = 1;
+  optional string project = 2;
+}
+
+message CustomExecutor {
+  string pythonVersion = 1;
+  string command = 2;
+}
+
+message PoetryExecutor {
+  string pythonVersion = 1;  
+  string command = 2;
+}
+
+
+message PipExecutor {
+  string pythonVersion = 1;  
+  string command = 2;
+}
+
+message MlflowExecutor {
+  
+}
+
+message PipEnvExecutor {
+  string command = 1;
+}
+
+// TODO: should there be an error case?
+message TaskStatus {
+  oneof phase {
+    TaskPhasePending pending = 2;  
+    TaskPhaseRunning running = 3;
+    TaskPhaseFailed failed = 4;
+    TaskPhaseSucceeded succeeded= 5;
+  }
+}
+
+message TaskPhasePending {
+  
+}
+
+message TaskPhaseRunning {
+  string workflowName= 1;
+}
+message TaskPhaseFailed {
+  string workflowName= 1;
+}
+message TaskPhaseSucceeded {
+  string workflowName= 1;
 }
 
 message DataSetCfg {
@@ -175,9 +254,81 @@ message DataSetCfg {
  optional string size = 4;
 }
 
+message ProjectCfg {
+  string name = 1;
+  repeated Model models = 2;
+  repeated DataSetCfg dataSets = 3;
+  repeated TaskCfg tasks = 4;
+  repeated TaskCfg templates = 5;
+  optional bool enableTriggers = 6;
+}
+
+message ProjectStatus {
+  map<string,ModelStatus> models = 1;
+}
+
+message ModelStatus {
+  optional string latestValidatedModelVersion = 1;
+}
+
+message CreateProjectRequest {
+  ProjectCfg cfg = 1;
+  optional bool enableTriggers = 2;
+}
+
+message ProjectId {
+  string name = 1;
+}
+
+message Model {
+  string name = 1;
+  optional TaskCfg validationTask = 2; 
+  optional ModelTrainingCfg training = 3;
+  optional ModelDeploymentCfg deployment = 4;
+}
+
+message ModelTrainingCfg {
+  TaskCfg task = 1;
+}
+
+message ModelDeploymentCfg {
+   map<string, string> ingressAnnotations = 1;
+   optional int32 replicas = 2;
+   optional string image = 3;
+   map<string,string> resources = 4;
+   optional bool enableTls = 5;
+}
+
+message RunTaskRequest {
+  ProjectId projectId= 1;
+  TaskCfg taskCfg = 2;
+}
+
+message TaskId {
+  string name = 1;
+}
+
+message ListTasksRequest {
+  
+}
+
+message ListTasksResponse {
+  map<string, TaskListEntry> tasks = 1;
+}
+
+message TaskListEntry {
+  TaskStatus status = 1;
+  string timeStamp = 2;
+}
+
+message RemoveTaskRequest {
+  string name = 1;
+  optional bool approve = 2;
+}
+
 service AmeService {
-  rpc GetTask(TaskIdentifier) returns (TaskTemplate) {}
-  rpc CreateTask(CreateTaskRequest) returns (TaskIdentifier) {}
+  rpc RunTask(RunTaskRequest) returns (TaskIdentifier) {}
+  rpc GetTask(TaskIdentifier) returns (TaskCfg) {}
   rpc DeleteTask(TaskIdentifier) returns (Empty) {}
   rpc CreateTaskProjectDirectory(TaskProjectDirectoryStructure) returns (Empty) {}
   rpc UploadProjectFile(stream ProjectFileChunk) returns (Empty) {}
@@ -196,4 +347,7 @@ service AmeService {
   rpc GetProjectSrcStatus(ProjectSourceId) returns (ProjectSourceStatus) {}
   rpc GetProjectSrcId(ProjectSrcIdRequest) returns (ProjectSourceId) {}
   rpc ListProjectSrcs(ProjectSourceListParams) returns (ListProjectSrcsResponse) {} 
-}
+  rpc CreateProject(CreateProjectRequest) returns (ProjectId) {}
+  rpc ListTasks(ListTasksRequest) returns (ListTasksResponse) {}
+  rpc RemoveTask(RemoveTaskRequest) returns (Empty) {}
+ }
diff --git a/lib/build.rs b/lib/build.rs
index 1f4f3c1a..a830df5c 100644
--- a/lib/build.rs
+++ b/lib/build.rs
@@ -1,6 +1,10 @@
+use prost_build::Config;
 use std::io;
 
 fn main() -> io::Result<()> {
+    let mut prost_cfg = Config::new();
+    prost_cfg.btree_map(["resources", "ingressAnnotations"]);
+
     tonic_build::configure()
         .build_server(true)
         .build_client(true)
@@ -8,6 +12,58 @@ fn main() -> io::Result<()> {
             ".",
             "#[derive(serde::Serialize, serde::Deserialize, schemars::JsonSchema )]",
         )
+        .type_attribute(".", "#[serde(rename_all = \"camelCase\")]")
+        .field_attribute(
+            "ProjectCfg.models",
+            "#[serde(skip_serializing_if = \"Vec::is_empty\", default)]",
+        )
+        .field_attribute(
+            "ProjectCfg.dataSets",
+            "#[serde(skip_serializing_if = \"Vec::is_empty\", default)]",
+        )
+        .field_attribute(
+            "TaskCfg.dataSets",
+            "#[serde(skip_serializing_if = \"Vec::is_empty\", default)]",
+        )
+        .field_attribute(
+            "ProjectCfg.templates",
+            "#[serde(skip_serializing_if = \"Vec::is_empty\", default)]",
+        )
+        .field_attribute(
+            "ProjectCfg.tasks",
+            "#[serde(default = \"Vec::<TaskCfg>::new\")]",
+        )
+        .field_attribute("paths", "#[serde(default = \"Vec::<String>::new\")]")
+        .field_attribute(
+            "TaskCfg.secrets",
+            "#[serde(default = \"Vec::<Secret>::new\")]",
+        )
+        .field_attribute(
+            "TaskCfg.taskRef",
+            "#[serde(skip_serializing_if = \"Option::is_none\")]",
+        )
+        .field_attribute(
+            "TaskCfg.fromTemplate",
+            "#[serde(skip_serializing_if = \"Option::is_none\")]",
+        )
+        .field_attribute(
+            "TaskCfg.artifactCfg",
+            "#[serde(skip_serializing_if = \"Option::is_none\")]",
+        )
+        .field_attribute(
+            "TaskCfg.triggers",
+            "#[serde(skip_serializing_if = \"Option::is_none\")]",
+        )
+        .field_attribute("TaskCfg.env", "#[serde(default = \"Vec::<EnvVar>::new\")]")
+        .field_attribute(
+            "resources",
+            "#[serde(default = \"std::collections::BTreeMap::<String, String>::new\")]",
+        )
+        .field_attribute(
+            "ingressAnnotations",
+            "#[serde(default = \"std::collections::BTreeMap::<String, String>::new\")]",
+        )
+        .field_attribute("Secret.variant", "#[serde(flatten)]")
         .protoc_arg("--experimental_allow_proto3_optional")
-        .compile(&["ame.proto"], &["./"])
+        .compile_with_config(prost_cfg, &["ame.proto"], &["./"])
 }
diff --git a/lib/src/ctrl.rs b/lib/src/ctrl.rs
index 4eb9551a..4a19e957 100644
--- a/lib/src/ctrl.rs
+++ b/lib/src/ctrl.rs
@@ -1,16 +1,18 @@
 use std::fmt::Debug;
 
 use crate::{
+    custom_resources::common::kind_is_project,
     error::AmeError,
     grpc::{resource_id, ProjectSourceId, ResourceId},
     Result,
 };
 use async_trait::async_trait;
 use futures::StreamExt;
+use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
 use kube::{
     api::{DeleteParams, ListParams, Patch, PatchParams, PostParams},
     runtime::{watcher, WatchStreamExt},
-    Api, Resource, ResourceExt,
+    Api, CustomResourceExt, Resource, ResourceExt,
 };
 use serde::{de::DeserializeOwned, Serialize};
 use serde_merge::omerge;
@@ -18,6 +20,43 @@ use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
 use tonic::Status;
 
+fn oref_is_project(oref: &OwnerReference) -> bool {
+    kind_is_project(&oref.kind)
+}
+
+impl<K: Resource<DynamicType = ()> + CustomResourceExt> AmeResource for K {
+    fn gen_owner_ref(&self) -> Option<OwnerReference> {
+        let mut oref = self.controller_owner_ref(&())?;
+        oref.controller = Some(false);
+        Some(oref)
+    }
+}
+
+pub trait AmeResource: ResourceExt {
+    fn parent_project_oref(&self) -> Result<OwnerReference> {
+        let potential_orefs: Vec<OwnerReference> = self
+            .meta()
+            .owner_references
+            .to_owned()
+            .map(|orefs| orefs.into_iter().filter(oref_is_project).collect())
+            .unwrap_or(vec![]);
+        if potential_orefs.len() != 1 {
+            return Err(AmeError::FailedToFindParentProjectOref(
+                self.name_any(),
+                potential_orefs.len(),
+            ));
+        }
+
+        Ok(potential_orefs[0].clone())
+    }
+
+    fn parent_project_name(&self) -> Result<String> {
+        Ok(self.parent_project_oref()?.name)
+    }
+
+    fn gen_owner_ref(&self) -> Option<OwnerReference>;
+}
+
 #[async_trait]
 pub trait AmeKubeResourceCtrl {
     type KubeResource: Resource
diff --git a/lib/src/custom_resources/argo.rs b/lib/src/custom_resources/argo.rs
index 5349a0f4..e46f2e8f 100644
--- a/lib/src/custom_resources/argo.rs
+++ b/lib/src/custom_resources/argo.rs
@@ -1,14 +1,27 @@
-// WARNING: generated by kopium - manual changes will be overwritten
-use k8s_openapi::api::core::v1::{
-    Container, LocalObjectReference, PersistentVolumeClaim, PodSecurityContext, Volume,
+use super::new_task::Task;
+use crate::{
+    grpc::{secret::Variant, AmeSecretVariant},
+    Result,
+};
+use k8s_openapi::{
+    api::core::v1::{
+        Container, EnvVar, EnvVarSource, LocalObjectReference, PersistentVolumeClaim,
+        PersistentVolumeClaimSpec, PersistentVolumeClaimStatus, PodSecurityContext,
+        ResourceRequirements, SecretKeySelector, Volume,
+    },
+    apimachinery::pkg::{
+        api::resource::Quantity,
+        apis::meta::v1::{ObjectMeta, OwnerReference},
+    },
 };
-use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference};
 use kube::CustomResource;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 use serde_tuple::*;
-use std::collections::BTreeMap;
-use std::default::Default;
+use std::{collections::BTreeMap, default::Default};
+
+use super::new_task::TaskContext;
 
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema, Default, PartialEq)]
 #[kube(
@@ -127,7 +140,6 @@ impl WorkflowTemplate {
 pub struct ArgoScriptTemplate {
     #[serde(flatten)]
     pub container: Container,
-
     pub source: String,
 }
 
@@ -148,6 +160,15 @@ pub struct WorkflowStep {
     pub inline: Option<Box<WorkflowTemplate>>,
 }
 
+impl WorkflowStep {
+    fn new_inline(name: String, template: WorkflowTemplate) -> Self {
+        Self {
+            name,
+            inline: Some(Box::new(template)),
+        }
+    }
+}
+
 impl Default for Workflow {
     fn default() -> Self {
         Workflow {
@@ -198,6 +219,12 @@ impl Workflow {
         self
     }
 
+    pub fn set_entrypoint(&mut self, template: WorkflowTemplate) -> &mut Workflow {
+        self.spec.entrypoint = template.name.clone();
+        self.add_template(template);
+        self
+    }
+
     pub fn add_template(&mut self, template: WorkflowTemplate) -> &mut Workflow {
         let mut templates = if let Some(templates) = self.spec.templates.clone() {
             templates
@@ -244,6 +271,221 @@ impl Workflow {
     }
 }
 
+pub struct WorkflowBuilder {
+    templates: Vec<WorkflowTemplate>,
+    task_name: String,
+    service_account: String,
+    owner_reference: Option<OwnerReference>,
+    volumes: Vec<(String, BTreeMap<String, Quantity>)>,
+}
+
+impl WorkflowBuilder {
+    pub fn new(task_name: String, service_account: String) -> Self {
+        Self {
+            templates: vec![],
+            task_name,
+            service_account,
+            owner_reference: None,
+            volumes: vec![],
+        }
+    }
+
+    pub fn add_owner_reference(&mut self, owner_reference: OwnerReference) -> &mut Self {
+        self.owner_reference = Some(owner_reference);
+        self
+    }
+
+    pub fn add_template(&mut self, template: WorkflowTemplate) -> &mut Self {
+        self.templates.push(template);
+
+        self
+    }
+
+    pub fn add_volume(&mut self, name: String, resources: BTreeMap<String, Quantity>) -> &mut Self {
+        self.volumes.push((name, resources));
+        self
+    }
+
+    pub fn build(self) -> Result<Workflow> {
+        let mut workflow = Workflow::default()
+            .set_name(self.task_name.clone())
+            .set_service_account(self.service_account)
+            .label("ame-task".to_string(), self.task_name.clone())
+            .clone();
+
+        let mut main_template = WorkflowTemplate::new("main".to_string());
+        for mut template in self.templates {
+            template.label("ame-task".to_string(), self.task_name.clone());
+            main_template.add_parallel_step(vec![WorkflowStep::new_inline(
+                template.name.clone(),
+                template,
+            )]);
+        }
+
+        if let Some(owner_reference) = self.owner_reference {
+            workflow.add_owner_reference(owner_reference);
+        }
+
+        workflow.set_entrypoint(main_template);
+        for (name, resources) in self.volumes {
+            workflow.add_volume_claim_template(new_pvc(
+                name,
+                vec!["ReadWriteOnce".to_string()],
+                ResourceRequirements {
+                    requests: Some(resources),
+                    limits: None,
+                },
+            ));
+        }
+        Ok(workflow)
+    }
+}
+
+fn new_pvc(
+    name: String,
+    access_mode: Vec<String>,
+    resources: ResourceRequirements,
+) -> PersistentVolumeClaim {
+    PersistentVolumeClaim {
+        metadata: ObjectMeta {
+            name: Some(name),
+            ..ObjectMeta::default()
+        },
+        spec: Some(PersistentVolumeClaimSpec {
+            // TODO: make access modes configurable.
+            access_modes: Some(access_mode),
+            resources: Some(resources),
+            ..PersistentVolumeClaimSpec::default()
+        }),
+
+        // Note that it is important to create the equivalent of an empty struct here
+        // and not just a None.
+        // Otherwise the Workflow controller will disagree with AME's controller on
+        // how an empty status should be specified.
+        status: Some(PersistentVolumeClaimStatus::default()),
+    }
+}
+
+pub struct WorkflowTemplateBuilder<'a> {
+    env: Vec<EnvVar>,
+    ctx: &'a TaskContext,
+    script: String,
+    name: String,
+}
+
+impl<'a> WorkflowTemplateBuilder<'a> {
+    pub fn new(ctx: &'a TaskContext, script: String, name: String) -> Result<Self> {
+        let required_env: Vec<EnvVar> = serde_json::from_value(json!([
+            {
+            "name":  "AWS_ACCESS_KEY_ID",
+            "valueFrom": {
+                "secretKeyRef":  {
+                    "key": "MINIO_ROOT_USER",
+                    "name": "ame-minio-secret",
+                    "optional": false,
+                }
+            },
+        },
+        {
+            "name":  "AWS_SECRET_ACCESS_KEY",
+            "valueFrom": {
+                "secretKeyRef":  {
+                    "key": "MINIO_ROOT_PASSWORD",
+                    "name": "ame-minio-secret",
+                    "optional": false
+                }
+            },
+        },
+        {
+            "name": "MLFLOW_TRACKING_URI",
+            "value": "http://mlflow.default.svc.cluster.local:5000"
+        },
+        {
+            "name":  "MINIO_URL",
+            "value": "http://ame-minio.ame-system.svc.cluster.local:9000",
+        },
+
+                    {
+                        "name":  "PIPENV_YES",
+                        "value": "1",
+                    },
+        ]))?;
+
+        Ok(Self {
+            env: required_env,
+            ctx,
+            script,
+            name,
+        })
+    }
+
+    fn add_env_var(&mut self, var: EnvVar) -> &mut Self {
+        self.env.push(var);
+        self
+    }
+
+    fn add_secret_env_var(&mut self, secret: Variant) -> &mut Self {
+        let Variant::Ame(AmeSecretVariant { key, inject_as }) = secret;
+        let var = EnvVar {
+            name: inject_as,
+            value_from: Some(EnvVarSource {
+                secret_key_ref: Some(SecretKeySelector {
+                    key: "secret".to_string(),
+                    name: Some(key),
+                    ..SecretKeySelector::default()
+                }),
+                ..EnvVarSource::default()
+            }),
+            ..EnvVar::default()
+        };
+
+        self.add_env_var(var)
+    }
+
+    pub fn build(mut self, task: &Task) -> Result<WorkflowTemplate> {
+        for var in task.spec.cfg.env.clone() {
+            self.add_env_var(EnvVar {
+                name: var.key,
+                value: Some(var.val),
+                value_from: None,
+            });
+        }
+
+        for secret in task.spec.cfg.secrets.clone() {
+            if let Some(variant) = secret.variant {
+                self.add_secret_env_var(variant);
+            }
+        }
+
+        let script_template = ArgoScriptTemplate {
+            source: self.script,
+            container: serde_json::from_value(json!(
+                    {
+                      "image": self.ctx.executor_image,
+                      "command": ["bash"],
+                      "volumeMounts": [{
+                          "name": self.ctx.task_volume,
+                          "mountPath": "/project",
+                      }],
+                      "env": self.env,
+                      "resources": {
+                         "limits":  task.spec.cfg.resources,
+                    }
+                    }
+            ))?,
+        };
+        Ok(WorkflowTemplate {
+            security_context: Some(serde_json::from_value(json!({
+                "runAsUser": 1001,
+                "fsGroup": 2000
+            }
+            ))?),
+            script: Some(script_template),
+            ..WorkflowTemplate::new(self.name)
+        })
+    }
+}
+
 #[cfg(test)]
 mod test {
 
diff --git a/lib/src/custom_resources/common.rs b/lib/src/custom_resources/common.rs
index 23d863f9..e78d565e 100644
--- a/lib/src/custom_resources/common.rs
+++ b/lib/src/custom_resources/common.rs
@@ -1,27 +1,40 @@
-use crate::custom_resources::argo::Workflow;
-use crate::custom_resources::task::Task;
+use crate::{
+    custom_resources::{argo::Workflow, data_set::DataSet, new_task::Task, project::Project},
+    error::AmeError,
+};
 use either::Either;
-use k8s_openapi::api::core::v1::{LoadBalancerStatus, Service, ServiceSpec, ServiceStatus};
+use k8s_openapi::{
+    api::core::v1::{LoadBalancerStatus, Service, ServiceSpec, ServiceStatus},
+    apimachinery::pkg::apis::meta::v1::OwnerReference,
+};
 use kube::{
-    api::{DeleteParams, ListParams},
-    Api, Client,
+    api::{DeleteParams, ListParams, PatchParams},
+    Api, Client, ResourceExt,
 };
+
 use std::time::Duration;
 
+use super::project_source::ProjectSource;
+
 pub async fn find_ame_endpoint(
     namespace: &str,
     service_name: &str,
 ) -> Result<String, Box<dyn std::error::Error>> {
+    return Ok("http://localhost:3342".to_string());
     let client = Client::try_default().await?;
     let services = Api::<Service>::namespaced(client.clone(), namespace);
     let service = services.get(service_name).await?;
 
-    let Service { spec: Some(ServiceSpec{
-       ports: Some(ports),
-       ..
-    }),
-    ..} = service else {
-        return Err(format!("failed to extract service ips and ports: {service:#?}"))?; 
+    let Service {
+        spec: Some(ServiceSpec {
+            ports: Some(ports), ..
+        }),
+        ..
+    } = service
+    else {
+        return Err(format!(
+            "failed to extract service ips and ports: {service:#?}"
+        ))?;
     };
 
     let port = ports
@@ -43,18 +56,24 @@ pub async fn find_service_endpoint(
     let services = Api::<Service>::namespaced(client.clone(), namespace);
     let service = services.get(service_name).await?;
 
-    let Service { spec: Some(ServiceSpec{
-       ports: Some(ports),
-       ..
-    }), status: Some(ServiceStatus{
-
-        load_balancer: Some(LoadBalancerStatus{
-            ingress: Some(ingress)
+    let Service {
+        spec: Some(ServiceSpec {
+            ports: Some(ports), ..
         }),
+        status:
+            Some(ServiceStatus {
+                load_balancer:
+                    Some(LoadBalancerStatus {
+                        ingress: Some(ingress),
+                    }),
+                ..
+            }),
         ..
-    }),
-    ..} = service else {
-        return Err(format!("failed to extract service ips and ports: {service:#?}"))?; 
+    } = service
+    else {
+        return Err(format!(
+            "failed to extract service ips and ports: {service:#?}"
+        ))?;
     };
 
     if ingress.len() != 1 {
@@ -75,11 +94,28 @@ pub async fn find_service_endpoint(
         ))?;
     }
 
-    Ok(format!(
-        "http://{}:{}",
-        ingress[0].ip.as_ref().unwrap(),
-        ports[0].port
-    ))
+    let Some(ip) = ingress[0].ip.as_ref() else {
+        return Err("could not find IP from ingress".to_string().into());
+    };
+
+    Ok(format!("http://{}:{}", ip, ports[0].port))
+}
+
+pub fn parent_project(owner_references: Vec<OwnerReference>) -> crate::Result<String> {
+    let projects: Vec<OwnerReference> = owner_references
+        .into_iter()
+        .filter(|o| kind_is_project(&o.kind))
+        .collect();
+
+    if projects.len() != 1 {
+        return Err(AmeError::MissingProject(projects.len()));
+    }
+
+    Ok(projects[0].name.clone())
+}
+
+pub fn kind_is_project(kind: &str) -> bool {
+    kind == "Project"
 }
 
 /// Prepare a cluster for tests, under the assumptions that the `just setup_cluster` recipe has been run successfully.
@@ -91,12 +127,124 @@ pub async fn setup_cluster(
     let client = Client::try_default().await?;
     let tasks = Api::<Task>::namespaced(client.clone(), namespace);
     let workflows = Api::<Workflow>::namespaced(client.clone(), namespace);
+    let project_srcs = Api::<ProjectSource>::namespaced(client.clone(), namespace);
+    let projects = Api::<Project>::namespaced(client.clone(), namespace);
+    let data_sets = Api::<DataSet>::namespaced(client.clone(), namespace);
 
     let dp = DeleteParams::default();
     let lp = ListParams::default();
+    let patch_params = PatchParams::apply("AME_TEST").force();
+
+    for mut project in projects.list(&lp).await?.into_iter() {
+        project.spec.deletion_approved = true;
+        project.metadata.managed_fields = None;
+
+        let res = projects
+            .patch(
+                &project.name_any(),
+                &patch_params,
+                &kube::api::Patch::Apply(project),
+            )
+            .await;
+
+        match res {
+            Err(kube::Error::Api(e)) => {
+                if e.code == 409 {
+                    return Ok((tasks, workflows));
+                }
+            }
+            Err(e) => return Err(Box::new(e)),
+            _ => (),
+        }
+    }
+
+    match project_srcs.delete_collection(&dp, &lp).await? {
+        Either::Left(_) => {
+            while !project_srcs.list(&lp).await?.items.is_empty() {
+                tokio::time::sleep(Duration::from_millis(100)).await;
+            }
+        }
+
+        Either::Right(status) => {
+            println!("Deleted collection of tasks: {status:?}");
+        }
+    };
+    for mut task in tasks.list(&lp).await?.into_iter() {
+        task.spec.deletion_approved = true;
+        task.metadata.managed_fields = None;
+
+        tasks
+            .patch(
+                &task.name_any(),
+                &patch_params,
+                &kube::api::Patch::Apply(task),
+            )
+            .await?;
+    }
+
+    match projects.delete_collection(&dp, &lp).await? {
+        Either::Left(_) => {
+            for mut project in projects.list(&lp).await?.into_iter() {
+                project.spec.deletion_approved = true;
+                project.metadata.managed_fields = None;
+
+                let _ = projects
+                    .patch(
+                        &project.name_any(),
+                        &patch_params,
+                        &kube::api::Patch::Apply(project),
+                    )
+                    .await;
+            }
+            while !projects.list(&lp).await?.items.is_empty() {
+                tokio::time::sleep(Duration::from_millis(100)).await;
+            }
+        }
+
+        Either::Right(status) => {
+            println!("Deleted collection of tasks: {status:?}");
+        }
+    };
+
+    match data_sets.delete_collection(&dp, &lp).await? {
+        Either::Left(_) => {
+            for mut data_set in data_sets.list(&lp).await?.into_iter() {
+                data_set.spec.deletion_approved = true;
+
+                data_sets
+                    .patch(
+                        &data_set.name_any(),
+                        &PatchParams::default(),
+                        &kube::api::Patch::Merge(data_set),
+                    )
+                    .await?;
+            }
+
+            while !data_sets.list(&lp).await?.items.is_empty() {
+                tokio::time::sleep(Duration::from_millis(100)).await;
+            }
+        }
+
+        Either::Right(status) => {
+            println!("Deleted collection of tasks: {status:?}");
+        }
+    };
 
     match tasks.delete_collection(&dp, &lp).await? {
         Either::Left(_) => {
+            for mut task in tasks.list(&lp).await?.into_iter() {
+                task.spec.deletion_approved = true;
+                task.metadata.managed_fields = None;
+
+                tasks
+                    .patch(
+                        &task.name_any(),
+                        &patch_params,
+                        &kube::api::Patch::Apply(task),
+                    )
+                    .await?;
+            }
+
             while !tasks.list(&lp).await?.items.is_empty() {
                 tokio::time::sleep(Duration::from_millis(100)).await;
             }
diff --git a/lib/src/custom_resources/data_set.rs b/lib/src/custom_resources/data_set.rs
index ca5209f4..15ba9d01 100644
--- a/lib/src/custom_resources/data_set.rs
+++ b/lib/src/custom_resources/data_set.rs
@@ -1,15 +1,12 @@
-use crate::grpc::TaskCfg;
-use crate::AmeError;
-use crate::{custom_resources::task::Task, Result};
+use crate::{custom_resources::new_task::Task, grpc::TaskCfg, AmeError, Result};
 
-use kube::core::ObjectMeta;
-use kube::{Client, CustomResource, Resource, ResourceExt};
+use kube::{core::ObjectMeta, Client, CustomResource, Resource, ResourceExt};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use crate::grpc::DataSetCfg;
+use crate::grpc::{task_status::Phase, DataSetCfg};
 
-use super::task::{TaskPhase, TaskSpec};
+use super::new_task::{ProjectSource, TaskSpec};
 
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
 #[kube(
@@ -24,11 +21,12 @@ pub struct DataSetSpec {
     #[serde(flatten)]
     pub cfg: DataSetCfg,
     pub deletion_approved: bool,
+    pub project: Option<String>,
 }
 
 #[derive(Debug, Clone, JsonSchema, Serialize, Deserialize, Default)]
 pub struct DataSetStatus {
-    pub phase: DataSetPhase,
+    pub phase: Option<DataSetPhase>,
 }
 
 #[derive(Debug, Clone, JsonSchema, Serialize, Deserialize)]
@@ -44,14 +42,15 @@ impl DataSetPhase {
     pub fn from_task(task: Task) -> DataSetPhase {
         let task_name = task.name_any();
 
-        let task_phase = task.status.unwrap_or_default().phase.unwrap_or_default();
+        // TODO: implement default trait for task phase.
+        let task_phase = task.status.unwrap_or_default().phase.unwrap_or(
+            crate::grpc::task_status::Phase::Pending(crate::grpc::TaskPhasePending {}),
+        );
 
         match task_phase {
-            TaskPhase::Running | TaskPhase::Error | TaskPhase::Pending => {
-                DataSetPhase::RunningTask { task_name }
-            }
-            TaskPhase::Succeeded => DataSetPhase::Ready { task_name },
-            TaskPhase::Failed => DataSetPhase::Failed { task_name },
+            Phase::Running(_) | Phase::Pending(_) => DataSetPhase::RunningTask { task_name },
+            Phase::Succeeded(_) => DataSetPhase::Ready { task_name },
+            Phase::Failed(_) => DataSetPhase::Failed { task_name },
         }
     }
 }
@@ -69,15 +68,31 @@ impl Default for &DataSetPhase {
 }
 
 impl DataSet {
+    pub fn from_cfg(name: &str, cfg: DataSetCfg) -> Self {
+        Self::new(
+            name,
+            DataSetSpec {
+                cfg,
+                deletion_approved: false,
+                project: None,
+            },
+        )
+    }
+
     pub async fn running_task(&self, _client: Client) -> Option<String> {
         self.status.as_ref().and_then(|s| match &s.phase {
-            DataSetPhase::RunningTask { task_name } => Some(task_name.clone()),
+            Some(DataSetPhase::RunningTask { task_name }) => Some(task_name.clone()),
             _ => None,
         })
     }
 
     pub fn phase(&self) -> &DataSetPhase {
-        self.status.as_ref().map(|s| &s.phase).unwrap_or_default()
+        self.status
+            .as_ref()
+            .map(|s| &s.phase)
+            .unwrap_or(&Some(DataSetPhase::Pending {}))
+            .as_ref()
+            .unwrap_or_default()
     }
 
     pub fn task_cfg(&self) -> &Option<TaskCfg> {
@@ -86,18 +101,27 @@ impl DataSet {
 
     pub fn generate_task(&self) -> Result<Task> {
         let Some(owner_ref) = self.controller_owner_ref(&()) else {
-        return Err(AmeError::MissingOwnerRef(self.name_any()));
-    };
+            return Err(AmeError::MissingOwnerRef(self.name_any()));
+        };
 
         let Some(ref task_cfg) = self.spec.cfg.task else {
-        return Err(AmeError::MissingTaskCfg(self.name_any()));
-    };
+            return Err(AmeError::MissingTaskCfg(self.name_any()));
+        };
 
         // TODO: this should be removed once task_cfg is fleshed out.
         let default_name = "datatask".to_string();
-        let name = task_cfg.name.as_ref().unwrap_or(&default_name);
+        let name = task_cfg
+            .name
+            .as_ref()
+            .unwrap_or(&default_name)
+            .replace('_', "-"); // TODO sanitize names
+
+        let mut spec = TaskSpec::from(task_cfg.clone());
 
-        let spec = TaskSpec::try_from(task_cfg.clone())?;
+        spec.project = self.spec.project.clone();
+        if let Some(repo) = self.annotations().get("gitrepository") {
+            spec.source = Some(ProjectSource::from_public_git_repo(repo.to_string()));
+        }
 
         let metadata = ObjectMeta {
             name: Some(format!("{}{}", self.name_any(), name)),
diff --git a/lib/src/custom_resources/mod.rs b/lib/src/custom_resources/mod.rs
index 5f34d3c3..f0ca6ecb 100644
--- a/lib/src/custom_resources/mod.rs
+++ b/lib/src/custom_resources/mod.rs
@@ -1,28 +1,24 @@
-use crate::error::AmeError;
 use crate::grpc::*;
-use k8s_openapi::chrono::OutOfRangeError;
-use k8s_openapi::chrono::ParseError;
-use kube::core::ObjectMeta;
-use kube::ResourceExt;
+use k8s_openapi::chrono::{OutOfRangeError, ParseError};
+use kube::{api::ListParams, core::ObjectMeta, Api, ResourceExt};
+use new_task::Task;
 use secrets::SecretError;
 use std::env::VarError;
-use task::Task;
 use thiserror::Error;
 
-use self::project_source::ProjectSource;
-use self::project_source::ProjectSourceSpec;
-
-use self::task::TaskSpec;
-use self::task::TaskType;
+use self::project_source::{ProjectSource, ProjectSourceSpec};
+use project::Project;
 
 pub mod argo;
 pub mod common;
 pub mod data_set;
+pub mod new_task;
 pub mod project;
 pub mod project_source;
 pub mod project_source_ctrl;
 pub mod secrets;
-pub mod task;
+
+pub mod task_ctrl;
 
 #[derive(Error, Debug)]
 pub enum Error {
@@ -35,7 +31,7 @@ pub enum Error {
     #[error("Received error from kube API: {0}")]
     KubeApiError(#[from] kube::Error),
 
-    #[error("Failed to find projet source: {0}")]
+    #[error("Failed to find project source: {0}")]
     MissingProjectSrc(String),
 
     #[error("libgit2 produced an error: {0}")]
@@ -103,42 +99,19 @@ pub enum Error {
 
     #[error("failed to find AME file project source with name : {0}")]
     MissingAmeFile(String),
-}
 
-pub type Result<T, E = Error> = std::result::Result<T, E>;
+    #[error("Task {0} is missing an executor")]
+    MissingExecutor(String),
 
-impl TryFrom<self::CreateTaskRequest> for Task {
-    type Error = Error;
+    #[error("Failed to find task cfg {0} referenced in {1}")]
+    MissingTaskCfg(String, String),
 
-    fn try_from(t: CreateTaskRequest) -> Result<Self> {
-        let CreateTaskRequest {
-            id: Some(TaskIdentifier { name: id }),
-            template: Some(template),
-        } = t else {
-            return Err(Error::ConversionError("Failed to extract id and template from CreateTaskRequest".to_string()))
-        };
-
-        Ok(Task {
-            metadata: ObjectMeta {
-                name: Some(id),
-                ..ObjectMeta::default()
-            },
-            spec: TaskSpec {
-                projectid: Some(template.projectid),
-                runcommand: Some(template.command),
-                image: template.image,
-                task_type: template.task_type.map(|t| match t {
-                    2 => TaskType::Poetry,
-                    1 => TaskType::Mlflow,
-                    _ => TaskType::PipEnv,
-                }),
-                ..TaskSpec::default()
-            },
-            status: None,
-        })
-    }
+    #[error("{0}")]
+    AmeError(#[from] crate::AmeError),
 }
 
+pub type Result<T, E = Error> = std::result::Result<T, E>;
+
 impl From<ProjectSourceCfg> for ProjectSource {
     fn from(project_src: ProjectSourceCfg) -> Self {
         ProjectSource {
@@ -152,58 +125,28 @@ impl From<ProjectSourceCfg> for ProjectSource {
     }
 }
 
-impl From<TaskTemplate> for Task {
-    fn from(t: TaskTemplate) -> Self {
-        Task {
-            metadata: ObjectMeta {
-                generate_name: Some("mytask".to_string()),
-                ..ObjectMeta::default()
-            },
-            spec: TaskSpec {
-                projectid: Some(t.projectid),
-                runcommand: Some(t.command),
-                image: t.image,
-                task_type: t.task_type.map(|t| {
-                    if t == 1 {
-                        TaskType::Mlflow
-                    } else {
-                        TaskType::PipEnv
-                    }
-                }),
-                ..TaskSpec::default()
-            },
-            status: None,
-        }
-    }
-}
-
-impl From<Task> for TaskTemplate {
-    fn from(t: Task) -> Self {
-        TaskTemplate {
-            name: "".to_string(),
-            command: t.spec.runcommand.unwrap_or("".to_string()),
-            projectid: t.spec.projectid.unwrap_or("".to_string()),
-            image: t.spec.image,
-            task_type: t.spec.task_type.map(|t| match t {
-                TaskType::Mlflow => 1,
-                TaskType::Poetry => 2,
-                TaskType::PipEnv => 0,
-            }),
-        }
-    }
-}
-
 impl From<Task> for TaskIdentifier {
     fn from(t: Task) -> Self {
         TaskIdentifier { name: t.name_any() }
     }
 }
 
-impl TryFrom<TaskCfg> for TaskSpec {
-    type Error = AmeError;
-    fn try_from(cfg: TaskCfg) -> crate::Result<Self> {
-        Ok(TaskSpec::from_ref(cfg.task_ref.ok_or(
-            AmeError::MissingTaskRef(cfg.name.unwrap_or_default()),
-        )?))
+pub async fn find_project(
+    projects: Api<Project>,
+    name: String,
+    _source: String,
+) -> Result<Project> {
+    let matches: Vec<Project> = projects
+        .list(&ListParams::default())
+        .await?
+        .items
+        .into_iter()
+        .filter(|p| p.spec.cfg.name == name)
+        .collect();
+
+    if matches.len() != 1 {
+        return Err(Error::MissingProject(name));
     }
+
+    Ok(matches[0].clone())
 }
diff --git a/lib/src/custom_resources/new_task.rs b/lib/src/custom_resources/new_task.rs
new file mode 100644
index 00000000..2a70ec37
--- /dev/null
+++ b/lib/src/custom_resources/new_task.rs
@@ -0,0 +1,507 @@
+use std::collections::BTreeMap;
+
+use super::{
+    argo::{Workflow, WorkflowBuilder, WorkflowTemplateBuilder},
+    common::parent_project,
+    data_set::DataSet,
+    project::{add_owner_reference, Project},
+    secrets::SecretReference,
+};
+use crate::{
+    custom_resources::{find_project, task_ctrl::resolve_data_set_path},
+    error::AmeError,
+    grpc::{task_status, ArtifactCfg, TemplateRef},
+    Result,
+};
+use k8s_openapi::apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::OwnerReference};
+use kube::{core::ObjectMeta, Api, CustomResource, Resource, ResourceExt};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use serde_merge::omerge;
+use tracing::debug;
+
+use crate::grpc::{TaskCfg, TaskPhasePending, TaskStatus};
+
+#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
+#[kube(
+    kind = "Task",
+    group = "ame.teainspace.com",
+    version = "v1alpha1",
+    namespaced
+)]
+#[kube(status = "TaskStatus", shortname = "tk")]
+#[serde(rename_all = "camelCase")]
+pub struct TaskSpec {
+    #[serde(flatten)]
+    pub cfg: TaskCfg,
+    pub deletion_approved: bool,
+    pub source: Option<ProjectSource>,
+    pub project: Option<String>,
+}
+
+impl TaskStatus {
+    pub fn pending() -> Self {
+        TaskStatus {
+            phase: Some(task_status::Phase::pending()),
+        }
+    }
+}
+
+impl task_status::Phase {
+    pub fn pending() -> Self {
+        Self::Pending(TaskPhasePending {})
+    }
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub enum ProjectSource {
+    Git {
+        repository: String,
+        reference: String,
+        user_name: String,
+        secret: Option<SecretReference>,
+    },
+    Ame {
+        path: String,
+    },
+}
+
+impl ProjectSource {
+    pub fn from_public_git_repo(repository: String) -> Self {
+        ProjectSource::Git {
+            repository,
+            reference: "".to_string(),
+            user_name: "".to_string(),
+            secret: None,
+        }
+    }
+
+    fn command(&self) -> String {
+        match self {
+            ProjectSource::Git {
+                repository,
+                reference: _,
+                user_name: _,
+                secret: None,
+            } => {
+                format!(
+                    "
+                    
+                git clone {repository} repo
+
+                cd repo
+
+                git fetch origin
+
+                git checkout origin/reference
+
+                cd ..
+
+                cp -r repo/* .
+
+                rm -rf repo
+
+                "
+                )
+            }
+            ProjectSource::Ame { path } => {
+                format!("s3cmd --no-ssl --region eu-central-1 --host=$MINIO_URL --host-bucket=$MINIO_URL get --recursive s3://{path} ./" )
+            }
+            _ => todo!("add tests for other git sources"),
+        }
+    }
+}
+
+pub fn build_workflow(task: Task, ctx: TaskContext) -> Result<Workflow> {
+    debug!("building task with context: {:?}", ctx);
+
+    let mut wf_builder = WorkflowBuilder::new(task.name_any(), ctx.service_account.clone());
+
+    let mut volume_resource_requirements = BTreeMap::new();
+    volume_resource_requirements.insert("storage".to_string(), Quantity("50Gi".to_string()));
+
+    wf_builder.add_volume(task.name_any(), volume_resource_requirements);
+
+    if let Some(oref) = task.controller_owner_ref(&()) {
+        wf_builder.add_owner_reference(oref);
+    };
+
+    let setup_template =
+        WorkflowTemplateBuilder::new(&ctx, task.load_command(&ctx)?, "setup".to_string())?
+            .build(&task)?;
+
+    wf_builder.add_template(setup_template);
+
+    let main_template =
+        WorkflowTemplateBuilder::new(&ctx, task.exec_command()?, task.name_any())?.build(&task)?;
+    wf_builder.add_template(main_template);
+
+    if task.should_save_artifacts() {
+        let artifact_save_template = WorkflowTemplateBuilder::new(
+            &ctx,
+            task.artifact_save_command()?,
+            "saveartifacts".to_string(),
+        )?
+        .build(&task)?;
+        wf_builder.add_template(artifact_save_template);
+    }
+
+    wf_builder.build()
+}
+
+impl Task {
+    pub fn approve_deletion_patch() -> Self {
+        Task {
+            metadata: ObjectMeta::default(),
+            spec: TaskSpec {
+                deletion_approved: true,
+                ..TaskSpec::default()
+            },
+            status: None,
+        }
+    }
+
+    pub fn project(&self) -> Result<String> {
+        parent_project(self.owner_references().to_vec())
+    }
+
+    fn should_save_artifacts(&self) -> bool {
+        self.spec.cfg.artifact_cfg.is_some()
+    }
+
+    fn artifact_save_command(&self) -> Result<String> {
+        match self.spec.cfg.artifact_cfg {
+            Some(ArtifactCfg{ save_changed_files, ..}) if save_changed_files => Ok(format!("save_artifacts {}", self.artifact_path()?)),
+            Some(ArtifactCfg { ref paths, .. }) => Ok(paths.iter().map(|p| format!("s3cmd --no-ssl --region us-east-1 --host=$MINIO_URL --host-bucket=$MINIO_URL put {p} s3://$ARTIFACT_STORAGE_PATH{p}") ).collect::<Vec<String>>().join("\n\n")),
+            None => Err(AmeError::EmptyArtifactCfg(self.spec.cfg.name.clone().unwrap_or_default()))
+        }
+    }
+
+    fn artifact_path(&self) -> Result<String> {
+        if let Some(ref name) = self.metadata.name {
+            Ok(format!("ame/tasks/{name}/artifacts/"))
+        } else {
+            Err(AmeError::MissingName)
+        }
+    }
+
+    fn load_command(&self, ctx: &TaskContext) -> Result<String> {
+        let mut cmd = String::new();
+
+        for ds in ctx.required_data_sets.iter() {
+            cmd = format!("cmd \n\n s3cmd --no-ssl --region eu-central-1 --host=$MINIO_URL --host-bucket=$MINIO_URL get --recursive s3://{} ./
+
+                \n\n", resolve_data_set_path(ds.clone())?)
+        }
+
+        let parent_project = parent_project(self.owner_references().to_vec())?;
+
+        let load_cmd = if let Some(ref source) = self.spec.source {
+            source.command()
+        } else {
+            ProjectSource::Ame {
+                path: self.project_dir_path(parent_project),
+            }
+            .command()
+        };
+
+        Ok(format!("{cmd} \n\n {load_cmd}"))
+    }
+
+    fn project_dir_path(&self, parent_project: String) -> String {
+        format!("ame/tasks/{parent_project}/projectfiles/")
+    }
+
+    fn exec_command(&self) -> Result<String> {
+        self.spec
+            .cfg
+            .executor
+            .as_ref()
+            .map(|executor| executor.command())
+            .ok_or(AmeError::MissingExecutor(self.name_any()))
+    }
+}
+
+#[derive(Clone)]
+pub struct TaskBuilder {
+    task: Task,
+}
+
+impl Default for TaskBuilder {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl TaskBuilder {
+    pub fn new() -> Self {
+        Self {
+            task: Task {
+                metadata: ObjectMeta::default(),
+                spec: TaskSpec::default(),
+                status: None,
+            },
+        }
+    }
+
+    pub fn set_project(&mut self, name: String) -> &mut Self {
+        self.task.spec.project = Some(name);
+        self
+    }
+
+    pub fn from_cfg(cfg: TaskCfg) -> Self {
+        Self {
+            task: Task {
+                metadata: ObjectMeta::default(),
+                spec: TaskSpec::from(cfg),
+                status: None,
+            },
+        }
+    }
+
+    pub fn set_name_prefix(&mut self, prefix: String) -> &mut Self {
+        self.task.metadata.generate_name = Some(prefix);
+        self
+    }
+
+    pub fn set_name(&mut self, name: String) -> &mut Self {
+        self.task.metadata.name = Some(name);
+        self
+    }
+
+    pub fn set_project_src(&mut self, src: ProjectSource) -> &mut Self {
+        self.task.spec.source = Some(src);
+
+        self
+    }
+
+    pub fn set_model_version(&mut self, src: String) -> &mut Self {
+        self.task
+            .annotations_mut()
+            .insert("model_source".to_string(), src);
+        self
+    }
+
+    pub fn build(mut self) -> Task {
+        if self.task.name_any().is_empty() {
+            let prefix = self
+                .task
+                .spec
+                .cfg
+                .name
+                .clone()
+                .unwrap_or("task".to_string());
+            self.set_name_prefix(prefix);
+        }
+
+        self.task.clone()
+    }
+
+    pub fn add_owner_reference(&mut self, owner_ref: OwnerReference) -> &mut Self {
+        self.task.metadata = add_owner_reference(self.task.meta().clone(), owner_ref);
+
+        self
+    }
+}
+
+impl From<TaskCfg> for TaskSpec {
+    fn from(cfg: TaskCfg) -> Self {
+        TaskSpec {
+            cfg,
+            ..TaskSpec::default()
+        }
+    }
+}
+
+pub async fn resolve_task_templates(
+    task: Task,
+    project: Project,
+    projects: Api<Project>,
+) -> Result<Task> {
+    let Some(ref template_ref) = task.spec.cfg.from_template else {
+        return Ok(task);
+    };
+
+    let template = match template_ref {
+        TemplateRef {
+            ref name,
+            project: None,
+        } => project.get_template(name).ok_or(AmeError::MissingTemplate(
+            name.clone(),
+            project.spec.cfg.name,
+        ))?,
+        TemplateRef {
+            name,
+            project: Some(project_name),
+        } => find_project(projects, project_name.clone(), "".to_string())
+            .await
+            .map_err(|_| AmeError::MissingProject(0))?
+            .get_template(name)
+            .ok_or(AmeError::MissingTemplate(
+                name.clone(),
+                project.spec.cfg.name,
+            ))?,
+    };
+
+    debug!("found template {:?}", template);
+
+    let task_spec = TaskSpec {
+        cfg: omerge(template, task.spec.cfg)?,
+        ..task.spec
+    };
+
+    Ok(Task {
+        spec: task_spec,
+        ..task
+    })
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TaskContext {
+    pub executor_image: String,
+    pub task_volume: String,
+    pub required_data_sets: Vec<DataSet>,
+    pub service_account: String,
+}
+
+#[cfg(test)]
+mod test {
+
+    use kube::core::ObjectMeta;
+
+    use crate::grpc::{
+        secret::Variant, task_cfg::Executor, AmeSecretVariant, EnvVar, PoetryExecutor, Secret,
+        TaskPhaseRunning, TaskRef,
+    };
+
+    use super::*;
+
+    #[test]
+    fn snap_shot_task_yaml() -> Result<()> {
+        let mut resources = BTreeMap::new();
+        resources.insert("cpu".to_string(), "2".to_string());
+
+        let task = Task {
+            metadata: ObjectMeta {
+                name: Some("mytask".to_string()),
+                ..ObjectMeta::default()
+            },
+
+            spec: TaskSpec {
+                cfg: TaskCfg {
+                    name: Some("mytask".to_string()),
+                    task_ref: Some(TaskRef {
+                        name: "othertask".to_string(),
+                        project: None,
+                    }),
+                    executor: None,
+                    resources,
+                    data_sets: Vec::new(),
+                    from_template: None,
+                    artifact_cfg: None,
+                    triggers: None,
+                    env: vec![EnvVar {
+                        key: "SOME_VAR".to_string(),
+                        val: "someval".to_string(),
+                    }],
+                    secrets: vec![Secret {
+                        variant: Some(Variant::Ame(AmeSecretVariant {
+                            key: "secretkey".to_string(),
+                            inject_as: "MY_SECRET".to_string(),
+                        })),
+                    }],
+                },
+                source: Some(ProjectSource::Ame {
+                    path: "test".to_string(),
+                }),
+                deletion_approved: false,
+                project: None,
+            },
+            status: Some(TaskStatus {
+                phase: Some(task_status::Phase::Running(TaskPhaseRunning {
+                    workflow_name: "someinfo".to_string(),
+                })),
+            }),
+        };
+
+        insta::assert_yaml_snapshot!(&task);
+
+        Ok(())
+    }
+    #[test]
+    fn snap_shot_workflow_yaml() -> Result<()> {
+        let mut resources = BTreeMap::new();
+        resources.insert("cpu".to_string(), "2".to_string());
+        resources.insert("memory".to_string(), "2Gi".to_string());
+        let task = Task {
+            metadata: ObjectMeta {
+                name: Some("mytask".to_string()),
+                owner_references: Some(vec![OwnerReference {
+                    block_owner_deletion: None,
+                    api_version: "sfsdd".to_string(),
+                    controller: None,
+                    kind: "Project".to_string(),
+                    name: "parentproject343".to_string(),
+                    uid: "sdsfdsf".to_string(),
+                }]),
+                ..ObjectMeta::default()
+            },
+
+            spec: TaskSpec {
+                cfg: TaskCfg {
+                    name: Some("mytask".to_string()),
+                    task_ref: Some(TaskRef {
+                        name: "othertask".to_string(),
+                        project: None,
+                    }),
+                    executor: Some(Executor::Poetry(PoetryExecutor {
+                        python_version: "3.11".to_string(),
+                        command: "python train.py".to_string(),
+                    })),
+                    data_sets: Vec::new(),
+                    resources,
+                    from_template: None,
+                    artifact_cfg: Some(ArtifactCfg {
+                        save_changed_files: true,
+                        paths: vec![],
+                    }),
+                    triggers: None,
+                    env: vec![EnvVar {
+                        key: "SOME_VAR".to_string(),
+                        val: "someval".to_string(),
+                    }],
+                    secrets: vec![Secret {
+                        variant: Some(Variant::Ame(AmeSecretVariant {
+                            key: "secretkey".to_string(),
+                            inject_as: "MY_SECRET".to_string(),
+                        })),
+                    }],
+                },
+                source: Some(ProjectSource::Ame {
+                    path: "test".to_string(),
+                }),
+                deletion_approved: false,
+                project: None,
+            },
+            status: Some(TaskStatus {
+                phase: Some(task_status::Phase::Running(TaskPhaseRunning {
+                    workflow_name: "someinfo".to_string(),
+                })),
+            }),
+        };
+
+        // TODO: "create data set for testing ");
+
+        let task_ctx = TaskContext {
+            executor_image: "myimage".to_string(),
+            task_volume: "myvolume".to_string(),
+            required_data_sets: vec![],
+            service_account: "ame-task".to_string(),
+        };
+
+        insta::assert_yaml_snapshot!(build_workflow(task, task_ctx)?);
+
+        Ok(())
+    }
+}
diff --git a/lib/src/custom_resources/project.rs b/lib/src/custom_resources/project.rs
index b189e512..d07ac28e 100644
--- a/lib/src/custom_resources/project.rs
+++ b/lib/src/custom_resources/project.rs
@@ -1,18 +1,17 @@
 use std::{
     collections::{BTreeMap, HashMap},
     default::Default,
-    sync::Arc,
-    time::Duration,
 };
 
 use crate::{
-    custom_resources::task,
-    custom_resources::task::{TaskPhase, TaskSpec},
-    custom_resources::{Error, Result},
+    ctrl::AmeResource,
+    custom_resources::{data_set::DataSet, Error, Result},
+    error::AmeError,
+    grpc::{resource_map_conv, DataSetCfg, Model, ProjectCfg, ProjectStatus, TaskCfg, TaskRef},
 };
 
-use crate::grpc::LogEntry;
-use futures::{future::BoxFuture, FutureExt, StreamExt};
+use super::new_task::{ProjectSource, Task, TaskBuilder};
+
 use k8s_openapi::{
     api::{
         apps::v1::{Deployment, DeploymentSpec},
@@ -26,21 +25,16 @@ use k8s_openapi::{
         },
     },
     apimachinery::pkg::{
-        apis::meta::v1::{LabelSelector, OwnerReference, Time},
+        apis::meta::v1::{LabelSelector, OwnerReference},
         util::intstr::IntOrString,
     },
 };
-use kube::{
-    api::{ListParams, Patch, PatchParams, PostParams},
-    core::ObjectMeta,
-    runtime::{controller::Action, Controller},
-    Api, Client, CustomResource, Resource, ResourceExt,
-};
-use reqwest::Url;
+use kube::{core::ObjectMeta, CustomResource, Resource, ResourceExt};
+
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use tracing::{debug, error, log::info};
+use tracing::debug;
 
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
 #[kube(
@@ -53,36 +47,12 @@ use tracing::{debug, error, log::info};
 )]
 #[serde(rename_all = "camelCase")]
 pub struct ProjectSpec {
-    #[serde(rename = "projectid")]
-    pub id: String,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub models: Option<Vec<Model>>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub tasks: Option<Vec<TaskSpec>>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub templates: Option<Vec<TaskSpec>>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub log_entry: Option<LogEntry>,
+    #[serde(flatten)]
+    pub cfg: ProjectCfg,
+    pub deletion_approved: bool,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub data_sets: Option<Vec<DataSet>>,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, PartialEq, Eq, Default)]
-pub struct ProjectStatus {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub models: Option<BTreeMap<String, ModelStatus>>,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
-pub struct DataSet {
-    pub name: String,
-    pub task: TaskSpec,
-    pub path: String,
+    #[serde(default)]
+    pub enable_triggers: Option<bool>,
 }
 
 pub fn local_name(name: String) -> String {
@@ -98,192 +68,159 @@ pub fn local_name(name: String) -> String {
     }
 }
 
-impl DataSet {
-    pub fn gen_task(&self) -> task::Task {
-        task::Task {
-            metadata: ObjectMeta {
-                // TODO: what are the impliciations of using a fixed name here instead of
-                // generating a random one?
-                name: Some(self.name.clone()),
-                ..ObjectMeta::default()
-            },
-            spec: self.task.clone(),
-            status: None,
-        }
+pub fn project_name(name: String) -> Option<String> {
+    if !name.contains('.') {
+        return None;
+    }
+
+    let splits: Vec<String> = name.split('.').map(String::from).collect();
+
+    if splits.len() > 1 {
+        return Some(splits[0].clone());
     }
+
+    None
 }
 
-impl ProjectStatus {
-    fn set_model_status(&mut self, name: &str, status: ModelStatus) {
-        if let Some(ref mut statuses) = self.models {
-            statuses.insert(name.to_string(), status);
-        } else {
-            let mut statuses: BTreeMap<String, ModelStatus> = BTreeMap::new();
-            statuses.insert(name.to_string(), status);
-            self.models = Some(statuses);
+impl From<ProjectCfg> for ProjectSpec {
+    fn from(cfg: ProjectCfg) -> Self {
+        Self {
+            cfg,
+            deletion_approved: false,
+            enable_triggers: Some(false),
         }
     }
+}
 
-    fn set_model_validation(&mut self, name: &str, validation: ModelValidationStatus) {
-        let mut default = ModelStatus::default();
-        let mut status = self.get_model_status(name).unwrap_or(&mut default).clone();
+pub fn generate_task_name(project_name: String, task_name: String) -> String {
+    format!("{project_name}{task_name}")
+}
 
-        status.validation = Some(validation);
+pub fn generate_data_set_task_name(project_name: String, data_set_name: String) -> String {
+    format!("{}{}_task", project_name, data_set_name)
+}
 
-        self.set_model_status(name, status);
-    }
+impl Project {
+    pub fn from_cfg(cfg: ProjectCfg) -> Self {
+        Self {
+            metadata: ObjectMeta {
+                generate_name: Some(cfg.name.clone()),
+                ..ObjectMeta::default()
+            },
 
-    pub fn get_model_status(&mut self, name: &str) -> Option<&mut ModelStatus> {
-        self.models.as_mut().and_then(|models| models.get_mut(name))
+            spec: ProjectSpec::from(cfg),
+            status: None,
+        }
     }
 
-    fn set_latest_valid_model_version(&mut self, name: &str, version: String) {
-        let mut status = self
-            .get_model_status(name)
-            .map(|s| s.to_owned())
-            .unwrap_or_default();
-
-        status.latest_valid_model_version = Some(version);
-        self.set_model_status(name, status)
+    pub fn deletion_approved(&self) -> bool {
+        self.spec.deletion_approved
     }
 
-    fn get_latest_valid_model_version(&mut self, name: &str) -> Option<String> {
-        self.get_model_status(name)
-            .as_ref()
-            .and_then(|model_status| model_status.latest_valid_model_version.clone())
+    pub fn approve_deletion(&mut self) {
+        self.spec.deletion_approved = true;
     }
-}
 
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
-#[serde(rename_all = "camelCase")]
-pub struct Model {
-    name: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    model_type: Option<ModelType>,
+    pub fn get_data_set(&self, data_set_name: String) -> Option<DataSetCfg> {
+        let data_set_name = local_name(data_set_name);
 
-    training: TrainingCfg,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    deployment: Option<ModelDeployment>,
+        self.clone()
+            .spec
+            .cfg
+            .data_sets
+            .into_iter()
+            .find(|ds| ds.name == data_set_name)
+    }
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    validation_task: Option<TaskSpec>,
-}
+    // TODO: documentat that this assumes a local data set.
+    pub fn generate_data_set(&self, data_set_name: String) -> crate::Result<DataSet> {
+        let Some(cfg) = self.get_data_set(data_set_name.clone()) else {
+            debug!("failed to get data set {}", data_set_name);
+            return Err(AmeError::MissingDataSet(data_set_name, self.name_any()));
+        };
 
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
-pub struct TrainingCfg {
-    task: TaskSpec,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    schedule: Option<String>,
-}
+        let mut data_set = DataSet::from_cfg(
+            &format!("dataset{}{}", self.spec.cfg.name, data_set_name),
+            cfg,
+        );
 
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
-pub struct ModelDeployment {
-    deploy: bool,
-    auto_train: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    image: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    resources: Option<ResourceRequirements>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    replicas: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    ingress: Option<Ingress>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    ingress_annotations: Option<BTreeMap<String, String>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    enable_tls: Option<bool>,
-}
+        if let Some(repo) = self.annotations().get("gitrepository") {
+            data_set
+                .annotations_mut()
+                .insert("gitrepository".to_string(), repo.to_string());
+        }
 
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-pub enum ModelType {
-    Mlflow,
-}
+        data_set.spec.project = Some(self.spec.cfg.name.clone());
 
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, PartialEq, Eq, Default)]
-#[serde(rename_all = "camelCase")]
-pub struct ModelStatus {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub latest_model_version: Option<Time>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub latest_validated_model_version: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub last_trained: Option<Time>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub last_deployed: Option<Time>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub validation: Option<ModelValidationStatus>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub latest_valid_model_version: Option<String>,
-}
+        let Some(project_oref) = self.gen_owner_ref() else {
+            return Err(AmeError::FailedToCreateOref(self.name_any()));
+        };
 
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, PartialEq, Eq)]
-#[serde(rename_all = "camelCase")]
-pub enum ModelValidationStatus {
-    UnValidated {}, // we cannot use a unit type here: see https://docs.rs/kube/latest/kube/derive.CustomResource.html#enums
-    Validating { task: String, model_version: String },
-    FailedValidation { model_version: String },
-    Validated { model_version: String },
-}
+        data_set.owner_references_mut().push(project_oref);
 
-impl Default for ModelValidationStatus {
-    fn default() -> Self {
-        ModelValidationStatus::UnValidated {}
+        Ok(data_set)
     }
-}
 
-impl Project {
-    pub fn get_data_set(&self, data_set_name: String) -> Option<DataSet> {
-        let data_set_name = local_name(data_set_name);
-        self.clone()
-            .spec
-            .data_sets
-            .and_then(|data_sets| data_sets.into_iter().find(|ds| ds.name == data_set_name))
-    }
-
-    pub fn generate_data_set_task(&self, data_set_name: String) -> Option<task::Task> {
-        let Some(mut task) = self.get_data_set(data_set_name).map(|ds| ds.gen_task()) else {
+    pub fn generate_data_set_task(&self, data_set_name: String) -> Option<Task> {
+        let Some(Some(task)) = self.get_data_set(data_set_name.clone()).map(|ds| ds.task) else {
             return None;
         };
 
+        let mut task_builder = TaskBuilder::from_cfg(task);
+
+        task_builder.set_name(generate_data_set_task_name(
+            self.spec.cfg.name.clone(),
+            data_set_name,
+        ));
+        task_builder.set_project(self.spec.cfg.name.clone());
+
         if let Some(repo) = self.annotations().get("gitrepository") {
-            task.spec.source = Some(task::ProjectSource {
-                gitrepository: Some(repo.to_owned()),
-                ..task::ProjectSource::default()
-            });
+            task_builder.set_project_src(ProjectSource::from_public_git_repo(repo.to_string()));
         }
 
-        Some(task)
+        Some(task_builder.build())
     }
 
-    fn gen_validation_task(&self, model: &Model) -> Result<task::Task> {
-        let Some(mut spec) =match model.validation_task.as_ref() {
-            Some(TaskSpec {
-                task_ref: Some(name),
-                ..
-            }) => self.find_task_spec(name),
-            Some(task_spec) => Some(task_spec),
-            None => None
-        }.map(|ts| ts.to_owned()) else {
+    pub fn generate_validation_task(&self, model: &Model, latest_version: String) -> Result<Task> {
+        let cfg = if let Some(cfg) = model.validation_task.as_ref() {
+            if let Some(ref task_ref) = cfg.task_ref {
+                self.find_task_cfg(&task_ref.name)
+                    .ok_or(Error::MissingTaskCfg(
+                        task_ref.name.clone(),
+                        self.spec.cfg.name.clone(),
+                    ))?
+            } else {
+                cfg.to_owned()
+            }
+        } else {
             return Err(Error::MissingValidationTask(model.name.clone()));
         };
 
+        let mut task_builder = TaskBuilder::from_cfg(cfg);
+
         if let Some(repo) = self.annotations().get("gitrepository") {
-            spec.source = Some(task::ProjectSource {
-                gitrepository: Some(repo.to_owned()),
-                ..task::ProjectSource::default()
-            });
+            task_builder.set_project_src(ProjectSource::from_public_git_repo(repo.to_string()));
         }
 
-        let metadata = ObjectMeta {
-            generate_name: Some(format!("validate-{}", model.name.clone())),
-            ..ObjectMeta::default()
-        };
+        task_builder.add_owner_reference(
+            self.controller_owner_ref(&())
+                .ok_or(AmeError::FailedToCreateOref(self.name_any()))?,
+        );
 
-        Ok(task::Task {
-            metadata,
-            spec,
-            status: None,
-        })
+        let validation_task = task_builder
+            .set_name(format!(
+                "validate-{}-{}",
+                model.name.clone(),
+                latest_version
+            ))
+            .clone()
+            .build();
+
+        if validation_task.spec.cfg.executor.is_none() {
+            return Err(Error::MissingExecutor(validation_task.name_any()));
+        }
+
+        Ok(validation_task)
     }
 
     pub fn add_owner_reference(&mut self, owner_reference: OwnerReference) -> &mut Project {
@@ -309,71 +246,85 @@ impl Project {
     }
 
     fn get_model(&self, name: &str) -> Option<Model> {
-        if let Some(ref models) = self.spec.models {
-            models.clone().into_iter().find(|m| m.name == name)
-        } else {
-            None
-        }
+        self.spec
+            .cfg
+            .models
+            .clone()
+            .into_iter()
+            .find(|m| m.name == name)
     }
 
-    pub fn get_template(&self, name: &str) -> Option<TaskSpec> {
-        if let Some(ref templates) = self.spec.templates {
-            templates.clone().into_iter().find(|m| {
-                if let Some(ref tname) = m.name {
-                    tname == name
-                } else {
-                    false
-                }
-            })
-        } else {
-            None
-        }
+    pub fn get_template(&self, name: &str) -> Option<TaskCfg> {
+        self.spec.cfg.templates.clone().into_iter().find(|m| {
+            if let Some(ref tname) = m.name {
+                tname == name
+            } else {
+                false
+            }
+        })
     }
 
-    fn find_task_spec(&self, name: &str) -> Option<&task::TaskSpec> {
-        self.spec.tasks.as_ref().and_then(|tasks| {
-            tasks
-                .iter()
-                .find(|task| task.name.as_ref().map(|n| n == name).unwrap_or(false))
-        })
+    fn find_task_cfg(&self, name: &str) -> Option<TaskCfg> {
+        self.spec
+            .cfg
+            .tasks
+            .iter()
+            .find(|task| task.name.as_ref().map(|n| n == name).unwrap_or(false))
+            .cloned()
     }
 
-    pub fn generate_model_training_task(&self, name: &str) -> Result<task::Task> {
-        let Some(model) = self.get_model(name) else {
-            return Err(Error::MissingProjectSrc("model".to_string()));
+    fn get_task_cfg_for_ref(&self, task_ref: TaskRef) -> Option<TaskCfg> {
+        self.spec.cfg.get_task_cfg(&task_ref.name)
+    }
+
+    fn get_model_training_cfg(&self, model: &str) -> Result<TaskCfg> {
+        let Some(model) = self.get_model(model) else {
+            return Err(
+                AmeError::MissingModelTrainingTaskCfg(model.to_string(), self.name_any()).into(),
+            );
         };
 
-        let task_ref = model.training.task.task_ref.clone().unwrap();
+        match model.get_training_task_cfg() {
+            Some(TaskCfg {
+                task_ref: Some(ref task_ref),
+                ..
+            }) => self
+                .get_task_cfg_for_ref(task_ref.clone())
+                .ok_or(AmeError::MissingTaskRef(task_ref.name.to_string()).into()),
+            Some(task_cfg @ TaskCfg { task_ref: None, .. }) => Ok(task_cfg),
+            None => Err(AmeError::MissingTrainingTaskCfg(model.name).into()),
+        }
+    }
+
+    pub fn generate_model_training_task(&self, name: &str) -> Result<Task> {
+        let task_cfg = self.get_model_training_cfg(name)?;
 
-        let project_spec = self.spec.clone();
-        let tasks = project_spec.tasks.unwrap();
-        let mut task_spec = tasks
-            .iter()
-            .find(|t| t.name.clone().unwrap() == task_ref)
-            .unwrap()
-            .to_owned();
+        let mut task_builder = TaskBuilder::from_cfg(task_cfg.clone());
 
         let medata = self.metadata.clone();
 
         let annotations = medata.annotations.unwrap();
 
         let repo = annotations.get("gitrepository").unwrap();
+        // NOTE: how is the data set repo set as src here?
+
+        let training_task = task_builder
+            .set_project_src(ProjectSource::from_public_git_repo(repo.to_string()))
+            .set_name(format!(
+                "{}{}{}{}",
+                self.name_any(),
+                name,
+                task_cfg.name(),
+                self.spec.cfg.name
+            ))
+            .add_owner_reference(
+                self.controller_owner_ref(&())
+                    .unwrap_or(OwnerReference::default()),
+            )
+            .clone()
+            .build();
 
-        task_spec.source = Some(task::ProjectSource {
-            gitrepository: Some(repo.to_owned()),
-            ..task::ProjectSource::default()
-        });
-
-        let metadata = ObjectMeta {
-            generate_name: Some(model.name),
-            ..ObjectMeta::default()
-        };
-
-        Ok(task::Task {
-            metadata: add_owner_reference(metadata, self.controller_owner_ref(&()).unwrap()),
-            spec: task_spec,
-            status: None,
-        })
+        Ok(training_task)
     }
 }
 
@@ -389,7 +340,39 @@ pub fn add_owner_reference(
     metadata
 }
 
+pub async fn get_latest_model_version(
+    model: &Model,
+    mlflow_url: String,
+) -> Result<MlflowModelVersion> {
+    let Some(model_version) = ({
+        let mut body = HashMap::new();
+        body.insert("name", model.name.clone());
+        let client = reqwest::Client::new();
+        let MlflowModelVersionsRes { model_versions } = client
+            .post(format!(
+                "{mlflow_url}/api/2.0/mlflow/registered-models/get-latest-versions"
+            ))
+            .json(&body)
+            .send()
+            .await?
+            .json()
+            .await?;
+
+        model_versions
+            .into_iter()
+            .max_by_key(|v| v.creation_timestamp)
+    }) else {
+        return Err(Error::MissingMlflowUrl());
+    };
+
+    Ok(model_version)
+}
+
 impl Model {
+    fn get_training_task_cfg(&self) -> Option<TaskCfg> {
+        self.training.as_ref().and_then(|t| t.task.clone())
+    }
+
     fn labels(&self) -> BTreeMap<String, String> {
         let mut labels: BTreeMap<String, String> = BTreeMap::new();
         labels.insert("ame-model".to_string(), self.name.clone());
@@ -399,18 +382,22 @@ impl Model {
     fn object_metadata(&self) -> ObjectMeta {
         ObjectMeta {
             name: Some(self.name.clone()),
-            labels: Some(self.labels()),
+            labels: Some(self.labels().to_owned()),
             ..ObjectMeta::default()
         }
     }
 
-    fn generate_model_ingress(&self, ctrl_cfg: &ProjectCtrlCfg) -> Result<Ingress> {
-        let Some(model_deployment) = self.deployment.clone() else {
-            return Err(Error::MissingDeployment())
+    pub fn generate_model_ingress(
+        &self,
+        ingress_host: String,
+        ingress_annotations: Option<BTreeMap<String, String>>,
+        project_name: String,
+    ) -> Result<Ingress> {
+        let Some(mut model_deployment) = self.deployment.clone() else {
+            return Err(Error::MissingDeployment());
         };
 
-        let mut ingress_annotations = ctrl_cfg
-            .model_ingress_annotations
+        let mut ingress_annotations = ingress_annotations
             .clone()
             .unwrap_or(BTreeMap::<String, String>::new());
 
@@ -419,9 +406,13 @@ impl Model {
             "false".to_string(),
         );
 
-        if let Some(mut annotations) = model_deployment.ingress_annotations {
-            ingress_annotations.append(&mut annotations);
-        }
+        ingress_annotations.append(&mut model_deployment.ingress_annotations);
+
+        // TODO: we need a better method of setting paths for models as this is can easily break.
+        ingress_annotations.insert(
+            "nginx.ingress.kubernetes.io/rewrite-target".to_string(),
+            "/$2".to_string(),
+        );
 
         let metadata = ObjectMeta {
             name: Some(self.name.clone()),
@@ -432,10 +423,7 @@ impl Model {
 
         let tls: Option<Vec<IngressTLS>> = match model_deployment.enable_tls {
             Some(true) | None => Some(vec![IngressTLS {
-                hosts: Some(vec![ctrl_cfg
-                    .clone()
-                    .model_ingress_host
-                    .unwrap_or("".to_string())]),
+                hosts: Some(vec![ingress_host.clone()]),
                 secret_name: Some(format!("{}-tls", self.name)),
             }]),
             _ => None,
@@ -446,12 +434,7 @@ impl Model {
             spec: Some(IngressSpec {
                 ingress_class_name: Some("nginx".to_string()),
                 rules: Some(vec![IngressRule {
-                    host: Some(
-                        ctrl_cfg
-                            .clone()
-                            .model_ingress_host
-                            .unwrap_or("".to_string()),
-                    ),
+                    host: Some(ingress_host),
                     http: Some(HTTPIngressRuleValue {
                         paths: vec![HTTPIngressPath {
                             backend: IngressBackend {
@@ -464,8 +447,11 @@ impl Model {
                                 }),
                                 ..IngressBackend::default()
                             },
-                            path_type: "ImplementationSpecific".to_string(),
-                            path: Some("/invocations".to_string()),
+                            path_type: "Prefix".to_string(),
+                            path: Some(format!(
+                                "/projects/{}/models/{}(/|$)(.*)",
+                                project_name, self.name
+                            )),
                         }],
                     }),
                 }]),
@@ -476,9 +462,9 @@ impl Model {
         })
     }
 
-    fn generate_model_service(&self, _ctrl_cfg: &ProjectCtrlCfg) -> Result<Service> {
+    pub fn generate_model_service(&self) -> Result<Service> {
         let Some(_model_deployment) = self.deployment.clone() else {
-            return Err(Error::MissingDeployment())
+            return Err(Error::MissingDeployment());
         };
 
         Ok(Service {
@@ -495,75 +481,17 @@ impl Model {
         })
     }
 
-    async fn get_model_version(
+    pub async fn get_model_version(
         &self,
-        ctrl_cfg: &ProjectCtrlCfg,
-        version: &str,
+        _ctrl_cfg: &ProjectCtrlCfg,
+        _version: &str,
     ) -> Result<MlflowModelVersion> {
-        let Some(ref mlflow_url) = ctrl_cfg.mlflow_url else {
-            return Err(Error::MissingMlflowUrl());
-        };
-
-        let model_version = {
-            let mut body = HashMap::new();
-            body.insert("name", self.name.clone());
-            body.insert("version", version.to_string());
-            let url = Url::parse_with_params(
-                &format!("{mlflow_url}/api/2.0/mlflow/model-versions/get"),
-                &[
-                    ("name", self.name.clone()),
-                    ("version", version.to_string()),
-                ],
-            )
-            .unwrap();
-
-            let client = reqwest::Client::new();
-            client
-                .get(url)
-                .send()
-                .await?
-                .json::<MlflowModelVersionResponse>()
-                .await?
-                .model_version
-        };
-        Ok(model_version)
+        todo!();
     }
 
-    async fn get_latest_model_version(
+    pub async fn generate_model_deployment(
         &self,
-        ctrl_cfg: &ProjectCtrlCfg,
-    ) -> Result<MlflowModelVersion> {
-        let Some(ref mlflow_url) = ctrl_cfg.mlflow_url else {
-            return Err(Error::MissingMlflowUrl());
-        };
-
-        let Some(model_version) = ({
-            let mut body = HashMap::new();
-            body.insert("name", self.name.clone());
-            let client = reqwest::Client::new();
-            let MlflowModelVersionsRes{model_versions } = client
-                .post(format!(
-                    "{mlflow_url}/api/2.0/mlflow/registered-models/get-latest-versions"
-                ))
-                .json(&body)
-                .send()
-                .await?
-                .json()
-                .await?;
-
-            model_versions
-                .into_iter()
-                .max_by_key(|v| v.creation_timestamp)
-        }) else {
-            return Err(Error::MissingMlflowUrl());
-        };
-
-        Ok(model_version)
-    }
-
-    async fn generate_model_deployment(
-        &self,
-        ctrl_cfg: &ProjectCtrlCfg,
+        deployment_image: String,
         model_source: String,
     ) -> Result<Deployment> {
         let Some(model_deployment) = self.deployment.clone() else {
@@ -588,7 +516,7 @@ impl Model {
                 replicas: Some(model_deployment.replicas.unwrap_or(1)),
                 template: PodTemplateSpec {
                     metadata: Some(ObjectMeta {
-                        labels: Some(labels),
+                        labels: Some(labels.clone()),
                         ..ObjectMeta::default()
                     }),
                     spec: Some(PodSpec {
@@ -601,14 +529,17 @@ impl Model {
                             image: Some(
                                 model_deployment
                                     .image
-                                    .unwrap_or(ctrl_cfg.clone().deployment_image),
+                                    .unwrap_or(deployment_image),
                             ),
                             command: Some(vec!["/bin/bash".to_string()]),
                             args: Some(vec![
                                 "-c".to_string(),
                                 format!("export PATH=$HOME/.pyenv/bin:$PATH; mlflow models serve -m {model_source} --host 0.0.0.0"),
                             ]),
-                            resources: model_deployment.resources,
+                            resources: Some(ResourceRequirements{
+                                limits: Some(resource_map_conv(model_deployment.resources)),
+                                requests: None,
+                            }),
                             env: Some(vec![EnvVar {
                                 name: "MLFLOW_TRACKING_URI".to_string(),
                                 value: Some(
@@ -671,11 +602,6 @@ impl ProjectCtrlCfg {
     }
 }
 
-struct Context {
-    client: Client,
-    config: ProjectCtrlCfg,
-}
-
 #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Default)]
 struct MlflowModelVersionsRes {
     model_versions: Vec<MlflowModelVersion>,
@@ -687,307 +613,15 @@ struct MlflowModelVersionResponse {
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Default)]
-struct MlflowModelVersion {
+pub struct MlflowModelVersion {
     name: String,
-    version: String,
+    pub version: String,
     current_stage: String,
     creation_timestamp: i64,
-    source: String,
+    pub source: String,
     run_id: String,
 }
 
-async fn reconcile(project: Arc<Project>, ctx: Arc<Context>) -> Result<Action> {
-    info!("reconciling projects");
-    let _projects = Api::<Project>::namespaced(ctx.client.clone(), &ctx.config.namespace);
-    let deployments = Api::<Deployment>::namespaced(ctx.client.clone(), &ctx.config.namespace);
-    let ingresses = Api::<Ingress>::namespaced(ctx.client.clone(), &ctx.config.namespace);
-    let services = Api::<Service>::namespaced(ctx.client.clone(), &ctx.config.namespace);
-    let tasks_cli = Api::<task::Task>::namespaced(ctx.client.clone(), &ctx.config.namespace);
-    let ctrl_cfg = ctx.config.clone();
-    let oref = if let Some(refe) = project.controller_owner_ref(&()) {
-        refe
-    } else {
-        OwnerReference::default()
-    };
-
-    let mut status = project.status.clone().unwrap_or_default();
-
-    if let Some(models) = project.spec.clone().models {
-        for model in models {
-            let model_version = model.get_latest_model_version(&ctrl_cfg).await;
-
-            if let Some(ModelDeployment {
-                auto_train: true,
-                deploy: true,
-                ..
-            }) = model.clone().deployment
-            {
-                info!("creating autotrain task {model_version:?}");
-                if model_version.is_err() {
-                    let task_ref = model.training.task.task_ref.clone().unwrap();
-
-                    let project_spec = project.spec.clone();
-                    let tasks = project_spec.tasks.unwrap();
-                    let mut task_spec = tasks
-                        .iter()
-                        .find(|t| t.name.clone().unwrap() == task_ref)
-                        .unwrap()
-                        .to_owned();
-
-                    let medata = project.metadata.clone();
-
-                    let annotations = medata.annotations.unwrap();
-
-                    let repo = annotations.get("gitrepository").unwrap();
-
-                    task_spec.source = Some(task::ProjectSource {
-                        gitrepository: Some(repo.to_owned()),
-                        ..task::ProjectSource::default()
-                    });
-
-                    let metadata = ObjectMeta {
-                        name: Some(model.name.clone()),
-                        ..ObjectMeta::default()
-                    };
-
-                    let task = task::Task {
-                        metadata: add_owner_reference(metadata, oref.clone()),
-                        spec: task_spec.clone(),
-                        status: None,
-                    };
-
-                    tasks_cli
-                        .patch(
-                            &task.name_any(),
-                            &PatchParams::apply("ame"),
-                            &Patch::Apply(&task),
-                        )
-                        .await?;
-                }
-            };
-
-            debug!(
-                "check model validation status {:?} {:?}",
-                model_version, model.validation_task
-            );
-
-            if model.validation_task.is_some() && model_version.is_ok() {
-                let validation = status
-                    .clone()
-                    .get_model_status(&model.name)
-                    .and_then(|s| s.validation.as_ref())
-                    .unwrap_or(&ModelValidationStatus::UnValidated {})
-                    .clone();
-
-                debug!("update model validation state");
-
-                match validation {
-                    ModelValidationStatus::UnValidated {} => {
-                        debug!(
-                            "model {} with version {} needs to be validated",
-                            &model.name,
-                            model_version
-                                .as_ref()
-                                .map(|m| m.version.clone())
-                                .unwrap_or("'no version found'".to_string())
-                        );
-
-                        let mut validation_task = project.gen_validation_task(&model)?; // TODO: the ? here could cause unwanted early exits.
-                        validation_task.owner_references_mut().push(oref.clone());
-
-                        let validation_task = tasks_cli
-                            .create(&PostParams::default(), &validation_task)
-                            .await?;
-
-                        status.set_model_validation(
-                            &model.name,
-                            ModelValidationStatus::Validating {
-                                task: validation_task.name_any(),
-                                model_version: model_version.as_ref().unwrap().version.clone(),
-                            },
-                        );
-                    }
-                    ModelValidationStatus::Validating {
-                        task,
-                        model_version,
-                    } => {
-                        // TODO: handle early cancelation if a new model version appears before
-                        // this validation completes.
-
-                        debug!("debug status: validating");
-
-                        let Ok(task_status) = tasks_cli.get_status(&task).await else {
-                            debug!("failed to find status for task {task}, resetting model status to unvalidatd");
-                            status.set_model_validation(&model.name, ModelValidationStatus::UnValidated {  });
-                            continue;
-                        };
-
-                        debug!("validation task status: {task_status:?}");
-
-                        if let Some(task_status) = task_status.status {
-                            match task_status.phase {
-                                Some(TaskPhase::Succeeded) => {
-                                    status.set_latest_valid_model_version(
-                                        &model.name,
-                                        model_version.clone(),
-                                    );
-                                    status.set_model_validation(
-                                        &model.name,
-                                        ModelValidationStatus::Validated {
-                                            model_version: model_version.to_string(),
-                                        },
-                                    );
-                                }
-                                Some(TaskPhase::Failed) => status.set_model_validation(
-                                    &model.name,
-                                    ModelValidationStatus::FailedValidation {
-                                        model_version: model_version.to_string(),
-                                    },
-                                ),
-                                _ => (),
-                            };
-
-                            debug!("set model validations status: {status:?}");
-                        }
-                    }
-
-                    ModelValidationStatus::Validated {
-                        model_version: validatd_version,
-                    } => {
-                        if *validatd_version == model_version.as_ref().unwrap().version {
-                            debug!("new model version requires validation");
-                            let mut validation_task = project.gen_validation_task(&model)?; // TODO: the ? here could cause unwanted early exits.
-                            validation_task.owner_references_mut().push(oref.clone());
-
-                            let validation_task = tasks_cli
-                                .create(&PostParams::default(), &validation_task)
-                                .await?;
-
-                            status.set_model_validation(
-                                &model.name,
-                                ModelValidationStatus::Validating {
-                                    task: validation_task.name_any(),
-                                    model_version: model_version.as_ref().unwrap().version.clone(),
-                                },
-                            );
-
-                            continue;
-                        }
-                    }
-                    ModelValidationStatus::FailedValidation {
-                        model_version: validatd_version,
-                    } => {
-                        if *validatd_version != model_version.as_ref().unwrap().version {
-                            debug!("failed to validate");
-                            let mut validation_task = project.gen_validation_task(&model)?; // TODO: the ? here could cause unwanted early exits.
-                            validation_task.owner_references_mut().push(oref.clone());
-
-                            let validation_task = tasks_cli
-                                .create(&PostParams::default(), &validation_task)
-                                .await?;
-
-                            status.set_model_validation(
-                                &model.name,
-                                ModelValidationStatus::Validating {
-                                    task: validation_task.name_any(),
-                                    model_version: model_version.as_ref().unwrap().version.clone(),
-                                },
-                            );
-                        }
-                        continue;
-                    }
-                }
-            }
-
-            debug!("preparing model deployment");
-
-            let Some(latest_version) = status.get_latest_valid_model_version(&model.name) else {
-                debug!("no valid model version to deploy");
-                continue;
-            };
-
-            let mlflow_version = model.get_model_version(&ctrl_cfg, &latest_version).await?;
-
-            let mut deployment = model
-                .generate_model_deployment(&ctrl_cfg, mlflow_version.source)
-                .await?;
-            let mut service = model.generate_model_service(&ctrl_cfg)?;
-            let mut ingress = model.generate_model_ingress(&ctrl_cfg)?;
-            let pp = PatchParams::apply("ame");
-
-            deployment.metadata = add_owner_reference(deployment.metadata, oref.clone());
-            service.metadata = add_owner_reference(service.metadata, oref.clone());
-            ingress.metadata = add_owner_reference(ingress.metadata, oref.clone());
-
-            deployments
-                .patch(&deployment.name_any(), &pp, &Patch::Apply(&deployment))
-                .await?;
-            services
-                .patch(&service.name_any(), &pp, &Patch::Apply(&service))
-                .await?;
-            ingresses
-                .patch(&ingress.name_any(), &pp, &Patch::Apply(&ingress))
-                .await?;
-        }
-    }
-
-    debug!("patching status");
-
-    let mut patch = Project {
-        metadata: project.metadata.clone(),
-        spec: ProjectSpec::default(),
-        status: Some(status),
-    };
-
-    patch.meta_mut().managed_fields = None;
-
-    _projects
-        .patch_status(
-            &project.name_any(),
-            &PatchParams::apply("ame").force(),
-            &Patch::Apply(patch),
-        )
-        .await?;
-
-    debug!("requeing");
-
-    Ok(Action::requeue(Duration::from_secs(300)))
-}
-
-fn error_policy(src: Arc<Project>, error: &Error, _ctx: Arc<Context>) -> Action {
-    error!("error: {}, for project: {}", error, src.name_any());
-    Action::requeue(Duration::from_secs(60))
-}
-
-pub async fn start_project_controller(config: ProjectCtrlCfg) -> BoxFuture<'static, ()> {
-    let client = Client::try_default().await.expect("Failed to create a K8S client, is the controller running in an environment with access to cluster credentials?");
-    let context = Arc::new(Context {
-        client: client.clone(),
-        config,
-    });
-
-    let projects = Api::<Project>::namespaced(client.clone(), &context.config.namespace);
-    projects
-        .list(&ListParams::default())
-        .await
-        .expect("Is the CRD installed?");
-
-    let services = Api::<Service>::namespaced(client.clone(), &context.config.namespace);
-    let ingresses = Api::<Ingress>::namespaced(client.clone(), &context.config.namespace);
-    let deployments = Api::<Deployment>::namespaced(client.clone(), &context.config.namespace);
-    let tasks = Api::<task::Task>::namespaced(client, &context.config.namespace);
-
-    Controller::new(projects.clone(), ListParams::default())
-        .owns(deployments, ListParams::default())
-        .owns(services, ListParams::default())
-        .owns(ingresses, ListParams::default())
-        .owns(tasks, ListParams::default())
-        .run(reconcile, error_policy, context)
-        .filter_map(|x| async move { std::result::Result::ok(x) })
-        .for_each(|_| futures::future::ready(()))
-        .boxed()
-}
-
 #[cfg(test)]
 mod test {
     use std::time::Duration;
@@ -1001,17 +635,25 @@ mod test {
         Ok(serde_json::from_value(json!({
             "apiVersion": "ame.teainspace.com/v1alpha1",
             "kind": "Project",
-            "metadata": { "name": "test-private" },
+            "metadata": { "name": "test-private", "annotations": {
+                "gitrepository": "my-git-repo"
+            } },
             "spec": {
-                "projectid": "myproject",
+                "deletionApproved": false,
+
+                "name": "myproject",
+                "deletionApproved": false,
                 "models": [
                 {
                     "name": "test",
                     "training": {
                         "task": {
-                            "taskRef": "trainingtask",
-                            "runcommand": "test",
-                            "projectid": "test",
+                            "name": "mytask",
+                            "executor": {
+                                 "pipEnv": {
+                                        "command": "test cmd"
+                                    }
+                                },
                         }
                     },
                     "deployment": {
@@ -1039,18 +681,26 @@ mod test {
         let project = test_project()?;
 
         insta::assert_yaml_snapshot!(
-            &project.spec.models.unwrap().clone()[0]
-                .generate_model_deployment(&ctrl_cfg, "model_source".to_string())
+            &project.spec.cfg.models.clone()[0]
+                .generate_model_deployment(ctrl_cfg.deployment_image, "model_source".to_string())
                 .await?
         );
 
         Ok(())
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn produces_valid_model_training_task() -> Result<()> {
+        let project = test_project().unwrap();
+        insta::assert_yaml_snapshot!(&project.generate_model_training_task("test")?);
+        Ok(())
+    }
+
     #[tokio::test]
     #[serial]
     async fn produces_valid_service() -> Result<()> {
-        let ctrl_cfg = ProjectCtrlCfg {
+        let _ctrl_cfg = ProjectCtrlCfg {
             namespace: "default".to_string(),
             deployment_image: "test_img".to_string(),
             model_ingress_host: Some("testhost".to_string()),
@@ -1060,9 +710,7 @@ mod test {
 
         let project = test_project()?;
 
-        insta::assert_yaml_snapshot!(
-            &project.spec.models.unwrap()[0].generate_model_service(&ctrl_cfg)?
-        );
+        insta::assert_yaml_snapshot!(&project.spec.cfg.models[0].generate_model_service()?);
 
         Ok(())
     }
@@ -1070,7 +718,7 @@ mod test {
     #[tokio::test]
     #[serial]
     async fn produces_valid_ingress() -> Result<()> {
-        let ctrl_cfg = ProjectCtrlCfg {
+        let _ctrl_cfg = ProjectCtrlCfg {
             namespace: "default".to_string(),
             deployment_image: "test_img".to_string(),
             model_ingress_host: Some("testhost".to_string()),
@@ -1080,9 +728,11 @@ mod test {
 
         let project = test_project()?;
 
-        insta::assert_yaml_snapshot!(
-            &project.spec.models.unwrap()[0].generate_model_ingress(&ctrl_cfg)?
-        );
+        insta::assert_yaml_snapshot!(&project.spec.cfg.models[0].generate_model_ingress(
+            "".to_string(),
+            None,
+            "projectname".to_string()
+        )?);
 
         Ok(())
     }
diff --git a/lib/src/custom_resources/project_source.rs b/lib/src/custom_resources/project_source.rs
index 203a63d0..ba91dae2 100644
--- a/lib/src/custom_resources/project_source.rs
+++ b/lib/src/custom_resources/project_source.rs
@@ -1,17 +1,18 @@
-use crate::custom_resources::project::{Project, ProjectSpec};
-use crate::custom_resources::secrets::SecretCtrl;
-use crate::custom_resources::{Error, Result};
-use crate::grpc::GitProjectSource;
-use crate::grpc::ProjectSourceState;
-use crate::grpc::ProjectSourceStatus;
+use crate::{
+    custom_resources::{
+        project::{Project, ProjectSpec},
+        secrets::SecretCtrl,
+        Error, Result,
+    },
+    grpc::{GitProjectSource, ProjectCfg, ProjectSourceState, ProjectSourceStatus},
+};
 use duration_string::DurationString;
 use envconfig::Envconfig;
 use futures::{future::BoxFuture, FutureExt, StreamExt};
-use git2::build::RepoBuilder;
-use git2::{Cred, FetchOptions};
-use k8s_openapi::api::core::v1::Secret;
-use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
-use k8s_openapi::chrono::DateTime;
+use git2::{build::RepoBuilder, Cred, FetchOptions};
+use k8s_openapi::{
+    api::core::v1::Secret, apimachinery::pkg::apis::meta::v1::OwnerReference, chrono::DateTime,
+};
 
 use kube::{
     api::{Api, ListParams, PatchParams, ResourceExt},
@@ -22,10 +23,13 @@ use kube::{
 };
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
-use std::path::Path;
-use std::time::SystemTime;
-use std::{fs, sync::Arc, time::Duration};
-use tracing::{error, info};
+use std::{
+    fs,
+    path::Path,
+    sync::Arc,
+    time::{Duration, SystemTime},
+};
+use tracing::{debug, error, info};
 
 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
 #[kube(
@@ -50,21 +54,31 @@ pub struct ProjectSrcCtrlCfg {
 
 impl ProjectSource {
     async fn git_secret(&self, secrets: Api<Secret>) -> Result<Option<String>> {
-        let Some(GitProjectSource {  secret: Some(secret_name), .. }) = self.spec.clone().cfg.git else {
-        return Ok(None);
-    };
+        let Some(GitProjectSource {
+            secret: Some(secret_name),
+            ..
+        }) = self.spec.clone().cfg.git
+        else {
+            return Ok(None);
+        };
         Ok(Some(
             SecretCtrl::from(secrets).get_secret(&secret_name).await?,
         ))
     }
 
     async fn extract_projects(&self, secrets: Api<Secret>) -> Result<Vec<ProjectSpec>> {
-        let Some(GitProjectSource{
-        repository,
-        username,
-        ..
-        }) = self.spec.clone().cfg.git else {
-            return Err(Error::MissingProjectSrc("git".to_string()))
+        debug!(
+            "Attempting to extract project file from repository: {:?}",
+            self.spec.cfg.git
+        );
+
+        let Some(GitProjectSource {
+            repository,
+            username,
+            ..
+        }) = self.spec.clone().cfg.git
+        else {
+            return Err(Error::MissingProjectSrc("git".to_string()));
         };
 
         let _ = fs::remove_dir_all("/tmp/".to_string() + &self.name_any());
@@ -77,12 +91,11 @@ impl ProjectSource {
             let mut callbacks = git2::RemoteCallbacks::new();
             callbacks.credentials(|_user, _, _| -> std::result::Result<Cred, git2::Error> {
                 let Some(git_secret) = git_secret.clone() else {
-                    return Err(git2::Error::from_str("git secret was empty"))
+                    return Err(git2::Error::from_str("git secret was empty"));
                 };
 
                 let Some(username) = username.clone() else {
-                return Err(git2::Error::from_str("missing username"));
-
+                    return Err(git2::Error::from_str("missing username"));
                 };
 
                 git2::Cred::userpass_plaintext(&username, &git_secret)
@@ -112,7 +125,16 @@ impl ProjectSource {
             return Err(Error::MissingAmeFile(self.name_any()));
         };
 
-        let project: ProjectSpec = serde_yaml::from_str(&ame_file)?;
+        debug!("Reading ame.yaml: {}", ame_file);
+
+        let project: ProjectCfg = serde_yaml::from_str(&ame_file)?;
+        let project = ProjectSpec {
+            cfg: project,
+            deletion_approved: false,
+            enable_triggers: Some(false),
+        };
+
+        debug!("Successfully extracted project: {:?}", project);
 
         fs::remove_dir_all("/tmp/".to_string() + &self.name_any())?;
 
@@ -139,9 +161,10 @@ impl ProjectSource {
             ..
         }) = self.status
         {
+            let sync_interva = self.sync_interval()?;
             Ok(std::time::SystemTime::now()
-                .duration_since(SystemTime::from(DateTime::parse_from_rfc2822(last_synced)?))?
-                > self.sync_interval()?)
+                .duration_since(SystemTime::from(DateTime::parse_from_rfc3339(last_synced)?))?
+                > sync_interva)
         } else {
             Ok(true)
         }
@@ -170,9 +193,15 @@ async fn reconcile(src: Arc<ProjectSource>, ctx: Arc<Context>) -> Result<Action>
     patch.metadata.managed_fields = None;
 
     if src.requires_sync()? {
+        info!("Synchronizing project source {}", src.name_any());
         let project_specs = match src.extract_projects(secrets).await {
             Ok(specs) => specs,
             Err(e) => {
+                error!(
+                    "Failed to find AME project in source {}: {}",
+                    src.name_any(),
+                    e
+                );
                 let status = ProjectSourceStatus {
                     state: ProjectSourceState::Error.into(),
                     reason: Some(e.to_string()),
@@ -193,9 +222,12 @@ async fn reconcile(src: Arc<ProjectSource>, ctx: Arc<Context>) -> Result<Action>
         };
 
         if project_specs.is_empty() {
+            debug!("no project specs were found");
             return Ok(Action::requeue(Duration::from_secs(50)));
         }
 
+        info!("Patch project {}", project_specs[0].cfg.name);
+
         let mut project = Project {
             metadata: ObjectMeta {
                 name: Some(src.name_any()),
@@ -211,6 +243,8 @@ async fn reconcile(src: Arc<ProjectSource>, ctx: Arc<Context>) -> Result<Action>
 
         let project = project.add_owner_reference(oref);
 
+        debug!("Patch project: {:?}", project);
+
         projects
             .patch(
                 &project.name_any(),
@@ -218,35 +252,35 @@ async fn reconcile(src: Arc<ProjectSource>, ctx: Arc<Context>) -> Result<Action>
                 &kube::api::Patch::Apply(project),
             )
             .await?;
-    }
-
-    let last_synced = Some(humantime::format_rfc3339(SystemTime::now()).to_string());
-    let mut patch: ProjectSource = _srcs.get_status(&src.name_any()).await?;
-    patch.metadata.managed_fields = None;
+        let last_synced = Some(humantime::format_rfc3339(SystemTime::now()).to_string());
+        let mut patch: ProjectSource = _srcs.get_status(&src.name_any()).await?;
+        patch.metadata.managed_fields = None;
+
+        if let Some(mut status) = patch.clone().status {
+            status.last_synced = last_synced;
+            status.reason = Some("project has been synced".to_string());
+            status.state = ProjectSourceState::Synchronized.into();
+            patch.status = Some(status);
+        } else {
+            patch.status = Some(ProjectSourceStatus {
+                last_synced,
+                reason: Some("project has been synced".to_string()),
+                state: ProjectSourceState::Synchronized.into(),
+                ..ProjectSourceStatus::default()
+            })
+        }
 
-    if let Some(mut status) = patch.clone().status {
-        status.last_synced = last_synced;
-        status.reason = Some("project has been synced".to_string());
-        status.state = ProjectSourceState::Synchronized.into();
-        patch.status = Some(status);
-    } else {
-        patch.status = Some(ProjectSourceStatus {
-            last_synced,
-            reason: Some("project has been synced".to_string()),
-            state: ProjectSourceState::Synchronized.into(),
-            ..ProjectSourceStatus::default()
-        })
+        _srcs
+            .patch_status(
+                &src.name_any(),
+                &PatchParams::apply("ame-controller"),
+                &kube::api::Patch::Apply(patch),
+            )
+            .await?;
     }
 
-    _srcs
-        .patch_status(
-            &src.name_any(),
-            &PatchParams::apply("ame-controller"),
-            &kube::api::Patch::Apply(patch),
-        )
-        .await?;
-
-    Ok(Action::requeue(Duration::from_secs(5 * 60)))
+    // TODO: how should we handle if project source needs to be reconciled but not synced?
+    Ok(Action::requeue(src.sync_interval()?))
 }
 
 fn error_policy(_src: Arc<ProjectSource>, error: &Error, _ctx: Arc<Context>) -> Action {
@@ -255,6 +289,7 @@ fn error_policy(_src: Arc<ProjectSource>, error: &Error, _ctx: Arc<Context>) ->
 }
 
 pub async fn start_project_source_controller(config: ProjectSrcCtrlCfg) -> BoxFuture<'static, ()> {
+    info!("Starting project source controller.");
     let client = Client::try_default().await.expect("Failed to create a K8S client, is the controller running in an environment with access to cluster credentials?");
     let context = Arc::new(Context {
         client: client.clone(),
@@ -285,8 +320,7 @@ mod test {
     use super::*;
     use crate::custom_resources::secrets::SecretCtrl;
     use assert_fs::prelude::*;
-    use futures::StreamExt;
-    use futures::TryStreamExt;
+    use futures::{StreamExt, TryStreamExt};
     use k8s_openapi::api::core::v1::Secret;
     use kube::api::DeleteParams;
 
@@ -401,6 +435,7 @@ mod test {
         let secret_ctrl = SecretCtrl::new(client, "default");
 
         let secret_name = "ghsecret";
+        let _ = secret_ctrl.delete_secret(secret_name).await;
         secret_ctrl
             .store_secret_if_empty(secret_name, private_repo_gh_pat().unwrap())
             .await
@@ -428,11 +463,9 @@ mod test {
             .await?
             .boxed();
         while let Ok(Some(e)) = stream.try_next().await {
-            if let WatchEvent::Added(project) = e {
+            if let WatchEvent::Added(project) = e.clone() {
                 if project
-                    .metadata
-                    .owner_references
-                    .unwrap()
+                    .owner_references()
                     .iter()
                     .any(|r| r.name == project_src.name_any())
                 {
diff --git a/lib/src/custom_resources/project_source_ctrl.rs b/lib/src/custom_resources/project_source_ctrl.rs
index 8a2ba20a..82aa063c 100644
--- a/lib/src/custom_resources/project_source_ctrl.rs
+++ b/lib/src/custom_resources/project_source_ctrl.rs
@@ -1,7 +1,9 @@
-use crate::ctrl::AmeKubeResourceCtrl;
-use crate::custom_resources::project_source::{ProjectSource, ProjectSourceSpec};
-use crate::error::AmeError;
-use crate::grpc::{ProjectSourceCfg, ProjectSourceId, ProjectSourceStatus};
+use crate::{
+    ctrl::AmeKubeResourceCtrl,
+    custom_resources::project_source::{ProjectSource, ProjectSourceSpec},
+    error::AmeError,
+    grpc::{ProjectSourceCfg, ProjectSourceId, ProjectSourceStatus},
+};
 use kube::{
     api::{DeleteParams, ListParams, Patch, PatchParams, PostParams},
     Api, Client, ResourceExt,
@@ -69,7 +71,9 @@ impl ProjectSrcCtrl {
 
     pub async fn create_project_src(&self, cfg: &ProjectSourceCfg) -> Result<ProjectSourceId> {
         let Some(repository) = cfg.git_repository() else {
-            return Err(Error::InvalidProjectSourceCfg("missing git configuration".to_string()));
+            return Err(Error::InvalidProjectSourceCfg(
+                "missing git configuration".to_string(),
+            ));
         };
 
         match self.get_project_src_for_repo(repository).await {
@@ -89,7 +93,9 @@ impl ProjectSrcCtrl {
 
     pub async fn update_project_src(&self, cfg: &ProjectSourceCfg) -> Result<ProjectSourceId> {
         let Some(repository) = cfg.git_repository() else {
-            return Err(Error::InvalidProjectSourceCfg("missing git configuration".to_string()));
+            return Err(Error::InvalidProjectSourceCfg(
+                "missing git configuration".to_string(),
+            ));
         };
 
         let mut project_src = self.get_project_src_for_repo(repository).await?;
@@ -178,7 +184,9 @@ impl AmeKubeResourceCtrl for ProjectSrcCtrl {
 
     async fn validate_resource(&self, cfg: &Self::KubeResource) -> crate::Result<()> {
         let Some(repository) = cfg.spec.cfg.git_repository() else {
-            return Err(AmeError::InvalidProjectSourceCfg("missing git configuration".to_string()));
+            return Err(AmeError::InvalidProjectSourceCfg(
+                "missing git configuration".to_string(),
+            ));
         };
 
         match self.get_project_src_for_repo(repository).await {
diff --git a/lib/src/custom_resources/secrets.rs b/lib/src/custom_resources/secrets.rs
index 1692cb36..fccc98c9 100644
--- a/lib/src/custom_resources/secrets.rs
+++ b/lib/src/custom_resources/secrets.rs
@@ -1,12 +1,14 @@
 use crate::grpc::AmeSecretId;
-use k8s_openapi::api::core::v1::Secret;
-use k8s_openapi::ByteString;
-use kube::api::{DeleteParams, ListParams};
-use kube::core::ObjectMeta;
-use kube::error::ErrorResponse;
-use kube::ResourceExt;
-use kube::{api::PostParams, Api, Client};
-use std::collections::BTreeMap;
+use k8s_openapi::{api::core::v1::Secret, ByteString};
+use kube::{
+    api::{DeleteParams, ListParams, PostParams},
+    core::ObjectMeta,
+    error::ErrorResponse,
+    Api, Client, ResourceExt,
+};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use std::{collections::BTreeMap, string::FromUtf8Error};
 use thiserror::Error;
 
 #[derive(Error, Debug)]
@@ -19,6 +21,9 @@ pub enum SecretError {
 
     #[error("secret with key: {0}, was misconfigured")]
     MissingSecretKey(String),
+
+    #[error("{0}")]
+    FailedToParseString(#[from] FromUtf8Error),
 }
 
 impl From<SecretError> for tonic::Status {
@@ -126,7 +131,7 @@ impl SecretCtrl {
         };
 
         if let Some(ByteString(v)) = secret_data.get("secret").to_owned() {
-            Ok(String::from_utf8(v.to_owned()).unwrap())
+            Ok(String::from_utf8(v.to_owned())?)
         } else {
             Err(SecretError::MissingSecretKey(key.to_string()))
         }
@@ -143,6 +148,11 @@ impl SecretCtrl {
     }
 }
 
+#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
+pub enum SecretReference {
+    AmeSecret(String),
+}
+
 #[cfg(test)]
 mod test {
     use super::{Result, SecretCtrl};
diff --git a/lib/src/custom_resources/snapshots/ame__custom_resources__new_task__test__snap_shot_task_yaml.snap b/lib/src/custom_resources/snapshots/ame__custom_resources__new_task__test__snap_shot_task_yaml.snap
new file mode 100644
index 00000000..b626be00
--- /dev/null
+++ b/lib/src/custom_resources/snapshots/ame__custom_resources__new_task__test__snap_shot_task_yaml.snap
@@ -0,0 +1,33 @@
+---
+source: lib/src/custom_resources/new_task.rs
+expression: "&task"
+---
+apiVersion: ame.teainspace.com/v1alpha1
+kind: Task
+metadata:
+  name: mytask
+spec:
+  name: mytask
+  taskRef:
+    name: othertask
+    project: ~
+  resources:
+    cpu: "2"
+  env:
+    - key: SOME_VAR
+      val: someval
+  secrets:
+    - ame:
+        key: secretkey
+        injectAs: MY_SECRET
+  executor: ~
+  deletionApproved: false
+  source:
+    ame:
+      path: test
+  project: ~
+status:
+  phase:
+    running:
+      workflowName: someinfo
+
diff --git a/lib/src/custom_resources/snapshots/ame__custom_resources__new_task__test__snap_shot_workflow_yaml.snap b/lib/src/custom_resources/snapshots/ame__custom_resources__new_task__test__snap_shot_workflow_yaml.snap
new file mode 100644
index 00000000..511c566b
--- /dev/null
+++ b/lib/src/custom_resources/snapshots/ame__custom_resources__new_task__test__snap_shot_workflow_yaml.snap
@@ -0,0 +1,190 @@
+---
+source: lib/src/custom_resources/new_task.rs
+expression: "build_workflow(task, task_ctx)?"
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  labels:
+    ame-task: mytask
+  name: mytask
+spec:
+  entrypoint: main
+  templates:
+    - name: main
+      metadata:
+        labels: ~
+        annotations: ~
+      steps:
+        - - name: setup
+            inline:
+              name: setup
+              metadata:
+                labels:
+                  ame-task: mytask
+                annotations: ~
+              steps: ~
+              securityContext:
+                fsGroup: 2000
+                runAsUser: 1001
+              script:
+                command:
+                  - bash
+                env:
+                  - name: AWS_ACCESS_KEY_ID
+                    valueFrom:
+                      secretKeyRef:
+                        key: MINIO_ROOT_USER
+                        name: ame-minio-secret
+                        optional: false
+                  - name: AWS_SECRET_ACCESS_KEY
+                    valueFrom:
+                      secretKeyRef:
+                        key: MINIO_ROOT_PASSWORD
+                        name: ame-minio-secret
+                        optional: false
+                  - name: MLFLOW_TRACKING_URI
+                    value: "http://mlflow.default.svc.cluster.local:5000"
+                  - name: MINIO_URL
+                    value: "http://ame-minio.ame-system.svc.cluster.local:9000"
+                  - name: PIPENV_YES
+                    value: "1"
+                  - name: SOME_VAR
+                    value: someval
+                  - name: MY_SECRET
+                    valueFrom:
+                      secretKeyRef:
+                        key: secret
+                        name: secretkey
+                image: myimage
+                name: ""
+                resources:
+                  limits:
+                    cpu: "2"
+                    memory: 2Gi
+                volumeMounts:
+                  - mountPath: /project
+                    name: myvolume
+                source: " \n\n s3cmd --no-ssl --region eu-central-1 --host=$MINIO_URL --host-bucket=$MINIO_URL get --recursive s3://test ./"
+              podSpecPatch: ~
+        - - name: mytask
+            inline:
+              name: mytask
+              metadata:
+                labels:
+                  ame-task: mytask
+                annotations: ~
+              steps: ~
+              securityContext:
+                fsGroup: 2000
+                runAsUser: 1001
+              script:
+                command:
+                  - bash
+                env:
+                  - name: AWS_ACCESS_KEY_ID
+                    valueFrom:
+                      secretKeyRef:
+                        key: MINIO_ROOT_USER
+                        name: ame-minio-secret
+                        optional: false
+                  - name: AWS_SECRET_ACCESS_KEY
+                    valueFrom:
+                      secretKeyRef:
+                        key: MINIO_ROOT_PASSWORD
+                        name: ame-minio-secret
+                        optional: false
+                  - name: MLFLOW_TRACKING_URI
+                    value: "http://mlflow.default.svc.cluster.local:5000"
+                  - name: MINIO_URL
+                    value: "http://ame-minio.ame-system.svc.cluster.local:9000"
+                  - name: PIPENV_YES
+                    value: "1"
+                  - name: SOME_VAR
+                    value: someval
+                  - name: MY_SECRET
+                    valueFrom:
+                      secretKeyRef:
+                        key: secret
+                        name: secretkey
+                image: myimage
+                name: ""
+                resources:
+                  limits:
+                    cpu: "2"
+                    memory: 2Gi
+                volumeMounts:
+                  - mountPath: /project
+                    name: myvolume
+                source: "\n                        source ~/.bashrc\n                        \n                        pyenv install 3.11\n\n                        pyenv global 3.11\n\n                        poetry install\n                    \n                        poetry run python train.py\n                    "
+              podSpecPatch: ~
+        - - name: saveartifacts
+            inline:
+              name: saveartifacts
+              metadata:
+                labels:
+                  ame-task: mytask
+                annotations: ~
+              steps: ~
+              securityContext:
+                fsGroup: 2000
+                runAsUser: 1001
+              script:
+                command:
+                  - bash
+                env:
+                  - name: AWS_ACCESS_KEY_ID
+                    valueFrom:
+                      secretKeyRef:
+                        key: MINIO_ROOT_USER
+                        name: ame-minio-secret
+                        optional: false
+                  - name: AWS_SECRET_ACCESS_KEY
+                    valueFrom:
+                      secretKeyRef:
+                        key: MINIO_ROOT_PASSWORD
+                        name: ame-minio-secret
+                        optional: false
+                  - name: MLFLOW_TRACKING_URI
+                    value: "http://mlflow.default.svc.cluster.local:5000"
+                  - name: MINIO_URL
+                    value: "http://ame-minio.ame-system.svc.cluster.local:9000"
+                  - name: PIPENV_YES
+                    value: "1"
+                  - name: SOME_VAR
+                    value: someval
+                  - name: MY_SECRET
+                    valueFrom:
+                      secretKeyRef:
+                        key: secret
+                        name: secretkey
+                image: myimage
+                name: ""
+                resources:
+                  limits:
+                    cpu: "2"
+                    memory: 2Gi
+                volumeMounts:
+                  - mountPath: /project
+                    name: myvolume
+                source: save_artifacts ame/tasks/mytask/artifacts/
+              podSpecPatch: ~
+      securityContext: ~
+      script: ~
+      podSpecPatch: ~
+  imagePullSecrets: ~
+  volumeClaimTemplates:
+    - apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
+        name: mytask
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 50Gi
+      status: {}
+  volumes: ~
+  serviceAccountName: ame-task
+
diff --git a/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_deployment.snap b/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_deployment.snap
index c7abee03..0cb6686f 100644
--- a/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_deployment.snap
+++ b/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_deployment.snap
@@ -1,6 +1,6 @@
 ---
 source: lib/src/custom_resources/project.rs
-expression: "&project.spec.models.unwrap().clone()[0].generate_model_deployment(&ctrl_cfg,\n                \"model_source\".to_string()).await?"
+expression: "&project.spec.cfg.models.clone()[0].generate_model_deployment(ctrl_cfg.deployment_image,\n                \"model_source\".to_string()).await?"
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -35,6 +35,8 @@ spec:
             httpGet:
               path: /health
               port: 5000
+          resources:
+            limits: {}
           securityContext:
             runAsUser: 1001
 
diff --git a/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_ingress.snap b/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_ingress.snap
index 56506df8..09012a3f 100644
--- a/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_ingress.snap
+++ b/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_ingress.snap
@@ -1,11 +1,12 @@
 ---
 source: lib/src/custom_resources/project.rs
-expression: "&project.spec.models.unwrap()[0].generate_model_ingress(&ctrl_cfg)?"
+expression: "&project.spec.cfg.models[0].generate_model_ingress(\"\".to_string(), None,\n            \"projectname\".to_string())?"
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
   labels:
     ame-model: test
@@ -13,7 +14,7 @@ metadata:
 spec:
   ingressClassName: nginx
   rules:
-    - host: testhost
+    - host: ""
       http:
         paths:
           - backend:
@@ -21,10 +22,10 @@ spec:
                 name: test
                 port:
                   number: 5000
-            path: /invocations
-            pathType: ImplementationSpecific
+            path: /projects/projectname/models/test(/|$)(.*)
+            pathType: Prefix
   tls:
     - hosts:
-        - testhost
+        - ""
       secretName: test-tls
 
diff --git a/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_model_training_task.snap b/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_model_training_task.snap
new file mode 100644
index 00000000..b57a4609
--- /dev/null
+++ b/lib/src/custom_resources/snapshots/ame__custom_resources__project__test__produces_valid_model_training_task.snap
@@ -0,0 +1,30 @@
+---
+source: lib/src/custom_resources/project.rs
+expression: "&project.generate_model_training_task(\"test\")?"
+---
+apiVersion: ame.teainspace.com/v1alpha1
+kind: Task
+metadata:
+  name: test-privatetestmytaskmyproject
+  ownerReferences:
+    - apiVersion: ""
+      kind: ""
+      name: ""
+      uid: ""
+spec:
+  name: mytask
+  resources: {}
+  env: []
+  secrets: []
+  executor:
+    pipEnv:
+      command: test cmd
+  deletionApproved: false
+  source:
+    git:
+      repository: my-git-repo
+      reference: ""
+      user_name: ""
+      secret: ~
+  project: ~
+
diff --git a/lib/src/custom_resources/snapshots/ame__custom_resources__project_source__test__can_extract_projects_from_public_git_repository.snap b/lib/src/custom_resources/snapshots/ame__custom_resources__project_source__test__can_extract_projects_from_public_git_repository.snap
index c3b58dd0..70f8ed42 100644
--- a/lib/src/custom_resources/snapshots/ame__custom_resources__project_source__test__can_extract_projects_from_public_git_repository.snap
+++ b/lib/src/custom_resources/snapshots/ame__custom_resources__project_source__test__can_extract_projects_from_public_git_repository.snap
@@ -2,13 +2,15 @@
 source: lib/src/custom_resources/project_source.rs
 expression: "&projects"
 ---
-- projectid: echo
+- name: echo
   tasks:
     - name: echo
-      runcommand: python echo.py myoutput
-      projectid: echo
-      source: {}
       resources:
         cpu: "2"
         memory: 4Gi
+      env: []
+      secrets: []
+      executor: ~
+  deletionApproved: false
+  enableTriggers: false
 
diff --git a/lib/src/custom_resources/task.rs b/lib/src/custom_resources/task.rs
deleted file mode 100644
index 1495d914..00000000
--- a/lib/src/custom_resources/task.rs
+++ /dev/null
@@ -1,1465 +0,0 @@
-use crate::custom_resources::argo::ArgoScriptTemplate;
-use crate::custom_resources::argo::Workflow;
-use crate::custom_resources::argo::WorkflowPhase;
-use crate::custom_resources::argo::WorkflowStep;
-use crate::custom_resources::argo::WorkflowTemplate;
-use crate::custom_resources::project::{add_owner_reference, DataSet, Project};
-use crate::custom_resources::{Error, Result};
-use futures::future::BoxFuture;
-use futures::FutureExt;
-use futures::StreamExt;
-
-use crate::grpc::TaskRef;
-
-use futures::future::join_all;
-use k8s_openapi::api::core::v1::EnvVar;
-use k8s_openapi::api::core::v1::EnvVarSource;
-
-use k8s_openapi::api::core::v1::PersistentVolumeClaim;
-use k8s_openapi::api::core::v1::PersistentVolumeClaimSpec;
-use k8s_openapi::api::core::v1::PersistentVolumeClaimStatus;
-use k8s_openapi::api::core::v1::ResourceRequirements;
-
-use k8s_openapi::api::core::v1::SecretKeySelector;
-use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
-use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
-use kube::api::Patch;
-use kube::api::PatchParams;
-use kube::core::object::HasStatus;
-use kube::core::ObjectMeta;
-use kube::error::ErrorResponse;
-use kube::{
-    api::{Api, ListParams, ResourceExt},
-    client::Client,
-    runtime::controller::{Action, Controller},
-    CustomResource, Resource,
-};
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-use serde_json::json;
-use std::collections::BTreeMap;
-use std::sync::Arc;
-use tokio::time::Duration;
-use tracing::debug;
-use tracing::error;
-use tracing::info;
-
-use envconfig::Envconfig;
-use serde_merge::omerge;
-
-use super::project::local_name;
-
-#[derive(Envconfig, Clone)]
-pub struct TaskControllerConfig {
-    #[envconfig(
-        from = "EXECUTOR_IMAGE",
-        default = "main.localhost:45373/ame-executor:latest"
-    )]
-    pub executor_image: String,
-
-    #[envconfig(from = "NAMESPACE", default = "ame-system")]
-    pub namespace: String,
-
-    #[envconfig(from = "BUCKET", default = "ame")]
-    pub bucket: String,
-}
-
-#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
-#[kube(
-    kind = "Task",
-    group = "ame.teainspace.com",
-    version = "v1alpha1",
-    namespaced
-)]
-#[kube(status = "TaskStatus", shortname = "task")]
-#[serde(rename_all = "camelCase")]
-pub struct TaskSpec {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub name: Option<String>,
-    // Runcommand defines the command AME will use to start this Task.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub runcommand: Option<String>,
-
-    // Projectid defines which project this Task belongs to.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub projectid: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub env: Option<Vec<TaskEnvVar>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub image: Option<String>,
-    // Secrets that will be made available to the Task during execution.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub secrets: Option<Vec<TaskSecret>>,
-
-    // Pipeline defines a sequence of tasks to execute.
-    // If a pipeline is specified the rest of the fields in this
-    // specification are ignored.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub pipeline: Option<Vec<PipelineStep>>,
-
-    // source defines where AME will pull the project from.
-    // This can either be AME's own object storage or a git repository.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub source: Option<ProjectSource>,
-
-    // Resources define what resources this Task requires.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub resources: Option<BTreeMap<String, Quantity>>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub task_type: Option<TaskType>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub task_ref: Option<String>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub template_ref: Option<String>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub data_set: Option<Vec<String>>,
-}
-
-impl Task {
-    pub fn new_gen_name(prefix: String, spec: TaskSpec) -> Self {
-        Self {
-            metadata: ObjectMeta {
-                generate_name: Some(prefix),
-                ..ObjectMeta::default()
-            },
-            spec,
-            status: None,
-        }
-    }
-}
-
-impl TaskSpec {
-    pub fn from_ref(TaskRef { name, project }: TaskRef) -> Self {
-        let task_ref = if let Some(project) = project {
-            format!("{project}.{name}")
-        } else {
-            name
-        };
-
-        Self {
-            task_ref: Some(task_ref),
-            ..Self::default()
-        }
-    }
-}
-
-#[derive(JsonSchema, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub enum TaskType {
-    PipEnv,
-    Mlflow,
-    Poetry,
-}
-
-/// The status object of `Task`
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, PartialEq, Eq, Default)]
-pub struct TaskStatus {
-    pub phase: Option<TaskPhase>,
-    pub reason: Option<String>,
-    pub data_set_tasks: Option<BTreeMap<String, DataSetStatus>>,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, PartialEq, Eq, Default)]
-pub struct DataSetStatus {
-    pub phase: TaskPhase,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, PartialEq, Eq, Default)]
-pub enum TaskPhase {
-    Running,
-    #[default]
-    Pending,
-    Failed,
-    Succeeded,
-    Error,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-pub struct TaskEnvVar {
-    name: String,
-    value: String,
-}
-
-impl From<&TaskEnvVar> for EnvVar {
-    fn from(te: &TaskEnvVar) -> EnvVar {
-        EnvVar {
-            name: te.name.clone(),
-            value: Some(te.value.clone()),
-            ..EnvVar::default()
-        }
-    }
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-pub struct AmeSecret {
-    pub name: String,
-    pub envkey: String,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct VaultSecret {
-    pub vault_name: String,
-    pub secret_key: String,
-    pub secret_path: String,
-    pub envkey: String,
-}
-
-impl VaultSecret {
-    fn annotations(&self, service_account: String) -> BTreeMap<String, String> {
-        let mut annotations = BTreeMap::new();
-
-        annotations.insert(
-            "vault.hashicorp.com/agent-inject".to_string(),
-            "true".to_string(),
-        );
-        annotations.insert("vault.hashicorp.com/role".to_string(), service_account);
-        annotations.insert(
-            "vault.hashicorp.com/agent-inject-secret-config".to_string(),
-            "internal/data/database/config".to_string(),
-        );
-        annotations.insert(
-            "vault.hashicorp.com/agent-inject-template-config".to_string(),
-            format!(
-                "{{{{- with secret \"{}\" -}}}}
-            export {}=\"{{{{ .Data.{} }}}}\"
-          {{{{- end -}}}}",
-                self.secret_path, self.envkey, self.secret_key
-            ),
-        );
-
-        annotations
-    }
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-pub enum TaskSecret {
-    AmeSecret(AmeSecret),
-    VaultSecret(VaultSecret),
-}
-
-impl From<&AmeSecret> for EnvVar {
-    fn from(ts: &AmeSecret) -> EnvVar {
-        EnvVar {
-            name: ts.envkey.clone(),
-            value_from: Some(EnvVarSource {
-                secret_key_ref: Some(SecretKeySelector {
-                    key: "secret".to_string(),
-                    name: Some(ts.name.clone()),
-                    ..SecretKeySelector::default()
-                }),
-                ..EnvVarSource::default()
-            }),
-            ..EnvVar::default()
-        }
-    }
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)]
-pub struct PipelineStep {
-    taskname: String,
-    runcommand: String,
-    env: Vec<TaskEnvVar>,
-    // Secrets that will be made available to the Task during execution.
-    secrets: Vec<TaskSecret>,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, Default)]
-pub struct ProjectSource {
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub gitrepository: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub gitreference: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub amestoragepath: Option<String>,
-}
-
-async fn resolve_template(
-    name: &str,
-    project_id: &str,
-    projects: Api<Project>,
-) -> Result<TaskSpec> {
-    let Some(project) = projects
-        .list(&ListParams::default())
-        .await?
-        .into_iter()
-        .find(|p| p.spec.id == project_id) else {
-        return Err(Error::MissingTemplate(
-            project_id.to_string(),
-            name.to_string(),
-        ));
-        };
-
-    if let Some(template) = project.get_template(name) {
-        Ok(template)
-    } else {
-        Err(Error::MissingTemplate(
-            project_id.to_string(),
-            name.to_string(),
-        ))
-    }
-}
-
-impl Task {
-    fn common_wf_template(
-        &self,
-        name: String,
-        scrict_src: String,
-        volume_name: &str,
-        addition_env: Option<Vec<EnvVar>>,
-        config: &TaskControllerConfig,
-    ) -> Result<WorkflowTemplate> {
-        let required_env: Vec<EnvVar> = serde_json::from_value(json!([
-            {
-            "name":  "AWS_ACCESS_KEY_ID",
-            "valueFrom": {
-                "secretKeyRef":  {
-                    "key": "MINIO_ROOT_USER",
-                    "name": "ame-minio-secret",
-                    "optional": false,
-                }
-            },
-        },
-        {
-            "name":  "AWS_SECRET_ACCESS_KEY",
-            "valueFrom": {
-                "secretKeyRef":  {
-                    "key": "MINIO_ROOT_PASSWORD",
-                    "name": "ame-minio-secret",
-                    "optional": false
-                }
-            },
-        },
-        {
-            "name": "MLFLOW_TRACKING_URI",
-            "value": "http://mlflow.default.svc.cluster.local:5000"
-        },
-        {
-            "name":  "MINIO_URL",
-            "value": "http://ame-minio.ame-system.svc.cluster.local:9000",
-        },
-
-                    {
-                        "name":  "PIPENV_YES",
-                        "value": "1",
-                    },
-        ]))?;
-
-        let final_env = if let Some(vars) = addition_env {
-            [required_env, vars].concat()
-        } else {
-            required_env
-        };
-
-        Ok(WorkflowTemplate {
-            security_context: Some(serde_json::from_value(json!({
-                "runAsUser": 1001,
-                "fsGroup": 2000
-            }
-            ))?),
-            script: Some(ArgoScriptTemplate {
-                source: scrict_src,
-                container: serde_json::from_value(json!(
-                        {
-                          "image": config.executor_image,
-                          "command": ["bash"],
-                          "volumeMounts": [{
-                              "name": volume_name,
-                              "mountPath": "/project",
-                          }],
-                          "env": final_env,
-                          "resources": {},
-                        }
-                ))?,
-            }),
-            ..WorkflowTemplate::new(name)
-        }
-        .label("ame-task".to_string(), self.name_any())
-        .clone())
-    }
-
-    fn generate_setup_template(
-        &self,
-        volume_name: &str,
-        config: &TaskControllerConfig,
-        data_sets: Option<Vec<DataSet>>,
-    ) -> Result<WorkflowTemplate> {
-        let data_set_download_command = data_sets.map(|data_sets| data_sets.into_iter().map(|ds| {
-            format!("s3cmd --no-ssl --region eu-central-1 --host=$MINIO_URL --host-bucket=$MINIO_URL get --recursive s3://ame/tasks/{}/artifacts/{} ./
-
-                ", local_name(ds.name), ds.path )
-            }).collect::<String>());
-
-        let project_pull_command = if let Some(ProjectSource {
-            gitrepository: Some(ref repo),
-            ..
-        }) = self.spec.source
-        {
-            format!(
-                "
-                git clone {repo} repo
-
-                cp -r repo/* .
-
-                rm -rf repo
-
-                ls
-                "
-            )
-        } else {
-            format!("s3cmd --no-ssl --region eu-central-1 --host=$MINIO_URL --host-bucket=$MINIO_URL get --recursive s3://{} ./", self.task_files_path())
-        };
-        let script_src = format!(
-            "
-            {}
-
-            {project_pull_command}
-
-                       echo \"0\" >> exit.status
-                        ",
-            data_set_download_command.unwrap_or_default()
-        );
-        self.common_wf_template("setup".to_string(), script_src, volume_name, None, config)
-    }
-
-    fn task_files_path(&self) -> String {
-        format!("ame/tasks/{}/projectfiles/", self.name_any())
-    }
-
-    fn task_artifacts_path(&self) -> String {
-        format!("ame/tasks/{}/artifacts/", self.name_any())
-    }
-
-    fn script_src(&self) -> String {
-        let task_exec_cmd = if let Some(TaskType::Mlflow) = self.spec.task_type {
-            "export PATH=$HOME/.pyenv/bin:$PATH
-
-             unset AWS_SECRET_ACCESS_KEY
-
-             unset AWS_ACCESS_KEY_ID
-
-             mlflow run ."
-                .to_string()
-        } else if let Some(TaskType::Poetry) = self.spec.task_type {
-            format!(
-                "
-                          poetry install
-
-                          poetry run {}
-
-                         save_artifacts {}
-                ",
-                self.spec
-                    .runcommand
-                    .clone()
-                    .unwrap_or("missing command".to_string()), // TODO: handle missing commands
-                self.task_artifacts_path()
-            )
-        } else {
-            format!(
-                "
-                pipenv sync
-
-                pipenv run {}
-
-                save_artifacts {}",
-                self.spec
-                    .runcommand
-                    .clone()
-                    .unwrap_or("missing command".to_string()), // TODO: handle missing commands
-                self.task_artifacts_path()
-            )
-        };
-
-        let src = format!("
-          set -e # It is important that the workflow exits with an error code if execute or save_artifacts fails, so AME can take action based on that information.
-
-          {task_exec_cmd}
-
-          echo \"0\" >> exit.status
-            ",);
-
-        if self.uses_vault() {
-            format!(
-                "
-                source /vault/secrets/config
-
-                {src}"
-            )
-        } else {
-            src
-        }
-    }
-
-    fn uses_vault(&self) -> bool {
-        if let Some(ref secrets) = self.spec.secrets {
-            secrets
-                .iter()
-                .any(|s| matches!(s, TaskSecret::VaultSecret(_)))
-        } else {
-            false
-        }
-    }
-
-    fn vault_secrets(&self) -> Vec<&VaultSecret> {
-        if let Some(secrets) = &self.spec.secrets {
-            secrets
-                .iter()
-                .filter_map(|s| {
-                    if let TaskSecret::VaultSecret(v) = s {
-                        Some(v)
-                    } else {
-                        None
-                    }
-                })
-                .collect()
-        } else {
-            vec![]
-        }
-    }
-
-    fn generate_wf_template(
-        &mut self,
-        volume_name: &str,
-        config: &TaskControllerConfig,
-    ) -> Result<WorkflowTemplate> {
-        let secret_env: Vec<EnvVar> = if let Some(secrets) = &self.spec.secrets {
-            secrets
-                .iter()
-                .filter_map(|v| {
-                    if let TaskSecret::AmeSecret(secret) = v {
-                        Some(secret)
-                    } else {
-                        None
-                    }
-                })
-                .map(EnvVar::from)
-                .collect()
-        } else {
-            vec![]
-        };
-
-        let task_env: Vec<EnvVar> = if let Some(vars) = &self.spec.env {
-            [secret_env, vars.iter().map(EnvVar::from).collect()].concat()
-        } else {
-            secret_env
-        };
-
-        let mut wf_template = WorkflowTemplate {
-            pod_spec_patch: Some(self.generate_pod_spec_patch()?),
-            ..self.common_wf_template(
-                self.name_any(),
-                self.script_src(),
-                volume_name,
-                Some(task_env),
-                &TaskControllerConfig {
-                    executor_image: self
-                        .spec
-                        .image
-                        .clone()
-                        .unwrap_or(config.clone().executor_image),
-                    ..config.clone()
-                },
-            )?
-        };
-
-        if self.uses_vault() {
-            let vault_secrets = self.vault_secrets();
-            for vs in vault_secrets {
-                wf_template = wf_template
-                    .bulk_annotate(vs.annotations("ame-task".to_string()))
-                    .clone();
-            }
-        }
-
-        Ok(wf_template)
-    }
-
-    fn generate_pod_spec_patch(&self) -> Result<String> {
-        Ok(format!(
-            "{{\"containers\":[{{\"name\":\"main\", \"resources\":{{\"limits\":{}}}}}]}}",
-            json!(self.spec.resources)
-        ))
-    }
-
-    fn generate_workflow(
-        &mut self,
-        config: &TaskControllerConfig,
-        _projects: Api<Project>,
-        data_sets: Option<Vec<DataSet>>,
-    ) -> Result<Workflow> {
-        let volume_name = format!("{}-volume", self.name_any());
-        let mut volume_resource_requirements = BTreeMap::new();
-        volume_resource_requirements.insert("storage".to_string(), Quantity("50Gi".to_string()));
-        let oref = if let Some(refe) = self.controller_owner_ref(&()) {
-            refe
-        } else {
-            OwnerReference::default()
-        };
-
-        Ok(Workflow::default()
-            .set_name(self.name_any())
-            .set_service_account("ame-task".to_string())
-            .label("ame-task".to_string(), self.name_any())
-            .add_template(
-                WorkflowTemplate::new("main".to_string())
-                    .add_parallel_step(vec![WorkflowStep {
-                        name: "setup".to_string(),
-                        inline: Some(Box::new(self.generate_setup_template(
-                            &volume_name,
-                            config,
-                            data_sets,
-                        )?)),
-                    }])
-                    .add_parallel_step(vec![WorkflowStep {
-                        name: "main".to_string(),
-                        inline: Some(Box::new(self.generate_wf_template(&volume_name, config)?)),
-                    }])
-                    .clone(),
-            )
-            .add_volume_claim_template(PersistentVolumeClaim {
-                metadata: ObjectMeta {
-                    name: Some(volume_name.clone()),
-                    ..ObjectMeta::default()
-                },
-                spec: Some(PersistentVolumeClaimSpec {
-                    access_modes: Some(vec!["ReadWriteOnce".to_string()]),
-                    resources: Some(ResourceRequirements {
-                        requests: Some(volume_resource_requirements),
-                        limits: None,
-                    }),
-                    ..PersistentVolumeClaimSpec::default()
-                }),
-
-                // Note that it is important to create the equivalent of an empty struct here
-                // and not just a None.
-                // Otherwise the Workflow controller will disagree with AME's controller on
-                // how an empty status should be specified.
-                status: Some(PersistentVolumeClaimStatus::default()),
-            })
-            .add_owner_reference(oref)
-            .clone())
-    }
-
-    async fn get_datasets(&self, projects: Api<Project>) -> Result<Vec<DataSet>> {
-        let TaskSpec {
-            data_set: Some(data_sets),
-            ..
-        } = self.spec.clone() else {
-            return Ok(vec![])
-        };
-
-        let (remote_data_sets, _local_data_sets): (Vec<String>, Vec<String>) = data_sets
-            .clone()
-            .into_iter()
-            .partition(|ds| ds.contains('.'));
-
-        // TODO: this is very brittle
-        let projects_ids: Vec<String> = remote_data_sets
-            .clone()
-            .into_iter()
-            .map(|ds| ds.split('.').map(String::from).collect::<Vec<String>>()[0].clone())
-            .collect();
-
-        // TODO: this will break if datasets reference there own project.
-        //
-
-        debug!("looking for project vals");
-
-        let project_vals: Vec<Project> = join_all(projects_ids.into_iter().map(|p| {
-            let projects = projects.clone();
-            async move {
-                projects
-                    .list(&ListParams::default())
-                    .await?
-                    .into_iter()
-                    .find(|project| project.spec.id == p)
-                    .ok_or(Error::MissingProject(p.to_string()))
-            }
-        }))
-        .await
-        .into_iter()
-        .collect::<Result<Vec<Project>>>()?;
-
-        debug!("mapping remote datasets");
-
-        // TODO: this is very brittle, we need more checks to ensure nothing went wrong.
-        let data_sets: Vec<DataSet> = data_sets
-            .clone()
-            .into_iter()
-            .map(|ref ds| {
-                project_vals
-                    .clone()
-                    .into_iter()
-                    .find_map(|pds| pds.get_data_set(ds.to_string()))
-                    .ok_or(Error::MissingDataSets(ds.to_string()))
-            })
-            .collect::<Result<Vec<DataSet>>>()?;
-
-        debug!("mapping local datasets");
-
-        if data_sets.len() != data_sets.len() {
-            return Err(Error::MissingDataSets(self.name_any())); // TODO: find a proper way of describing this error.
-        }
-
-        Ok(data_sets)
-    }
-
-    async fn get_data_set_tasks(&self, projects: Api<Project>) -> Result<Vec<Task>> {
-        let TaskSpec {
-            data_set: Some(data_sets),
-            ..
-        } = self.spec.clone() else {
-            return Ok(vec![])
-        };
-
-        let (remote_data_sets, _local_data_sets): (Vec<String>, Vec<String>) = data_sets
-            .clone()
-            .into_iter()
-            .partition(|ds| ds.contains('.'));
-
-        // TODO: this is very brittle
-        let projects_ids: Vec<String> = remote_data_sets
-            .clone()
-            .into_iter()
-            .map(|ds| ds.split('.').map(String::from).collect::<Vec<String>>()[0].clone())
-            .collect();
-
-        // TODO: this will break if datasets reference there own project.
-        //
-
-        debug!("looking for project vals");
-
-        let project_vals: Vec<Project> = join_all(projects_ids.into_iter().map(|p| {
-            let projects = projects.clone();
-            async move {
-                projects
-                    .list(&ListParams::default())
-                    .await?
-                    .into_iter()
-                    .find(|project| project.spec.id == p)
-                    .ok_or(Error::MissingProject(p.to_string()))
-            }
-        }))
-        .await
-        .into_iter()
-        .collect::<Result<Vec<Project>>>()?;
-
-        debug!("mapping remote datasets");
-
-        // TODO: this is very brittle, we need more checks to ensure nothing went wrong.
-        let data_set_tasks: Vec<Task> = data_sets
-            .clone()
-            .into_iter()
-            .map(|ref ds| {
-                project_vals
-                    .clone()
-                    .into_iter()
-                    .find_map(|pds| pds.generate_data_set_task(ds.to_string()))
-                    .ok_or(Error::MissingDataSets(ds.to_string()))
-            })
-            .collect::<Result<Vec<Task>>>()?;
-
-        debug!("mapping local datasets");
-
-        if data_set_tasks.len() != data_sets.len() {
-            return Err(Error::MissingDataSets(self.name_any())); // TODO: find a proper way of describing this error.
-        }
-
-        Ok(data_set_tasks)
-    }
-
-    async fn solve_template(&mut self, projects: Api<Project>) -> Result<()> {
-        let TaskSpec {
-            template_ref: Some(ref template_ref),
-            ..
-        } = self.spec else {
-            return Ok(())
-        };
-
-        let (template_name, project_id) = {
-            let vals: Vec<&str> = template_ref.split('.').collect();
-            if vals.len() != 2 {
-                return Err(Error::MissingTemplate("".to_string(), "".to_string()));
-            }
-
-            (vals[1], vals[0])
-        };
-
-        let template = resolve_template(template_name, project_id, projects).await?;
-
-        self.spec = omerge(template, self.spec.clone())?;
-
-        Ok(())
-    }
-}
-
-#[derive(Clone)]
-pub struct Context {
-    client: Client,
-    config: TaskControllerConfig,
-}
-
-pub async fn reconcile(task: Arc<Task>, ctx: Arc<Context>) -> Result<Action> {
-    let tasks = Api::<Task>::namespaced(ctx.client.clone(), &ctx.config.namespace);
-    let workflows = Api::<Workflow>::namespaced(ctx.client.clone(), &ctx.config.namespace);
-    let projects = Api::<Project>::namespaced(ctx.client.clone(), &ctx.config.namespace);
-
-    let mut task: Task = Task::clone(&task);
-
-    let data_sets = task.clone().spec.data_set;
-
-    let mut mut_task = task.clone();
-
-    let Some(ref mut status) = mut_task.status_mut() else {
-        let mut new_task = Task {
-            metadata: task.metadata.clone(),
-            spec: TaskSpec::default(),
-            status: Some(TaskStatus {
-                phase: Some(TaskPhase::Pending),
-                reason: None,
-                data_set_tasks: None,
-            }),
-        };
-
-        new_task.metadata.managed_fields = None;
-
-
-        tasks
-            .patch_status(
-                &task.name_any(),
-                &PatchParams::apply("taskmanager.teainspace.com"),
-                &kube::api::Patch::Apply(new_task),
-            )
-            .await?;
-
-        return Ok(Action::requeue(Duration::from_secs(50)));
-    };
-
-    let task_phase = status.clone().phase.unwrap_or_default();
-    if task_phase == TaskPhase::Failed || task_phase == TaskPhase::Succeeded {
-        return Ok(Action::requeue(Duration::from_secs(50)));
-    }
-
-    if let Some(_data_sets) = data_sets.as_ref() {
-        debug!("reconciling datasets");
-
-        // TODO: how do we make this idempotent?
-        let data_sets = task.get_data_set_tasks(projects.clone()).await?;
-
-        let data_set_statuses: Vec<(String, DataSetStatus)> =
-            join_all(data_sets.iter().map(|data_task: &Task| async {
-                let mut data_task = data_task.to_owned();
-
-                data_task.metadata = add_owner_reference(
-                    data_task.metadata.clone(),
-                    task.controller_owner_ref(&()).unwrap_or_default(),
-                );
-
-                let data_task: Task = tasks
-                    .patch(
-                        &data_task.name_any(),
-                        &PatchParams::apply("ame-controller"),
-                        &Patch::Apply(&data_task),
-                    )
-                    .await?;
-
-                Ok((
-                    data_task.name_any(),
-                    DataSetStatus {
-                        phase: data_task
-                            .status
-                            .and_then(|status| status.phase)
-                            .unwrap_or_default(),
-                    },
-                )) as Result<(String, DataSetStatus)>
-            }))
-            .await
-            .into_iter()
-            .collect::<Result<_>>()?;
-
-        let mut data_set_status_map: BTreeMap<String, DataSetStatus> = BTreeMap::new();
-
-        for (name, status) in data_set_statuses.clone() {
-            data_set_status_map.insert(name, status);
-        }
-
-        // TODO: what if status is None?
-        match &mut status.data_set_tasks {
-            None => {
-                status.data_set_tasks = Some(data_set_status_map);
-            }
-            Some(data_set_tasks) => {
-                for (k, v) in data_set_status_map.into_iter() {
-                    data_set_tasks.insert(k, v);
-                }
-            }
-        };
-
-        if data_set_statuses
-            .clone()
-            .into_iter()
-            .any(|(_name, dst)| dst.phase == TaskPhase::Failed)
-        {
-            status.phase = Some(TaskPhase::Failed);
-            status.reason = Some("one or more data sources have failed".to_string());
-
-            task.meta_mut().managed_fields = None;
-
-            let res = tasks
-                .patch_status(
-                    &task.name_any(),
-                    &PatchParams::apply("taskmanager.teainspace.com"),
-                    &kube::api::Patch::Apply(task),
-                )
-                .await;
-
-            info!("one or more data sources have failed");
-
-            if let Err(e) = res {
-                return Err(Error::KubeApiError(e));
-            }
-
-            return Ok(Action::requeue(Duration::from_secs(50)));
-        }
-
-        if data_set_statuses
-            .into_iter()
-            .any(|(_, dst)| dst.phase != TaskPhase::Succeeded)
-        {
-            status.phase = Some(TaskPhase::Pending);
-            status.reason = Some("one or more data sources are not ready".to_string());
-            // TODO: should we really be setting managed_fields to None?
-            task.meta_mut().managed_fields = None;
-            let res = tasks
-                .patch_status(
-                    &task.name_any(),
-                    &PatchParams::apply("taskmanager.teainspace.com"),
-                    &kube::api::Patch::Apply(task),
-                )
-                .await;
-
-            info!("one or more data sources are not ready");
-
-            if let Err(e) = res {
-                return Err(Error::KubeApiError(e));
-            }
-
-            return Ok(Action::requeue(Duration::from_secs(50)));
-        }
-    }
-
-    task.solve_template(projects.clone()).await?;
-
-    let data_sets = task.get_datasets(projects.clone()).await?;
-
-    let correct_wf = task.generate_workflow(&ctx.config, projects, Some(data_sets))?;
-    let wf = match workflows
-        .patch(
-            &correct_wf.name_any(),
-            &PatchParams::apply("taskmanager.teainspace.com"),
-            &kube::api::Patch::Apply(&correct_wf),
-        )
-        .await
-    {
-        Ok(wf) => wf,
-        Err(kube::Error::Api(ErrorResponse { code: 409, .. })) => {
-            workflows
-                .patch(
-                    &correct_wf.name_any(),
-                    &PatchParams::apply("taskmanager.teainspace.com").force(),
-                    &kube::api::Patch::Apply(correct_wf),
-                )
-                .await?
-        }
-        Err(e) => {
-            let wf = workflows.get(&correct_wf.name_any()).await?;
-            similar_asserts::assert_eq!(wf.spec, correct_wf.spec);
-            return Err(e)?;
-        }
-    };
-
-    if let Some(wf_status) = &wf.status {
-        let correct_task_phase = match wf_status.phase {
-            WorkflowPhase::Running => TaskPhase::Running,
-            WorkflowPhase::Pending => TaskPhase::Pending,
-            WorkflowPhase::Succeeded => TaskPhase::Succeeded,
-            WorkflowPhase::Failed => TaskPhase::Failed,
-            WorkflowPhase::Error => TaskPhase::Error,
-        };
-
-        if correct_task_phase != task_phase {
-            let mut new_task = task.clone();
-            let original_status = new_task.status.unwrap_or(TaskStatus::default());
-
-            new_task.metadata.managed_fields = None;
-            new_task.status = Some(TaskStatus {
-                phase: Some(correct_task_phase),
-                ..original_status
-            });
-
-            let res = tasks
-                .patch_status(
-                    &new_task.name_any(),
-                    &PatchParams::apply("taskmanager.teainspace.com"),
-                    &kube::api::Patch::Apply(new_task),
-                )
-                .await;
-
-            match res {
-                Ok(_) => println!("Patched status for task: {} ", task.name_any()),
-                Err(e) => {
-                    return Err(Error::KubeApiError(e));
-                }
-            };
-        }
-    }
-
-    Ok(Action::requeue(Duration::from_secs(50)))
-}
-
-fn error_policy(task: Arc<Task>, error: &Error, _ctx: Arc<Context>) -> Action {
-    error!("error: {}, for task: {}", error, task.name_any());
-    Action::requeue(Duration::from_secs(5))
-}
-
-pub async fn start_task_controller(config: TaskControllerConfig) -> BoxFuture<'static, ()> {
-    let client = Client::try_default().await.expect("Failed to create a K8S client, is the controller running in an environment with access to cluster credentials?");
-    let context = Arc::new(Context {
-        client: client.clone(),
-        config,
-    });
-
-    let tasks = Api::<Task>::namespaced(client.clone(), &context.config.namespace);
-    let _ = tasks
-        .list(&ListParams::default())
-        .await
-        .expect("Is the CRD installed?");
-
-    let workflows = Api::<Workflow>::namespaced(client, &context.config.namespace);
-
-    Controller::new(tasks.clone(), ListParams::default())
-        .owns(workflows, ListParams::default())
-        .owns(tasks, ListParams::default())
-        .run(reconcile, error_policy, context)
-        .filter_map(|x| async move { std::result::Result::ok(x) })
-        .for_each(|_| futures::future::ready(()))
-        .boxed()
-}
-
-#[cfg(test)]
-mod test {
-
-    use super::*;
-    use either::Either;
-    use insta;
-    use kube::api::{DeleteParams, PostParams, WatchEvent};
-    use serial_test::serial;
-
-    fn gen_test_config() -> TaskControllerConfig {
-        // It is important to explicitly set the struct values, so the environment cannot impact
-        // test runs.
-        TaskControllerConfig {
-            namespace: String::from("default"),
-            executor_image: "ghcr.io/teainspace/ame/ame-executor:0.0.3".to_string(),
-            bucket: String::from("ame"),
-        }
-    }
-
-    /// Prepare a cluster for tests, under the assumptions that the `just setup_cluster` recipe has been run successfully.
-    /// This implies that all required custom resource definitions are installed in the cluster.
-    /// This function will generate clients and clear all Task and `Workflow` objects in the cluster.
-    async fn setup_cluster(
-    ) -> Result<(TaskControllerConfig, Api<Task>, Api<Workflow>), Box<dyn std::error::Error>> {
-        let config = gen_test_config();
-        let client = Client::try_default().await?;
-        let tasks = Api::<Task>::namespaced(client.clone(), &config.namespace);
-        let workflows = Api::<Workflow>::namespaced(client.clone(), &config.namespace);
-
-        let dp = DeleteParams::default();
-        let lp = ListParams::default();
-
-        match tasks.delete_collection(&dp, &lp).await? {
-            Either::Left(_) => {
-                while !tasks.list(&lp).await?.items.is_empty() {
-                    tokio::time::sleep(Duration::from_millis(100)).await;
-                }
-            }
-
-            Either::Right(status) => {
-                println!("Deleted collection of tasks: {status:?}");
-            }
-        };
-
-        match workflows.delete_collection(&dp, &lp).await? {
-            Either::Left(_) => {
-                while !workflows.list(&lp).await?.items.is_empty() {
-                    tokio::time::sleep(Duration::from_millis(100)).await;
-                }
-            }
-
-            Either::Right(status) => {
-                println!("Deleted collection of tasks: {status:?}");
-            }
-        };
-
-        // TODO: How do we expose controller errors in tests?
-        let controller = start_task_controller(config.clone()).await;
-        tokio::spawn(controller);
-
-        Ok((config, tasks, workflows))
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn can_create_workflow() -> Result<(), Box<dyn std::error::Error>> {
-        let (_, tasks, workflows) = setup_cluster().await?;
-
-        let all_wfs = workflows.list(&ListParams::default()).await.unwrap();
-        assert_eq!(all_wfs.items.len(), 0);
-
-        let t: Task = serde_json::from_value(json!({
-            "apiVersion": "ame.teainspace.com/v1alpha1",
-            "kind": "Task",
-            "metadata": { "generateName": "training" },
-            "spec": {
-                "runcommand": "python train.py",
-                "projectid": "myproject",
-            }
-        }))
-        .unwrap();
-
-        let pp = PostParams::default();
-        let task_in_cluster = tasks.create(&pp, &t).await?;
-
-        let lp = ListParams::default()
-            .labels(&format!("ame-task={}", task_in_cluster.name_any()))
-            .timeout(5);
-
-        let mut stream = workflows.watch(&lp, "0").await?.boxed();
-        while let Some(status) = stream.next().await {
-            if let WatchEvent::Added(_) = status? {
-                return Ok(());
-            }
-        }
-
-        panic!("Did not find workflow");
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn controller_adds_status_to_new_task() -> Result<(), Box<dyn std::error::Error>> {
-        let (_, tasks, _) = setup_cluster().await?;
-
-        let t: Task = serde_json::from_value(json!({
-            "apiVersion": "ame.teainspace.com/v1alpha1",
-            "kind": "Task",
-            "metadata": { "generateName": "training" },
-            "spec": {
-                "runcommand": "python train.py",
-                "projectid": "myproject",
-            }
-        }))
-        .unwrap();
-
-        let pp = PostParams::default();
-        let task_in_cluster = tasks.create(&pp, &t).await?;
-
-        let lp = ListParams::default()
-            .fields(&format!("metadata.name={}", task_in_cluster.name_any()))
-            .timeout(5);
-
-        let mut stream = tasks.watch(&lp, "0").await?.boxed();
-        while let Some(status) = stream.next().await {
-            if let WatchEvent::Modified(t) = status? {
-                if let Some(status) = t.status {
-                    assert_eq!(status.phase, Some(TaskPhase::Pending));
-                    return Ok(());
-                }
-            }
-        }
-
-        panic!("Did not add status");
-    }
-
-    // TODO: Should we test some fields explicitly?
-    #[tokio::test]
-    async fn task_can_generate_workflow() {
-        let mut t: Task = serde_json::from_value(json!({
-            "apiVersion": "ame.teainspace.com/v1alpha1",
-            "kind": "Task",
-            "metadata": { "name": "training" },
-            "spec": {
-                "runcommand": "python train.py",
-                "projectid": "myproject",
-                "env": [
-                {
-                    "name": "VAR1",
-                    "value": "val1"
-                },
-                {
-                    "name": "VAR2",
-                    "value": "val2"
-                },
-                {
-                    "name": "VAR3",
-                    "value": "val3"
-                }
-                ],
-                "secrets": [
-                {
-                    "AmeSecret" : {
-                    "name": "secret1",
-                    "envkey": "KEY1"
-                    }
-                },
-                {
-                    "AmeSecret" : {
-                    "name": "secret2",
-                    "envkey": "KEY2"
-                    }
-                },
-                ]
-            }
-        }))
-        .unwrap();
-
-        let client = Client::try_default().await.unwrap();
-        let projects = Api::<Project>::default_namespaced(client);
-        let wf: Workflow = t
-            .generate_workflow(
-                &gen_test_config(),
-                projects,
-                Some(vec![DataSet {
-                    name: "mydataset".to_string(),
-                    task: TaskSpec::default(),
-                    path: "path_to_data".to_string(),
-                }]),
-            )
-            .unwrap();
-
-        insta::assert_yaml_snapshot!(&wf);
-    }
-
-    #[tokio::test]
-    async fn task_can_override_workflow_image() {
-        let mut t: Task = serde_json::from_value(json!({
-            "apiVersion": "ame.teainspace.com/v1alpha1",
-            "kind": "Task",
-            "metadata": { "name": "training" },
-            "spec": {
-                "runcommand": "python train.py",
-                "projectid": "myproject",
-                "env": [
-                {
-                    "name": "VAR1",
-                    "value": "val1"
-                },
-                {
-                    "name": "VAR2",
-                    "value": "val2"
-                }
-                ],
-                "image": "very-different-image",
-                "secrets": [
-                {
-                    "AmeSecret" : {
-                    "name": "secret1",
-                    "envkey": "KEY1"
-                    }
-                },
-                {
-                    "AmeSecret" : {
-                    "name": "secret2",
-                    "envkey": "KEY2"
-                    }
-                },
-                ]
-            }
-        }))
-        .unwrap();
-
-        let client = Client::try_default().await.unwrap();
-        let projects = Api::<Project>::default_namespaced(client);
-        let wf: Workflow = t
-            .generate_workflow(&gen_test_config(), projects, None)
-            .unwrap();
-
-        insta::assert_yaml_snapshot!(&wf);
-    }
-
-    #[tokio::test]
-    async fn task_can_have_git_src() {
-        let mut t: Task = serde_json::from_value(json!({
-            "apiVersion": "ame.teainspace.com/v1alpha1",
-            "kind": "Task",
-            "metadata": { "name": "training" },
-            "spec": {
-                "runcommand": "python train.py",
-                "projectid": "myproject",
-                "env": [
-                {
-                    "name": "VAR1",
-                    "value": "val1"
-                },
-                {
-                    "name": "VAR2",
-                    "value": "val2"
-                }
-                ],
-                "secrets": [
-                {
-                    "AmeSecret" : {
-                    "name": "secret1",
-                    "envkey": "KEY1"
-                    }
-                },
-                {
-                    "AmeSecret" : {
-                    "name": "secret2",
-                    "envkey": "KEY2"
-                    }
-                },
-                ],
-                "source": {
-                    "gitrepository": "gitrepo",
-                },
-            }
-        }))
-        .unwrap();
-
-        let client = Client::try_default().await.unwrap();
-        let projects = Api::<Project>::default_namespaced(client);
-        let wf: Workflow = t
-            .generate_workflow(&gen_test_config(), projects, None)
-            .unwrap();
-
-        insta::assert_yaml_snapshot!(&wf);
-    }
-
-    #[tokio::test]
-    async fn task_can_have_vault_secret() {
-        let mut t: Task = serde_json::from_value(json!({
-            "apiVersion": "ame.teainspace.com/v1alpha1",
-            "kind": "Task",
-            "metadata": { "name": "training" },
-            "spec": {
-                "runcommand": "python train.py",
-                "projectid": "myproject",
-                "env": [
-                {
-                    "name": "VAR1",
-                    "value": "val1"
-                },
-                {
-                    "name": "VAR2",
-                    "value": "val2"
-                }
-                ],
-                "secrets": [
-                {
-                    "VaultSecret" : {
-                    "vaultName": "myvault",
-                    "secretKey": "mysecret",
-                    "secretPath": "/my/secret/path",
-                    "envkey": "KEY1",
-                    }
-                },
-                {
-                    "AmeSecret" : {
-                    "name": "secret2",
-                    "envkey": "KEY2"
-                    }
-                },
-                ],
-            }
-        }))
-        .unwrap();
-
-        let client = Client::try_default().await.unwrap();
-        let projects = Api::<Project>::default_namespaced(client);
-        let wf: Workflow = t
-            .generate_workflow(&gen_test_config(), projects, None)
-            .unwrap();
-
-        insta::assert_yaml_snapshot!(&wf);
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn can_correct_workflow() -> Result<(), Box<dyn std::error::Error>> {
-        let (_, tasks, workflows) = setup_cluster().await?;
-
-        let all_wfs = workflows.list(&ListParams::default()).await.unwrap();
-        assert_eq!(all_wfs.items.len(), 0);
-
-        let all_tasks = tasks.list(&ListParams::default()).await.unwrap();
-        assert_eq!(all_tasks.items.len(), 0);
-
-        let t: Task = serde_json::from_value(json!({
-            "apiVersion": "ame.teainspace.com/v1alpha1",
-            "kind": "Task",
-            "metadata": { "generateName": "training" },
-            "spec": {
-                "runcommand": "python train.py",
-                "projectid": "myproject",
-            }
-        }))
-        .unwrap();
-
-        let task_in_cluster = tasks.create(&PostParams::default(), &t).await?;
-
-        let lp = ListParams::default()
-            .labels(&format!("ame-task={}", task_in_cluster.name_any()))
-            .timeout(5);
-
-        let mut correct_labels = BTreeMap::new();
-        let mut stream = workflows.watch(&lp, "0").await?.boxed();
-        while let Some(e) = stream.next().await {
-            if let WatchEvent::Added(wf) = e? {
-                correct_labels = wf.metadata.labels.clone().unwrap();
-                let mut labels = wf.metadata.labels.clone().unwrap();
-                labels.insert("ame-task".to_string(), "test".to_string());
-                workflows
-                    .patch(
-                        &wf.name_any(),
-                        &PatchParams::apply("taskmanager.teainspace.com"),
-                        &kube::api::Patch::Apply(Workflow {
-                            metadata: ObjectMeta {
-                                labels: Some(labels),
-                                managed_fields: None,
-                                ..wf.metadata
-                            },
-                            ..wf
-                        }),
-                    )
-                    .await?;
-
-                break;
-            }
-        }
-
-        // TODO: make the event streams in this test run in parallel so
-        // so we can track that the controller produces the correct events without
-        // relying on timing to workout.
-
-        // TODO: Determine why we received Added and not Modified events.
-        let mut stream = workflows.watch(&lp, "0").await?.boxed();
-        while let Some(wf) = stream.next().await {
-            match wf {
-                Ok(WatchEvent::Added(wf)) => {
-                    println!("wf labels {:?} ", wf.metadata.labels);
-                    assert_eq!(wf.metadata.labels.unwrap(), correct_labels);
-                    return Ok(());
-                }
-
-                e => println!("event: {e:?}"),
-            }
-        }
-
-        panic!("workflow was not corrected")
-    }
-}
diff --git a/lib/src/custom_resources/task_ctrl.rs b/lib/src/custom_resources/task_ctrl.rs
new file mode 100644
index 00000000..f3b92995
--- /dev/null
+++ b/lib/src/custom_resources/task_ctrl.rs
@@ -0,0 +1,118 @@
+use futures::future::join_all;
+use kube::{
+    api::{ListParams, Patch, PatchParams},
+    Api, ResourceExt,
+};
+use tracing::debug;
+
+use super::{
+    common::parent_project,
+    data_set::DataSet,
+    new_task::{Task, TaskContext},
+    project::{generate_data_set_task_name, Project},
+};
+
+use crate::custom_resources::project::{local_name, project_name};
+
+use crate::Result;
+
+pub async fn approve_deletion(tasks: &Api<Task>, name: &str) -> Result<()> {
+    let patch: Task = Task::approve_deletion_patch();
+
+    tasks
+        .patch(
+            name,
+            &PatchParams::apply("ame-server").force(), // TODO: should we be forcing here?
+            &Patch::Apply(patch),
+        )
+        .await?;
+
+    Ok(())
+}
+
+pub struct TaskCtrl {
+    data_sets: Api<DataSet>,
+    projects: Api<Project>,
+}
+
+impl TaskCtrl {
+    pub fn new(data_sets: Api<DataSet>, projects: Api<Project>) -> Self {
+        Self {
+            data_sets,
+            projects,
+        }
+    }
+
+    pub async fn gather_task_ctx(
+        &self,
+        task: &Task,
+        executor_image: String,
+        service_account: String,
+    ) -> Result<TaskContext> {
+        debug!("gathering task context");
+        // NOTE: we at least have to get datasets which this task depends on.
+        let parent_project = task.project()?;
+
+        let dependent_data_sets: Result<Vec<DataSet>> =
+            join_all(task.spec.cfg.data_sets.clone().iter().map(|ds| async {
+                self.resolve_data_set_ref(ds.clone(), parent_project.clone())
+                    .await
+            }))
+            .await
+            .into_iter()
+            .collect();
+
+        Ok(TaskContext {
+            executor_image,
+            task_volume: task.name_any(),
+            required_data_sets: dependent_data_sets?,
+            service_account,
+        })
+    }
+
+    async fn resolve_data_set_ref(&self, ds_ref: String, root_project: String) -> Result<DataSet> {
+        let project_name = if let Some(project_name) = project_name(ds_ref.clone()) {
+            project_name
+        } else {
+            self.projects.get(&root_project).await?.spec.cfg.name
+        };
+
+        let project_objs = self.projects.list(&ListParams::default()).await?;
+
+        let local_name = local_name(ds_ref.clone());
+
+        let potential_ds: Vec<DataSet> = self
+            .data_sets
+            .list(&ListParams::default())
+            .await?
+            .items
+            .into_iter()
+            .filter(|ds| ds.spec.cfg.name == local_name)
+            .filter(|ds| {
+                project_objs.items.clone().iter().any(|po| {
+                    ds.owner_references().iter().any(|oref| {
+                        oref.kind == "Project"
+                            && oref.name == po.name_any()
+                            && po.spec.cfg.name == project_name
+                    })
+                })
+            })
+            .collect();
+
+        if potential_ds.len() != 1 {
+            todo!("len: {} {ds_ref}", potential_ds.len())
+        }
+
+        Ok(potential_ds[0].clone())
+    }
+}
+
+pub fn resolve_data_set_path(data_set: DataSet) -> Result<String> {
+    let project_name = parent_project(data_set.owner_references().to_vec())?;
+
+    Ok(format!(
+        "ame/tasks/{}/artifacts/{}",
+        generate_data_set_task_name(project_name, data_set.spec.cfg.name),
+        data_set.spec.cfg.path
+    ))
+}
diff --git a/lib/src/error.rs b/lib/src/error.rs
index 3110922a..716f6880 100644
--- a/lib/src/error.rs
+++ b/lib/src/error.rs
@@ -67,6 +67,9 @@ pub enum AmeError {
     #[error("missing task config for resource: {0}")]
     MissingTaskCfg(String),
 
+    #[error("missing task config for model: {0}")]
+    MissingTrainingTaskCfg(String),
+
     #[error("failed to find a task ref for: {0}")]
     MissingTaskRef(String),
 
@@ -75,6 +78,51 @@ pub enum AmeError {
 
     #[error("missing status field for: {0}")]
     MissingTaskStatus(String),
+
+    #[error("failed to parse json: {0}")]
+    SerdeJsonError(#[from] serde_json::Error),
+
+    #[error("failed to parse yaml: {0}")]
+    SerdeYamlError(#[from] serde_yaml::Error),
+
+    #[error("missing executor in task: {0}")]
+    MissingExecutor(String),
+
+    #[error("deletion not approved for resource: {0}")]
+    DeletionNotApproved(String),
+
+    #[error("project configuration is missing from request")]
+    MissingProjectcfg,
+
+    #[error("the {0} controller failed to reconile {1}, reason: {2}")]
+    ReconcilitationFailure(String, String, String),
+
+    #[error("No name was present on the resource")]
+    MissingName,
+
+    #[error("failed to find a single owner project instead found: {0}]")]
+    MissingProject(usize),
+
+    #[error("failed to find template {1} in project {0}")]
+    MissingTemplate(String, String),
+
+    #[error("An empty artifact configuration was supplied for task {0}")]
+    EmptyArtifactCfg(String),
+
+    #[error("Failed to parse cron schedule: {0} ")]
+    CronError(#[from] cron_parser::ParseError),
+
+    #[error("Could not generate an owner reference for {0}")]
+    FailedToCreateOref(String),
+
+    #[error("Failed to find an owner reference for {0}, reason: expected to find 1 but found {1} instead")]
+    FailedToFindParentProjectOref(String, usize),
+
+    #[error("Could not find data {0} in project {1}")]
+    MissingDataSet(String, String),
+
+    #[error("Missing model training cfg {0} {1}")]
+    MissingModelTrainingTaskCfg(String, String),
 }
 
 impl From<Status> for AmeError {
diff --git a/lib/src/lib.rs b/lib/src/lib.rs
index 86672dc7..f5f1541a 100644
--- a/lib/src/lib.rs
+++ b/lib/src/lib.rs
@@ -15,13 +15,46 @@ pub mod web;
 #[cfg(any(feature = "web-components", feature = "native-client"))]
 pub mod client;
 
+#[cfg(feature = "project-tools")]
+pub mod project;
+
 pub mod grpc {
     #![allow(clippy::all)]
 
-    use std::fmt::{self, Display};
+    use std::{
+        collections::BTreeMap,
+        fmt::{self, Display},
+    };
+
+    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
+
+    use self::task_cfg::Executor;
 
     tonic::include_proto!("ame.v1");
 
+    impl Display for task_status::Phase {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            let output = match self {
+                task_status::Phase::Pending(_) => "Pending",
+                task_status::Phase::Succeeded(_) => "Succeeded",
+                task_status::Phase::Running(_) => "Running",
+                task_status::Phase::Failed(_) => "Failed",
+            };
+
+            write!(f, "{output}")
+        }
+    }
+
+    impl self::task_status::Phase {
+        pub fn success(&self) -> bool {
+            if let self::task_status::Phase::Succeeded(_) = self {
+                true
+            } else {
+                false
+            }
+        }
+    }
+
     impl Display for ResourceId {
         fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
             match self.id {
@@ -53,31 +86,21 @@ pub mod grpc {
         }
     }
 
-    impl TaskLogRequest {
-        pub fn stream_from_beginning(task_name: &str, watch: bool) -> Self {
-            Self {
-                taskid: Some(TaskIdentifier::from(task_name)),
-                start_from: Some(1),
-                watch: Some(watch),
-            }
+    pub fn resource_map_conv(resources: BTreeMap<String, String>) -> BTreeMap<String, Quantity> {
+        let mut new_resources = BTreeMap::<String, Quantity>::new();
+        for (k, v) in resources.into_iter() {
+            new_resources.insert(k, Quantity(v));
         }
-    }
 
-    impl CreateTaskRequest {
-        pub fn new(task_name: &str, task_template: TaskTemplate) -> Self {
-            Self {
-                id: Some(TaskIdentifier::from(task_name)),
-                template: Some(task_template),
-            }
-        }
+        new_resources
     }
 
-    impl TaskProjectDirectoryStructure {
-        pub fn new(task_name: &str, project_id: &str, paths: Vec<String>) -> Self {
+    impl TaskLogRequest {
+        pub fn stream_from_beginning(task_name: &str, watch: bool) -> Self {
             Self {
                 taskid: Some(TaskIdentifier::from(task_name)),
-                projectid: project_id.to_string(),
-                paths,
+                start_from: Some(1),
+                watch: Some(watch),
             }
         }
     }
@@ -114,6 +137,86 @@ pub mod grpc {
             }
         }
     }
+
+    impl Executor {
+        pub fn command(&self) -> String {
+            match self {
+                Executor::Poetry(PoetryExecutor {
+                    python_version,
+                    command,
+                }) => {
+                    format!(
+                        "
+                        source ~/.bashrc
+                        
+                        pyenv install {python_version}
+
+                        pyenv global {python_version}
+
+                        poetry install
+                    
+                        poetry run {command}
+                    "
+                    )
+                }
+                Executor::Mlflow(_) => "export PATH=$HOME/.pyenv/bin:$PATH
+
+             unset AWS_SECRET_ACCESS_KEY
+
+             unset AWS_ACCESS_KEY_ID
+
+             mlflow run ."
+                    .to_string(),
+                Executor::PipEnv(PipEnvExecutor { command }) => {
+                    format!(
+                        "
+                            pipenv sync
+
+                            pipenv run {command}
+                           
+                        "
+                    )
+                }
+
+                Executor::Pip(PipExecutor {
+                    python_version,
+                    command,
+                }) => {
+                    format!(
+                        "
+
+                        source ~/.bashrc
+                        
+                        pyenv install {python_version}
+
+                        pyenv global {python_version}
+
+                        pip install -r requirements.txt
+                    
+                        {command}
+                    "
+                    )
+                }
+
+                Executor::Custom(CustomExecutor {
+                    python_version,
+                    command,
+                }) => {
+                    format!(
+                        "
+                        source ~/.bashrc
+                        
+                        pyenv install {python_version}
+
+                        pyenv global {python_version}
+                    
+                        {command}
+                    "
+                    )
+                }
+            }
+        }
+    }
 }
 
 pub mod proto;
diff --git a/lib/src/project.rs b/lib/src/project.rs
new file mode 100644
index 00000000..13a41dac
--- /dev/null
+++ b/lib/src/project.rs
@@ -0,0 +1,32 @@
+use anyhow::{Context, Result};
+use std::fs;
+
+use crate::grpc::{ProjectCfg, TaskCfg};
+
+impl ProjectCfg {
+    pub fn try_from_working_dir() -> Result<Self> {
+        serde_yaml::from_str(
+            &fs::read_to_string("ame.yaml")
+                .context("Could not read ame.yaml, are you in an AME project?")?,
+        )
+        .context("Could not parse ame.yaml :(")
+    }
+
+    pub fn try_from_dir(dir: &str) -> Result<Self> {
+        Ok(serde_yaml::from_str(&fs::read_to_string(format!(
+            "{}/ame.yaml",
+            dir
+        ))?)?)
+    }
+
+    pub fn task_names(&self) -> Vec<String> {
+        self.tasks.iter().filter_map(|t| t.name.clone()).collect()
+    }
+
+    pub fn get_task_cfg(&self, name: &str) -> Option<TaskCfg> {
+        self.tasks
+            .iter()
+            .find(|t| t.name.clone().map(|n| n == name).unwrap_or(false))
+            .cloned()
+    }
+}
diff --git a/lib/tests/e2e_test.rs b/lib/tests/e2e_test.rs
new file mode 100644
index 00000000..220b4849
--- /dev/null
+++ b/lib/tests/e2e_test.rs
@@ -0,0 +1,555 @@
+use ame::custom_resources::{
+    data_set::{DataSet, DataSetPhase, DataSetStatus},
+    new_task::Task,
+};
+
+use ame::{
+    custom_resources::{
+        common::setup_cluster, project::Project, project_source_ctrl::ProjectSrcCtrl,
+        secrets::SecretCtrl,
+    },
+    grpc::{task_status::Phase, ProjectCfg, TaskCfg, TaskStatus},
+};
+use assert_cmd::prelude::*;
+use fs_extra::dir::CopyOptions;
+
+use futures_util::StreamExt;
+
+use k8s_openapi::api::{apps::v1::Deployment, core::v1::Service, networking::v1::Ingress};
+use kube::{
+    api::{DeleteParams, ListParams, PatchParams},
+    runtime::{watcher, WatchStreamExt},
+    Api, Client,
+};
+use rstest::*;
+use serial_test::serial;
+use std::collections::HashMap;
+
+use time::Instant;
+
+use std::{
+    path::{Path, PathBuf},
+    process::Command,
+};
+use tracing::{debug, instrument};
+use tracing_subscriber::EnvFilter;
+
+static INGRESS_NAMESPACE: &str = "ingress-nginx";
+static INGRESS_SERVICE: &str = "ingress-nginx-controller";
+static AME_NAMESPACE: &str = "ame-system";
+
+async fn test_setup() -> Result<(), Box<dyn std::error::Error>> {
+    // std::env::set_var(
+    //     "AME_ENDPOINT",
+    //     find_ame_endpoint(INGRESS_NAMESPACE, INGRESS_SERVICE).await?,
+    // );
+    std::env::set_var("AME_ENDPOINT", "http://localhost:3342");
+
+    Ok(())
+}
+
+fn prepare_test_project(path_from_root: &str) -> Result<PathBuf, Box<dyn std::error::Error>> {
+    let mut test_project_path = PathBuf::from(path_from_root);
+
+    for i in 0..10 {
+        if test_project_path.is_dir() && i < 9 {
+            break;
+        }
+
+        if i < 9 {
+            test_project_path = Path::new("..").join(test_project_path);
+            continue;
+        }
+
+        return Err(format!(
+            "failed to find test project directory: {}",
+            test_project_path.display()
+        ))?;
+    }
+
+    let temp_dir = assert_fs::TempDir::new()?.into_persistent();
+
+    fs_extra::copy_items(
+        &[test_project_path],
+        temp_dir.path(),
+        &CopyOptions::default(),
+    )?;
+
+    Ok(temp_dir
+        .path()
+        .join(Path::new(path_from_root).file_name().unwrap()))
+}
+
+// TODO add concurrent tests
+// TODO Test failure handling
+
+#[rstest]
+#[trace]
+#[case("test_data/test_projects/executors/poetry", "training")]
+#[case("test_data/test_projects/executors/pipenv", "training")]
+#[case("test_data/test_projects/executors/pip", "training")]
+#[case("test_data/test_projects/executors/mlflow", "training")]
+#[case(
+    "test_data/test_projects/executors/mlflow",
+    "training-with-local-template"
+)]
+#[case(
+    "test_data/test_projects/executors/mlflow",
+    "training-with-remote-template"
+)]
+#[case("test_data/test_projects/executors/custom", "training")]
+#[case("test_data/test_projects/env", "testenv")]
+#[case("test_data/test_projects/env", "testsecret")]
+#[ignore]
+#[tokio::test]
+async fn ame_run_task(
+    #[case] project_dir: &str,
+    #[case] task_id: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    init_test_tracing();
+    let _ = setup_cluster("ame-system").await; // TODO: async tests are affecting each other here.
+    debug!("running task {task_id} in {project_dir}");
+    let projects: Api<Project> = Api::namespaced(kube_client().await?, AME_NAMESPACE);
+    let project: Project = serde_json::from_value(serde_json::json!(
+        {
+            "metadata": {
+            "name": "shared-templates-project"
+                },
+            "spec": {
+                "name": "shared-templates",
+                "templates": [{
+
+                "name": "mlflow-template",
+                "executor": {
+                    "mlflow": {}
+                }}],
+
+                "deletionApproved": false
+            }
+        }
+    ))
+    .unwrap();
+
+    projects
+        .patch(
+            "shared-templates-project",
+            &PatchParams::apply("AME-TEST").force(),
+            &kube::api::Patch::Apply(project),
+        )
+        .await
+        .unwrap();
+
+    let temp = prepare_test_project(project_dir)?;
+    let mut cmd = Command::cargo_bin("ame")?;
+    test_setup().await?;
+
+    let res = cmd
+        .current_dir(temp.clone())
+        .arg("task")
+        .arg("run")
+        .arg("--logs")
+        .arg(task_id)
+        .assert()
+        .success();
+
+    let mut settings = insta::Settings::clone_current();
+    settings.add_filter("time=\".*\".*\n", "");
+    settings.add_filter(
+        "created virtual environment .*",
+        "created virtual environment \"redacted\"",
+    );
+    settings.add_filter("creator .*", "creator \"redacted\"");
+    settings.add_filter(
+        "\\d\\d\\d\\d/\\d\\d/\\d\\d \\d\\d:\\d\\d:\\d\\d",
+        "\"redacted timestamp\"",
+    );
+    settings.add_filter("run .*", "\"redacted run ID\"");
+    settings.add_filter("ID '.*'", "\"redacted run ID\"");
+    settings.add_filter("tmp/tmp.*\\s", "\"redacted temporary directory\"");
+    settings.add_filter("mlflow-.*/", "\"redacted MLflow env ID\"");
+    settings.add_filter(".*: UserWarning: .*\\n", "");
+    settings.add_filter("warnings\\.warn.*\\n", "");
+    settings.add_filter("  \"redacted timestamp", "\"redacted timestamp");
+    settings.add_filter("  Score:", "Score:");
+    settings.add_filter("added seed packages.*", "redacted");
+    settings.add_filter("Registered model '.*' already exists.*", "redacted");
+    settings.add_filter("Created version '.'.*\\n", "");
+    settings.add_filter(", version .", ", version %");
+    settings.add_filter("Successfully registered model .*", "redacted");
+    // TODO: why does \\. not work here?
+    settings.add_filter("\\d+\\.\\d+ MB", "redacted");
+    settings.add_filter("\\d+\\.\\d+ KB", "redacted");
+    settings.add_filter("tasks/.+/projectfiles", "tasks/redacted/projectfiles");
+    let _guard = settings.bind_to_scope();
+
+    // TODO: how can we incorporate snapshots in a non brittle way?
+    // insta::assert_snapshot!(&String::from_utf8(res.get_output().stdout.clone())?);
+
+    let project = ProjectCfg::try_from_dir(temp.to_str().unwrap()).unwrap();
+
+    debug!(
+        "stdout: {}",
+        String::from_utf8(res.get_output().stdout.clone()).unwrap()
+    );
+
+    // TODO: should be use the ame API server to monitor the status here?
+    let tasks: Api<Task> = Api::namespaced(kube_client().await?.clone(), AME_NAMESPACE);
+    let task_list = tasks.list(&ListParams::default()).await?;
+    let task_obs: Vec<Task> = task_list
+        .items
+        .into_iter()
+        .filter(|t| {
+            t.spec.cfg.name.as_ref().unwrap() == task_id
+                && t.metadata
+                    .owner_references
+                    .as_ref()
+                    .unwrap()
+                    .iter()
+                    .any(|oref| oref.name.contains(&project.name))
+        })
+        .collect();
+
+    assert_eq!(
+        task_obs.len(),
+        1,
+        "expected to find a single matching task, instead found {}",
+        task_obs.len()
+    );
+
+    // TODO: we need to identify each task uniqely here
+    assert!(
+        task_obs[0]
+            .status
+            .as_ref()
+            .unwrap()
+            .phase
+            .as_ref()
+            .unwrap()
+            .success(),
+        "task {} was not successful",
+        { task_id }
+    );
+
+    Ok(())
+}
+#[rstest]
+#[trace]
+#[case("test_data/test_projects/executors/poetry", "crontraining")]
+#[ignore]
+#[tokio::test]
+#[serial]
+async fn ame_trigger_tasks(
+    #[case] project_dir: &str,
+    #[case] task_id: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    init_test_tracing();
+    let _ = setup_cluster("ame-system").await; // TODO: async tests are affecting each other here.
+    let _projects: Api<Project> = Api::namespaced(kube_client().await?, AME_NAMESPACE);
+    let tasks: Api<Task> = Api::namespaced(kube_client().await?, AME_NAMESPACE);
+
+    let temp = prepare_test_project(project_dir)?;
+    let mut cmd = Command::cargo_bin("ame")?;
+    test_setup().await?;
+
+    let mut project = ProjectCfg::try_from_dir(temp.to_str().unwrap()).unwrap();
+
+    let now = time::OffsetDateTime::now_utc();
+
+    let target = now.checked_add(time::Duration::seconds(270)).unwrap();
+
+    let mut project_file_path = temp.clone();
+    project_file_path.push("ame.yaml");
+
+    project.tasks[0].triggers = Some(ame::grpc::TriggerCfg {
+        schedule: Some(format!("{} {} * * *", target.minute(), target.hour())),
+    });
+
+    let f = std::fs::File::create(project_file_path)?;
+    serde_yaml::to_writer(f, &project).unwrap();
+
+    let res = cmd
+        .current_dir(temp.clone())
+        .arg("project")
+        .arg("push")
+        .arg("--triggers")
+        .assert()
+        .success();
+
+    let start_t = Instant::now();
+
+    loop {
+        tokio::time::sleep(std::time::Duration::from_secs(2)).await;
+        if start_t.elapsed() > time::Duration::seconds(360) {
+            return Err("task took too long to complete".to_string().into());
+        }
+        let task_list = tasks.list(&ListParams::default()).await?;
+        let task_obs: Vec<Task> = task_list
+            .items
+            .into_iter()
+            .filter(|t| {
+                t.spec.cfg.name.as_ref().unwrap() == task_id
+                    && t.metadata
+                        .owner_references
+                        .as_ref()
+                        .unwrap()
+                        .iter()
+                        .any(|oref| oref.name.contains(&project.name))
+            })
+            .collect();
+
+        assert!(
+            task_obs.len() < 2,
+            "expected to find a single or no matching task, instead found {}",
+            task_obs.len(),
+        );
+
+        let Some(task) = task_obs.get(0) else {
+            continue;
+        };
+
+        let _now = time::OffsetDateTime::now_utc();
+
+        if start_t.elapsed() < time::Duration::seconds(120) {
+            return Err(format!(
+                "we do not expect to see the schedule task a head of time {}",
+                start_t.elapsed()
+            )
+            .to_string()
+            .into());
+        }
+
+        if task
+            .status
+            .as_ref()
+            .unwrap()
+            .phase
+            .as_ref()
+            .unwrap()
+            .success()
+        {
+            break;
+        }
+    }
+
+    debug!(
+        "stdout: {}",
+        String::from_utf8(res.get_output().stdout.clone()).unwrap()
+    );
+
+    Ok(())
+}
+
+// TODO get rid of this, we don't need a fixture for something so simple
+#[fixture]
+async fn kube_client() -> Result<Client, kube::Error> {
+    Client::try_default().await
+}
+
+pub async fn delete_registered_model(name: &str, mlflow_url: &str) -> Result<(), reqwest::Error> {
+    let mut map = HashMap::new();
+    map.insert("name", &name);
+
+    let res = reqwest::Client::new()
+        .delete(format!(
+            "{mlflow_url}/api/2.0/mlflow/registered-models/delete"
+        ))
+        .json(&map);
+
+    res.send().await?.error_for_status()?;
+
+    Ok(())
+}
+
+fn init_test_tracing() {
+    let _ = tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::from_default_env())
+        .try_init();
+}
+
+#[tokio::test]
+#[serial]
+#[ignore]
+#[instrument]
+async fn can_use_data_set_train_validate_and_deploy_model() -> Result<(), Box<dyn std::error::Error>>
+{
+    debug!("can use data, preparing cluster");
+    init_test_tracing();
+    let _ = setup_cluster(AME_NAMESPACE).await;
+    test_setup().await?;
+
+    debug!("Cluster is ready!");
+
+    // The template repo is required as the ame-demo requires it to train.
+    let template_repo = "https://github.com/TeaInSpace/ame-template-demo.git";
+    let data_set_repo = "https://github.com/TeaInSpace/ame-dataset-demo.git";
+    let project_repo = "https://github.com/TeaInSpace/ame-demo.git";
+    let model_name = "logreg"; // this name is from the ame-demo repo.
+    let project_src_ctrl = ProjectSrcCtrl::new(kube_client().await?, AME_NAMESPACE);
+
+    let kube_client = kube_client().await?;
+    let deployments: Api<Deployment> = Api::namespaced(kube_client.clone(), AME_NAMESPACE);
+    let datasets: Api<DataSet> = Api::namespaced(kube_client.clone(), AME_NAMESPACE);
+    let projects: Api<Project> = Api::namespaced(kube_client.clone(), AME_NAMESPACE);
+    let ingress: Api<Ingress> = Api::namespaced(kube_client.clone(), AME_NAMESPACE);
+    let services: Api<Service> = Api::namespaced(kube_client.clone(), AME_NAMESPACE);
+    let tasks: Api<Task> = Api::namespaced(kube_client.clone(), AME_NAMESPACE);
+
+    // TODO: whyid this not throw an error with AME_FILE_NAME.
+    let secret_ctrl = SecretCtrl::try_default(AME_NAMESPACE).await?;
+    let s3_secret = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG";
+
+    secret_ctrl
+        .store_secret_if_empty("s3secretkey", s3_secret.to_string())
+        .await?;
+
+    let res = delete_registered_model("logreg", "http://localhost:5000").await;
+
+    // If a model was not present mlflow should return a 404. In this case we can
+    // proceed as that does not indicate a problem. Any other error code indicates
+    // that something went wrong and the test therefor aborts.
+    if let Err(e) = res {
+        if let Some(status) = e.status() {
+            if status != 404 {
+                return Err(format!("received unexpected error from mlflow server {e}").into());
+            }
+        }
+    }
+
+    // TODO: this is very brittle, should be replaced by proper resource ownership.
+    let _ = deployments.delete("logreg", &DeleteParams::default()).await;
+    let _ = ingress.delete("logreg", &DeleteParams::default()).await;
+    let _ = services.delete("logreg", &DeleteParams::default()).await;
+
+    // TODO: test when controller is blocked from reconciling due to a deletion request.
+
+    // Create each project src using the CLI.
+    // This should be the only interaction necessary to deploy a model.
+    debug!("create project sources with the CLI");
+    for repo in [template_repo, data_set_repo, project_repo] {
+        let mut cmd = Command::cargo_bin("ame")?;
+        let _output = cmd
+            .arg("projectsrc")
+            .arg("create")
+            .arg(repo)
+            .assert()
+            .success();
+    }
+
+    // It is important that we only have 3 projects as the watcher
+    // does not perform any filtering. If other projects are present
+    // something is interfering with the test.
+    assert_eq!(
+        projects
+            .list_metadata(&ListParams::default())
+            .await?
+            .items
+            .len(),
+        3
+    );
+
+    // No deployment for the model should be present.
+    // TODO: this should be replaced by ownershop somehow.
+    assert!(deployments.get(model_name).await.is_err());
+
+    let timeout = std::time::Duration::from_secs(300);
+    let start = std::time::Instant::now();
+
+    let mut data_set_watcher = watcher(datasets, ListParams::default())
+        .applied_objects()
+        .boxed();
+
+    debug!("waiting for dataset to be ready");
+    // Before the model can be trained we expect a data set to be prepared.
+    while let Some(data_set) = data_set_watcher.next().await {
+        if let DataSet {
+            status:
+                Some(DataSetStatus {
+                    phase: Some(DataSetPhase::Ready { .. }),
+                }),
+            ..
+        } = data_set?
+        {
+            break;
+        }
+
+        if start.elapsed() > timeout {
+            return Err("failed to prepare dataset within timeout, with event"
+                .to_string()
+                .into());
+        }
+    }
+
+    let mut task_watcher = watcher(tasks, ListParams::default())
+        .applied_objects()
+        .boxed();
+
+    // Before the model can be deployed we expect it to be trained and then validated.
+    for target_name in ["training", "mlflow_validation"] {
+        let start = std::time::Instant::now();
+        debug!("waiting for task {target_name}");
+        while let Some(e) = task_watcher.next().await {
+            let task = e?;
+
+            let TaskCfg {
+                name: Some(task_name),
+                ..
+            } = task.spec.cfg
+            else {
+                return Err(format!("{target_name} task was missing a name in the cfg").into());
+            };
+
+            if task_name != target_name {
+                continue;
+            }
+
+            if let Some(
+                TaskStatus {
+                    phase: Some(Phase::Succeeded(_)),
+                },
+                ..,
+            ) = task.status
+            {
+                break;
+            }
+
+            if start.elapsed() > timeout {
+                return Err(format!("failed to start task {} within timeout", task_name).into());
+            }
+        }
+    }
+
+    let mut deployment_watcher = watcher(
+        deployments,
+        ListParams::default().fields(&format!("metadata.name=={model_name}")),
+    )
+    .applied_objects()
+    .boxed();
+
+    debug!("waiting for deployment");
+    let start = std::time::Instant::now();
+    while let Some(e) = deployment_watcher.next().await {
+        let Some(status) = e?.status else {
+            return Err("missing deployment status".into());
+        };
+
+        if let Some(1) = status.ready_replicas {
+            break;
+        }
+
+        if start.elapsed() > timeout {
+            return Err("failed to deploy model within timeout".into());
+        }
+    }
+
+    project_src_ctrl
+        .delete_project_src_for_repo(project_repo)
+        .await?;
+    project_src_ctrl
+        .delete_project_src_for_repo(data_set_repo)
+        .await?;
+    project_src_ctrl
+        .delete_project_src_for_repo(template_repo)
+        .await?;
+
+    Ok(())
+}
diff --git a/lib/tests/snapshots/e2e_test__ame_run_task.snap b/lib/tests/snapshots/e2e_test__ame_run_task.snap
new file mode 100644
index 00000000..8469ef72
--- /dev/null
+++ b/lib/tests/snapshots/e2e_test__ame_run_task.snap
@@ -0,0 +1,38 @@
+---
+source: lib/tests/e2e_test.rs
+expression: "&String::from_utf8(res.get_output().stdout.clone())?"
+---
+Uploading project!
+download: 's3://ame/tasks/redacted/projectfiles/ame.yaml' -> './ame.yaml' (149 bytes in 0.0 seconds, redacted/s)
+download: 's3://ame/tasks/redacted/projectfiles/nn.py' -> './nn.py' (58 bytes in 0.0 seconds, redacted/s)
+download: 's3://ame/tasks/redacted/projectfiles/poetry.lock' -> './poetry.lock' (4587 bytes in 0.0 seconds, redacted/s)
+download: 's3://ame/tasks/redacted/projectfiles/pyproject.toml' -> './pyproject.toml' (297 bytes in 0.0 seconds, redacted/s)
+Installing Python-3.10.9...
+Installed Python-3.10.9 to /home/ame/.pyenv/versions/3.10.9
+The currently activated Python version 3.11.4 is not supported by the project (3.10.9).
+Trying to find and use a compatible version. 
+Using python3 (3.10.9)
+Creating virtualenv test-6gE1vKXj-py3.10 in /home/ame/.cache/pypoetry/virtualenvs
+Installing dependencies from lock file
+
+Package operations: 1 install, 0 updates, 0 removals
+
+  • Installing numpy (1.24.3)
+The currently activated Python version 3.11.4 is not supported by the project (3.10.9).
+Trying to find and use a compatible version. 
+Using python3 (3.10.9)
+np array:  [1 2 3]
+WARNING: Exiting now because of --dry-run
+Uploading artifacts: './ame.yaml'
+'./nn.py'
+'./poetry.lock'
+'./pyproject.toml'
+WARNING: Module python-magic is not available. Guessing MIME types based on file extensions.
+upload: 'ame.yaml' -> 's3://ame/tasks/trainingclzjj/artifacts/ame.yaml' (149 bytes in 0.0 seconds, redacted/s) [1 of 1]
+WARNING: Module python-magic is not available. Guessing MIME types based on file extensions.
+upload: 'nn.py' -> 's3://ame/tasks/trainingclzjj/artifacts/nn.py' (58 bytes in 0.0 seconds, redacted/s) [1 of 1]
+WARNING: Module python-magic is not available. Guessing MIME types based on file extensions.
+upload: 'poetry.lock' -> 's3://ame/tasks/trainingclzjj/artifacts/poetry.lock' (4587 bytes in 0.0 seconds, redacted/s) [1 of 1]
+WARNING: Module python-magic is not available. Guessing MIME types based on file extensions.
+upload: 'pyproject.toml' -> 's3://ame/tasks/trainingclzjj/artifacts/pyproject.toml' (297 bytes in 0.0 seconds, redacted/s) [1 of 1]
+
diff --git a/lib/tests/snapshots/e2e_test__can_list_project_srcs.snap b/lib/tests/snapshots/e2e_test__can_list_project_srcs.snap
new file mode 100644
index 00000000..dc377dd1
--- /dev/null
+++ b/lib/tests/snapshots/e2e_test__can_list_project_srcs.snap
@@ -0,0 +1,8 @@
+---
+source: lib/tests/e2e_test.rs
+expression: "&String::from_utf8(output.get_output().stdout.clone())?"
+---
+Project Sources:
+https://github.com/TeaInSpace/ame-demo.git  
+https://github.com/TeaInSpace/ame-template-demo.git  
+
diff --git a/manifests/controller/base/resources/controller.yaml b/manifests/controller/base/resources/controller.yaml
index fb37ca48..9759e3b8 100644
--- a/manifests/controller/base/resources/controller.yaml
+++ b/manifests/controller/base/resources/controller.yaml
@@ -35,7 +35,7 @@ spec:
             cpu: 10m
             memory: 250Mi
         env:
-          - name: EXECUTOR_IMAGE
+          - name: AME_EXECUTOR_IMAGE
             valueFrom:
               configMapKeyRef:
                 name: ame-controller-configmap
@@ -47,7 +47,7 @@ spec:
           - name: AME_MODEL_INGRESS_HOST
             value: "20.4.90.180.nip.io"
           - name: RUST_LOG
-            value: controller=debug
+            value: info
           - name: AME_MODEL_DEPLOYMENT_INGRESS
             value: |
                 apiVersion: networking.k8s.io/v1
diff --git a/manifests/crd.yaml b/manifests/crd.yaml
index 2b8ac4a2..4f08b954 100644
--- a/manifests/crd.yaml
+++ b/manifests/crd.yaml
@@ -9,7 +9,7 @@ spec:
     kind: Task
     plural: tasks
     shortNames:
-    - task
+    - tk
     singular: task
   scope: Namespaced
   versions:
@@ -21,232 +21,232 @@ spec:
         properties:
           spec:
             properties:
-              dataSet:
+              artifactCfg:
+                nullable: true
+                properties:
+                  paths:
+                    default: []
+                    items:
+                      type: string
+                    type: array
+                  saveChangedFiles:
+                    type: boolean
+                required:
+                - saveChangedFiles
+                type: object
+              dataSets:
                 items:
                   type: string
-                nullable: true
                 type: array
+              deletionApproved:
+                type: boolean
               env:
+                default: []
                 items:
                   properties:
-                    name:
+                    key:
                       type: string
-                    value:
+                    val:
                       type: string
                   required:
-                  - name
-                  - value
+                  - key
+                  - val
                   type: object
-                nullable: true
                 type: array
-              image:
+              executor:
                 nullable: true
-                type: string
+                oneOf:
+                - required:
+                  - poetry
+                - required:
+                  - mlflow
+                - required:
+                  - pipEnv
+                - required:
+                  - pip
+                - required:
+                  - custom
+                properties:
+                  custom:
+                    properties:
+                      command:
+                        type: string
+                      pythonVersion:
+                        type: string
+                    required:
+                    - command
+                    - pythonVersion
+                    type: object
+                  mlflow:
+                    type: object
+                  pip:
+                    properties:
+                      command:
+                        type: string
+                      pythonVersion:
+                        type: string
+                    required:
+                    - command
+                    - pythonVersion
+                    type: object
+                  pipEnv:
+                    properties:
+                      command:
+                        type: string
+                    required:
+                    - command
+                    type: object
+                  poetry:
+                    properties:
+                      command:
+                        type: string
+                      pythonVersion:
+                        type: string
+                    required:
+                    - command
+                    - pythonVersion
+                    type: object
+                type: object
+              fromTemplate:
+                nullable: true
+                properties:
+                  name:
+                    type: string
+                  project:
+                    nullable: true
+                    type: string
+                required:
+                - name
+                type: object
               name:
                 nullable: true
                 type: string
-              pipeline:
-                items:
-                  properties:
-                    env:
-                      items:
-                        properties:
-                          name:
-                            type: string
-                          value:
-                            type: string
-                        required:
-                        - name
-                        - value
-                        type: object
-                      type: array
-                    runcommand:
-                      type: string
-                    secrets:
-                      items:
-                        oneOf:
-                        - required:
-                          - AmeSecret
-                        - required:
-                          - VaultSecret
-                        properties:
-                          AmeSecret:
-                            properties:
-                              envkey:
-                                type: string
-                              name:
-                                type: string
-                            required:
-                            - envkey
-                            - name
-                            type: object
-                          VaultSecret:
-                            properties:
-                              envkey:
-                                type: string
-                              secretKey:
-                                type: string
-                              secretPath:
-                                type: string
-                              vaultName:
-                                type: string
-                            required:
-                            - envkey
-                            - secretKey
-                            - secretPath
-                            - vaultName
-                            type: object
-                        type: object
-                      type: array
-                    taskname:
-                      type: string
-                  required:
-                  - env
-                  - runcommand
-                  - secrets
-                  - taskname
-                  type: object
-                nullable: true
-                type: array
-              projectid:
+              project:
                 nullable: true
                 type: string
               resources:
                 additionalProperties:
-                  description: |-
-                    Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.
-
-                    The serialization format is:
-
-                    <quantity>        ::= <signedNumber><suffix>
-                      (Note that <suffix> may be empty, from the "" case in <decimalSI>.)
-                    <digit>           ::= 0 | 1 | ... | 9 <digits>          ::= <digit> | <digit><digits> <number>          ::= <digits> | <digits>.<digits> | <digits>. | .<digits> <sign>            ::= "+" | "-" <signedNumber>    ::= <number> | <sign><number> <suffix>          ::= <binarySI> | <decimalExponent> | <decimalSI> <binarySI>        ::= Ki | Mi | Gi | Ti | Pi | Ei
-                      (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
-                    <decimalSI>       ::= m | "" | k | M | G | T | P | E
-                      (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)
-                    <decimalExponent> ::= "e" <signedNumber> | "E" <signedNumber>
-
-                    No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.
-
-                    When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.
-
-                    Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:
-                      a. No precision is lost
-                      b. No fractional digits will be emitted
-                      c. The exponent (or suffix) is as large as possible.
-                    The sign will be omitted unless the number is negative.
-
-                    Examples:
-                      1.5 will be serialized as "1500m"
-                      1.5Gi will be serialized as "1536Mi"
-
-                    Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.
-
-                    Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)
-
-                    This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation.
                   type: string
-                nullable: true
+                default: {}
                 type: object
-              runcommand:
-                nullable: true
-                type: string
               secrets:
+                default: []
                 items:
                   oneOf:
                   - required:
-                    - AmeSecret
-                  - required:
-                    - VaultSecret
+                    - ame
                   properties:
-                    AmeSecret:
-                      properties:
-                        envkey:
-                          type: string
-                        name:
-                          type: string
-                      required:
-                      - envkey
-                      - name
-                      type: object
-                    VaultSecret:
+                    ame:
                       properties:
-                        envkey:
-                          type: string
-                        secretKey:
-                          type: string
-                        secretPath:
+                        injectAs:
                           type: string
-                        vaultName:
+                        key:
                           type: string
                       required:
-                      - envkey
-                      - secretKey
-                      - secretPath
-                      - vaultName
+                      - injectAs
+                      - key
                       type: object
                   type: object
-                nullable: true
                 type: array
               source:
                 nullable: true
+                oneOf:
+                - required:
+                  - git
+                - required:
+                  - ame
                 properties:
-                  amestoragepath:
-                    nullable: true
+                  ame:
+                    properties:
+                      path:
+                        type: string
+                    required:
+                    - path
+                    type: object
+                  git:
+                    properties:
+                      reference:
+                        type: string
+                      repository:
+                        type: string
+                      secret:
+                        nullable: true
+                        oneOf:
+                        - required:
+                          - AmeSecret
+                        properties:
+                          AmeSecret:
+                            type: string
+                        type: object
+                      user_name:
+                        type: string
+                    required:
+                    - reference
+                    - repository
+                    - user_name
+                    type: object
+                type: object
+              taskRef:
+                nullable: true
+                properties:
+                  name:
                     type: string
-                  gitreference:
+                  project:
                     nullable: true
                     type: string
-                  gitrepository:
+                required:
+                - name
+                type: object
+              triggers:
+                nullable: true
+                properties:
+                  schedule:
                     nullable: true
                     type: string
                 type: object
-              taskRef:
-                nullable: true
-                type: string
-              taskType:
-                enum:
-                - PipEnv
-                - Mlflow
-                - Poetry
-                nullable: true
-                type: string
-              templateRef:
-                nullable: true
-                type: string
+            required:
+            - deletionApproved
             type: object
           status:
-            description: The status object of `Task`
+            description: 'TODO: should there be an error case?'
             nullable: true
             properties:
-              data_set_tasks:
-                additionalProperties:
-                  properties:
-                    phase:
-                      enum:
-                      - Running
-                      - Pending
-                      - Failed
-                      - Succeeded
-                      - Error
-                      type: string
-                  required:
-                  - phase
-                  type: object
-                nullable: true
-                type: object
               phase:
-                enum:
-                - Running
-                - Pending
-                - Failed
-                - Succeeded
-                - Error
                 nullable: true
-                type: string
-              reason:
-                nullable: true
-                type: string
+                oneOf:
+                - required:
+                  - pending
+                - required:
+                  - running
+                - required:
+                  - failed
+                - required:
+                  - succeeded
+                properties:
+                  failed:
+                    properties:
+                      workflowName:
+                        type: string
+                    required:
+                    - workflowName
+                    type: object
+                  pending:
+                    type: object
+                  running:
+                    properties:
+                      workflowName:
+                        type: string
+                    required:
+                    - workflowName
+                    type: object
+                  succeeded:
+                    properties:
+                      workflowName:
+                        type: string
+                    required:
+                    - workflowName
+                    type: object
+                type: object
             type: object
         required:
         - spec
diff --git a/manifests/data_set_crd.yaml b/manifests/data_set_crd.yaml
index b00b378c..baae9fdb 100644
--- a/manifests/data_set_crd.yaml
+++ b/manifests/data_set_crd.yaml
@@ -21,17 +21,144 @@ spec:
         properties:
           spec:
             properties:
+              deletionApproved:
+                type: boolean
               name:
                 type: string
               path:
                 type: string
+              project:
+                nullable: true
+                type: string
+              size:
+                nullable: true
+                type: string
               task:
                 nullable: true
                 properties:
+                  artifactCfg:
+                    nullable: true
+                    properties:
+                      paths:
+                        default: []
+                        items:
+                          type: string
+                        type: array
+                      saveChangedFiles:
+                        type: boolean
+                    required:
+                    - saveChangedFiles
+                    type: object
+                  dataSets:
+                    items:
+                      type: string
+                    type: array
+                  env:
+                    default: []
+                    items:
+                      properties:
+                        key:
+                          type: string
+                        val:
+                          type: string
+                      required:
+                      - key
+                      - val
+                      type: object
+                    type: array
+                  executor:
+                    nullable: true
+                    oneOf:
+                    - required:
+                      - poetry
+                    - required:
+                      - mlflow
+                    - required:
+                      - pipEnv
+                    - required:
+                      - pip
+                    - required:
+                      - custom
+                    properties:
+                      custom:
+                        properties:
+                          command:
+                            type: string
+                          pythonVersion:
+                            type: string
+                        required:
+                        - command
+                        - pythonVersion
+                        type: object
+                      mlflow:
+                        type: object
+                      pip:
+                        properties:
+                          command:
+                            type: string
+                          pythonVersion:
+                            type: string
+                        required:
+                        - command
+                        - pythonVersion
+                        type: object
+                      pipEnv:
+                        properties:
+                          command:
+                            type: string
+                        required:
+                        - command
+                        type: object
+                      poetry:
+                        properties:
+                          command:
+                            type: string
+                          pythonVersion:
+                            type: string
+                        required:
+                        - command
+                        - pythonVersion
+                        type: object
+                    type: object
+                  fromTemplate:
+                    nullable: true
+                    properties:
+                      name:
+                        type: string
+                      project:
+                        nullable: true
+                        type: string
+                    required:
+                    - name
+                    type: object
                   name:
                     nullable: true
                     type: string
-                  task_ref:
+                  resources:
+                    additionalProperties:
+                      type: string
+                    default: {}
+                    type: object
+                  secrets:
+                    default: []
+                    items:
+                      oneOf:
+                      - required:
+                        - ame
+                      properties:
+                        ame:
+                          properties:
+                            injectAs:
+                              type: string
+                            key:
+                              type: string
+                          required:
+                          - injectAs
+                          - key
+                          type: object
+                      type: object
+                    type: array
+                  taskRef:
                     nullable: true
                     properties:
                       name:
@@ -42,8 +169,16 @@ spec:
                     required:
                     - name
                     type: object
+                  triggers:
+                    nullable: true
+                    properties:
+                      schedule:
+                        nullable: true
+                        type: string
+                    type: object
                 type: object
             required:
+            - deletionApproved
             - name
             - path
             type: object
@@ -51,6 +186,7 @@ spec:
             nullable: true
             properties:
               phase:
+                nullable: true
                 oneOf:
                 - required:
                   - pending
@@ -85,8 +221,6 @@ spec:
                     - task_name
                     type: object
                 type: object
-            required:
-            - phase
             type: object
         required:
         - spec
diff --git a/manifests/project_crd.yaml b/manifests/project_crd.yaml
index 328263b0..bad1897e 100644
--- a/manifests/project_crd.yaml
+++ b/manifests/project_crd.yaml
@@ -28,1487 +28,784 @@ spec:
                       type: string
                     path:
                       type: string
+                    size:
+                      nullable: true
+                      type: string
                     task:
+                      nullable: true
                       properties:
-                        dataSet:
+                        artifactCfg:
+                          nullable: true
+                          properties:
+                            paths:
+                              default: []
+                              items:
+                                type: string
+                              type: array
+                            saveChangedFiles:
+                              type: boolean
+                          required:
+                          - saveChangedFiles
+                          type: object
+                        dataSets:
                           items:
                             type: string
-                          nullable: true
                           type: array
                         env:
+                          default: []
                           items:
                             properties:
-                              name:
+                              key:
                                 type: string
-                              value:
+                              val:
                                 type: string
                             required:
-                            - name
-                            - value
+                            - key
+                            - val
                             type: object
-                          nullable: true
                           type: array
-                        image:
+                        executor:
+                          nullable: true
+                          oneOf:
+                          - required:
+                            - poetry
+                          - required:
+                            - mlflow
+                          - required:
+                            - pipEnv
+                          - required:
+                            - pip
+                          - required:
+                            - custom
+                          properties:
+                            custom:
+                              properties:
+                                command:
+                                  type: string
+                                pythonVersion:
+                                  type: string
+                              required:
+                              - command
+                              - pythonVersion
+                              type: object
+                            mlflow:
+                              type: object
+                            pip:
+                              properties:
+                                command:
+                                  type: string
+                                pythonVersion:
+                                  type: string
+                              required:
+                              - command
+                              - pythonVersion
+                              type: object
+                            pipEnv:
+                              properties:
+                                command:
+                                  type: string
+                              required:
+                              - command
+                              type: object
+                            poetry:
+                              properties:
+                                command:
+                                  type: string
+                                pythonVersion:
+                                  type: string
+                              required:
+                              - command
+                              - pythonVersion
+                              type: object
+                          type: object
+                        fromTemplate:
                           nullable: true
-                          type: string
+                          properties:
+                            name:
+                              type: string
+                            project:
+                              nullable: true
+                              type: string
+                          required:
+                          - name
+                          type: object
                         name:
                           nullable: true
                           type: string
-                        pipeline:
-                          items:
-                            properties:
-                              env:
-                                items:
-                                  properties:
-                                    name:
-                                      type: string
-                                    value:
-                                      type: string
-                                  required:
-                                  - name
-                                  - value
-                                  type: object
-                                type: array
-                              runcommand:
-                                type: string
-                              secrets:
-                                items:
-                                  oneOf:
-                                  - required:
-                                    - AmeSecret
-                                  - required:
-                                    - VaultSecret
-                                  properties:
-                                    AmeSecret:
-                                      properties:
-                                        envkey:
-                                          type: string
-                                        name:
-                                          type: string
-                                      required:
-                                      - envkey
-                                      - name
-                                      type: object
-                                    VaultSecret:
-                                      properties:
-                                        envkey:
-                                          type: string
-                                        secretKey:
-                                          type: string
-                                        secretPath:
-                                          type: string
-                                        vaultName:
-                                          type: string
-                                      required:
-                                      - envkey
-                                      - secretKey
-                                      - secretPath
-                                      - vaultName
-                                      type: object
-                                  type: object
-                                type: array
-                              taskname:
-                                type: string
-                            required:
-                            - env
-                            - runcommand
-                            - secrets
-                            - taskname
-                            type: object
-                          nullable: true
-                          type: array
-                        projectid:
-                          nullable: true
-                          type: string
                         resources:
                           additionalProperties:
-                            description: |-
-                              Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.
-
-                              The serialization format is:
-
-                              <quantity>        ::= <signedNumber><suffix>
-                                (Note that <suffix> may be empty, from the "" case in <decimalSI>.)
-                              <digit>           ::= 0 | 1 | ... | 9 <digits>          ::= <digit> | <digit><digits> <number>          ::= <digits> | <digits>.<digits> | <digits>. | .<digits> <sign>            ::= "+" | "-" <signedNumber>    ::= <number> | <sign><number> <suffix>          ::= <binarySI> | <decimalExponent> | <decimalSI> <binarySI>        ::= Ki | Mi | Gi | Ti | Pi | Ei
-                                (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
-                              <decimalSI>       ::= m | "" | k | M | G | T | P | E
-                                (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)
-                              <decimalExponent> ::= "e" <signedNumber> | "E" <signedNumber>
-
-                              No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.
-
-                              When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.
-
-                              Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:
-                                a. No precision is lost
-                                b. No fractional digits will be emitted
-                                c. The exponent (or suffix) is as large as possible.
-                              The sign will be omitted unless the number is negative.
-
-                              Examples:
-                                1.5 will be serialized as "1500m"
-                                1.5Gi will be serialized as "1536Mi"
-
-                              Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.
-
-                              Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)
-
-                              This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation.
                             type: string
-                          nullable: true
+                          default: {}
                           type: object
-                        runcommand:
-                          nullable: true
-                          type: string
                         secrets:
+                          default: []
                           items:
                             oneOf:
                             - required:
-                              - AmeSecret
-                            - required:
-                              - VaultSecret
+                              - ame
                             properties:
-                              AmeSecret:
+                              ame:
                                 properties:
-                                  envkey:
+                                  injectAs:
                                     type: string
-                                  name:
+                                  key:
                                     type: string
                                 required:
-                                - envkey
-                                - name
-                                type: object
-                              VaultSecret:
-                                properties:
-                                  envkey:
-                                    type: string
-                                  secretKey:
-                                    type: string
-                                  secretPath:
-                                    type: string
-                                  vaultName:
-                                    type: string
-                                required:
-                                - envkey
-                                - secretKey
-                                - secretPath
-                                - vaultName
+                                - injectAs
+                                - key
                                 type: object
                             type: object
-                          nullable: true
                           type: array
-                        source:
+                        taskRef:
                           nullable: true
                           properties:
-                            amestoragepath:
-                              nullable: true
+                            name:
                               type: string
-                            gitreference:
+                            project:
                               nullable: true
                               type: string
-                            gitrepository:
+                          required:
+                          - name
+                          type: object
+                        triggers:
+                          nullable: true
+                          properties:
+                            schedule:
                               nullable: true
                               type: string
                           type: object
-                        taskRef:
-                          nullable: true
-                          type: string
-                        taskType:
-                          enum:
-                          - PipEnv
-                          - Mlflow
-                          - Poetry
-                          nullable: true
-                          type: string
-                        templateRef:
-                          nullable: true
-                          type: string
                       type: object
                   required:
                   - name
                   - path
-                  - task
                   type: object
-                nullable: true
                 type: array
-              logEntry:
+              deletionApproved:
+                type: boolean
+              enableTriggers:
                 nullable: true
-                properties:
-                  contents:
-                    items:
-                      format: uint8
-                      minimum: 0.0
-                      type: integer
-                    type: array
-                required:
-                - contents
-                type: object
+                type: boolean
               models:
                 items:
                   properties:
                     deployment:
                       nullable: true
                       properties:
-                        auto_train:
-                          type: boolean
-                        deploy:
-                          type: boolean
-                        enable_tls:
+                        enableTls:
                           nullable: true
                           type: boolean
                         image:
                           nullable: true
                           type: string
-                        ingress:
-                          description: Ingress is a collection of rules that allow inbound connections to reach the endpoints defined by a backend. An Ingress can be configured to give services externally-reachable urls, load balance traffic, terminate SSL, offer name based virtual hosting etc.
-                          nullable: true
-                          properties:
-                            apiVersion:
-                              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-                              type: string
-                            kind:
-                              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-                              type: string
-                            metadata:
-                              description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
-                              properties:
-                                annotations:
-                                  additionalProperties:
-                                    type: string
-                                  description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
-                                  type: object
-                                clusterName:
-                                  description: The name of the cluster which the object belongs to. This is used to distinguish resources with same name and namespace in different clusters. This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.
-                                  type: string
-                                creationTimestamp:
-                                  description: |-
-                                    CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.
-
-                                    Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
-                                  format: date-time
-                                  type: string
-                                deletionGracePeriodSeconds:
-                                  description: Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.
-                                  format: int64
-                                  type: integer
-                                deletionTimestamp:
-                                  description: |-
-                                    DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This field is set by the server when a graceful deletion is requested by the user, and is not directly settable by a client. The resource is expected to be deleted (no longer visible from resource lists, and not reachable by name) after the time in this field, once the finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. Once the deletionTimestamp is set, this value may not be unset or be set further into the future, although it may be shortened or the resource may be deleted prior to this time. For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react by sending a graceful termination signal to the containers in the pod. After that 30 seconds, the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, remove the pod from the API. In the presence of network partitions, this object may still exist after this timestamp, until an administrator or automated process can determine the resource is fully terminated. If not set, graceful deletion of the object has not been requested.
-
-                                    Populated by the system when a graceful deletion is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
-                                  format: date-time
-                                  type: string
-                                finalizers:
-                                  description: Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order.  Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.
-                                  items:
-                                    type: string
-                                  type: array
-                                generateName:
-                                  description: |-
-                                    GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.
-
-                                    If this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).
-
-                                    Applied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency
-                                  type: string
-                                generation:
-                                  description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.
-                                  format: int64
-                                  type: integer
-                                labels:
-                                  additionalProperties:
-                                    type: string
-                                  description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
-                                  type: object
-                                managedFields:
-                                  description: ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like "ci-cd". The set of fields is always in the version that the workflow used when modifying the object.
-                                  items:
-                                    description: ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.
-                                    properties:
-                                      apiVersion:
-                                        description: APIVersion defines the version of this resource that this field set applies to. The format is "group/version" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.
-                                        type: string
-                                      fieldsType:
-                                        description: 'FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: "FieldsV1"'
-                                        type: string
-                                      fieldsV1:
-                                        description: FieldsV1 holds the first JSON version format as described in the "FieldsV1" type.
-                                        type: object
-                                      manager:
-                                        description: Manager is an identifier of the workflow managing these fields.
-                                        type: string
-                                      operation:
-                                        description: Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.
-                                        type: string
-                                      subresource:
-                                        description: Subresource is the name of the subresource used to update that object, or empty string if the object was updated through the main resource. The value of this field is used to distinguish between managers, even if they share the same name. For example, a status update will be distinct from a regular update using the same manager name. Note that the APIVersion field is not related to the Subresource field and it always corresponds to the version of the main resource.
-                                        type: string
-                                      time:
-                                        description: Time is timestamp of when these fields were set. It should always be empty if Operation is 'Apply'
-                                        format: date-time
-                                        type: string
-                                    type: object
-                                  type: array
-                                name:
-                                  description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
-                                  type: string
-                                namespace:
-                                  description: |-
-                                    Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the "default" namespace, but "default" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.
-
-                                    Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces
-                                  type: string
-                                ownerReferences:
-                                  description: List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.
-                                  items:
-                                    description: OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.
-                                    properties:
-                                      apiVersion:
-                                        description: API version of the referent.
-                                        type: string
-                                      blockOwnerDeletion:
-                                        description: If true, AND if the owner has the "foregroundDeletion" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. Defaults to false. To set this field, a user needs "delete" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.
-                                        type: boolean
-                                      controller:
-                                        description: If true, this reference points to the managing controller.
-                                        type: boolean
-                                      kind:
-                                        description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-                                        type: string
-                                      name:
-                                        description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
-                                        type: string
-                                      uid:
-                                        description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids'
-                                        type: string
-                                    required:
-                                    - apiVersion
-                                    - kind
-                                    - name
-                                    - uid
-                                    type: object
-                                  type: array
-                                resourceVersion:
-                                  description: |-
-                                    An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.
-
-                                    Populated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
-                                  type: string
-                                selfLink:
-                                  description: |-
-                                    SelfLink is a URL representing this object. Populated by the system. Read-only.
-
-                                    DEPRECATED Kubernetes will stop propagating this field in 1.20 release and the field is planned to be removed in 1.21 release.
-                                  type: string
-                                uid:
-                                  description: |-
-                                    UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.
-
-                                    Populated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids
-                                  type: string
-                              type: object
-                            spec:
-                              description: 'Spec is the desired state of the Ingress. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'
-                              properties:
-                                defaultBackend:
-                                  description: DefaultBackend is the backend that should handle requests that don't match any rule. If Rules are not specified, DefaultBackend must be specified. If DefaultBackend is not set, the handling of requests that do not match any of the rules will be up to the Ingress controller.
-                                  properties:
-                                    resource:
-                                      description: Resource is an ObjectRef to another Kubernetes resource in the namespace of the Ingress object. If resource is specified, a service.Name and service.Port must not be specified. This is a mutually exclusive setting with "Service".
-                                      properties:
-                                        apiGroup:
-                                          description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.
-                                          type: string
-                                        kind:
-                                          description: Kind is the type of resource being referenced
-                                          type: string
-                                        name:
-                                          description: Name is the name of resource being referenced
-                                          type: string
-                                      required:
-                                      - kind
-                                      - name
-                                      type: object
-                                    service:
-                                      description: Service references a Service as a Backend. This is a mutually exclusive setting with "Resource".
-                                      properties:
-                                        name:
-                                          description: Name is the referenced service. The service must exist in the same namespace as the Ingress object.
-                                          type: string
-                                        port:
-                                          description: Port of the referenced service. A port name or port number is required for a IngressServiceBackend.
-                                          properties:
-                                            name:
-                                              description: Name is the name of the port on the Service. This is a mutually exclusive setting with "Number".
-                                              type: string
-                                            number:
-                                              description: Number is the numerical port number (e.g. 80) on the Service. This is a mutually exclusive setting with "Name".
-                                              format: int32
-                                              type: integer
-                                          type: object
-                                      required:
-                                      - name
-                                      type: object
-                                  type: object
-                                ingressClassName:
-                                  description: IngressClassName is the name of the IngressClass cluster resource. The associated IngressClass defines which controller will implement the resource. This replaces the deprecated `kubernetes.io/ingress.class` annotation. For backwards compatibility, when that annotation is set, it must be given precedence over this field. The controller may emit a warning if the field and annotation have different values. Implementations of this API should ignore Ingresses without a class specified. An IngressClass resource may be marked as default, which can be used to set a default value for this field. For more information, refer to the IngressClass documentation.
-                                  type: string
-                                rules:
-                                  description: A list of host rules used to configure the Ingress. If unspecified, or no rule matches, all traffic is sent to the default backend.
-                                  items:
-                                    description: IngressRule represents the rules mapping the paths under a specified host to the related backend services. Incoming requests are first evaluated for a host match, then routed to the backend associated with the matching IngressRuleValue.
-                                    properties:
-                                      host:
-                                        description: "Host is the fully qualified domain name of a network host, as defined by RFC 3986. Note the following deviations from the \"host\" part of the URI as defined in RFC 3986: 1. IPs are not allowed. Currently an IngressRuleValue can only apply to\n   the IP in the Spec of the parent Ingress.\n2. The `:` delimiter is not respected because ports are not allowed.\n\t  Currently the port of an Ingress is implicitly :80 for http and\n\t  :443 for https.\nBoth these may change in the future. Incoming requests are matched against the host before the IngressRuleValue. If the host is unspecified, the Ingress routes all traffic based on the specified IngressRuleValue.\n\nHost can be \"precise\" which is a domain name without the terminating dot of a network host (e.g. \"foo.bar.com\") or \"wildcard\", which is a domain name prefixed with a single wildcard label (e.g. \"*.foo.com\"). The wildcard character '*' must appear by itself as the first DNS label and matches only a single label. You cannot have a wildcard label by itself (e.g. Host == \"*\"). Requests will be matched against the Host field in the following way: 1. If Host is precise, the request matches this rule if the http host header is equal to Host. 2. If Host is a wildcard, then the request matches this rule if the http host header is to equal to the suffix (removing the first label) of the wildcard rule."
-                                        type: string
-                                      http:
-                                        description: 'HTTPIngressRuleValue is a list of http selectors pointing to backends. In the example: http://<host>/<path>?<searchpart> -> backend where where parts of the url correspond to RFC 3986, this resource will be used to match against everything after the last ''/'' and before the first ''?'' or ''#''.'
-                                        properties:
-                                          paths:
-                                            description: A collection of paths that map requests to backends.
-                                            items:
-                                              description: HTTPIngressPath associates a path with a backend. Incoming urls matching the path are forwarded to the backend.
-                                              properties:
-                                                backend:
-                                                  description: Backend defines the referenced service endpoint to which the traffic will be forwarded to.
-                                                  properties:
-                                                    resource:
-                                                      description: Resource is an ObjectRef to another Kubernetes resource in the namespace of the Ingress object. If resource is specified, a service.Name and service.Port must not be specified. This is a mutually exclusive setting with "Service".
-                                                      properties:
-                                                        apiGroup:
-                                                          description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.
-                                                          type: string
-                                                        kind:
-                                                          description: Kind is the type of resource being referenced
-                                                          type: string
-                                                        name:
-                                                          description: Name is the name of resource being referenced
-                                                          type: string
-                                                      required:
-                                                      - kind
-                                                      - name
-                                                      type: object
-                                                    service:
-                                                      description: Service references a Service as a Backend. This is a mutually exclusive setting with "Resource".
-                                                      properties:
-                                                        name:
-                                                          description: Name is the referenced service. The service must exist in the same namespace as the Ingress object.
-                                                          type: string
-                                                        port:
-                                                          description: Port of the referenced service. A port name or port number is required for a IngressServiceBackend.
-                                                          properties:
-                                                            name:
-                                                              description: Name is the name of the port on the Service. This is a mutually exclusive setting with "Number".
-                                                              type: string
-                                                            number:
-                                                              description: Number is the numerical port number (e.g. 80) on the Service. This is a mutually exclusive setting with "Name".
-                                                              format: int32
-                                                              type: integer
-                                                          type: object
-                                                      required:
-                                                      - name
-                                                      type: object
-                                                  type: object
-                                                path:
-                                                  description: Path is matched against the path of an incoming request. Currently it can contain characters disallowed from the conventional "path" part of a URL as defined by RFC 3986. Paths must begin with a '/' and must be present when using PathType with value "Exact" or "Prefix".
-                                                  type: string
-                                                pathType:
-                                                  description: |-
-                                                    PathType determines the interpretation of the Path matching. PathType can be one of the following values: * Exact: Matches the URL path exactly. * Prefix: Matches based on a URL path prefix split by '/'. Matching is
-                                                      done on a path element by element basis. A path element refers is the
-                                                      list of labels in the path split by the '/' separator. A request is a
-                                                      match for path p if every p is an element-wise prefix of p of the
-                                                      request path. Note that if the last element of the path is a substring
-                                                      of the last element in request path, it is not a match (e.g. /foo/bar
-                                                      matches /foo/bar/baz, but does not match /foo/barbaz).
-                                                    * ImplementationSpecific: Interpretation of the Path matching is up to
-                                                      the IngressClass. Implementations can treat this as a separate PathType
-                                                      or treat it identically to Prefix or Exact path types.
-                                                    Implementations are required to support all path types.
-                                                  type: string
-                                              required:
-                                              - backend
-                                              - pathType
-                                              type: object
-                                            type: array
-                                        required:
-                                        - paths
-                                        type: object
-                                    type: object
-                                  type: array
-                                tls:
-                                  description: TLS configuration. Currently the Ingress only supports a single TLS port, 443. If multiple members of this list specify different hosts, they will be multiplexed on the same port according to the hostname specified through the SNI TLS extension, if the ingress controller fulfilling the ingress supports SNI.
-                                  items:
-                                    description: IngressTLS describes the transport layer security associated with an Ingress.
-                                    properties:
-                                      hosts:
-                                        description: Hosts are a list of hosts included in the TLS certificate. The values in this list must match the name/s used in the tlsSecret. Defaults to the wildcard host setting for the loadbalancer controller fulfilling this Ingress, if left unspecified.
-                                        items:
-                                          type: string
-                                        type: array
-                                      secretName:
-                                        description: SecretName is the name of the secret used to terminate TLS traffic on port 443. Field is left optional to allow TLS routing based on SNI hostname alone. If the SNI host in a listener conflicts with the "Host" header field used by an IngressRule, the SNI host is used for termination and value of the Host header is used for routing.
-                                        type: string
-                                    type: object
-                                  type: array
-                              type: object
-                            status:
-                              description: 'Status is the current state of the Ingress. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'
-                              properties:
-                                loadBalancer:
-                                  description: LoadBalancer contains the current status of the load-balancer.
-                                  properties:
-                                    ingress:
-                                      description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points.
-                                      items:
-                                        description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.'
-                                        properties:
-                                          hostname:
-                                            description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers)
-                                            type: string
-                                          ip:
-                                            description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers)
-                                            type: string
-                                          ports:
-                                            description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it
-                                            items:
-                                              properties:
-                                                error:
-                                                  description: |-
-                                                    Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use
-                                                      CamelCase names
-                                                    - cloud provider specific error values must have names that comply with the
-                                                      format foo.example.com/CamelCase.
-                                                  type: string
-                                                port:
-                                                  description: Port is the port number of the service port of which status is recorded here
-                                                  format: int32
-                                                  type: integer
-                                                protocol:
-                                                  description: |+
-                                                    Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"
-
-                                                  type: string
-                                              required:
-                                              - port
-                                              - protocol
-                                              type: object
-                                            type: array
-                                        type: object
-                                      type: array
-                                  type: object
-                              type: object
-                          required:
-                          - metadata
-                          type: object
-                        ingress_annotations:
+                        ingressAnnotations:
                           additionalProperties:
                             type: string
-                          nullable: true
+                          default: {}
                           type: object
                         replicas:
                           format: int32
                           nullable: true
                           type: integer
                         resources:
-                          description: ResourceRequirements describes the compute resource requirements.
-                          nullable: true
-                          properties:
-                            limits:
-                              additionalProperties:
-                                description: |-
-                                  Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.
-
-                                  The serialization format is:
-
-                                  <quantity>        ::= <signedNumber><suffix>
-                                    (Note that <suffix> may be empty, from the "" case in <decimalSI>.)
-                                  <digit>           ::= 0 | 1 | ... | 9 <digits>          ::= <digit> | <digit><digits> <number>          ::= <digits> | <digits>.<digits> | <digits>. | .<digits> <sign>            ::= "+" | "-" <signedNumber>    ::= <number> | <sign><number> <suffix>          ::= <binarySI> | <decimalExponent> | <decimalSI> <binarySI>        ::= Ki | Mi | Gi | Ti | Pi | Ei
-                                    (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
-                                  <decimalSI>       ::= m | "" | k | M | G | T | P | E
-                                    (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)
-                                  <decimalExponent> ::= "e" <signedNumber> | "E" <signedNumber>
-
-                                  No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.
-
-                                  When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.
-
-                                  Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:
-                                    a. No precision is lost
-                                    b. No fractional digits will be emitted
-                                    c. The exponent (or suffix) is as large as possible.
-                                  The sign will be omitted unless the number is negative.
-
-                                  Examples:
-                                    1.5 will be serialized as "1500m"
-                                    1.5Gi will be serialized as "1536Mi"
-
-                                  Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.
-
-                                  Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)
-
-                                  This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation.
-                                type: string
-                              description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
-                              type: object
-                            requests:
-                              additionalProperties:
-                                description: |-
-                                  Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.
-
-                                  The serialization format is:
-
-                                  <quantity>        ::= <signedNumber><suffix>
-                                    (Note that <suffix> may be empty, from the "" case in <decimalSI>.)
-                                  <digit>           ::= 0 | 1 | ... | 9 <digits>          ::= <digit> | <digit><digits> <number>          ::= <digits> | <digits>.<digits> | <digits>. | .<digits> <sign>            ::= "+" | "-" <signedNumber>    ::= <number> | <sign><number> <suffix>          ::= <binarySI> | <decimalExponent> | <decimalSI> <binarySI>        ::= Ki | Mi | Gi | Ti | Pi | Ei
-                                    (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
-                                  <decimalSI>       ::= m | "" | k | M | G | T | P | E
-                                    (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)
-                                  <decimalExponent> ::= "e" <signedNumber> | "E" <signedNumber>
-
-                                  No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.
-
-                                  When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.
-
-                                  Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:
-                                    a. No precision is lost
-                                    b. No fractional digits will be emitted
-                                    c. The exponent (or suffix) is as large as possible.
-                                  The sign will be omitted unless the number is negative.
-
-                                  Examples:
-                                    1.5 will be serialized as "1500m"
-                                    1.5Gi will be serialized as "1536Mi"
-
-                                  Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.
-
-                                  Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)
-
-                                  This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation.
-                                type: string
-                              description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
-                              type: object
+                          additionalProperties:
+                            type: string
+                          default: {}
                           type: object
-                      required:
-                      - auto_train
-                      - deploy
                       type: object
-                    modelType:
-                      enum:
-                      - Mlflow
-                      nullable: true
-                      type: string
                     name:
                       type: string
                     training:
+                      nullable: true
                       properties:
-                        schedule:
-                          nullable: true
-                          type: string
                         task:
+                          nullable: true
                           properties:
-                            dataSet:
+                            artifactCfg:
+                              nullable: true
+                              properties:
+                                paths:
+                                  default: []
+                                  items:
+                                    type: string
+                                  type: array
+                                saveChangedFiles:
+                                  type: boolean
+                              required:
+                              - saveChangedFiles
+                              type: object
+                            dataSets:
                               items:
                                 type: string
-                              nullable: true
                               type: array
                             env:
+                              default: []
                               items:
                                 properties:
-                                  name:
+                                  key:
                                     type: string
-                                  value:
+                                  val:
                                     type: string
                                 required:
-                                - name
-                                - value
+                                - key
+                                - val
                                 type: object
-                              nullable: true
                               type: array
-                            image:
-                              nullable: true
-                              type: string
-                            name:
+                            executor:
                               nullable: true
-                              type: string
-                            pipeline:
-                              items:
-                                properties:
-                                  env:
-                                    items:
-                                      properties:
-                                        name:
-                                          type: string
-                                        value:
-                                          type: string
-                                      required:
-                                      - name
-                                      - value
-                                      type: object
-                                    type: array
-                                  runcommand:
-                                    type: string
-                                  secrets:
-                                    items:
-                                      oneOf:
-                                      - required:
-                                        - AmeSecret
-                                      - required:
-                                        - VaultSecret
-                                      properties:
-                                        AmeSecret:
-                                          properties:
-                                            envkey:
-                                              type: string
-                                            name:
-                                              type: string
-                                          required:
-                                          - envkey
-                                          - name
-                                          type: object
-                                        VaultSecret:
-                                          properties:
-                                            envkey:
-                                              type: string
-                                            secretKey:
-                                              type: string
-                                            secretPath:
-                                              type: string
-                                            vaultName:
-                                              type: string
-                                          required:
-                                          - envkey
-                                          - secretKey
-                                          - secretPath
-                                          - vaultName
-                                          type: object
-                                      type: object
-                                    type: array
-                                  taskname:
-                                    type: string
-                                required:
-                                - env
-                                - runcommand
-                                - secrets
-                                - taskname
-                                type: object
+                              oneOf:
+                              - required:
+                                - poetry
+                              - required:
+                                - mlflow
+                              - required:
+                                - pipEnv
+                              - required:
+                                - pip
+                              - required:
+                                - custom
+                              properties:
+                                custom:
+                                  properties:
+                                    command:
+                                      type: string
+                                    pythonVersion:
+                                      type: string
+                                  required:
+                                  - command
+                                  - pythonVersion
+                                  type: object
+                                mlflow:
+                                  type: object
+                                pip:
+                                  properties:
+                                    command:
+                                      type: string
+                                    pythonVersion:
+                                      type: string
+                                  required:
+                                  - command
+                                  - pythonVersion
+                                  type: object
+                                pipEnv:
+                                  properties:
+                                    command:
+                                      type: string
+                                  required:
+                                  - command
+                                  type: object
+                                poetry:
+                                  properties:
+                                    command:
+                                      type: string
+                                    pythonVersion:
+                                      type: string
+                                  required:
+                                  - command
+                                  - pythonVersion
+                                  type: object
+                              type: object
+                            fromTemplate:
                               nullable: true
-                              type: array
-                            projectid:
+                              properties:
+                                name:
+                                  type: string
+                                project:
+                                  nullable: true
+                                  type: string
+                              required:
+                              - name
+                              type: object
+                            name:
                               nullable: true
                               type: string
                             resources:
                               additionalProperties:
-                                description: |-
-                                  Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.
-
-                                  The serialization format is:
-
-                                  <quantity>        ::= <signedNumber><suffix>
-                                    (Note that <suffix> may be empty, from the "" case in <decimalSI>.)
-                                  <digit>           ::= 0 | 1 | ... | 9 <digits>          ::= <digit> | <digit><digits> <number>          ::= <digits> | <digits>.<digits> | <digits>. | .<digits> <sign>            ::= "+" | "-" <signedNumber>    ::= <number> | <sign><number> <suffix>          ::= <binarySI> | <decimalExponent> | <decimalSI> <binarySI>        ::= Ki | Mi | Gi | Ti | Pi | Ei
-                                    (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
-                                  <decimalSI>       ::= m | "" | k | M | G | T | P | E
-                                    (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)
-                                  <decimalExponent> ::= "e" <signedNumber> | "E" <signedNumber>
-
-                                  No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.
-
-                                  When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.
-
-                                  Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:
-                                    a. No precision is lost
-                                    b. No fractional digits will be emitted
-                                    c. The exponent (or suffix) is as large as possible.
-                                  The sign will be omitted unless the number is negative.
-
-                                  Examples:
-                                    1.5 will be serialized as "1500m"
-                                    1.5Gi will be serialized as "1536Mi"
-
-                                  Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.
-
-                                  Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)
-
-                                  This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation.
                                 type: string
-                              nullable: true
+                              default: {}
                               type: object
-                            runcommand:
-                              nullable: true
-                              type: string
                             secrets:
+                              default: []
                               items:
                                 oneOf:
                                 - required:
-                                  - AmeSecret
-                                - required:
-                                  - VaultSecret
+                                  - ame
                                 properties:
-                                  AmeSecret:
-                                    properties:
-                                      envkey:
-                                        type: string
-                                      name:
-                                        type: string
-                                    required:
-                                    - envkey
-                                    - name
-                                    type: object
-                                  VaultSecret:
+                                  ame:
                                     properties:
-                                      envkey:
-                                        type: string
-                                      secretKey:
-                                        type: string
-                                      secretPath:
+                                      injectAs:
                                         type: string
-                                      vaultName:
+                                      key:
                                         type: string
                                     required:
-                                    - envkey
-                                    - secretKey
-                                    - secretPath
-                                    - vaultName
+                                    - injectAs
+                                    - key
                                     type: object
                                 type: object
-                              nullable: true
                               type: array
-                            source:
+                            taskRef:
                               nullable: true
                               properties:
-                                amestoragepath:
-                                  nullable: true
+                                name:
                                   type: string
-                                gitreference:
+                                project:
                                   nullable: true
                                   type: string
-                                gitrepository:
+                              required:
+                              - name
+                              type: object
+                            triggers:
+                              nullable: true
+                              properties:
+                                schedule:
                                   nullable: true
                                   type: string
                               type: object
-                            taskRef:
-                              nullable: true
-                              type: string
-                            taskType:
-                              enum:
-                              - PipEnv
-                              - Mlflow
-                              - Poetry
-                              nullable: true
-                              type: string
-                            templateRef:
-                              nullable: true
-                              type: string
                           type: object
-                      required:
-                      - task
                       type: object
                     validationTask:
                       nullable: true
                       properties:
-                        dataSet:
+                        artifactCfg:
+                          nullable: true
+                          properties:
+                            paths:
+                              default: []
+                              items:
+                                type: string
+                              type: array
+                            saveChangedFiles:
+                              type: boolean
+                          required:
+                          - saveChangedFiles
+                          type: object
+                        dataSets:
                           items:
                             type: string
-                          nullable: true
                           type: array
                         env:
+                          default: []
                           items:
                             properties:
-                              name:
+                              key:
                                 type: string
-                              value:
+                              val:
                                 type: string
                             required:
-                            - name
-                            - value
+                            - key
+                            - val
                             type: object
-                          nullable: true
                           type: array
-                        image:
+                        executor:
+                          nullable: true
+                          oneOf:
+                          - required:
+                            - poetry
+                          - required:
+                            - mlflow
+                          - required:
+                            - pipEnv
+                          - required:
+                            - pip
+                          - required:
+                            - custom
+                          properties:
+                            custom:
+                              properties:
+                                command:
+                                  type: string
+                                pythonVersion:
+                                  type: string
+                              required:
+                              - command
+                              - pythonVersion
+                              type: object
+                            mlflow:
+                              type: object
+                            pip:
+                              properties:
+                                command:
+                                  type: string
+                                pythonVersion:
+                                  type: string
+                              required:
+                              - command
+                              - pythonVersion
+                              type: object
+                            pipEnv:
+                              properties:
+                                command:
+                                  type: string
+                              required:
+                              - command
+                              type: object
+                            poetry:
+                              properties:
+                                command:
+                                  type: string
+                                pythonVersion:
+                                  type: string
+                              required:
+                              - command
+                              - pythonVersion
+                              type: object
+                          type: object
+                        fromTemplate:
                           nullable: true
-                          type: string
+                          properties:
+                            name:
+                              type: string
+                            project:
+                              nullable: true
+                              type: string
+                          required:
+                          - name
+                          type: object
                         name:
                           nullable: true
                           type: string
-                        pipeline:
-                          items:
-                            properties:
-                              env:
-                                items:
-                                  properties:
-                                    name:
-                                      type: string
-                                    value:
-                                      type: string
-                                  required:
-                                  - name
-                                  - value
-                                  type: object
-                                type: array
-                              runcommand:
-                                type: string
-                              secrets:
-                                items:
-                                  oneOf:
-                                  - required:
-                                    - AmeSecret
-                                  - required:
-                                    - VaultSecret
-                                  properties:
-                                    AmeSecret:
-                                      properties:
-                                        envkey:
-                                          type: string
-                                        name:
-                                          type: string
-                                      required:
-                                      - envkey
-                                      - name
-                                      type: object
-                                    VaultSecret:
-                                      properties:
-                                        envkey:
-                                          type: string
-                                        secretKey:
-                                          type: string
-                                        secretPath:
-                                          type: string
-                                        vaultName:
-                                          type: string
-                                      required:
-                                      - envkey
-                                      - secretKey
-                                      - secretPath
-                                      - vaultName
-                                      type: object
-                                  type: object
-                                type: array
-                              taskname:
-                                type: string
-                            required:
-                            - env
-                            - runcommand
-                            - secrets
-                            - taskname
-                            type: object
-                          nullable: true
-                          type: array
-                        projectid:
-                          nullable: true
-                          type: string
                         resources:
                           additionalProperties:
-                            description: |-
-                              Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.
-
-                              The serialization format is:
-
-                              <quantity>        ::= <signedNumber><suffix>
-                                (Note that <suffix> may be empty, from the "" case in <decimalSI>.)
-                              <digit>           ::= 0 | 1 | ... | 9 <digits>          ::= <digit> | <digit><digits> <number>          ::= <digits> | <digits>.<digits> | <digits>. | .<digits> <sign>            ::= "+" | "-" <signedNumber>    ::= <number> | <sign><number> <suffix>          ::= <binarySI> | <decimalExponent> | <decimalSI> <binarySI>        ::= Ki | Mi | Gi | Ti | Pi | Ei
-                                (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
-                              <decimalSI>       ::= m | "" | k | M | G | T | P | E
-                                (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)
-                              <decimalExponent> ::= "e" <signedNumber> | "E" <signedNumber>
-
-                              No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.
-
-                              When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.
-
-                              Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:
-                                a. No precision is lost
-                                b. No fractional digits will be emitted
-                                c. The exponent (or suffix) is as large as possible.
-                              The sign will be omitted unless the number is negative.
-
-                              Examples:
-                                1.5 will be serialized as "1500m"
-                                1.5Gi will be serialized as "1536Mi"
-
-                              Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.
-
-                              Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)
-
-                              This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation.
                             type: string
-                          nullable: true
+                          default: {}
                           type: object
-                        runcommand:
-                          nullable: true
-                          type: string
                         secrets:
+                          default: []
                           items:
                             oneOf:
                             - required:
-                              - AmeSecret
-                            - required:
-                              - VaultSecret
+                              - ame
                             properties:
-                              AmeSecret:
-                                properties:
-                                  envkey:
-                                    type: string
-                                  name:
-                                    type: string
-                                required:
-                                - envkey
-                                - name
-                                type: object
-                              VaultSecret:
+                              ame:
                                 properties:
-                                  envkey:
-                                    type: string
-                                  secretKey:
-                                    type: string
-                                  secretPath:
+                                  injectAs:
                                     type: string
-                                  vaultName:
+                                  key:
                                     type: string
                                 required:
-                                - envkey
-                                - secretKey
-                                - secretPath
-                                - vaultName
+                                - injectAs
+                                - key
                                 type: object
                             type: object
-                          nullable: true
                           type: array
-                        source:
+                        taskRef:
                           nullable: true
                           properties:
-                            amestoragepath:
-                              nullable: true
+                            name:
                               type: string
-                            gitreference:
+                            project:
                               nullable: true
                               type: string
-                            gitrepository:
+                          required:
+                          - name
+                          type: object
+                        triggers:
+                          nullable: true
+                          properties:
+                            schedule:
                               nullable: true
                               type: string
                           type: object
-                        taskRef:
-                          nullable: true
-                          type: string
-                        taskType:
-                          enum:
-                          - PipEnv
-                          - Mlflow
-                          - Poetry
-                          nullable: true
-                          type: string
-                        templateRef:
-                          nullable: true
-                          type: string
                       type: object
                   required:
                   - name
-                  - training
                   type: object
-                nullable: true
                 type: array
-              projectid:
+              name:
                 type: string
               tasks:
+                default: []
                 items:
                   properties:
-                    dataSet:
+                    artifactCfg:
+                      nullable: true
+                      properties:
+                        paths:
+                          default: []
+                          items:
+                            type: string
+                          type: array
+                        saveChangedFiles:
+                          type: boolean
+                      required:
+                      - saveChangedFiles
+                      type: object
+                    dataSets:
                       items:
                         type: string
-                      nullable: true
                       type: array
                     env:
+                      default: []
                       items:
                         properties:
-                          name:
+                          key:
                             type: string
-                          value:
+                          val:
                             type: string
                         required:
-                        - name
-                        - value
+                        - key
+                        - val
                         type: object
-                      nullable: true
                       type: array
-                    image:
+                    executor:
                       nullable: true
-                      type: string
-                    name:
-                      nullable: true
-                      type: string
-                    pipeline:
-                      items:
-                        properties:
-                          env:
-                            items:
-                              properties:
-                                name:
-                                  type: string
-                                value:
-                                  type: string
-                              required:
-                              - name
-                              - value
-                              type: object
-                            type: array
-                          runcommand:
-                            type: string
-                          secrets:
-                            items:
-                              oneOf:
-                              - required:
-                                - AmeSecret
-                              - required:
-                                - VaultSecret
-                              properties:
-                                AmeSecret:
-                                  properties:
-                                    envkey:
-                                      type: string
-                                    name:
-                                      type: string
-                                  required:
-                                  - envkey
-                                  - name
-                                  type: object
-                                VaultSecret:
-                                  properties:
-                                    envkey:
-                                      type: string
-                                    secretKey:
-                                      type: string
-                                    secretPath:
-                                      type: string
-                                    vaultName:
-                                      type: string
-                                  required:
-                                  - envkey
-                                  - secretKey
-                                  - secretPath
-                                  - vaultName
-                                  type: object
-                              type: object
-                            type: array
-                          taskname:
-                            type: string
-                        required:
-                        - env
-                        - runcommand
-                        - secrets
-                        - taskname
-                        type: object
+                      oneOf:
+                      - required:
+                        - poetry
+                      - required:
+                        - mlflow
+                      - required:
+                        - pipEnv
+                      - required:
+                        - pip
+                      - required:
+                        - custom
+                      properties:
+                        custom:
+                          properties:
+                            command:
+                              type: string
+                            pythonVersion:
+                              type: string
+                          required:
+                          - command
+                          - pythonVersion
+                          type: object
+                        mlflow:
+                          type: object
+                        pip:
+                          properties:
+                            command:
+                              type: string
+                            pythonVersion:
+                              type: string
+                          required:
+                          - command
+                          - pythonVersion
+                          type: object
+                        pipEnv:
+                          properties:
+                            command:
+                              type: string
+                          required:
+                          - command
+                          type: object
+                        poetry:
+                          properties:
+                            command:
+                              type: string
+                            pythonVersion:
+                              type: string
+                          required:
+                          - command
+                          - pythonVersion
+                          type: object
+                      type: object
+                    fromTemplate:
                       nullable: true
-                      type: array
-                    projectid:
+                      properties:
+                        name:
+                          type: string
+                        project:
+                          nullable: true
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    name:
                       nullable: true
                       type: string
                     resources:
                       additionalProperties:
-                        description: |-
-                          Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.
-
-                          The serialization format is:
-
-                          <quantity>        ::= <signedNumber><suffix>
-                            (Note that <suffix> may be empty, from the "" case in <decimalSI>.)
-                          <digit>           ::= 0 | 1 | ... | 9 <digits>          ::= <digit> | <digit><digits> <number>          ::= <digits> | <digits>.<digits> | <digits>. | .<digits> <sign>            ::= "+" | "-" <signedNumber>    ::= <number> | <sign><number> <suffix>          ::= <binarySI> | <decimalExponent> | <decimalSI> <binarySI>        ::= Ki | Mi | Gi | Ti | Pi | Ei
-                            (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
-                          <decimalSI>       ::= m | "" | k | M | G | T | P | E
-                            (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)
-                          <decimalExponent> ::= "e" <signedNumber> | "E" <signedNumber>
-
-                          No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.
-
-                          When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.
-
-                          Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:
-                            a. No precision is lost
-                            b. No fractional digits will be emitted
-                            c. The exponent (or suffix) is as large as possible.
-                          The sign will be omitted unless the number is negative.
-
-                          Examples:
-                            1.5 will be serialized as "1500m"
-                            1.5Gi will be serialized as "1536Mi"
-
-                          Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.
-
-                          Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)
-
-                          This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation.
                         type: string
-                      nullable: true
+                      default: {}
                       type: object
-                    runcommand:
-                      nullable: true
-                      type: string
                     secrets:
+                      default: []
                       items:
                         oneOf:
                         - required:
-                          - AmeSecret
-                        - required:
-                          - VaultSecret
+                          - ame
                         properties:
-                          AmeSecret:
+                          ame:
                             properties:
-                              envkey:
+                              injectAs:
                                 type: string
-                              name:
+                              key:
                                 type: string
                             required:
-                            - envkey
-                            - name
-                            type: object
-                          VaultSecret:
-                            properties:
-                              envkey:
-                                type: string
-                              secretKey:
-                                type: string
-                              secretPath:
-                                type: string
-                              vaultName:
-                                type: string
-                            required:
-                            - envkey
-                            - secretKey
-                            - secretPath
-                            - vaultName
+                            - injectAs
+                            - key
                             type: object
                         type: object
-                      nullable: true
                       type: array
-                    source:
+                    taskRef:
                       nullable: true
                       properties:
-                        amestoragepath:
-                          nullable: true
+                        name:
                           type: string
-                        gitreference:
+                        project:
                           nullable: true
                           type: string
-                        gitrepository:
+                      required:
+                      - name
+                      type: object
+                    triggers:
+                      nullable: true
+                      properties:
+                        schedule:
                           nullable: true
                           type: string
                       type: object
-                    taskRef:
-                      nullable: true
-                      type: string
-                    taskType:
-                      enum:
-                      - PipEnv
-                      - Mlflow
-                      - Poetry
-                      nullable: true
-                      type: string
-                    templateRef:
-                      nullable: true
-                      type: string
                   type: object
-                nullable: true
                 type: array
               templates:
                 items:
                   properties:
-                    dataSet:
+                    artifactCfg:
+                      nullable: true
+                      properties:
+                        paths:
+                          default: []
+                          items:
+                            type: string
+                          type: array
+                        saveChangedFiles:
+                          type: boolean
+                      required:
+                      - saveChangedFiles
+                      type: object
+                    dataSets:
                       items:
                         type: string
-                      nullable: true
                       type: array
                     env:
+                      default: []
                       items:
                         properties:
-                          name:
+                          key:
                             type: string
-                          value:
+                          val:
                             type: string
                         required:
-                        - name
-                        - value
+                        - key
+                        - val
                         type: object
-                      nullable: true
                       type: array
-                    image:
-                      nullable: true
-                      type: string
-                    name:
+                    executor:
                       nullable: true
-                      type: string
-                    pipeline:
-                      items:
-                        properties:
-                          env:
-                            items:
-                              properties:
-                                name:
-                                  type: string
-                                value:
-                                  type: string
-                              required:
-                              - name
-                              - value
-                              type: object
-                            type: array
-                          runcommand:
-                            type: string
-                          secrets:
-                            items:
-                              oneOf:
-                              - required:
-                                - AmeSecret
-                              - required:
-                                - VaultSecret
-                              properties:
-                                AmeSecret:
-                                  properties:
-                                    envkey:
-                                      type: string
-                                    name:
-                                      type: string
-                                  required:
-                                  - envkey
-                                  - name
-                                  type: object
-                                VaultSecret:
-                                  properties:
-                                    envkey:
-                                      type: string
-                                    secretKey:
-                                      type: string
-                                    secretPath:
-                                      type: string
-                                    vaultName:
-                                      type: string
-                                  required:
-                                  - envkey
-                                  - secretKey
-                                  - secretPath
-                                  - vaultName
-                                  type: object
-                              type: object
-                            type: array
-                          taskname:
-                            type: string
-                        required:
-                        - env
-                        - runcommand
-                        - secrets
-                        - taskname
-                        type: object
+                      oneOf:
+                      - required:
+                        - poetry
+                      - required:
+                        - mlflow
+                      - required:
+                        - pipEnv
+                      - required:
+                        - pip
+                      - required:
+                        - custom
+                      properties:
+                        custom:
+                          properties:
+                            command:
+                              type: string
+                            pythonVersion:
+                              type: string
+                          required:
+                          - command
+                          - pythonVersion
+                          type: object
+                        mlflow:
+                          type: object
+                        pip:
+                          properties:
+                            command:
+                              type: string
+                            pythonVersion:
+                              type: string
+                          required:
+                          - command
+                          - pythonVersion
+                          type: object
+                        pipEnv:
+                          properties:
+                            command:
+                              type: string
+                          required:
+                          - command
+                          type: object
+                        poetry:
+                          properties:
+                            command:
+                              type: string
+                            pythonVersion:
+                              type: string
+                          required:
+                          - command
+                          - pythonVersion
+                          type: object
+                      type: object
+                    fromTemplate:
                       nullable: true
-                      type: array
-                    projectid:
+                      properties:
+                        name:
+                          type: string
+                        project:
+                          nullable: true
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    name:
                       nullable: true
                       type: string
                     resources:
                       additionalProperties:
-                        description: |-
-                          Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.
-
-                          The serialization format is:
-
-                          <quantity>        ::= <signedNumber><suffix>
-                            (Note that <suffix> may be empty, from the "" case in <decimalSI>.)
-                          <digit>           ::= 0 | 1 | ... | 9 <digits>          ::= <digit> | <digit><digits> <number>          ::= <digits> | <digits>.<digits> | <digits>. | .<digits> <sign>            ::= "+" | "-" <signedNumber>    ::= <number> | <sign><number> <suffix>          ::= <binarySI> | <decimalExponent> | <decimalSI> <binarySI>        ::= Ki | Mi | Gi | Ti | Pi | Ei
-                            (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)
-                          <decimalSI>       ::= m | "" | k | M | G | T | P | E
-                            (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)
-                          <decimalExponent> ::= "e" <signedNumber> | "E" <signedNumber>
-
-                          No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.
-
-                          When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.
-
-                          Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:
-                            a. No precision is lost
-                            b. No fractional digits will be emitted
-                            c. The exponent (or suffix) is as large as possible.
-                          The sign will be omitted unless the number is negative.
-
-                          Examples:
-                            1.5 will be serialized as "1500m"
-                            1.5Gi will be serialized as "1536Mi"
-
-                          Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.
-
-                          Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)
-
-                          This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation.
                         type: string
-                      nullable: true
+                      default: {}
                       type: object
-                    runcommand:
-                      nullable: true
-                      type: string
                     secrets:
+                      default: []
                       items:
                         oneOf:
                         - required:
-                          - AmeSecret
-                        - required:
-                          - VaultSecret
+                          - ame
                         properties:
-                          AmeSecret:
+                          ame:
                             properties:
-                              envkey:
+                              injectAs:
                                 type: string
-                              name:
+                              key:
                                 type: string
                             required:
-                            - envkey
-                            - name
-                            type: object
-                          VaultSecret:
-                            properties:
-                              envkey:
-                                type: string
-                              secretKey:
-                                type: string
-                              secretPath:
-                                type: string
-                              vaultName:
-                                type: string
-                            required:
-                            - envkey
-                            - secretKey
-                            - secretPath
-                            - vaultName
+                            - injectAs
+                            - key
                             type: object
                         type: object
-                      nullable: true
                       type: array
-                    source:
+                    taskRef:
                       nullable: true
                       properties:
-                        amestoragepath:
-                          nullable: true
+                        name:
                           type: string
-                        gitreference:
+                        project:
                           nullable: true
                           type: string
-                        gitrepository:
+                      required:
+                      - name
+                      type: object
+                    triggers:
+                      nullable: true
+                      properties:
+                        schedule:
                           nullable: true
                           type: string
                       type: object
-                    taskRef:
-                      nullable: true
-                      type: string
-                    taskType:
-                      enum:
-                      - PipEnv
-                      - Mlflow
-                      - Poetry
-                      nullable: true
-                      type: string
-                    templateRef:
-                      nullable: true
-                      type: string
                   type: object
-                nullable: true
                 type: array
             required:
-            - projectid
+            - deletionApproved
+            - name
             type: object
           status:
             nullable: true
@@ -1516,69 +813,13 @@ spec:
               models:
                 additionalProperties:
                   properties:
-                    lastDeployed:
-                      description: Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON.  Wrappers are provided for many of the factory methods that the time package offers.
-                      format: date-time
-                      nullable: true
-                      type: string
-                    lastTrained:
-                      description: Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON.  Wrappers are provided for many of the factory methods that the time package offers.
-                      format: date-time
-                      nullable: true
-                      type: string
-                    latestModelVersion:
-                      description: Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON.  Wrappers are provided for many of the factory methods that the time package offers.
-                      format: date-time
-                      nullable: true
-                      type: string
-                    latestValidModelVersion:
-                      nullable: true
-                      type: string
                     latestValidatedModelVersion:
                       nullable: true
                       type: string
-                    validation:
-                      nullable: true
-                      oneOf:
-                      - required:
-                        - unValidated
-                      - required:
-                        - validating
-                      - required:
-                        - failedValidation
-                      - required:
-                        - validated
-                      properties:
-                        failedValidation:
-                          properties:
-                            model_version:
-                              type: string
-                          required:
-                          - model_version
-                          type: object
-                        unValidated:
-                          type: object
-                        validated:
-                          properties:
-                            model_version:
-                              type: string
-                          required:
-                          - model_version
-                          type: object
-                        validating:
-                          properties:
-                            model_version:
-                              type: string
-                            task:
-                              type: string
-                          required:
-                          - model_version
-                          - task
-                          type: object
-                      type: object
                   type: object
-                nullable: true
                 type: object
+            required:
+            - models
             type: object
         required:
         - spec
diff --git a/manifests/project_src_crd.yaml b/manifests/project_src_crd.yaml
index ca5dd78a..6f1099a1 100644
--- a/manifests/project_src_crd.yaml
+++ b/manifests/project_src_crd.yaml
@@ -29,7 +29,7 @@ spec:
                   secret:
                     nullable: true
                     type: string
-                  sync_interval:
+                  syncInterval:
                     nullable: true
                     type: string
                   username:
@@ -48,14 +48,14 @@ spec:
                     explanation:
                       nullable: true
                       type: string
-                    issue_type:
+                    issueType:
                       format: int32
                       type: integer
                   required:
-                  - issue_type
+                  - issueType
                   type: object
                 type: array
-              last_synced:
+              lastSynced:
                 nullable: true
                 type: string
               reason:
diff --git a/rustfmt.toml b/rustfmt.toml
new file mode 100644
index 00000000..d3db3454
--- /dev/null
+++ b/rustfmt.toml
@@ -0,0 +1 @@
+imports_granularity="Crate"
\ No newline at end of file
diff --git a/service/Cargo.toml b/service/Cargo.toml
index 9425ad16..3d447f9d 100644
--- a/service/Cargo.toml
+++ b/service/Cargo.toml
@@ -12,31 +12,32 @@ tonic = "0.8"
 prost = "0.11"
 futures-core = "0.3"
 futures-util = "0.3"
-tokio = { version = "1.0", features = ["rt-multi-thread", "macros", "sync", "time"] }
+tokio = { version = "1.29", features = ["rt-multi-thread", "macros", "sync", "time"] }
 tokio-stream = "0.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 k8s-openapi = { version = "0.17.0", features = ["v1_23", "schemars"] }
 kube = { version = "0.80.0", features = ["runtime", "client" ] }
-thiserror = "1.0.37"
+thiserror = "1.0.44"
 tonic-health = "0.7.1"
-log = "0.4.17"
+log = "0.4.19"
 tower =  { version = "0.4.13", features = ["filter"] }
-hyper = "0.14.23"
-tower-http = { version = "0.3.4", features = ["compression-full", "decompression-full", "sensitive-headers", "set-header"] }
+hyper = "0.14.27"
+tower-http = { version = "0.3.5", features = ["compression-full", "decompression-full", "sensitive-headers", "set-header"] }
 tracing = { version = "0.1.37", features = ["log-always", "log"] }
-tracing-subscriber  = {version = "0.3.16", features = ["env-filter"] }
-tracing-attributes = "0.1.23"
-url = "2.3.1"
+tracing-subscriber  = {version = "0.3.17", features = ["env-filter"] }
+tracing-attributes = "0.1.26"
+url = "2.4.0"
 rust-s3 = "0.32.3"
-async-trait = "0.1.59"
+async-trait = "0.1.72"
 envconfig = "0.10.0"
 tonic-web = "0.4.0"
 ame = { version = "0.1.0", path = "../lib", features = ["ame-control"] }
+either = "1.9.0"
 
 [build-dependencies]
 tonic-build = "0.8"
-prost-build = "0.11.2"
+prost-build = "0.11.9"
 serde = "1.0"
 
 [dev-dependencies]
diff --git a/service/src/ameservice.rs b/service/src/ameservice.rs
index e126c4f8..632bbefe 100644
--- a/service/src/ameservice.rs
+++ b/service/src/ameservice.rs
@@ -1,33 +1,38 @@
 use crate::{Error, Result};
 
-use ame::custom_resources::project::{self, Project};
-use ame::custom_resources::project_source_ctrl::ProjectSrcCtrl;
-use ame::custom_resources::secrets::SecretCtrl;
-use ame::custom_resources::task::{Task, TaskPhase};
-use ame::error::AmeError;
+use ame::{
+    custom_resources::{
+        new_task::{self, Task, TaskBuilder},
+        project::{self, Project},
+        project_source_ctrl::ProjectSrcCtrl,
+        secrets::SecretCtrl,
+        task_ctrl::approve_deletion,
+    },
+    error::AmeError,
+};
+use either::Either;
 use futures_util::StreamExt;
 use k8s_openapi::api::core::v1::Pod;
-use kube::api::{DeleteParams, ListParams, LogParams, PostParams};
-use kube::runtime::wait::await_condition;
-use kube::runtime::wait::conditions;
+use kube::{
+    api::{DeleteParams, ListParams, LogParams, PostParams},
+    runtime::wait::{await_condition, conditions},
+};
 
-use kube::{Api, Client, ResourceExt};
-use tokio::sync::mpsc;
-use tokio::task::JoinHandle;
+use kube::{Api, Client, Resource, ResourceExt};
+use tokio::{sync::mpsc, task::JoinHandle};
 use tokio_stream::wrappers::ReceiverStream;
 
-use ame::grpc::ame_service_server::AmeService;
-use ame::grpc::project_file_chunk::Messages;
+use ame::grpc::{ame_service_server::AmeService, project_file_chunk::Messages};
 
 use ame::grpc::*;
 
 use crate::storage::{AmeFile, ObjectStorage, S3Config, S3StorageDriver};
-use tracing::{debug, info, instrument};
+use tracing::{debug, instrument};
 
 use ame::ctrl::AmeKubeResourceCtrl;
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 use tokio::time::{sleep, Duration};
-use tonic::{Request, Response, Status, Streaming};
+use tonic::{Code, Request, Response, Status, Streaming};
 
 #[derive(Debug, Clone)]
 pub struct AmeServiceConfig {
@@ -43,6 +48,7 @@ pub struct Service {
     pods: Arc<Api<Pod>>,
     secret_ctrl: Arc<SecretCtrl>,
     project_src_ctrl: Arc<ProjectSrcCtrl>,
+    new_tasks: Arc<Api<new_task::Task>>,
 }
 
 #[tonic::async_trait]
@@ -50,51 +56,99 @@ impl AmeService for Service {
     async fn train_model(&self, request: Request<TrainRequest>) -> Result<Response<Empty>, Status> {
         let tr = request.into_inner();
 
-        let Some(project) = self.projects.list(&ListParams::default()).await.unwrap().into_iter().find(|p| p.spec.id == tr.projectid) else {
-            return Err(Status::from_error(Box::new(Error::MissingModel(tr.model_name.clone()))));
-        };
-
-        self.tasks
-            .create(
-                &PostParams::default(),
-                &project
-                    .generate_model_training_task(&tr.model_name)
-                    .unwrap(),
-            )
+        let Some(_project) = self
+            .projects
+            .list(&ListParams::default())
             .await
-            .unwrap();
+            .unwrap()
+            .into_iter()
+            .find(|p| p.spec.cfg.name == tr.projectid)
+        else {
+            return Err(Status::from_error(Box::new(Error::MissingModel(
+                tr.model_name.clone(),
+            ))));
+        };
 
-        Ok(Response::new(Empty {}))
+        todo!();
     }
 
     async fn get_task(
         &self,
         request: Request<TaskIdentifier>,
-    ) -> Result<Response<TaskTemplate>, Status> {
+    ) -> Result<Response<TaskCfg>, Status> {
         let task = self
             .tasks
             .get(&request.get_ref().name)
             .await
             .map_err(|e| Status::from_error(Box::new(e)))?;
 
-        Ok(Response::new(TaskTemplate::from(task)))
+        Ok(Response::new(task.spec.cfg))
     }
 
-    async fn create_task(
+    async fn run_task(
         &self,
-        request: Request<CreateTaskRequest>,
+        request: Request<RunTaskRequest>,
     ) -> Result<Response<TaskIdentifier>, Status> {
+        let RunTaskRequest {
+            project_id: Some(ref project_id),
+            task_cfg: Some(task_cfg),
+        } = request.into_inner()
+        else {
+            todo!();
+        };
+
+        let parent_project = self.projects.get(&project_id.name).await.unwrap();
+        let mut oref = parent_project.controller_owner_ref(&()).unwrap();
+        oref.controller = Some(false);
+
+        let mut task_builder = TaskBuilder::from_cfg(task_cfg.clone());
+        let task = task_builder
+            .set_project(project_id.name.clone())
+            .add_owner_reference(oref)
+            .set_name(format!(
+                "{}{}local",
+                project_id.name,
+                task_cfg.name.unwrap()
+            ))
+            .clone()
+            .build();
+
         let task_in_cluster = self
-            .tasks
-            .create(
-                &PostParams::default(),
-                &Task::try_from(request.into_inner())
-                    .map_err(|e| Status::from_error(Box::new(e)))?,
-            )
+            .new_tasks
+            .create(&PostParams::default(), &task)
             .await
             .map_err(|e| Status::from_error(Box::new(e)))?;
 
-        Ok(Response::new(TaskIdentifier::from(task_in_cluster)))
+        Ok(Response::new(TaskIdentifier {
+            name: task_in_cluster.name_any(),
+        }))
+    }
+
+    async fn list_tasks(
+        &self,
+        _request: Request<ListTasksRequest>,
+    ) -> Result<Response<ListTasksResponse>, Status> {
+        let tasks = self
+            .new_tasks
+            .list(&ListParams::default())
+            .await
+            .unwrap()
+            .items;
+
+        let mut task_statuses: HashMap<String, TaskListEntry> = HashMap::new();
+
+        for task in tasks {
+            let entry = TaskListEntry {
+                status: Some(task.clone().status.unwrap_or_default()),
+                time_stamp: serde_json::json!(task.creation_timestamp().unwrap()).to_string(),
+            };
+
+            task_statuses.insert(task.name_any(), entry);
+        }
+
+        Ok(Response::new(ListTasksResponse {
+            tasks: task_statuses,
+        }))
     }
 
     async fn delete_task(
@@ -115,8 +169,12 @@ impl AmeService for Service {
         request: Request<TaskProjectDirectoryStructure>,
     ) -> Result<Response<Empty>, Status> {
         let structure = request.into_inner();
-        let TaskProjectDirectoryStructure{taskid: Some(task_id), ..} = structure.clone() else {
-            return Err(Status::invalid_argument("missing Task identifier"))
+        let TaskProjectDirectoryStructure {
+            taskid: Some(task_id),
+            ..
+        } = structure.clone()
+        else {
+            return Err(Status::invalid_argument("missing Task identifier"));
         };
 
         self.storage
@@ -128,6 +186,7 @@ impl AmeService for Service {
             )
     }
 
+    #[instrument]
     async fn upload_project_file(
         &self,
         request: Request<Streaming<ProjectFileChunk>>,
@@ -161,7 +220,7 @@ impl AmeService for Service {
             }
         }
 
-        let Some(task_id) =  task_id_option  else {
+        let Some(task_id) = task_id_option else {
             return Err(Status::invalid_argument("missing TaskIdentifier in stream"));
         };
 
@@ -172,6 +231,7 @@ impl AmeService for Service {
         }
 
         //TODO: is an empty file valid?
+        debug!("uploading file: {}", file.key);
 
         self.storage
             .write_task_project_file(&task_id, file)
@@ -191,76 +251,119 @@ impl AmeService for Service {
         let (log_sender, log_receiver) = mpsc::channel(1);
         let task_name = request.get_ref().clone().taskid.unwrap().name;
         let pods = self.pods.clone();
-        let tasks = self.tasks.clone();
+        let tasks = self.new_tasks.clone();
 
-        tracing::info!("streaming logs for: {}", &task_name);
+        debug!("streaming logs for: {}", &task_name);
         let _handle: JoinHandle<Result<()>> = tokio::spawn(async move {
+            let mut seen_pods: Vec<String> = vec![];
             loop {
+                // TODO: move away from polling
+                sleep(Duration::from_secs(5)).await;
                 let task_pods = pods
                     .list(&ListParams::default().labels(&format!("ame-task={task_name}")))
                     .await
                     .unwrap();
 
                 // TODO: ensure the pod is actually done before  giving up on logging.
-                if !task_pods.items.is_empty() {
-                    let pod = &task_pods.items[0];
-                    info!("found pod {} for task {}", &pod.name_any(), &task_name);
-
-                    for pod in &task_pods.items {
-                        let status = pods.get_status(&pod.name_any()).await.unwrap();
-                        if status.status.unwrap().phase.unwrap() != "Running" {
-                            continue;
-                        }
+                for pod in &task_pods.items {
+                    debug!("found pod {} for task {}", &pod.name_any(), &task_name,);
+
+                    if seen_pods.contains(&pod.name_any()) {
+                        continue;
+                    }
+                    seen_pods.push(pod.name_any());
+
+                    let status = pods.get_status(&pod.name_any()).await.unwrap_or_default();
+
+                    let phase = status.status.unwrap_or_default().phase.unwrap_or_default();
+                    let mut log_params = LogParams::default();
+                    log_params.container = Some("main".to_string());
+
+                    if phase == "Failed" || phase == "Succeeded" {
+                        let logs = pods.logs(&pod.name_any(), &log_params).await?;
+                        debug!("sending full logs");
+                        let log_entry = LogEntry {
+                            contents: logs.into_bytes(),
+                        };
+                        log_sender
+                            .send(Ok(log_entry))
+                            .await
+                            .or(Err(Error::TokioSendError(format!(
+                                "failed to send log entry: {}",
+                                pod.name_any()
+                            ))))?;
+
+                        continue;
+                    }
+
+                    debug!("waiting for pod to start running {}", pod.name_any());
+                    // TODO: we end up waiting here in cases where we shouldn't. After the task has already completed.
+                    if let Err(_e) = tokio::time::timeout(
+                        std::time::Duration::from_secs(2),
                         await_condition(
                             Api::<Pod>::clone(&pods),
                             &pod.name_any(),
                             conditions::is_pod_running(),
+                        ),
+                    )
+                    .await
+                    {
+                        debug!(
+                            "failed to wait for running pod within timeout {}",
+                            pod.name_any()
+                        )
+                    };
+
+                    let mut pod_log_stream = pods
+                        .log_stream(
+                            &pod.name_any(),
+                            &LogParams {
+                                container: Some("main".to_string()),
+                                follow: true,
+                                since_seconds: Some(1),
+
+                                ..LogParams::default()
+                            },
                         )
                         .await?;
 
-                        info!("pod is running!");
-
-                        let mut pod_log_stream = pods
-                            .log_stream(
-                                &pod.name_any(),
-                                &LogParams {
-                                    container: Some("main".to_string()),
-                                    follow: true,
-                                    since_seconds: Some(1),
-
-                                    ..LogParams::default()
-                                },
-                            )
-                            .await?;
-
-                        while let Some(e) = pod_log_stream.next().await {
-                            debug!("sent log entry: {:?}", &e);
-                            let log_entry = LogEntry {
-                                contents: e.unwrap().to_vec(),
-                            };
-                            log_sender.send(Ok(log_entry.clone())).await.or(Err(
-                                Error::TokioSendError(format!(
-                                    "failed to send log entry: {}",
-                                    String::from_utf8(log_entry.contents)?
-                                )),
-                            ))?
-                        }
+                    debug!("Streaming logs for {}", pod.name_any());
+                    while let Some(e) = pod_log_stream.next().await {
+                        let log_entry = LogEntry {
+                            contents: e.unwrap().to_vec(),
+                        };
+                        log_sender.send(Ok(log_entry.clone())).await.or(Err(
+                            Error::TokioSendError(format!(
+                                "failed to send log entry: {}",
+                                String::from_utf8(log_entry.contents)?
+                            )),
+                        ))?
                     }
+
+                    debug!("finished streaming logs for {}", pod.name_any());
                 }
 
-                let Ok(task) = tasks
-                    .get_status(&task_name)
-                    .await else {
-                        sleep(Duration::from_secs(1)).await;
-                        continue;
-                    };
+                let Ok(task) = tasks.get_status(&task_name).await else {
+                    sleep(Duration::from_secs(1)).await;
+                    continue;
+                };
+
+                debug!("checking task status {} {:?}", task.name_any(), task.status);
 
-                if task.status.clone().unwrap().phase.unwrap() != TaskPhase::Running
-                    && task.status.unwrap().phase.unwrap() != TaskPhase::Pending
+                match task
+                    .status
+                    .clone()
+                    .unwrap_or_default()
+                    .phase
+                    .unwrap_or(task_status::Phase::Pending(TaskPhasePending {}))
                 {
-                    return Ok(());
+                    task_status::Phase::Failed(TaskPhaseFailed { .. })
+                    | task_status::Phase::Succeeded(TaskPhaseSucceeded { .. }) => {
+                        debug!("task has finished, stopping stream");
+                        return Ok(());
+                    }
+                    _ => (),
                 }
-
                 sleep(Duration::from_millis(50)).await;
             }
         });
@@ -268,6 +371,69 @@ impl AmeService for Service {
         Ok(Response::new(ReceiverStream::new(log_receiver)))
     }
 
+    #[instrument]
+    async fn remove_task(
+        &self,
+        request: Request<RemoveTaskRequest>,
+    ) -> Result<Response<Empty>, Status> {
+        let RemoveTaskRequest { name, approve } = request.into_inner();
+
+        if approve.unwrap_or(false) {
+            approve_deletion(&self.tasks, &name).await?;
+        }
+
+        let res = self
+            .tasks
+            .delete(&name, &DeleteParams::default())
+            .await
+            .map_err(AmeError::KubeApi)?;
+
+        match res {
+            Either::Left(t) => {
+                if t.spec.deletion_approved {
+                    return Ok(Response::new(Empty {}));
+                }
+
+                return Err(Status::new(
+                    Code::FailedPrecondition,
+                    "Task is not approved for deletion",
+                ));
+            }
+            Either::Right(e) => {
+                if e.is_failure() {
+                    return Err(Status::failed_precondition("sfsd"));
+                }
+            }
+        };
+
+        Ok(Response::new(Empty {}))
+    }
+
+    #[instrument]
+    async fn create_project(
+        &self,
+        request: Request<CreateProjectRequest>,
+    ) -> Result<Response<ProjectId>, Status> {
+        let request = request.into_inner();
+
+        let Some(cfg) = request.cfg else {
+            return Err(AmeError::MissingProjectcfg.into());
+        };
+
+        let mut project = Project::from_cfg(cfg);
+
+        project.spec.enable_triggers = Some(request.enable_triggers.unwrap_or(false));
+
+        let name = self
+            .projects
+            .create(&PostParams::default(), &project)
+            .await
+            .map_err(AmeError::KubeApi)?
+            .name_any();
+
+        Ok(Response::new(ProjectId { name }))
+    }
+
     #[instrument]
     async fn create_project_src(
         &self,
@@ -451,12 +617,14 @@ impl Service {
         let client = Client::try_default().await?;
         let target_namespace = "ame-system";
         let tasks = Api::<Task>::namespaced(client.clone(), target_namespace);
+        let new_tasks = Api::<new_task::Task>::namespaced(client.clone(), target_namespace);
         let pods = Api::<Pod>::namespaced(client.clone(), target_namespace);
         let projects = Api::<Project>::namespaced(client.clone(), target_namespace);
 
         let task_service = Service {
             tasks: Arc::new(tasks),
             pods: Arc::new(pods),
+            new_tasks: Arc::new(new_tasks),
             projects: Arc::new(projects),
             storage: Arc::new(ObjectStorage::<S3StorageDriver>::new_s3_storage(
                 &cfg.bucket,
diff --git a/service/src/lib.rs b/service/src/lib.rs
index d01e095f..b4c84097 100644
--- a/service/src/lib.rs
+++ b/service/src/lib.rs
@@ -1,14 +1,10 @@
 pub mod ameservice;
 pub mod storage;
 
-use ame::custom_resources::task::Task;
-use kube::api::ListParams;
-use kube::runtime::wait;
-use kube::{Api, Client};
+use ame::custom_resources::new_task::Task;
+use kube::{api::ListParams, runtime::wait, Api, Client};
 use s3::{creds::error::CredentialsError, error::S3Error};
-use std::net::AddrParseError;
-use std::string::FromUtf8Error;
-use std::time::Duration;
+use std::{net::AddrParseError, string::FromUtf8Error, time::Duration};
 use storage::S3Config;
 use thiserror::Error;
 
@@ -61,6 +57,9 @@ pub enum Error {
 
     #[error("Failed to find project source for repository: {0}")]
     MissingProjectSrc(String),
+
+    #[error("Project config is missing from request")]
+    MissingProjectCfg,
 }
 pub type Result<T, E = Error> = std::result::Result<T, E>;
 
@@ -90,16 +89,20 @@ pub async fn health_check<S: NamedService>(
             reporter.set_not_serving::<S>().await;
         }
 
-        tokio::time::sleep(Duration::from_secs(5)).await;
+        tokio::time::sleep(Duration::from_secs(50)).await;
     }
 }
 
 #[cfg(test)]
 mod test {
-    use super::storage::{AmeFile, ObjectStorage, ObjectStorageDriver, S3Config, S3StorageDriver};
-    use super::Result;
-    use ame::custom_resources::common::find_service_endpoint;
-    use ame::grpc::{TaskIdentifier, TaskProjectDirectoryStructure};
+    use super::{
+        storage::{AmeFile, ObjectStorage, ObjectStorageDriver, S3Config, S3StorageDriver},
+        Result,
+    };
+    use ame::{
+        custom_resources::common::find_service_endpoint,
+        grpc::{TaskIdentifier, TaskProjectDirectoryStructure},
+    };
     use serial_test::serial;
 
     async fn get_fresh_storage() -> Result<ObjectStorage<S3StorageDriver>> {
diff --git a/service/src/server.rs b/service/src/server.rs
index b79c42a7..7311823a 100644
--- a/service/src/server.rs
+++ b/service/src/server.rs
@@ -1,10 +1,11 @@
 use ame::grpc::ame_service_server::AmeServiceServer;
 use envconfig::Envconfig;
-use service::ameservice::AmeServiceConfig;
-use service::ameservice::Service;
-use service::health_check;
-use service::storage::S3Config;
-use service::Result;
+use service::{
+    ameservice::{AmeServiceConfig, Service},
+    health_check,
+    storage::S3Config,
+    Result,
+};
 use tonic::transport::Server;
 
 use tracing_subscriber::{prelude::*, EnvFilter, Registry};
@@ -47,210 +48,3 @@ async fn main() -> Result<()> {
 
     Ok(())
 }
-
-#[cfg(test)]
-mod test {
-    use std::time::Duration;
-
-    use ame::custom_resources::common::{find_service_endpoint, setup_cluster};
-    use kube::api::PostParams;
-    use kube::ResourceExt;
-    use kube::{Api, Client};
-
-    use ame::grpc::{CreateTaskRequest, TaskIdentifier, TaskTemplate};
-    use tokio::net::TcpListener;
-    use tonic::transport::Channel;
-    use tonic::Request;
-
-    use super::AmeServiceConfig;
-    use super::*;
-
-    use ame::custom_resources::task::Task;
-    use ame::grpc::ame_service_client::AmeServiceClient;
-    use serial_test::serial;
-    use service::storage::S3Config;
-    use service::Result;
-
-    async fn start_server() -> Result<AmeServiceClient<Channel>> {
-        setup_cluster("ame-system").await.unwrap();
-
-        // Request a free port from the os.
-        let addr = TcpListener::bind("127.0.0.1:0")
-            .await
-            .unwrap()
-            .local_addr()
-            .unwrap();
-
-        let s3config = S3Config {
-            region: "eu-central-1".to_string(),
-            endpoint: find_service_endpoint("ame-system", "ame-minio")
-                .await
-                .unwrap(),
-            access_id: "minio".to_string(),
-            secret: "minio123".to_string(),
-        };
-
-        let svc = Service::try_init(AmeServiceConfig {
-            s3config,
-            bucket: "ame".to_string(),
-        })
-        .await?;
-        tokio::spawn(
-            Server::builder()
-                .add_service(AmeServiceServer::new(svc))
-                .serve(addr),
-        );
-
-        // The server needs time to start serving requests.
-        for _ in 0..2 {
-            if let Ok(client) = AmeServiceClient::connect(format!("http://{addr}")).await {
-                return Ok(client);
-            }
-
-            tokio::time::sleep(Duration::from_millis(100)).await;
-        }
-
-        panic!("failed to start AME's server");
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn can_get_task() -> Result<()> {
-        let mut service_client = start_server().await?;
-        let client = Client::try_default().await?;
-        let tasks = Api::<Task>::namespaced(client, "ame-system");
-
-        let task = TaskTemplate {
-            command: "test".to_string(),
-            projectid: "myproject".to_string(),
-            ..TaskTemplate::default()
-        };
-
-        let task_in_cluster = tasks
-            .create(&PostParams::default(), &Task::from(task.clone()))
-            .await?;
-
-        let task_identifier = TaskIdentifier {
-            name: task_in_cluster.name_any(),
-        };
-
-        let new_task = service_client
-            .get_task(Request::new(task_identifier))
-            .await?;
-
-        assert_eq!(new_task.get_ref(), &task);
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn can_create_task() -> Result<()> {
-        let mut service_client = start_server().await?;
-        let client = Client::try_default().await?;
-        let tasks = Api::<Task>::namespaced(client, "ame-system");
-
-        let task = TaskTemplate {
-            command: "test".to_string(),
-            projectid: "myproject".to_string(),
-            ..TaskTemplate::default()
-        };
-
-        let create_task_req = CreateTaskRequest {
-            id: Some(TaskIdentifier {
-                name: "mytask".to_string(),
-            }),
-            template: Some(task.clone()),
-        };
-
-        let new_task = service_client
-            .create_task(Request::new(create_task_req.clone()))
-            .await?;
-        let task_in_cluster = tasks.get(&new_task.get_ref().name).await?;
-
-        assert_eq!(TaskTemplate::from(task_in_cluster), task);
-
-        service_client
-            .delete_task(Request::new(create_task_req.id.unwrap()))
-            .await?;
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn can_create_task_with_image_override() -> Result<()> {
-        let mut service_client = start_server().await?;
-        let client = Client::try_default().await?;
-        let tasks = Api::<Task>::namespaced(client, "ame-system");
-
-        let task = TaskTemplate {
-            name: "template_name".to_string(),
-            command: "test".to_string(),
-            projectid: "myproject".to_string(),
-            image: Some("my-new-image".to_string()),
-            task_type: None,
-        };
-
-        let create_task_req = CreateTaskRequest {
-            id: Some(TaskIdentifier {
-                name: "mytask2".to_string(),
-            }),
-            template: Some(task.clone()),
-        };
-
-        let _new_task = service_client
-            .create_task(Request::new(create_task_req.clone()))
-            .await?;
-
-        let task_in_cluster = tasks
-            .get(&create_task_req.id.as_ref().unwrap().name)
-            .await?;
-
-        assert_eq!(
-            task_in_cluster.spec.image,
-            Task::try_from(create_task_req.clone()).unwrap().spec.image
-        );
-
-        service_client
-            .delete_task(Request::new(create_task_req.id.unwrap()))
-            .await?;
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn can_delete_task() -> Result<()> {
-        let mut service_client = start_server().await?;
-        let client = Client::try_default().await?;
-        let tasks = Api::<Task>::namespaced(client, "ame-system");
-
-        let task = TaskTemplate {
-            name: "template_name".to_string(),
-            command: "test".to_string(),
-            projectid: "myproject".to_string(),
-            ..TaskTemplate::default()
-        };
-
-        let task_in_cluster = tasks
-            .create(&PostParams::default(), &Task::from(task))
-            .await?;
-
-        service_client
-            .delete_task(Request::new(TaskIdentifier::from(task_in_cluster.clone())))
-            .await?;
-
-        let res = tasks.get(&task_in_cluster.name_any()).await.err().unwrap();
-
-        assert!(
-            matches!(
-                &res,
-                kube::Error::Api(e) if e.code == 404
-            ),
-            "failed to match code: {res}"
-        );
-
-        Ok(())
-    }
-}
diff --git a/shell.nix b/shell.nix
index dbc9ac27..31e8b56a 100644
--- a/shell.nix
+++ b/shell.nix
@@ -18,12 +18,14 @@ pkgs.mkShell {
     s3cmd
     minio-client
     argo
-    python310Packages.mlflow
     kubernetes-helm
     grpcurl
     sass
     rust-analyzer
     jq
+    python311Packages.mkdocs-material
+    python311Packages.mkdocs
+    ripgrep
   ];
 
   # Certain Rust tools won't work without this
diff --git a/test_data/test_projects/env/ame.yaml b/test_data/test_projects/env/ame.yaml
index 2e766200..ad3581e7 100755
--- a/test_data/test_projects/env/ame.yaml
+++ b/test_data/test_projects/env/ame.yaml
@@ -1,23 +1,27 @@
-projectname: env
+name: env
 tasks:
-  env:
+  - name: testenv
     env:
-    - name: VAR1
-      value: VAL1
-    - name: VAR2
-      value: VAL2
+    - key: VAR1
+      val: VAL1
+    - key: VAR2
+      val: VAL2
     resources:
       cpu: "2"
       memory: 4Gi
-    runcommand: python env.py
-    source: {}
-  secret:
-    projectid: env
+    executor:
+     !pipEnv
+     command: python env.py
+  - name: testsecret
     resources:
       cpu: "2"
       memory: 4Gi
-    runcommand: python env.py VAR1=mysecretval
+    executor:
+     !pipEnv
+     command: python env.py VAR1=mysecretval
     secrets:
-    - envkey: VAR1
-      name: mysecret
-    source: {}
+      - ame:
+          injectAs: VAR1
+          key: mysecret
+
+# TODO:, why is it ame: and not !ame here?
diff --git a/test_data/test_projects/executors/custom/ame.yaml b/test_data/test_projects/executors/custom/ame.yaml
new file mode 100755
index 00000000..a5f6dab8
--- /dev/null
+++ b/test_data/test_projects/executors/custom/ame.yaml
@@ -0,0 +1,10 @@
+name: customtest
+models: 
+data_sets:
+templates:
+tasks:
+ - name: training
+   executor:
+     !custom
+     command: python nn.py
+     pythonVersion: 3.10
diff --git a/test_data/test_projects/executors/custom/nn.py b/test_data/test_projects/executors/custom/nn.py
new file mode 100644
index 00000000..1bfb9b6f
--- /dev/null
+++ b/test_data/test_projects/executors/custom/nn.py
@@ -0,0 +1 @@
+print("custom output array: ", [1,2,3])
\ No newline at end of file
diff --git a/test_data/test_projects/executors/mlflow/MLProject b/test_data/test_projects/executors/mlflow/MLProject
new file mode 100644
index 00000000..41e13c77
--- /dev/null
+++ b/test_data/test_projects/executors/mlflow/MLProject
@@ -0,0 +1,8 @@
+name: sklearn_logistic_example
+
+python_env: python_env.yaml
+
+entry_points:
+  main:
+    command: "python nn.py"
+
diff --git a/test_data/test_projects/executors/mlflow/Pipfile b/test_data/test_projects/executors/mlflow/Pipfile
new file mode 100644
index 00000000..bdb51e6b
--- /dev/null
+++ b/test_data/test_projects/executors/mlflow/Pipfile
@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+numpy = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"
diff --git a/test_data/test_projects/executors/mlflow/Pipfile.lock b/test_data/test_projects/executors/mlflow/Pipfile.lock
new file mode 100644
index 00000000..9eef0802
--- /dev/null
+++ b/test_data/test_projects/executors/mlflow/Pipfile.lock
@@ -0,0 +1,52 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "56f4dd913f06092b5800190ed38cbd47460bfb927fff151956e2efb228a6e810"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.10"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "numpy": {
+            "hashes": [
+                "sha256:012097b5b0d00a11070e8f2e261128c44157a8689f7dedcf35576e525893f4fe",
+                "sha256:0d3fe3dd0506a28493d82dc3cf254be8cd0d26f4008a417385cbf1ae95b54004",
+                "sha256:0def91f8af6ec4bb94c370e38c575855bf1d0be8a8fbfba42ef9c073faf2cf19",
+                "sha256:1a180429394f81c7933634ae49b37b472d343cccb5bb0c4a575ac8bbc433722f",
+                "sha256:1d5d3c68e443c90b38fdf8ef40e60e2538a27548b39b12b73132456847f4b631",
+                "sha256:20e1266411120a4f16fad8efa8e0454d21d00b8c7cee5b5ccad7565d95eb42dd",
+                "sha256:247d3ffdd7775bdf191f848be8d49100495114c82c2bd134e8d5d075fb386a1c",
+                "sha256:35a9527c977b924042170a0887de727cd84ff179e478481404c5dc66b4170009",
+                "sha256:38eb6548bb91c421261b4805dc44def9ca1a6eef6444ce35ad1669c0f1a3fc5d",
+                "sha256:3d7abcdd85aea3e6cdddb59af2350c7ab1ed764397f8eec97a038ad244d2d105",
+                "sha256:41a56b70e8139884eccb2f733c2f7378af06c82304959e174f8e7370af112e09",
+                "sha256:4a90725800caeaa160732d6b31f3f843ebd45d6b5f3eec9e8cc287e30f2805bf",
+                "sha256:6b82655dd8efeea69dbf85d00fca40013d7f503212bc5259056244961268b66e",
+                "sha256:6c6c9261d21e617c6dc5eacba35cb68ec36bb72adcff0dee63f8fbc899362588",
+                "sha256:77d339465dff3eb33c701430bcb9c325b60354698340229e1dff97745e6b3efa",
+                "sha256:791f409064d0a69dd20579345d852c59822c6aa087f23b07b1b4e28ff5880fcb",
+                "sha256:9a3a9f3a61480cc086117b426a8bd86869c213fc4072e606f01c4e4b66eb92bf",
+                "sha256:c1516db588987450b85595586605742879e50dcce923e8973f79529651545b57",
+                "sha256:c40571fe966393b212689aa17e32ed905924120737194b5d5c1b20b9ed0fb171",
+                "sha256:d412c1697c3853c6fc3cb9751b4915859c7afe6a277c2bf00acf287d56c4e625",
+                "sha256:d5154b1a25ec796b1aee12ac1b22f414f94752c5f94832f14d8d6c9ac40bcca6",
+                "sha256:d736b75c3f2cb96843a5c7f8d8ccc414768d34b0a75f466c05f3a739b406f10b",
+                "sha256:e8f6049c4878cb16960fbbfb22105e49d13d752d4d8371b55110941fb3b17800",
+                "sha256:f76aebc3358ade9eacf9bc2bb8ae589863a4f911611694103af05346637df1b7",
+                "sha256:fd67b306320dcadea700a8f79b9e671e607f8696e98ec255915c0c6d6b818503"
+            ],
+            "index": "pypi",
+            "version": "==1.25.1"
+        }
+    },
+    "develop": {}
+}
diff --git a/test_data/test_projects/executors/mlflow/ame.yaml b/test_data/test_projects/executors/mlflow/ame.yaml
new file mode 100755
index 00000000..3fddbea6
--- /dev/null
+++ b/test_data/test_projects/executors/mlflow/ame.yaml
@@ -0,0 +1,25 @@
+name: mlflowtest
+models: 
+data_sets:
+templates:
+  - name: resource_alloc
+    executor:
+      !mlflow
+tasks:
+ - name: training
+   executor:
+     !mlflow
+ - name: training-with-local-template
+   fromTemplate: 
+     name: resource_alloc
+   resources:
+      cpu: 2
+      memory: 1Gi
+ - name: training-with-remote-template
+   fromTemplate: 
+     name: mlflow-template
+     project: shared-templates
+   resources:
+      cpu: 2
+      memory: 1Gi
+  
diff --git a/test_data/test_projects/executors/mlflow/nn.py b/test_data/test_projects/executors/mlflow/nn.py
new file mode 100644
index 00000000..dd72adb2
--- /dev/null
+++ b/test_data/test_projects/executors/mlflow/nn.py
@@ -0,0 +1,3 @@
+import numpy as np
+
+print("np array: ", np.array([1,2,3]))
\ No newline at end of file
diff --git a/test_data/test_projects/executors/mlflow/python_env.yaml b/test_data/test_projects/executors/mlflow/python_env.yaml
new file mode 100644
index 00000000..ce0ccc80
--- /dev/null
+++ b/test_data/test_projects/executors/mlflow/python_env.yaml
@@ -0,0 +1,9 @@
+python: "3.11"
+build_dependencies:
+  - pip
+dependencies:
+  - mlflow>=2.5
+  - scipy
+  - scikit-learn
+  - boto3
+
diff --git a/test_data/test_projects/executors/pip/ame.yaml b/test_data/test_projects/executors/pip/ame.yaml
new file mode 100755
index 00000000..c0e07039
--- /dev/null
+++ b/test_data/test_projects/executors/pip/ame.yaml
@@ -0,0 +1,10 @@
+name: piptest
+models: 
+data_sets:
+templates:
+tasks:
+ - name: training
+   executor:
+     !pip
+     command: python nn.py
+     pythonVersion: 3.10
diff --git a/test_data/test_projects/executors/pip/nn.py b/test_data/test_projects/executors/pip/nn.py
new file mode 100644
index 00000000..dd72adb2
--- /dev/null
+++ b/test_data/test_projects/executors/pip/nn.py
@@ -0,0 +1,3 @@
+import numpy as np
+
+print("np array: ", np.array([1,2,3]))
\ No newline at end of file
diff --git a/test_data/test_projects/executors/pip/requirements.txt b/test_data/test_projects/executors/pip/requirements.txt
new file mode 100644
index 00000000..7545ef88
--- /dev/null
+++ b/test_data/test_projects/executors/pip/requirements.txt
@@ -0,0 +1,2 @@
+-i https://pypi.org/simple
+numpy==1.25.1
diff --git a/test_data/test_projects/executors/pipenv/Pipfile b/test_data/test_projects/executors/pipenv/Pipfile
new file mode 100644
index 00000000..bdb51e6b
--- /dev/null
+++ b/test_data/test_projects/executors/pipenv/Pipfile
@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+numpy = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"
diff --git a/test_data/test_projects/executors/pipenv/Pipfile.lock b/test_data/test_projects/executors/pipenv/Pipfile.lock
new file mode 100644
index 00000000..9eef0802
--- /dev/null
+++ b/test_data/test_projects/executors/pipenv/Pipfile.lock
@@ -0,0 +1,52 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "56f4dd913f06092b5800190ed38cbd47460bfb927fff151956e2efb228a6e810"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.10"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "numpy": {
+            "hashes": [
+                "sha256:012097b5b0d00a11070e8f2e261128c44157a8689f7dedcf35576e525893f4fe",
+                "sha256:0d3fe3dd0506a28493d82dc3cf254be8cd0d26f4008a417385cbf1ae95b54004",
+                "sha256:0def91f8af6ec4bb94c370e38c575855bf1d0be8a8fbfba42ef9c073faf2cf19",
+                "sha256:1a180429394f81c7933634ae49b37b472d343cccb5bb0c4a575ac8bbc433722f",
+                "sha256:1d5d3c68e443c90b38fdf8ef40e60e2538a27548b39b12b73132456847f4b631",
+                "sha256:20e1266411120a4f16fad8efa8e0454d21d00b8c7cee5b5ccad7565d95eb42dd",
+                "sha256:247d3ffdd7775bdf191f848be8d49100495114c82c2bd134e8d5d075fb386a1c",
+                "sha256:35a9527c977b924042170a0887de727cd84ff179e478481404c5dc66b4170009",
+                "sha256:38eb6548bb91c421261b4805dc44def9ca1a6eef6444ce35ad1669c0f1a3fc5d",
+                "sha256:3d7abcdd85aea3e6cdddb59af2350c7ab1ed764397f8eec97a038ad244d2d105",
+                "sha256:41a56b70e8139884eccb2f733c2f7378af06c82304959e174f8e7370af112e09",
+                "sha256:4a90725800caeaa160732d6b31f3f843ebd45d6b5f3eec9e8cc287e30f2805bf",
+                "sha256:6b82655dd8efeea69dbf85d00fca40013d7f503212bc5259056244961268b66e",
+                "sha256:6c6c9261d21e617c6dc5eacba35cb68ec36bb72adcff0dee63f8fbc899362588",
+                "sha256:77d339465dff3eb33c701430bcb9c325b60354698340229e1dff97745e6b3efa",
+                "sha256:791f409064d0a69dd20579345d852c59822c6aa087f23b07b1b4e28ff5880fcb",
+                "sha256:9a3a9f3a61480cc086117b426a8bd86869c213fc4072e606f01c4e4b66eb92bf",
+                "sha256:c1516db588987450b85595586605742879e50dcce923e8973f79529651545b57",
+                "sha256:c40571fe966393b212689aa17e32ed905924120737194b5d5c1b20b9ed0fb171",
+                "sha256:d412c1697c3853c6fc3cb9751b4915859c7afe6a277c2bf00acf287d56c4e625",
+                "sha256:d5154b1a25ec796b1aee12ac1b22f414f94752c5f94832f14d8d6c9ac40bcca6",
+                "sha256:d736b75c3f2cb96843a5c7f8d8ccc414768d34b0a75f466c05f3a739b406f10b",
+                "sha256:e8f6049c4878cb16960fbbfb22105e49d13d752d4d8371b55110941fb3b17800",
+                "sha256:f76aebc3358ade9eacf9bc2bb8ae589863a4f911611694103af05346637df1b7",
+                "sha256:fd67b306320dcadea700a8f79b9e671e607f8696e98ec255915c0c6d6b818503"
+            ],
+            "index": "pypi",
+            "version": "==1.25.1"
+        }
+    },
+    "develop": {}
+}
diff --git a/test_data/test_projects/executors/pipenv/ame.yaml b/test_data/test_projects/executors/pipenv/ame.yaml
new file mode 100755
index 00000000..711ec757
--- /dev/null
+++ b/test_data/test_projects/executors/pipenv/ame.yaml
@@ -0,0 +1,9 @@
+name: pipenvtest
+models: 
+data_sets:
+templates:
+tasks:
+ - name: training
+   executor:
+     !pipEnv
+     command: python nn.py
diff --git a/test_data/test_projects/executors/pipenv/nn.py b/test_data/test_projects/executors/pipenv/nn.py
new file mode 100644
index 00000000..dd72adb2
--- /dev/null
+++ b/test_data/test_projects/executors/pipenv/nn.py
@@ -0,0 +1,3 @@
+import numpy as np
+
+print("np array: ", np.array([1,2,3]))
\ No newline at end of file
diff --git a/test_data/test_projects/executors/poetry/ame.yaml b/test_data/test_projects/executors/poetry/ame.yaml
new file mode 100755
index 00000000..a8a87e2c
--- /dev/null
+++ b/test_data/test_projects/executors/poetry/ame.yaml
@@ -0,0 +1,17 @@
+name: poetry
+models: 
+data_sets:
+templates:
+tasks:
+ - name: crontraining
+   triggers:
+     schedule: "* * * * *"
+   executor:
+     !poetry
+     pythonVersion: 3.10.9
+     command: python nn.py
+ - name: training
+   executor:
+     !poetry
+     pythonVersion: 3.10.9
+     command: python nn.py
diff --git a/test_data/test_projects/executors/poetry/nn.py b/test_data/test_projects/executors/poetry/nn.py
new file mode 100644
index 00000000..dd72adb2
--- /dev/null
+++ b/test_data/test_projects/executors/poetry/nn.py
@@ -0,0 +1,3 @@
+import numpy as np
+
+print("np array: ", np.array([1,2,3]))
\ No newline at end of file
diff --git a/test_data/test_projects/executors/poetry/poetry.lock b/test_data/test_projects/executors/poetry/poetry.lock
new file mode 100644
index 00000000..df5c76c5
--- /dev/null
+++ b/test_data/test_projects/executors/poetry/poetry.lock
@@ -0,0 +1,44 @@
+# This file is automatically @generated by Poetry and should not be changed by hand.
+
+[[package]]
+name = "numpy"
+version = "1.24.3"
+description = "Fundamental package for array computing in Python"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"},
+    {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"},
+    {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"},
+    {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"},
+    {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"},
+    {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"},
+    {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"},
+    {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"},
+    {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"},
+    {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"},
+    {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"},
+    {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"},
+    {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"},
+    {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"},
+    {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"},
+    {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"},
+    {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"},
+    {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"},
+    {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"},
+    {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"},
+    {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"},
+    {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"},
+    {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"},
+    {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"},
+    {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"},
+]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "3.10.9"
+content-hash = "67f8ddb6608be4d81bcac374a5d2cc3f9f06264deea74c15f46404cef30493c5"
diff --git a/test_data/test_projects/executors/poetry/pyproject.toml b/test_data/test_projects/executors/poetry/pyproject.toml
new file mode 100644
index 00000000..5b2c8bf6
--- /dev/null
+++ b/test_data/test_projects/executors/poetry/pyproject.toml
@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "test"
+version = "0.1.0"
+description = ""
+authors = ["Jessie Chatham Spencer <--global>"]
+
+[tool.poetry.dependencies]
+python = "3.10.9"
+numpy = "^1.22"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
diff --git a/test_data/test_projects/invalid_project/ame.yaml b/test_data/test_projects/invalid_project/ame.yaml
new file mode 100755
index 00000000..3e57ffc2
--- /dev/null
+++ b/test_data/test_projects/invalid_project/ame.yaml
@@ -0,0 +1,17 @@
+name: [poetry]
+models: 
+data_sets:
+templates:
+tasks:
+ - name: crontraining
+   triggers:
+     schedule: "* * * * *"
+   executor:
+     !poetry
+     pythonVersion: 3.10.9
+     command: python nn.py
+ - name: training
+   executor:
+     !poetry
+     pythonVersion: 3.10.9
+     command: python nn.py
diff --git a/test_data/test_projects/invalid_project/nn.py b/test_data/test_projects/invalid_project/nn.py
new file mode 100644
index 00000000..dd72adb2
--- /dev/null
+++ b/test_data/test_projects/invalid_project/nn.py
@@ -0,0 +1,3 @@
+import numpy as np
+
+print("np array: ", np.array([1,2,3]))
\ No newline at end of file
diff --git a/test_data/test_projects/invalid_project/poetry.lock b/test_data/test_projects/invalid_project/poetry.lock
new file mode 100644
index 00000000..df5c76c5
--- /dev/null
+++ b/test_data/test_projects/invalid_project/poetry.lock
@@ -0,0 +1,44 @@
+# This file is automatically @generated by Poetry and should not be changed by hand.
+
+[[package]]
+name = "numpy"
+version = "1.24.3"
+description = "Fundamental package for array computing in Python"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"},
+    {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"},
+    {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"},
+    {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"},
+    {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"},
+    {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"},
+    {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"},
+    {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"},
+    {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"},
+    {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"},
+    {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"},
+    {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"},
+    {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"},
+    {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"},
+    {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"},
+    {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"},
+    {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"},
+    {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"},
+    {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"},
+    {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"},
+    {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"},
+    {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"},
+    {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"},
+    {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"},
+    {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"},
+]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "3.10.9"
+content-hash = "67f8ddb6608be4d81bcac374a5d2cc3f9f06264deea74c15f46404cef30493c5"
diff --git a/test_data/test_projects/invalid_project/pyproject.toml b/test_data/test_projects/invalid_project/pyproject.toml
new file mode 100644
index 00000000..5b2c8bf6
--- /dev/null
+++ b/test_data/test_projects/invalid_project/pyproject.toml
@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "test"
+version = "0.1.0"
+description = ""
+authors = ["Jessie Chatham Spencer <--global>"]
+
+[tool.poetry.dependencies]
+python = "3.10.9"
+numpy = "^1.22"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
diff --git a/test_data/test_projects/missing_project/nn.py b/test_data/test_projects/missing_project/nn.py
new file mode 100644
index 00000000..dd72adb2
--- /dev/null
+++ b/test_data/test_projects/missing_project/nn.py
@@ -0,0 +1,3 @@
+import numpy as np
+
+print("np array: ", np.array([1,2,3]))
\ No newline at end of file
diff --git a/test_data/test_projects/missing_project/poetry.lock b/test_data/test_projects/missing_project/poetry.lock
new file mode 100644
index 00000000..df5c76c5
--- /dev/null
+++ b/test_data/test_projects/missing_project/poetry.lock
@@ -0,0 +1,44 @@
+# This file is automatically @generated by Poetry and should not be changed by hand.
+
+[[package]]
+name = "numpy"
+version = "1.24.3"
+description = "Fundamental package for array computing in Python"
+category = "main"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"},
+    {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"},
+    {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"},
+    {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"},
+    {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"},
+    {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"},
+    {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"},
+    {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"},
+    {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"},
+    {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"},
+    {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"},
+    {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"},
+    {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"},
+    {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"},
+    {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"},
+    {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"},
+    {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"},
+    {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"},
+    {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"},
+    {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"},
+    {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"},
+    {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"},
+    {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"},
+    {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"},
+    {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"},
+    {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"},
+]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "3.10.9"
+content-hash = "67f8ddb6608be4d81bcac374a5d2cc3f9f06264deea74c15f46404cef30493c5"
diff --git a/test_data/test_projects/missing_project/pyproject.toml b/test_data/test_projects/missing_project/pyproject.toml
new file mode 100644
index 00000000..5b2c8bf6
--- /dev/null
+++ b/test_data/test_projects/missing_project/pyproject.toml
@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "test"
+version = "0.1.0"
+description = ""
+authors = ["Jessie Chatham Spencer <--global>"]
+
+[tool.poetry.dependencies]
+python = "3.10.9"
+numpy = "^1.22"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
diff --git a/test_data/test_projects/new_echo/ame.yaml b/test_data/test_projects/new_echo/ame.yaml
index 0874d79a..e111d8ab 100644
--- a/test_data/test_projects/new_echo/ame.yaml
+++ b/test_data/test_projects/new_echo/ame.yaml
@@ -1,6 +1,9 @@
-projectid: echo
+name: echo
 tasks:
   - name: echo
+    executor:
+      pythonVersion: 3.11
+      command: python train.py
     resources:
       cpu: "2"
       memory: 4Gi
diff --git a/test_data/test_projects/pipenvnn/forreadme2.cast b/test_data/test_projects/pipenvnn/forreadme2.cast
deleted file mode 100644
index b21f3159..00000000
--- a/test_data/test_projects/pipenvnn/forreadme2.cast
+++ /dev/null
@@ -1,309 +0,0 @@
-{"version": 2, "width": 100, "height": 30, "timestamp": 1665911384, "idle_time_limit": 1.0, "env": {"SHELL": "/nix/store/fq9y3qc1vczbs2wqslz0cxccrrk71iy1-bash-interactive-5.1-p16/bin/bash", "TERM": "screen"}}
-[0.019399, "o", "\u001b[?644h\r\r\n\u001b[1;32m[\u001b]0;jmintb@nixos: ~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn\u0007jmintb@nixos:~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn]$\u001b[0m "]
-[0.995054, "o", "."]
-[1.130027, "o", "/"]
-[1.989028, "o", "a"]
-[2.072047, "o", "m"]
-[2.206208, "o", "e"]
-[2.28602, "o", " "]
-[2.399001, "o", "c"]
-[2.63506, "o", "r"]
-[2.703129, "o", "e"]
-[2.798909, "o", "a"]
-[2.881242, "o", "t"]
-[2.924838, "o", "e"]
-[3.009951, "o", " "]
-[3.184071, "o", "p"]
-[3.308066, "o", "r"]
-[3.388091, "o", "o"]
-[3.556948, "o", "j"]
-[3.632972, "o", "e"]
-[3.685963, "o", "c"]
-[3.912977, "o", "t"]
-[4.175091, "o", "\r\n"]
-[4.175224, "o", "\u001b[?644l\r"]
-[4.206127, "o", "\u001b[0G\u001b[2K"]
-[4.206398, "o", "\u001b[1;92m? \u001b[0m\u001b[1;99mProject name: \u001b[0m\u001b[36m[? for help]\u001b[0m \u001b[37m(pipenvnn) \u001b[0m\u001b[?25l\u001b[?25l\u001b7\u001b[999;999f\u001b[6n"]
-[4.206509, "o", "\u001b8\u001b[?25h\u001b[6n"]
-[5.18098, "o", "\r\n\u001b[1B\u001b[0G"]
-[5.181093, "o", "\u001b[1A\u001b[?25h\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mProject name: \u001b[0m\u001b[36mpipenvnn\u001b[0m\r\n"]
-[5.18473, "o", "\u001b[?644h\r\r\n\u001b[1;32m[\u001b]0;jmintb@nixos: ~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn\u0007jmintb@nixos:~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn]$\u001b[0m "]
-[6.913014, "o", "./ame create project"]
-[7.349643, "o", "\b\u001b[K"]
-[7.491997, "o", "\b\u001b[K"]
-[7.658895, "o", "\b\u001b[K"]
-[7.817077, "o", "\b\u001b[K"]
-[7.983921, "o", "\b\u001b[K"]
-[8.140934, "o", "\b\u001b[K"]
-[8.303003, "o", "\b\u001b[K"]
-[8.35287, "o", "t"]
-[8.472887, "o", "a"]
-[8.559833, "o", "s"]
-[9.257088, "o", "k"]
-[9.392905, "o", "\r\n"]
-[9.39301, "o", "\u001b[?644l\r"]
-[9.401588, "o", "\u001b7\u001b[?25l\u001b8\u001b[0G\u001b[2K"]
-[9.410619, "o", "\u001b[1;92m? \u001b[0m\u001b[1;99mPlease select what type of Task you want to create:\u001b[0m  \u001b[36m[Use arrows to move, type to filter]\u001b[0m\r\n\u001b[1;36m> Single step Task\u001b[0m\r\n\u001b[39m  Multi step Task pipeline\u001b[0m\r\n\u001b7"]
-[9.410883, "o", "\u001b[1A\u001b[0G\u001b[1A\u001b[0G"]
-[10.538111, "o", "\u001b8\u001b[?25h\u001b8"]
-[10.538297, "o", "\u001b[0G\u001b[2K\u001b[1A\u001b[0G\u001b[2K\u001b[1A\u001b[0G\u001b[2K\u001b[1A\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mPlease select what type of Task you want to create:\u001b[0m\u001b[36m Single step Task\u001b[0m\r\n\u001b[0G\u001b[2K"]
-[10.538677, "o", "\u001b[1;92m? \u001b[0m\u001b[1;99mTask name: \u001b[0m"]
-[10.538715, "o", "\u001b[?25l\u001b[?25l\u001b7\u001b[999;999f\u001b[6n"]
-[10.538803, "o", "\u001b8\u001b[?25h\u001b[6n"]
-[11.721886, "o", "t"]
-[11.831358, "o", "r"]
-[12.034965, "o", "a"]
-[12.183822, "o", "i"]
-[12.365905, "o", "n"]
-[12.530883, "o", "i"]
-[12.616504, "o", "n"]
-[12.709812, "o", "g"]
-[13.071855, "o", "\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D"]
-[13.07648, "o", "\u001b[1D\u001b[1D\u001b[1B\u001b[0G\u001b[1A\u001b[?25h\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mTask name: \u001b[0m\u001b[36mtraining\u001b[0m\r\n\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mCommand: \u001b[0m"]
-[13.072089, "o", "\u001b[?25l\u001b[?25l\u001b7\u001b[999;999f\u001b[6n"]
-[13.072184, "o", "\u001b8\u001b[?25h\u001b[6n"]
-[14.211912, "o", "p"]
-[14.501903, "o", "y"]
-[14.63241, "o", "t"]
-[14.725697, "o", "h"]
-[14.879743, "o", "o"]
-[15.0488, "o", "n"]
-[15.15284, "o", " "]
-[15.367713, "o", "n"]
-[15.476841, "o", "n"]
-[16.126361, "o", "."]
-[16.345828, "o", "p"]
-[16.58283, "o", "y"]
-[17.513824, "o", "\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1D\u001b[1B\u001b[0G"]
-[17.513866, "o", "\u001b[1A\u001b[?25h\u001b[0G\u001b[2K"]
-[17.513998, "o", "\u001b[1;92m? \u001b[0m\u001b[1;99mCommand: \u001b[0m\u001b[36mpython nn.py\u001b[0m\r\n\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mCpu:  \u001b[0m\u001b[37m(2) \u001b[0m\u001b[?25l\u001b[?25l\u001b7\u001b[999;999f\u001b[6n"]
-[17.514105, "o", "\u001b8\u001b[?25h\u001b[6n"]
-[18.744686, "o", "4"]
-[19.268802, "o", "\u001b[1D\r\n\u001b[1B\u001b[0G"]
-[19.268932, "o", "\u001b[1A\u001b[?25h\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mCpu:  \u001b[0m\u001b[36m4\u001b[0m\r\n\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mMemory:  \u001b[0m\u001b[37m(4Gi) \u001b[0m"]
-[19.268987, "o", "\u001b[?25l\u001b[?25l\u001b7\u001b[999;999f\u001b[6n"]
-[19.269072, "o", "\u001b8\u001b[?25h\u001b[6n"]
-[20.677706, "o", "8"]
-[20.918727, "o", "G"]
-[21.125887, "o", "i"]
-[21.629351, "o", "\u001b[1D\u001b[1D\u001b[1D\r\n\u001b[1B\u001b[0G"]
-[21.62948, "o", "\u001b[1A\u001b[?25h\u001b[0G\u001b[2K"]
-[21.629495, "o", "\u001b[1;92m? \u001b[0m\u001b[1;99mMemory:  \u001b[0m\u001b[36m8Gi\u001b[0m\r\n\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mGpu:  \u001b[0m\u001b[37m(0) \u001b[0m\u001b[?25l\u001b[?25l\u001b7\u001b[999;999f\u001b[6n"]
-[21.629578, "o", "\u001b8\u001b[?25h\u001b[6n"]
-[22.408883, "o", "\r\n\u001b[1B"]
-[22.409093, "o", "\u001b[0G\u001b[1A\u001b[?25h\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mGpu:  \u001b[0m\u001b[36m0\u001b[0m\r\n\u001b[0G\u001b[2K"]
-[22.409275, "o", "\u001b[1;92m? \u001b[0m\u001b[1;99mWould you like to add an environment variable? \u001b[0m\u001b[37m(y/N) \u001b[0m\u001b[?25l"]
-[22.409327, "o", "\u001b7\u001b[999;999f\u001b[6n"]
-[22.409411, "o", "\u001b8\u001b[?25h\u001b[6n"]
-[23.213708, "o", "\r\n\u001b[1B\u001b[0G\u001b[1A"]
-[23.213862, "o", "\u001b[0G\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mWould you like to add an environment variable? \u001b[0m\u001b[36mNo\u001b[0m\r\n\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mWould you like to add a secret? \u001b[0m\u001b[37m(y/N) \u001b[0m\u001b[?25l\u001b7\u001b[999;999f"]
-[23.213884, "o", "\u001b[6n"]
-[23.213981, "o", "\u001b8\u001b[?25h\u001b[6n"]
-[23.397658, "o", "\r\n\u001b[1B\u001b[0G\u001b[1A\u001b[0G"]
-[23.397811, "o", "\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mWould you like to add a secret? \u001b[0m\u001b[36mNo\u001b[0m\r\n"]
-[23.397889, "o", "This task will be saved in the project file (ame.yaml):\r\nName:  training\r\nresources:\r\n  cpu: \"4\"\r\n  memory: 8Gi\r\nruncommand: python nn.py\r\n\r\n\r\n"]
-[23.402566, "o", "\u001b[?644h\r\r\n\u001b[1;32m[\u001b]0;jmintb@nixos: ~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn\u0007jmintb@nixos:~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn]$\u001b[0m "]
-[24.761708, "o", "."]
-[24.911676, "o", "/"]
-[25.170847, "o", "a"]
-[25.343344, "o", "m"]
-[25.489957, "o", "e"]
-[25.594552, "o", " "]
-[25.676615, "o", "r"]
-[25.793537, "o", "u"]
-[25.809491, "o", "i"]
-[25.815351, "o", "n"]
-[26.324689, "o", "\b\u001b[K"]
-[26.479695, "o", "\b\u001b[K"]
-[26.690677, "o", "n"]
-[27.74509, "o", "\r\n"]
-[27.74528, "o", "\u001b[?644l\r"]
-[27.757662, "o", "\u001b7\u001b[?25l\u001b8\u001b[0G\u001b[2K"]
-[27.758125, "o", "\u001b[1;92m? \u001b[0m\u001b[1;99mPlease select a task:\u001b[0m  \u001b[36m[Use arrows to move, type to filter]\u001b[0m\r\n\u001b[1;36m> training\u001b[0m\r\n\u001b7"]
-[27.758413, "o", "\u001b[1A\u001b[0G"]
-[29.475574, "o", "\u001b8\u001b[?25h\u001b8"]
-[29.475684, "o", "\u001b[0G\u001b[2K\u001b[1A\u001b[0G\u001b[2K\u001b[1A\u001b[0G\u001b[2K\u001b[1;92m? \u001b[0m\u001b[1;99mPlease select a task:\u001b[0m\u001b[36m training\u001b[0m\r\n"]
-[29.475697, "o", "\r\n\u001b[?25l"]
-[29.475783, "o", "\r\u001b[K"]
-[29.475823, "o", "\r\u001b[37m⠋\u001b[0m Uploading project: pipenvnn"]
-[29.575936, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Uploading project: pipenvnn"]
-[29.676177, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Uploading project: pipenvnn"]
-[29.776985, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Uploading project: pipenvnn"]
-[29.877257, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Uploading project: pipenvnn"]
-[29.97829, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Uploading project: pipenvnn"]
-[30.078576, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Uploading project: pipenvnn"]
-[30.178653, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Uploading project: pipenvnn"]
-[30.278878, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Uploading project: pipenvnn"]
-[30.379088, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Uploading project: pipenvnn"]
-[30.479292, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Uploading project: pipenvnn"]
-[30.579495, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Uploading project: pipenvnn"]
-[30.679718, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Uploading project: pipenvnn"]
-[30.779917, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Uploading project: pipenvnn"]
-[30.880122, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Uploading project: pipenvnn"]
-[30.980549, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[31.080754, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[31.180965, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[31.281178, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[31.381384, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[31.481596, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[31.581778, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[31.681974, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[31.78218, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[31.882376, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[31.982668, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[32.082881, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[32.183083, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[32.283931, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[32.384112, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[32.484323, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[32.584524, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[32.684725, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[32.784937, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[32.885125, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[32.985326, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[33.085537, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[33.18575, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[33.285944, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[33.386125, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[33.486328, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[33.586445, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[33.686656, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[33.78686, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[33.887078, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[33.987315, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[34.087501, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[34.187704, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[34.287914, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[34.388117, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[34.48833, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[34.588539, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[34.688743, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[34.788944, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[34.889132, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[34.989331, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[35.089547, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[35.18975, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[35.289953, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[35.390121, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[35.490324, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[59.098004, "o", "\u001b[?25h\r\u001b[KWarning: the environment variable LANG is not set!\r\n\u001b[?25l\u001b[?25h\r\u001b[KWe recommend setting this in ~/.profile (or equivalent) for proper expected behavior.\r\n\u001b[?25l"]
-[59.13719, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Executing"]
-[59.1958, "o", "\u001b[?25h\r\u001b[KWarning: Python 3.9 was not found on your system...\r\n\u001b[?25l"]
-[59.221433, "o", "\u001b[?25h\r\u001b[KInstalling CPython 3.9.15 with /home/ame/.pyenv/bin/pyenv (this may take a few minutes)...\r\n\u001b[?25l"]
-[59.237627, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Executing"]
-[59.337926, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Executing"]
-[59.438121, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Executing"]
-[59.538338, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Executing"]
-[59.638542, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Executing"]
-[59.738764, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Executing"]
-[59.838981, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Executing"]
-[59.939197, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Executing"]
-[60.039389, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Executing"]
-[60.139601, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Executing"]
-[60.239762, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Executing"]
-[60.33997, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Executing"]
-[61.101972, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Executing"]
-[61.66179, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Executing"]
-[61.275633, "o", "\u001b[?25h\r\u001b[K\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b"]
-[61.27581, "o", "\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Insta"]
-[61.275848, "o", "lling python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing pyth"]
-[61.275877, "o", "on...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧"]
-[61.275903, "o", " Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installin"]
-[61.275927, "o", "g python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python.."]
-[61.27595, "o", ".\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Ins"]
-[61.275978, "o", "talling python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing py"]
-[61.275998, "o", "thon...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r"]
-[61.27602, "o", "⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Install"]
-[61.276042, "o", "ing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python"]
-[61.27607, "o", "...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ I"]
-[61.276094, "o", "nstalling python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing "]
-[61.276119, "o", "python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installin"]
-[61.276144, "o", "g python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python.."]
-[61.276166, "o", ".\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Ins"]
-[61.276195, "o", "talling python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing py"]
-[61.276226, "o", "thon...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r"]
-[61.276252, "o", "⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Install"]
-[61.276279, "o", "ing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python"]
-[61.276302, "o", "...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b✔ Success! \r\n\u001b[?25l\u001b[?25h\r\u001b[K\r\n\u001b[?25l"]
-[61.302436, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Executing"]
-[61.319827, "o", "\u001b[?25h\r\u001b[KCreating a virtualenv for this project...\r\n\u001b[?25l\u001b[?25h\r\u001b[KPipfile: /project/pipenvnn/Pipfile\r\n\u001b[?25l\u001b[?25h\r\u001b[KUsing /home/ame/.pyenv/versions/3.9.15/bin/python3.9 (3.9.15) to create virtualenv...\r\n\u001b[?25l"]
-[61.403073, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Executing"]
-[61.50322, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Executing"]
-[61.603374, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Executing"]
-[61.659804, "o", "\u001b[?25h\r\u001b[K\r⠋ Creating virtual environment...\b\r⠙ Creating virtual environment...\b\r⠹ Creating virtual environment...\b\r⠸ Creating virtual environment...\b\r⠼ Creating virtual environment...created virtual environment CPython3.9.15.final.0-64 in 257ms\r\n\u001b[?25l\u001b[?25h\r\u001b[K  creator Venv(dest=/home/ame/.local/share/virtualenvs/pipenvnn-1o51VEpE, clear=False, no_vcs_ignore=False, global=False, describe=CPython3Posix)\r\n\u001b[?25l\u001b[?25h\r\u001b[K"]
-[61.659922, "o", "  seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/home/ame/.local/share/virtualenv)\r\n\u001b[?25l\u001b[?25h\r\u001b[K    added seed packages: pip==22.2.2, setuptools==65.3.0, wheel==0.37.1\r\n\u001b[?25l\u001b[?25h\r\u001b[K  activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator\r\n\u001b[?25l\u001b[?25h\r\u001b[K\r\n\u001b[?25l"]
-[61.704078, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Executing"]
-[61.722302, "o", "\u001b[?25h\r\u001b[K\b✔ Successfully created virtual environment! \r\n\u001b[?25l"]
-[61.771352, "o", "\u001b[?25h\r\u001b[KVirtualenv location: /home/ame/.local/share/virtualenvs/pipenvnn-1o51VEpE\r\n\u001b[?25l"]
-[61.791217, "o", "\u001b[?25h\r\u001b[KInstalling dependencies from Pipfile.lock (0ef8d2)...\r\n\u001b[?25l"]
-[61.804446, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Executing"]
-[64.344184, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Executing"]
-[64.444389, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Executing"]
-[64.544596, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Executing"]
-[64.6448, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Executing"]
-[64.744994, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Executing"]
-[64.845197, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Executing"]
-[64.945401, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Executing"]
-[64.964799, "o", "\u001b[?25h\r\u001b[KDownloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\r\n"]
-[64.964892, "o", "\u001b[?25l"]
-[65.046023, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Executing"]
-[65.146183, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Executing"]
-[65.246375, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Executing"]
-[65.346572, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Executing"]
-[65.446775, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Executing"]
-[65.57084, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Executing"]
-[65.647176, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Executing"]
-[65.747363, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Executing"]
-[65.84757, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Executing"]
-[65.947769, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Executing"]
-[66.047968, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Executing"]
-[66.057659, "o", "\u001b[?25h\r\u001b[K\r   16384/11490434 [..............................] - ETA: 2s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r  61072/11490434 [..............................] - ETA: 4s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r  598016/11490434 [>.............................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 1163264/11490434 [==>...........................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 1523712/11490434 [==>...........................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 2375680/11490434 [=====>........................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 2621440/11490434 [=====>........................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 3555328/11490434 [========>.....................] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 4128768/11490434 [=========>................"]
-[66.057753, "o", "....] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 722208/11490434 [===========>..................] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 529672/11490434 [============>.................] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 5865472/11490434 [==============>...............] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 6455296/11490434 [===============>..............] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 7045120/11490434 [=================>............] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 7634944/11490434 [==================>...........] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 819640/11490434 [====================>.........] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 8781824/11490434 [=====================>........] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"]
-[66.057797, "o", "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 9322496/11490434 [=======================>......] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 9879552/11490434 [========================>.....] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r10452992/11490434 [==========================>...] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r11010048/11490434 [===========================>..] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r11493376/11490434 [==============================] - 1s 0us/step\r\n\u001b[?25l\u001b[?25h\r\u001b[K\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r11501568/11490434 [==============================] - 1s 0us/step\r\n\u001b[?25l"]
-[66.148889, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Executing"]
-[66.249101, "o", "\r\u001b[K"]
-[66.249247, "o", "\r\u001b[37m⠼\u001b[0m Executing"]
-[66.349489, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Executing"]
-[66.449685, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Executing"]
-[66.549924, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Executing"]
-[66.650141, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Executing"]
-[66.750361, "o", "\r\u001b[K"]
-[66.750527, "o", "\r\u001b[37m⠏\u001b[0m Executing"]
-[66.850724, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Executing"]
-[66.950933, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Executing"]
-[67.051128, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Executing"]
-[67.088187, "o", "\u001b[?25h\r\u001b[K662-10-16 09:13:07.336898: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\r\n\u001b[?25l\u001b[?25h\r\u001b[K2022-10-16 09:13:07.336929: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\r\n\u001b[?25l\u001b[?25h\r\u001b[K2022-10-16 09:13:07.336955: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:163] no NVIDIA GPU device is present: /dev/nvidia0 does not exist\r\n\u001b[?25l"]
-[67.088354, "o", "\u001b[?25h\r\u001b[K662-10-16 09:13:07.337352: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\r\n\u001b[?25l\u001b[?25h\r\u001b[KTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\r\n\u001b[?25l"]
-[67.090015, "o", "\u001b[?25h\r\u001b[K662-10-16 09:13:07.339317: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 188160000 exceeds 10% of free system memory.\r\n\u001b[?25l"]
-[67.152187, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Executing"]
-[67.182952, "o", "\u001b[?25h\r\u001b[K662-10-16 09:13:07.431891: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 188160000 exceeds 10% of free system memory.\r\n\u001b[?25l"]
-[67.253119, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Executing"]
-[67.353242, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Executing"]
-[67.438629, "o", "\u001b[?25h\r\u001b[K662-10-16 09:13:07.687716: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\r\n\u001b[?25l"]
-[67.453839, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Executing"]
-[67.554057, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Executing"]
-[70.740619, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Executing"]
-[70.840823, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Executing"]
-[70.941034, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Executing"]
-[72.0412, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Executing"]
-[72.141421, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Executing"]
-[72.241618, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Executing"]
-[72.33866, "o", "\u001b[?25h\r\u001b[KEpoch 1, Loss: 0.13371674954666, Accuracy: 95.913330078125, Test Loss: 0.059672582894563675, Test Accuracy: 98.04999542236328\r\n\u001b[?25l"]
-[72.338968, "o", "\u001b[?25h\r\u001b[KEpoch 2, Loss: 0.04030441492795944, Accuracy: 98.7550048828125, Test Loss: 0.054502055048942566, Test Accuracy: 98.2640122070312\r\n\u001b[?25l\u001b[?25h\r\u001b[KEpoch 3, Loss: 0.021926309913396835, Accuracy: 99.25167083740234, Test Loss: 0.05565335601568222, Test Accuracy: 98.3499984741211\r\n\u001b[?25l"]
-[72.33901, "o", "\u001b[?25h\r\u001b[KEpoch 4, Loss: 0.01225259993225336, Accuracy: 99.57666015625, Test Loss: 0.05713661387562752, Test Accuracy: 98.41999816894531\r\n\u001b[?25l\u001b[?25h\r\u001b[KEpoch 5, Loss: 0.009215598925948143, Accuracy: 99.67500305175781, Test Loss: 0.08434005081653595, Test Accuracy: 97.93000030517578\r\n\u001b[?25l"]
-[72.342467, "o", "\r\u001b[K"]
-[72.34255, "o", "\r\u001b[37m⠇\u001b[0m Executing"]
-[72.442724, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Executing"]
-[72.542974, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Executing"]
-[72.643145, "o", "\r\u001b[K"]
-[72.643316, "o", "\r\u001b[37m⠙\u001b[0m Executing"]
-[72.743339, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Executing"]
-[72.843574, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Executing"]
-[72.943742, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Executing\n"]
-[73.554018, "o", "\rThe task finished successfully!\r\n"]
-[73.561973, "o", "\u001b[?25h\r\u001b[K✓ Done"]
-[73.56729, "o", "\u001b[?644h\r\r\n\u001b[1;32m[\u001b]0;jmintb@nixos: ~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn\u0007jmintb@nixos:~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn]$\u001b[0m "]
-[74.9919, "o", "^C\u001b[?644l\r"]
-[74.99643, "o", "\u001b[?2004h\u001b[?2004l\r\r\n"]
-[74.99419, "o", "\u001b[?644h\r\r\n\u001b[1;32m[\u001b]0;jmintb@nixos: ~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn\u0007jmintb@nixos:~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn]$\u001b[0m "]
-[75.091084, "o", "\u001b[?644l\r\r\nexit\r\n"]
diff --git a/test_data/test_projects/pipenvnn/nn.cast b/test_data/test_projects/pipenvnn/nn.cast
deleted file mode 100644
index 7cf1555b..00000000
--- a/test_data/test_projects/pipenvnn/nn.cast
+++ /dev/null
@@ -1,218 +0,0 @@
-{"version": 2, "width": 319, "height": 76, "timestamp": 1664094312, "env": {"SHELL": "/nix/store/fq9y3qc1vczbs2wqslz0cxccrrk71iy1-bash-interactive-5.1-p16/bin/bash", "TERM": "screen"}}
-[0.01387, "o", "\u001b[?2004h\r\r\n\u001b[1;32m[\u001b]0;jmintb@nixos: ~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn\u0007jmintb@nixos:~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn]$\u001b[0m "]
-[0.694687, "o", "./ame run \"python artifacts.py\""]
-[2.720683, "o", "\b"]
-[2.860532, "o", "\b"]
-[3.021523, "o", "\b"]
-[3.175453, "o", "\b"]
-[3.593582, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[3.730428, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[3.878541, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[4.032525, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[4.183607, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[4.335454, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[4.489719, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[4.64642, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[4.806397, "o", "\b\u001b[1P.py\"\b\b\b\b"]
-[5.116509, "o", "n.py\"\b\b\b\b"]
-[5.2264, "o", "n.py\"\b\b\b\b"]
-[5.779528, "o", "\r\n"]
-[5.779637, "o", "\u001b[?2004l\r"]
-[5.787646, "o", "\r\n\u001b[?25l"]
-[5.787703, "o", "\r\u001b[K"]
-[5.787734, "o", "\r\u001b[37m⠋\u001b[0m Uploading project: pipenvnn"]
-[5.887957, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Uploading project: pipenvnn"]
-[5.988169, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Uploading project: pipenvnn"]
-[6.0892, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Uploading project: pipenvnn"]
-[6.189397, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Uploading project: pipenvnn"]
-[6.289585, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Uploading project: pipenvnn"]
-[6.389797, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Uploading project: pipenvnn"]
-[6.490272, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[6.590463, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[6.690669, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[6.79085, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[6.890998, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[6.9912, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[7.091407, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[7.191603, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[7.291769, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[7.391973, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[7.492175, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[7.592369, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[7.692578, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[7.79277, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[7.892975, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[7.993176, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[8.093376, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[8.193532, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[8.2937, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[8.393892, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[8.494082, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[8.594223, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[8.69443, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[8.79463, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[8.894793, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[8.994985, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[9.095169, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[9.195369, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[9.295556, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[9.395707, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[9.495903, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[9.596111, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[9.69622, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[9.79641, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[9.896609, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[9.996778, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[10.096968, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[10.197165, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[10.297354, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[10.397546, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[10.497699, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[10.597901, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[10.698099, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[10.798291, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[10.898495, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[10.998702, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[11.098898, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[11.199098, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[11.299236, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[11.399428, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[11.499649, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[11.599768, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[11.699965, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[11.800159, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[11.90037, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[12.000571, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[12.100767, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[12.200961, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[12.301157, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[12.401355, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[12.501544, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[12.601717, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[12.701924, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[12.802121, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[12.902321, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[13.002512, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[13.102708, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[13.202899, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[13.303094, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[13.403285, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[13.503483, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[13.603688, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[13.703887, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[13.804076, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[13.904267, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[14.004468, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[14.104664, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[14.204867, "o", "\r\u001b[K\r\u001b[37m⠼\u001b[0m Preparing execution environment"]
-[14.305053, "o", "\r\u001b[K\r\u001b[37m⠴\u001b[0m Preparing execution environment"]
-[14.405261, "o", "\r\u001b[K\r\u001b[37m⠦\u001b[0m Preparing execution environment"]
-[14.505453, "o", "\r\u001b[K\r\u001b[37m⠧\u001b[0m Preparing execution environment"]
-[14.605657, "o", "\r\u001b[K\r\u001b[37m⠇\u001b[0m Preparing execution environment"]
-[14.705785, "o", "\r\u001b[K\r\u001b[37m⠏\u001b[0m Preparing execution environment"]
-[14.805976, "o", "\r\u001b[K\r\u001b[37m⠋\u001b[0m Preparing execution environment"]
-[14.906161, "o", "\r\u001b[K\r\u001b[37m⠙\u001b[0m Preparing execution environment"]
-[15.006369, "o", "\r\u001b[K\r\u001b[37m⠹\u001b[0m Preparing execution environment"]
-[15.10657, "o", "\r\u001b[K\r\u001b[37m⠸\u001b[0m Preparing execution environment"]
-[15.108041, "o", "\u001b[?25h\r\u001b[KYour task will be executed! python nn.py\r\n2022-09-25T08:25:27.164462907Z time=\"2022-09-25T08:25:27.164Z\" level=info msg=\"capturing logs\" argo=true\r\n"]
-[15.308544, "o", "2022-09-25T08:25:27.227537169Z download: 's3://ameprojectstorage/pipenvnn/Pipfile' -> './pipenvnn/Pipfile' (238 bytes in 0.0 seconds, 298.12 KB/s)\r\n"]
-[15.507748, "o", "2022-09-25T08:25:27.228434732Z download: 's3://ameprojectstorage/pipenvnn/Pipfile.lock' -> './pipenvnn/Pipfile.lock' (26247 bytes in 0.0 seconds, 32.10 MB/s)\r\n"]
-[15.708059, "o", "2022-09-25T08:25:27.297470989Z download: 's3://ameprojectstorage/pipenvnn/ame' -> './pipenvnn/ame' (43337073 bytes in 0.1 seconds, 600.84 MB/s)\r\n"]
-[15.908638, "o", "2022-09-25T08:25:27.298547441Z download: 's3://ameprojectstorage/pipenvnn/nn.cast' -> './pipenvnn/nn.cast' (1208 bytes in 0.0 seconds, 1338.74 KB/s)\r\n"]
-[16.107713, "o", "2022-09-25T08:25:27.299402534Z download: 's3://ameprojectstorage/pipenvnn/nn.py' -> './pipenvnn/nn.py' (2872 bytes in 0.0 seconds, 3.99 MB/s)\r\n"]
-[16.308307, "o", "2022-09-25T08:25:27.300188138Z download: 's3://ameprojectstorage/pipenvnn/test.cast' -> './pipenvnn/test.cast' (7413 bytes in 0.0 seconds, 11.16 MB/s)\r\n"]
-[16.507603, "o", "2022-09-25T08:25:27.534571450Z Warning: the environment variable LANG is not set!\r\n"]
-[16.708301, "o", "2022-09-25T08:25:27.534582400Z We recommend setting this in ~/.profile (or equivalent) for proper expected behavior.\r\n"]
-[16.90744, "o", "2022-09-25T08:25:27.664203267Z Warning: Python 3.9 was not found on your system...\r\n"]
-[17.107916, "o", "2022-09-25T08:25:27.684239775Z Installing CPython 3.9.14 with /home/ame/.pyenv/bin/pyenv (this may take a few minutes)...\r\n"]
-[57.841968, "o", "2022-09-25T08:26:10.480209782Z \r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ "]
-[57.842131, "o", "Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing"]
-[57.842183, "o", " python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python..."]
-[57.842214, "o", "\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Inst"]
-[57.842238, "o", "alling python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing pyt"]
-[57.842258, "o", "hon...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r"]
-[57.842289, "o", "⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installi"]
-[57.842325, "o", "ng python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python."]
-[57.842349, "o", "..\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ In"]
-[57.842373, "o", "stalling python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing p"]
-[57.842398, "o", "ython...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r"]
-[57.842419, "o", "⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Instal"]
-[57.842442, "o", "ling python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing pytho"]
-[57.84246, "o", "n...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b\r⠸ Installing python...\b\r⠼ Installing python...\b\r⠴ Installing python...\b\r⠦ Installing python...\b\r⠧ Installing python...\b\r⠇ Installing python...\b\r⠏ Installing python...\b\r⠋ Installing python...\b\r⠙ Installing python...\b\r⠹ Installing python...\b✔ Success! \r\n"]
-[57.844658, "o", "2022-09-25T08:26:10.480320091Z \r\n"]
-[57.880728, "o", "2022-09-25T08:26:10.519369047Z Creating a virtualenv for this project...\r\n"]
-[57.882525, "o", "2022-09-25T08:26:10.519388707Z Pipfile: /project/pipenvnn/Pipfile\r\n"]
-[57.883933, "o", "2022-09-25T08:26:10.519453636Z Using /home/ame/.pyenv/versions/3.9.14/bin/python3.9 (3.9.14) to create virtualenv...\r\n"]
-[58.268856, "o", "2022-09-25T08:26:10.903973314Z \r⠋ Creating virtual environment...\b\r⠙ Creating virtual environment...\b\r⠹ Creating virtual environment...\b\r⠸ Creating virtual environment...\b\r⠼ Creating virtual environment...created virtual environment CPython3.9.14.final.0-64 in 285ms\r\n"]
-[58.272899, "o", "2022-09-25T08:26:10.904046324Z   creator CPython3Posix(dest=/home/ame/.local/share/virtualenvs/pipenvnn-1o51VEpE, clear=False, no_vcs_ignore=False, global=False)\r\n"]
-[58.281004, "o", "2022-09-25T08:26:10.904051494Z   seeder FromAppData(download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/home/ame/.local/share/virtualenv)\r\n"]
-[58.286048, "o", "2022-09-25T08:26:10.904058734Z     added seed packages: pip==22.2.2, setuptools==65.3.0, wheel==0.37.1\r\n"]
-[58.29183, "o", "2022-09-25T08:26:10.904086014Z   activators BashActivator,CShellActivator,FishActivator,NushellActivator,PowerShellActivator,PythonActivator\r\n"]
-[58.294333, "o", "2022-09-25T08:26:10.904411331Z \r\n"]
-[58.297001, "o", "2022-09-25T08:26:10.923834030Z \b✔ Successfully created virtual environment! \r\n"]
-[58.390215, "o", "2022-09-25T08:26:11.027922792Z Virtualenv location: /home/ame/.local/share/virtualenvs/pipenvnn-1o51VEpE\r\n"]
-[58.450036, "o", "2022-09-25T08:26:11.075036339Z Installing dependencies from Pipfile.lock (0ef8d2)...\r\n"]
-[179.671043, "o", "2022-09-25T08:28:12.307476588Z To activate this project's virtualenv, run pipenv shell.\r\n"]
-[179.684964, "o", "2022-09-25T08:28:12.307575378Z Alternatively, run a command inside the virtualenv with pipenv run.\r\n"]
-[179.692087, "o", "2022-09-25T08:28:12.307916705Z All dependencies are now up-to-date!\r\n"]
-[181.524306, "o", "2022-09-25T08:28:14.160540174Z 2022-09-25 08:28:14.160311: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\r\n"]
-[181.529711, "o", "2022-09-25T08:28:14.160594744Z 2022-09-25 08:28:14.160340: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\r\n"]
-[184.047019, "o", "2022-09-25T08:28:16.685284814Z Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\r\n"]
-[185.12115, "o", "2022-09-25T08:28:17.756881508Z \r   16384/11490434 [..............................] - ETA: 4s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r   65536/11490434 [..............................] - ETA: 9s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r  229376/11490434 [..............................] - ETA: 5s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r  729088/11490434 [>.............................] - ETA: 2s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 1294336/11490434 [==>...........................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 1441792/11490434 [==>...........................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 2768896/11490434 [======>.......................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 2899968/11490434 [======>.......................] - ETA: 1s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 4210688/11490434 [====="]
-[185.121306, "o", "====>....................] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 4800512/11490434 [===========>..................] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 5357568/11490434 [============>.................] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 5980160/11490434 [==============>...............] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 6553600/11490434 [================>.............] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 7159808/11490434 [=================>............] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 7733248/11490434 [===================>..........] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 8347648/11490434 [====================>.........] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 8945664/11490434 [======================>.......] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"]
-[185.121342, "o", "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r 9535488/11490434 [=======================>......] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r10108928/11490434 [=========================>....] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r10715136/11490434 [==========================>...] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r11296768/11490434 [============================>.] - ETA: 0s\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r11493376/11490434 [==============================] - 1s 0us/step\r\n"]
-[185.129339, "o", "2022-09-25T08:28:17.757007097Z \b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r11501568/11490434 [==============================] - 1s 0us/step\r\n"]
-[188.186426, "o", "2022-09-25T08:28:20.824609713Z 2022-09-25 08:28:20.824502: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\r\n"]
-[188.191468, "o", "2022-09-25T08:28:20.824631643Z 2022-09-25 08:28:20.824522: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\r\n"]
-[188.195534, "o", "2022-09-25T08:28:20.824634873Z 2022-09-25 08:28:20.824542: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:163] no NVIDIA GPU device is present: /dev/nvidia0 does not exist\r\n"]
-[188.202437, "o", "2022-09-25T08:28:20.824848462Z 2022-09-25 08:28:20.824801: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\r\n"]
-[188.205709, "o", "2022-09-25T08:28:20.824855562Z To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\r\n"]
-[188.209493, "o", "2022-09-25T08:28:20.826603251Z 2022-09-25 08:28:20.826487: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 188160000 exceeds 10% of free system memory.\r\n"]
-[188.525866, "o", "2022-09-25T08:28:21.163901882Z 2022-09-25 08:28:21.163102: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 188160000 exceeds 10% of free system memory.\r\n"]
-[189.209728, "o", "2022-09-25T08:28:21.847352249Z 2022-09-25 08:28:21.847229: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\r\n"]
-[218.588485, "o", "2022-09-25T08:28:51.224029009Z 2022-09-25 08:28:51.223852: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 188160000 exceeds 10% of free system memory.\r\n"]
-[235.033965, "o", "2022-09-25T08:29:07.672423394Z 2022-09-25 08:29:07.672319: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 188160000 exceeds 10% of free system memory.\r\n"]
-[248.133773, "o", "2022-09-25T08:29:20.772174383Z 2022-09-25 08:29:20.772070: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 188160000 exceeds 10% of free system memory.\r\n"]
-[274.608607, "o", "2022-09-25T08:29:47.247089599Z Epoch 1, Loss: 0.13676388561725616, Accuracy: 95.90332794189453, Test Loss: 0.06667815893888474, Test Accuracy: 97.73999786376953\r\n"]
-[274.610847, "o", "2022-09-25T08:29:47.247109219Z Epoch 2, Loss: 0.04383760318160057, Accuracy: 98.6383285522461, Test Loss: 0.05425187945365906, Test Accuracy: 98.25999450683594\r\n"]
-[274.612228, "o", "2022-09-25T08:29:47.247112219Z Epoch 3, Loss: 0.021431904286146164, Accuracy: 99.32666778564453, Test Loss: 0.057288240641355515, Test Accuracy: 98.19999694824219\r\n"]
-[274.613992, "o", "2022-09-25T08:29:47.247113999Z Epoch 4, Loss: 0.013719640672206879, Accuracy: 99.5566635131836, Test Loss: 0.06239854544401169, Test Accuracy: 98.3499984741211\r\n"]
-[274.615359, "o", "2022-09-25T08:29:47.247115539Z Epoch 5, Loss: 0.008981593884527683, Accuracy: 99.7066650390625, Test Loss: 0.060237348079681396, Test Accuracy: 98.5\r\n"]
-[274.945054, "o", "2022-09-25T08:29:47.583605594Z WARNING: Exiting now because of --dry-run\r\n"]
-[274.952842, "o", "2022-09-25T08:29:47.591535515Z Uploading artifacts: \r\n"]
-[275.778333, "o", "2022-09-25T08:29:48.416940946Z time=\"2022-09-25T08:29:48.416Z\" level=info msg=\"sub-process exited\" argo=true error=\"<nil>\"\r\n"]
-[276.783259, "o", "\r\n"]
-[276.785202, "o", "\r\n"]
-[276.786612, "o", "\r\n"]
-[276.787762, "o", "\r\n"]
-[276.78894, "o", "\r\n"]
-[276.790062, "o", "\r\n"]
-[276.791236, "o", "\r\n"]
-[276.792463, "o", "\r\n"]
-[276.794796, "o", "\r\n"]
-[276.795948, "o", "\r\n"]
-[276.79708, "o", "\r\n"]
-[276.985071, "o", "\r\n"]
-[277.185409, "o", "\r\n"]
-[277.385999, "o", "\r\n"]
-[277.585493, "o", "\r\n"]
-[277.785944, "o", "\r\n"]
-[277.985257, "o", "\r\n"]
-[278.185495, "o", "\r\n"]
-[278.385663, "o", "\r\n"]
-[278.585799, "o", "\r\n"]
-[278.786259, "o", "\r\n"]
-[278.985908, "o", "\r\n"]
-[279.185505, "o", "\r\n"]
-[279.385788, "o", "\r\n"]
-[279.586164, "o", "\r\n"]
-[279.786128, "o", "\r\n2022/09/25 10:29:52 Fetching artifacts produced during task execution.\r\n"]
-[279.790227, "o", "2022/09/25 10:29:52 Writing artifacts to disk. \r\n2022/09/25 10:29:52 Artifacts successfully saved\r\n"]
-[279.794433, "o", "\u001b[?2004h\r\r\n"]
-[279.7945, "o", "\u001b[1;32m[\u001b]0;jmintb@nixos: ~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn\u0007jmintb@nixos:~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn]$\u001b[0m "]
-[283.3118, "o", "^C\u001b[?2004l\r"]
-[283.311896, "o", "\u001b[?2004h\u001b[?2004l\r\r\n"]
-[283.31428, "o", "\u001b[?2004h\r\r\n"]
-[283.314327, "o", "\u001b[1;32m[\u001b]0;jmintb@nixos: ~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn\u0007jmintb@nixos:~/Sync/lab/tea-in-space/ame/test_data/test_projects/pipenvnn]$\u001b[0m "]
-[286.271737, "o", "\u001b[?2004l\r\r\n"]
-[286.271841, "o", "exit\r\n"]
diff --git a/test_data/test_projects/poetry/nn.py b/test_data/test_projects/poetry/nn.py
index b87752ae..dd72adb2 100644
--- a/test_data/test_projects/poetry/nn.py
+++ b/test_data/test_projects/poetry/nn.py
@@ -1,95 +1,3 @@
-import tensorflow as tf
+import numpy as np
 
-from tensorflow.keras.layers import Dense, Flatten, Conv2D
-from tensorflow.keras import Model
-
-mnist = tf.keras.datasets.mnist
-
-(x_train, y_train), (x_test, y_test) = mnist.load_data()
-x_train, x_test = x_train / 255.0, x_test / 255.0
-
-# Add a channels dimension
-x_train = x_train[..., tf.newaxis].astype("float32")
-x_test = x_test[..., tf.newaxis].astype("float32")
-
-train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(10000).batch(32)
-
-test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)
-
-
-class MyModel(Model):
-    def __init__(self):
-        super(MyModel, self).__init__()
-        self.conv1 = Conv2D(32, 3, activation="relu")
-        self.flatten = Flatten()
-        self.d1 = Dense(128, activation="relu")
-        self.d2 = Dense(10)
-
-    def call(self, x):
-        x = self.conv1(x)
-        x = self.flatten(x)
-        x = self.d1(x)
-        return self.d2(x)
-
-
-# Create an instance of the model
-model = MyModel()
-
-loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
-
-optimizer = tf.keras.optimizers.Adam()
-
-train_loss = tf.keras.metrics.Mean(name="train_loss")
-train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name="train_accuracy")
-
-test_loss = tf.keras.metrics.Mean(name="test_loss")
-test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name="test_accuracy")
-
-
-@tf.function
-def train_step(images, labels):
-    with tf.GradientTape() as tape:
-        # training=True is only needed if there are layers with different
-        # behavior during training versus inference (e.g. Dropout).
-        predictions = model(images, training=True)
-        loss = loss_object(labels, predictions)
-    gradients = tape.gradient(loss, model.trainable_variables)
-    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
-
-    train_loss(loss)
-    train_accuracy(labels, predictions)
-
-
-@tf.function
-def test_step(images, labels):
-    # training=False is only needed if there are layers with different
-    # behavior during training versus inference (e.g. Dropout).
-    predictions = model(images, training=False)
-    t_loss = loss_object(labels, predictions)
-
-    test_loss(t_loss)
-    test_accuracy(labels, predictions)
-
-
-EPOCHS = 5
-
-for epoch in range(EPOCHS):
-    # Reset the metrics at the start of the next epoch
-    train_loss.reset_states()
-    train_accuracy.reset_states()
-    test_loss.reset_states()
-    test_accuracy.reset_states()
-
-    for images, labels in train_ds:
-        train_step(images, labels)
-
-    for test_images, test_labels in test_ds:
-        test_step(test_images, test_labels)
-
-    print(
-        f"Epoch {epoch + 1}, "
-        f"Loss: {train_loss.result()}, "
-        f"Accuracy: {train_accuracy.result() * 100}, "
-        f"Test Loss: {test_loss.result()}, "
-        f"Test Accuracy: {test_accuracy.result() * 100}"
-    )
+print("np array: ", np.array([1,2,3]))
\ No newline at end of file
diff --git a/test_data/test_projects/sklearn_logistic_regression/ame.yaml b/test_data/test_projects/sklearn_logistic_regression/ame.yaml
index 20c2d483..9d7c83ab 100755
--- a/test_data/test_projects/sklearn_logistic_regression/ame.yaml
+++ b/test_data/test_projects/sklearn_logistic_regression/ame.yaml
@@ -1,10 +1,9 @@
-projectid: sklearn_logistic_regression
+name: sklearnlogisticregression
 tasks:
   - name: training
     resources:
       cpu: "4"
       memory: 4Gi
-    projectid: sklearn_logistic_regression
     command: mlflow run .
     task_type: 1
     source: {}
diff --git a/test_data/test_projects/sklearn_logistic_regression/mlruns/0/meta.yaml b/test_data/test_projects/sklearn_logistic_regression/mlruns/0/meta.yaml
new file mode 100644
index 00000000..b0c0370b
--- /dev/null
+++ b/test_data/test_projects/sklearn_logistic_regression/mlruns/0/meta.yaml
@@ -0,0 +1,4 @@
+artifact_location: ./mlruns/0
+experiment_id: '0'
+lifecycle_stage: active
+name: Default
diff --git a/web/src/app.rs b/web/src/app.rs
index 45a3edb0..72ec8496 100644
--- a/web/src/app.rs
+++ b/web/src/app.rs
@@ -1,14 +1,17 @@
-use ame::client::wasm_client::{build_ame_client, AmeClient};
-use ame::client::AmeCtrl;
-use ame::grpc::*;
-use ame::web::{Header, HeaderProps};
-use ame::AmeServiceClientCfg;
-use leptos::html::Input;
+use ame::{
+    client::{
+        wasm_client::{build_ame_client, AmeClient},
+        AmeCtrl,
+    },
+    grpc::*,
+    web::{Header, HeaderProps},
+    AmeServiceClientCfg,
+};
 use leptos::{
     component, create_action, create_local_resource, create_node_ref, create_rw_signal,
-    create_signal, create_signal_from_stream, log, provide_context, use_context, view, Children,
-    For, ForProps, IntoView, ReadSignal, RwSignal, Scope, SignalGet, SignalSet, SignalWith,
-    WriteSignal,
+    create_signal, create_signal_from_stream, html::Input, log, provide_context, use_context, view,
+    Children, For, ForProps, IntoView, ReadSignal, RwSignal, Scope, SignalGet, SignalSet,
+    SignalWith, WriteSignal,
 };
 
 #[allow(unused)]
@@ -20,7 +23,7 @@ use tonic::Request;
 use tonic_web_wasm_client::Client;
 
 pub fn gen_client(_endpoint: String) -> AmeClient {
-    let ame_endpoint = "http://ame.local:32662".to_string();
+    let ame_endpoint = "http://ame.local:30735".to_string();
     build_ame_client(AmeServiceClientCfg {
         id_token: None,
         endpoint: ame_endpoint,
diff --git a/web/src/main.rs b/web/src/main.rs
index ad64cefc..c6678594 100644
--- a/web/src/main.rs
+++ b/web/src/main.rs
@@ -5,8 +5,7 @@ async fn main() {
     use leptos::*;
     use leptos_axum::{generate_route_list, LeptosRoutes};
     use std::sync::Arc;
-    use web::app::*;
-    use web::fileserv::file_and_error_handler;
+    use web::{app::*, fileserv::file_and_error_handler};
 
     simple_logger::init_with_level(log::Level::Debug).expect("couldn't initialize logging");