diff --git a/assets/src/generated/graphql.ts b/assets/src/generated/graphql.ts index 0cf21608a6..32f0d016ed 100644 --- a/assets/src/generated/graphql.ts +++ b/assets/src/generated/graphql.ts @@ -1176,6 +1176,12 @@ export type ConsoleConfiguration = { vpnEnabled?: Maybe; }; +export enum ConstraintEnforcement { + Deny = 'DENY', + DryRun = 'DRY_RUN', + Warn = 'WARN' +} + export type ConstraintRef = { __typename?: 'ConstraintRef'; kind: Scalars['String']['output']; @@ -1832,6 +1838,7 @@ export type HelmConfigAttributes = { /** pointer to a Plural GitRepository */ repositoryId?: InputMaybe; set?: InputMaybe; + url?: InputMaybe; values?: InputMaybe; valuesFiles?: InputMaybe>>; version?: InputMaybe; @@ -1876,6 +1883,8 @@ export type HelmSpec = { repositoryId?: Maybe; /** a list of helm name/value pairs to precisely set individual values */ set?: Maybe>>; + /** the helm repository url to use */ + url?: Maybe; /** a helm values file to use with this service, requires auth and so is heavy to query */ values?: Maybe; /** a list of relative paths to values files to use for helm applies */ @@ -3196,6 +3205,7 @@ export type PolicyConstraint = { __typename?: 'PolicyConstraint'; cluster?: Maybe; description?: Maybe; + enforcement?: Maybe; id: Scalars['ID']['output']; insertedAt?: Maybe; name: Scalars['String']['output']; @@ -3212,6 +3222,7 @@ export type PolicyConstraint = { /** inputs to add constraint data from an OPA gatekeeper constraint CRD */ export type PolicyConstraintAttributes = { description?: InputMaybe; + enforcement?: InputMaybe; name: Scalars['String']['input']; recommendation?: InputMaybe; /** pointer to the group/name for the CR */ diff --git a/lib/console/application.ex b/lib/console/application.ex index db2a63f31a..4cf766e3eb 100644 --- a/lib/console/application.ex +++ b/lib/console/application.ex @@ -26,11 +26,13 @@ defmodule Console.Application do {Registry, [keys: :unique, name: Console.Deployments.Git.Agent.registry()]}, {Registry, [keys: :unique, name: Console.Deployments.Pipelines.Supervisor.registry()]}, {Registry, [keys: :unique, name: Console.Deployments.Stacks.Worker.registry()]}, + {Registry, [keys: :unique, name: Console.Deployments.Helm.Agent.registry()]}, {Cluster.Supervisor, [topologies, [name: Console.ClusterSupervisor]]}, Console.Deployments.Git.Supervisor, Console.Deployments.Stacks.Supervisor, Console.Deployments.Helm.Server, Console.Deployments.Pipelines.Supervisor, + Console.Deployments.Helm.Supervisor, Console.Deployments.Git.Kick, Console.Deployments.Deprecations.Table, Console.Deployments.Compatibilities.Table, diff --git a/lib/console/deployments/compatibilities/table.ex b/lib/console/deployments/compatibilities/table.ex index 789d1d5739..03e75c8f44 100644 --- a/lib/console/deployments/compatibilities/table.ex +++ b/lib/console/deployments/compatibilities/table.ex @@ -39,7 +39,7 @@ defmodule Console.Deployments.Compatibilities.Table do end def handle_info(:poll, %State{table: table, static: true} = state) do - table = Enum.reduce(Static.compatibilities(), table, &KeyValueSet.put(&2, &1.name, &1)) + table = Enum.reduce(Static.compatibilities(), table, &KeyValueSet.put!(&2, &1.name, &1)) {:noreply, %{state | table: table}} end diff --git a/lib/console/deployments/git.ex b/lib/console/deployments/git.ex index 00bc3cec2a..9f1606ff19 100644 --- a/lib/console/deployments/git.ex +++ b/lib/console/deployments/git.ex @@ -14,13 +14,15 @@ defmodule Console.Deployments.Git do ScmWebhook, PrAutomation, PullRequest, - DependencyManagementService + DependencyManagementService, + HelmRepository } @cache Console.conf(:cache_adapter) @ttl :timer.minutes(30) @type repository_resp :: {:ok, GitRepository.t} | Console.error + @type helm_resp :: {:ok, HelmRepository.t} | Console.error @type connection_resp :: {:ok, ScmConnection.t} | Console.error @type webhook_resp :: {:ok, ScmWebhook.t} | Console.error @type automation_resp :: {:ok, PrAutomation.t} | Console.error @@ -30,6 +32,8 @@ defmodule Console.Deployments.Git do def get_repository!(id), do: Repo.get!(GitRepository, id) + def get_helm_repository(url), do: Repo.get_by(HelmRepository, url: url) + def get_by_url!(url), do: Repo.get_by!(GitRepository, url: url) def get_by_url(url), do: Repo.get_by(GitRepository, url: url) @@ -327,6 +331,16 @@ defmodule Console.Deployments.Git do end end + @spec upsert_helm_repository(binary) :: helm_resp + def upsert_helm_repository(url) do + case Console.Repo.get_by(HelmRepository, url: url) do + %HelmRepository{} = repo -> repo + nil -> %HelmRepository{url: url} + end + |> HelmRepository.changeset() + |> Console.Repo.insert_or_update() + end + @doc """ Sets up a service to run the renovate cron given an scm connection and target repositories """ diff --git a/lib/console/deployments/helm/agent.ex b/lib/console/deployments/helm/agent.ex new file mode 100644 index 0000000000..84cfb9cc9c --- /dev/null +++ b/lib/console/deployments/helm/agent.ex @@ -0,0 +1,86 @@ +defmodule Console.Deployments.Helm.Agent do + use GenServer, restart: :transient + alias Console.Deployments.Git + alias Console.Deployments.Helm.{AgentCache, Discovery} + alias Console.Schema.HelmRepository + require Logger + + defmodule State, do: defstruct [:repo, :cache] + + @poll :timer.minutes(2) + @jitter 15 + + def registry(), do: __MODULE__ + + def fetch(pid, chart, vsn), do: GenServer.call(pid, {:fetch, chart, vsn}) + + def start(url) do + GenServer.start(__MODULE__, url, name: via(url)) + end + + def start_link([url]), do: start_link(url) + def start_link(url) do + GenServer.start_link(__MODULE__, url, name: via(url)) + end + + defp via(url), do: {:via, Registry, {registry(), {:helm, url}}} + + def init(url) do + {:ok, repo} = Git.upsert_helm_repository(url) + schedule_pull() + :timer.send_interval(@poll, :move) + send self(), :pull + {:ok, %State{repo: repo, cache: AgentCache.new(repo)}} + end + + def handle_call({:fetch, c, v}, _, %State{cache: cache} = state) do + with {:ok, l, cache} <- AgentCache.fetch(cache, c, v), + {:ok, f} <- File.open(l.file) do + {:reply, {:ok, f, l.digest}, %{state | cache: cache}} + else + err -> {:reply, err, state} + end + end + + def handle_info(:pull, %State{repo: repo, cache: cache} = state) do + with {:ok, repo} <- Git.upsert_helm_repository(repo.url), + {:ok, cache} <- AgentCache.refresh(cache), + {:ok, repo} <- refresh(repo) do + schedule_pull() + {:noreply, %{state | cache: cache, repo: repo}} + else + err -> + schedule_pull() + Logger.error "Failed to resync helm repo: #{inspect(err)}" + {:noreply, state} + end + end + + def handle_info({:refresh, c, v}, %State{cache: cache} = state) do + case AgentCache.write(cache, c, v) do + {:ok, _, cache} -> {:noreply, %{state | cache: cache}} + _ -> {:noreply, state} + end + end + + def handle_info(:move, %State{repo: repo} = state) do + case Discovery.local?(repo.url) do + true -> {:noreply, state} + false -> {:stop, {:shutdown, :moved}, state} + end + end + + def handle_info(_, state), do: {:noreply, state} + + defp refresh(%HelmRepository{} = repo) do + HelmRepository.changeset(repo, %{pulled_at: Timex.now(), health: :pullable}) + |> Console.Repo.update() + end + + defp schedule_pull(), do: Process.send_after(self(), :pull, @poll + jitter()) + + defp jitter() do + :rand.uniform(@jitter) + |> :timer.seconds() + end +end diff --git a/lib/console/deployments/helm/agent_cache.ex b/lib/console/deployments/helm/agent_cache.ex new file mode 100644 index 0000000000..bc95cba0fe --- /dev/null +++ b/lib/console/deployments/helm/agent_cache.ex @@ -0,0 +1,67 @@ +defmodule Console.Deployments.Helm.AgentCache do + alias Console.Helm.Client + alias Console.Deployments.Helm.Utils + require Logger + + defstruct [:repo, :index, :dir, cache: %{}] + + defmodule Line do + @expiry [minutes: -10] + defstruct [:file, :chart, :vsn, :digest, :touched] + + def new(file, chart, vsn, digest) do + %__MODULE__{file: file, chart: chart, vsn: vsn, digest: digest, touched: Timex.now()} + end + + def touch(%__MODULE__{} = mod), do: %{mod | touched: Timex.now()} + + def expire(%__MODULE__{file: f}), do: File.rm(f) + + def expired?(%__MODULE__{touched: touched}) do + Timex.now() + |> Timex.shift(@expiry) + |> Timex.after?(touched) + end + end + + def new(repo) do + {:ok, dir} = Briefly.create(directory: true) + %__MODULE__{repo: repo, dir: dir, cache: %{}} + end + + def refresh(%__MODULE__{} = cache) do + case Client.index(cache.repo.url) do + {:ok, indx} -> {:ok, sweep(%{cache | index: indx})} + _ -> {:error, "could not fetch index"} + end + end + + def fetch(%__MODULE__{index: nil} = cache, chart, vsn) do + with {:ok, cache} <- refresh(cache), + do: fetch(cache, chart, vsn) + end + + def fetch(%__MODULE__{cache: lines} = cache, chart, vsn) do + case lines[{chart, vsn}] do + %Line{} = l -> {:ok, l, put_in(cache.cache[{chart, vsn}], Line.touch(l))} + nil -> write(cache, chart, vsn) + end + end + + def write(%__MODULE__{} = cache, chart, vsn) do + path = Path.join(cache.dir, "#{chart}.#{vsn}.tgz") + with {:ok, url, digest} <- Client.chart(cache.index, chart, vsn), + {:ok, tmp} <- Briefly.create(), + {:ok, _} <- Client.download(url, File.stream!(tmp)), + :ok <- Utils.clean_chart(tmp, path, chart), + line <- Line.new(path, chart, vsn, digest), + do: {:ok, line, put_in(cache.cache[{chart, vsn}], line)} + end + + defp sweep(%__MODULE__{cache: lines} = cache) do + {keep, expire} = Enum.split_with(lines, fn {_, l} -> !Line.expired?(l) end) + Enum.each(expire, &Line.expire/1) + Enum.each(keep, fn l -> send(self(), {:refresh, l.chart, l.vsn}) end) + %{cache | cache: Map.new(keep)} + end +end diff --git a/lib/console/deployments/helm/cache.ex b/lib/console/deployments/helm/cache.ex index 326ae7b986..997753987f 100644 --- a/lib/console/deployments/helm/cache.ex +++ b/lib/console/deployments/helm/cache.ex @@ -2,7 +2,7 @@ defmodule Console.Deployments.Helm.Cache do require Logger alias Kube.HelmChart alias Kube.HelmChart.Status - alias Console.Deployments.Tar + alias Console.Deployments.{Helm.Utils, Tar} defstruct [:dir, :touched] @@ -48,12 +48,17 @@ defmodule Console.Deployments.Helm.Cache do end defp build_tarball(url, cache, path, chart) do - with {:ok, f} <- Tar.from_url(url), - {:ok, contents} <- Tar.tar_stream(f), - :ok <- Tar.tarball(path, remove_prefix(contents, chart)), + with {:ok, path} <- download_to(url, path, chart), do: open(cache, path) end + def download_to(url, path, chart) do + with {:ok, tmp} <- Tar.from_url(url), + :ok <- File.open!(tmp) |> Utils.clean_chart(path, chart), + :ok <- File.rm(tmp), + do: {:ok, path} + end + def refresh(%__MODULE__{touched: touched} = cache) do Logger.info "expiring helm chart cache..." expires = Timex.now() |> Timex.shift(minutes: -30) diff --git a/lib/console/deployments/helm/discovery.ex b/lib/console/deployments/helm/discovery.ex new file mode 100644 index 0000000000..65283d3342 --- /dev/null +++ b/lib/console/deployments/helm/discovery.ex @@ -0,0 +1,33 @@ +defmodule Console.Deployments.Helm.Discovery do + alias Console.Deployments.Helm.{Supervisor, Agent} + + def agent(url) do + case maybe_rpc(url, Supervisor, :start_child, [url]) do + {:ok, pid} -> {:ok, pid} + {:error, {:already_started, pid}} -> {:ok, pid} + err -> err + end + end + + def fetch(url, chart, vsn) do + with {:ok, pid} <- agent(url), + do: Agent.fetch(pid, chart, vsn) + end + + defp maybe_rpc(id, module, func, args) do + me = node() + case worker_node(id) do + ^me -> apply(module, func, args) + node -> :rpc.call(node, module, func, args) + end + end + + def worker_node(url), do: HashRing.key_to_node(ring(), url) + + def local?(url), do: worker_node(url) == node() + + defp ring() do + HashRing.new() + |> HashRing.add_nodes([node() | Node.list()]) + end +end diff --git a/lib/console/deployments/helm/supervisor.ex b/lib/console/deployments/helm/supervisor.ex new file mode 100644 index 0000000000..9508178d0e --- /dev/null +++ b/lib/console/deployments/helm/supervisor.ex @@ -0,0 +1,17 @@ +defmodule Console.Deployments.Helm.Supervisor do + use DynamicSupervisor + alias Console.Deployments.Helm.Agent + + def start_link(init_arg \\ :ok) do + DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + def start_child(url) do + DynamicSupervisor.start_child(__MODULE__, {Agent, url}) + end + + @impl true + def init(_init_arg) do + DynamicSupervisor.init(strategy: :one_for_one) + end +end diff --git a/lib/console/deployments/helm/utils.ex b/lib/console/deployments/helm/utils.ex new file mode 100644 index 0000000000..76d6b39b02 --- /dev/null +++ b/lib/console/deployments/helm/utils.ex @@ -0,0 +1,23 @@ +defmodule Console.Deployments.Helm.Utils do + alias Console.Deployments.Tar + + def clean_chart(path, to, chart) when is_binary(path) do + file = File.open!(path) + try do + clean_chart(file, to, chart) + after + File.close(file) + end + end + + def clean_chart(f, to, chart) do + with {:ok, contents} <- Tar.tar_stream(f), + do: Tar.tarball(to, remove_prefix(contents, chart)) + end + + defp remove_prefix(contents, chart) do + Enum.map(contents, fn {path, content} -> + {String.trim_leading(path, "#{chart}/"), content} + end) + end +end diff --git a/lib/console/deployments/services.ex b/lib/console/deployments/services.ex index a28b18a153..e5360d3f7a 100644 --- a/lib/console/deployments/services.ex +++ b/lib/console/deployments/services.ex @@ -109,7 +109,16 @@ defmodule Console.Deployments.Services do Git.get_repository!(id) |> Git.Discovery.fetch(git) end - defp tarfile(%Service{helm: %Service.Helm{chart: c, version: v}} = svc) when is_binary(c) and is_binary(v) do + + defp tarfile(%Service{helm: %Service.Helm{chart: c, version: v, url: url}} = svc) + when is_binary(c) and is_binary(v) and is_binary(url) do + with {:ok, f, sha} <- Helm.Discovery.fetch(url, c, v), + {:ok, _} <- update_sha_without_revision(svc, sha), + do: {:ok, f} + end + + defp tarfile(%Service{helm: %Service.Helm{chart: c, version: v}} = svc) + when is_binary(c) and is_binary(v) do with {:ok, f, sha} <- Helm.Charts.artifact(svc), {:ok, _} <- update_sha_without_revision(svc, sha), do: {:ok, f} diff --git a/lib/console/deployments/tar.ex b/lib/console/deployments/tar.ex index d5d1614053..88a10599d8 100644 --- a/lib/console/deployments/tar.ex +++ b/lib/console/deployments/tar.ex @@ -6,10 +6,9 @@ defmodule Console.Deployments.Tar do """ @spec from_url(binary) :: {:ok, File.t} | err def from_url(url) do - stream = HTTPStream.get(url) with {:ok, tmp} <- Briefly.create(), - :ok <- Stream.into(stream, File.stream!(tmp)) |> Stream.run(), - do: File.open(tmp) + {:ok, _} <- Req.get(url, into: File.stream!(tmp)), + do: {:ok, tmp} end @doc """ @@ -51,6 +50,7 @@ defmodule Console.Deployments.Tar do with {:ok, tmp} <- Briefly.create(), _ <- IO.binstream(tar_file, 1024) |> Enum.into(File.stream!(tmp)), {:ok, res} <- :erl_tar.extract(tmp, [:compressed, :memory]), + _ <- File.rm(tmp), do: {:ok, Enum.map(res, fn {name, content} -> {to_string(name), to_string(content)} end)} after File.close(tar_file) diff --git a/lib/console/graphql/deployments/policy.ex b/lib/console/graphql/deployments/policy.ex index 2dd2a43cf3..f64ff387ee 100644 --- a/lib/console/graphql/deployments/policy.ex +++ b/lib/console/graphql/deployments/policy.ex @@ -2,6 +2,8 @@ defmodule Console.GraphQl.Deployments.Policy do use Console.GraphQl.Schema.Base alias Console.GraphQl.Resolvers.Deployments + ecto_enum :constraint_enforcement, Console.Schema.PolicyConstraint.Enforcement + enum :constraint_violation_field do value :namespace value :kind @@ -15,6 +17,7 @@ defmodule Console.GraphQl.Deployments.Policy do field :violation_count, :integer field :ref, :constraint_ref_attributes, description: "pointer to the group/name for the CR" field :violations, list_of(:violation_attributes) + field :enforcement, :constraint_enforcement end input_object :constraint_ref_attributes do @@ -38,6 +41,7 @@ defmodule Console.GraphQl.Deployments.Policy do field :description, :string field :recommendation, :string field :violation_count, :integer + field :enforcement, :constraint_enforcement field :object, :kubernetes_unstructured, description: "Fetches the live constraint object from K8s, this is an expensive query and should not be done in list endpoints", diff --git a/lib/console/graphql/deployments/service.ex b/lib/console/graphql/deployments/service.ex index ce1e3bf6a5..2383ebf62b 100644 --- a/lib/console/graphql/deployments/service.ex +++ b/lib/console/graphql/deployments/service.ex @@ -39,6 +39,7 @@ defmodule Console.GraphQl.Deployments.Service do field :chart, :string field :version, :string field :release, :string + field :url, :string field :set, :helm_value_attributes field :repository, :namespaced_name field :git, :git_ref_attributes @@ -224,6 +225,7 @@ defmodule Console.GraphQl.Deployments.Service do object :helm_spec do field :chart, :string, description: "the name of the chart this service is using" + field :url, :string, description: "the helm repository url to use" field :values, :string, description: "a helm values file to use with this service, requires auth and so is heavy to query", resolve: &Deployments.helm_values/3 diff --git a/lib/console/schema/helm_repository.ex b/lib/console/schema/helm_repository.ex new file mode 100644 index 0000000000..ca8877b1a4 --- /dev/null +++ b/lib/console/schema/helm_repository.ex @@ -0,0 +1,25 @@ +defmodule Console.Schema.HelmRepository do + use Piazza.Ecto.Schema + alias Console.Schema.GitRepository + + defenum Provider, basic: 0, bearer: 1, gcp: 2, azure: 3, aws: 4 + + schema "helm_repositories" do + field :url, :string + field :provider, Provider + field :health, GitRepository.Health + field :pulled_at, :utc_datetime_usec + + field :index, :map, virtual: true + + timestamps() + end + + @valid ~w(url provider health pulled_at)a + + def changeset(model, attrs \\ %{}) do + model + |> cast(attrs, @valid) + |> validate_required([:url]) + end +end diff --git a/lib/console/schema/policy_constraint.ex b/lib/console/schema/policy_constraint.ex index 60baa82961..032bcb7916 100644 --- a/lib/console/schema/policy_constraint.ex +++ b/lib/console/schema/policy_constraint.ex @@ -2,11 +2,14 @@ defmodule Console.Schema.PolicyConstraint do use Piazza.Ecto.Schema alias Console.Schema.{Cluster, ConstraintViolation} + defenum Enforcement, warn: 0, deny: 1, dry_run: 2 + schema "policy_constraints" do field :name, :string field :description, :string field :recommendation, :string field :violation_count, :integer + field :enforcement, Enforcement embeds_one :ref, Ref, on_replace: :update do field :kind, :string @@ -91,7 +94,7 @@ defmodule Console.Schema.PolicyConstraint do from(p in query, order_by: ^order) end - @valid ~w(name description recommendation violation_count)a + @valid ~w(name description recommendation violation_count enforcement)a def changeset(model, attrs \\ %{}) do model diff --git a/lib/console/schema/service.ex b/lib/console/schema/service.ex index 378606dd12..6bdb4ea7c1 100644 --- a/lib/console/schema/service.ex +++ b/lib/console/schema/service.ex @@ -47,6 +47,7 @@ defmodule Console.Schema.Service do field :chart, :string field :version, :string field :release, :string + field :url, :string field :values_files, {:array, :string} field :repository_id, :binary_id diff --git a/lib/helm/client.ex b/lib/helm/client.ex new file mode 100644 index 0000000000..c286b997fd --- /dev/null +++ b/lib/helm/client.ex @@ -0,0 +1,26 @@ +defmodule Console.Helm.Client do + alias Console.Helm.{Index, Chart} + + def index(url) do + with {:ok, %Req.Response{status: 200, body: body}} <- Req.get("#{url}/index.yaml"), + {:ok, yaml} <- YamlElixir.read_from_string(body) do + {:ok, Index.transform(%Index{entries: yaml["entries"]})} + else + _ -> {:error, "fould not fetch helm index"} + end + end + + @spec chart(Index.t, binary, binary) :: {:ok, binary, binary} | Console.error + def chart(%Index{entries: entries}, chart, vsn) do + entries = Map.new(entries, & {&1.name, &1.versions}) + with {:chart, %{^chart => chart}} <- {:chart, entries}, + {:version, %Chart{} = chart} <- {:version, Enum.find(chart, & &1.version == vsn)} do + {:ok, hd(chart.urls), chart.digest} + else + {:chart, _} -> {:error, "could not find chart #{chart}"} + {:version, _} -> {:error, "could not find version #{vsn}"} + end + end + + def download(url, to), do: Req.get(url, into: to) +end diff --git a/lib/helm/schema.ex b/lib/helm/schema.ex new file mode 100644 index 0000000000..df8c4d5138 --- /dev/null +++ b/lib/helm/schema.ex @@ -0,0 +1,30 @@ +defmodule Console.Helm.Chart do + @type t :: %__MODULE__{} + defstruct [:app_version, :version, :urls, :name, :type, :application, :digest] + + def build(map) do + %__MODULE__{ + app_version: map["appVersion"], + version: map["version"], + name: map["name"], + type: map["type"], + urls: map["urls"], + application: map["application"], + digest: map["digest"] + } + end +end + +defmodule Console.Helm.Index do + alias Console.Helm.Chart + + defstruct [:oci, :entries, :apiVersion] + + def transform(%__MODULE__{entries: %{} = entries} = repo) do + entries = Enum.map(entries, fn {name, charts} -> + %{name: name, versions: Enum.map(charts, &Chart.build/1)} + end) + %{repo | entries: entries} + end + def transform(pass), do: pass +end diff --git a/priv/repo/migrations/20240502133731_add_helm_repositories.exs b/priv/repo/migrations/20240502133731_add_helm_repositories.exs new file mode 100644 index 0000000000..4a257d7842 --- /dev/null +++ b/priv/repo/migrations/20240502133731_add_helm_repositories.exs @@ -0,0 +1,38 @@ +defmodule Console.Repo.Migrations.AddHelmRepositories do + use Ecto.Migration + + def change do + create table(:helm_repositories, primary_key: false) do + add :id, :uuid, primary_key: true + add :url, :string + add :status, :integer + add :provider, :integer + add :auth, :map + add :pulled_at, :utc_datetime_usec + add :health, :integer + + timestamps() + end + + create unique_index(:helm_repositories, [:url]) + + alter table(:policy_constraints) do + add :enforcement, :integer + end + + alter table(:service_templates) do + modify :repository_id, references(:git_repositories, type: :uuid, on_delete: :delete_all), + from: references(:git_repositories, type: :uuid) + end + + alter table(:global_services) do + modify :template_id, references(:service_templates, type: :uuid), + from: references(:service_templates, type: :uuid, on_delete: :delete_all) + end + + alter table(:managed_namespaces) do + modify :service_id, references(:service_templates, type: :uuid), + from: references(:service_templates, type: :uuid, on_delete: :delete_all) + end + end +end diff --git a/schema/schema.graphql b/schema/schema.graphql index 553262cdb3..003d07a8c6 100644 --- a/schema/schema.graphql +++ b/schema/schema.graphql @@ -1447,6 +1447,12 @@ type ManagedNamespaceConnection { edges: [ManagedNamespaceEdge] } +enum ConstraintEnforcement { + WARN + DENY + DRY_RUN +} + enum ConstraintViolationField { NAMESPACE KIND @@ -1466,6 +1472,8 @@ input PolicyConstraintAttributes { ref: ConstraintRefAttributes violations: [ViolationAttributes] + + enforcement: ConstraintEnforcement } input ConstraintRefAttributes { @@ -1494,6 +1502,8 @@ type PolicyConstraint { violationCount: Int + enforcement: ConstraintEnforcement + "Fetches the live constraint object from K8s, this is an expensive query and should not be done in list endpoints" object: KubernetesUnstructured @@ -2292,6 +2302,8 @@ input HelmConfigAttributes { release: String + url: String + set: HelmValueAttributes repository: NamespacedName @@ -2564,6 +2576,9 @@ type HelmSpec { "the name of the chart this service is using" chart: String + "the helm repository url to use" + url: String + "a helm values file to use with this service, requires auth and so is heavy to query" values: String diff --git a/test/console/deployments/helm/agent_test.exs b/test/console/deployments/helm/agent_test.exs new file mode 100644 index 0000000000..cb9c2962ed --- /dev/null +++ b/test/console/deployments/helm/agent_test.exs @@ -0,0 +1,28 @@ +defmodule Console.Deployments.Helm.AgentTest do + use Console.DataCase, async: false + alias Console.Deployments.Helm.Agent + + describe "#start/1" do + test "it can fetch a chart and version" do + repo = "https://pluralsh.github.io/console" + {:ok, pid} = Agent.start(repo) + + {:ok, f, _} = Agent.fetch(pid, "console", "0.3.15") + + files = stream_and_untar(f) + assert files["Chart.yaml"] + + assert Console.Deployments.Git.get_helm_repository(repo).health == :pullable + end + end + + defp stream_and_untar(f) do + {:ok, tmp} = Briefly.create() + IO.binstream(f, 1024) + |> Enum.into(File.stream!(tmp)) + File.close(f) + + {:ok, res} = :erl_tar.extract(tmp, [:compressed, :memory]) + Enum.into(res, %{}, fn {name, content} -> {to_string(name), to_string(content)} end) + end +end diff --git a/test/console/deployments/services_test.exs b/test/console/deployments/services_test.exs index 9ff5eb049c..9ad3736729 100644 --- a/test/console/deployments/services_test.exs +++ b/test/console/deployments/services_test.exs @@ -1116,6 +1116,23 @@ defmodule Console.Deployments.ServicesAsyncTest do assert refetch(svc).sha == "sha" end + test "it can fetch a chart for a helm service by url" do + svc = insert(:service, + helm: %{ + url: "https://stefanprodan.github.io/podinfo", + chart: "podinfo", + version: "6.5.2" + } + ) + + {:ok, f} = Services.tarstream(svc) + {:ok, content} = Tar.tar_stream(f) + content = Map.new(content) + assert content["Chart.yaml"] + + assert refetch(svc).sha == "98eeab2a630dbe6605266b635d0dfa0ce595bfe019b843f628c775ed1c588838" + end + test "it can splice in a new values.yaml.tpl" do git = insert(:git_repository, url: "https://github.com/pluralsh/console.git") svc = insert(:service, helm: %{values: "value: test"}, repository: git, git: %{ref: "master", folder: "charts/console"})