Troubleshoot vector indexing

Need to figure out what's going on while e2e testing

.github/workflows/ai-proxy-cd.yaml

@@ -27,7 +27,7 @@ env:
 jobs:
   test:
     name: Unit test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -44,7 +44,7 @@ jobs:
 
   publish-docker:
     name: Build and push ai-proxy container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/ai-proxy-ci.yaml

@@ -34,7 +34,7 @@ jobs:
 
   check:
     name: Check
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/assets.yaml

@@ -16,7 +16,7 @@ on:
 jobs:
   # e2e:
   #   name: End-to-end test
-  #   runs-on: ubuntu-20.04
+  #   runs-on: ubuntu-latest
   #   env:
   #     CYPRESS_EMAIL: ${{ secrets.CYPRESS_EMAIL }}
   #     CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }}
@@ -50,7 +50,7 @@ jobs:
 
   test:
     name: Unit test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -64,7 +64,7 @@ jobs:
       - run: yarn test
   lint:
     name: Lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/codeql.yml

@@ -17,7 +17,7 @@ jobs:
       contents: read
       security-events: write
     name: CodeQL
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:

.github/workflows/controller-cd.yaml

@@ -21,7 +21,7 @@ env:
 jobs:
   test:
     name: Unit test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -41,7 +41,7 @@ jobs:
       run: PATH=$PATH:$GOPATH/bin make test
   publish-docker:
     name: Build and push controller container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/controller-ci.yaml

@@ -37,7 +37,7 @@ jobs:
       - run: PATH=$PATH:$GOPATH/bin make build
   unit-test:
     name: Unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/demo-cd.yaml

@@ -7,9 +7,6 @@ on:
     paths:
       - ".github/workflows/flaky-service-cd.yaml"
       - "go/demo/**"
-  push:
-    tags:
-      - 'go/demo/v*.*.*'
 
 permissions:
   contents: read
@@ -22,7 +19,7 @@ env:
 jobs:
   publish-flaky-service-docker:
     name: Build and push flaky-service container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -35,18 +32,18 @@ jobs:
       - uses: actions/[email protected]
         with:
           fetch-depth: 0
-      - id: meta-flaky-service
-        uses: docker/metadata-action@v5
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
         with:
+          # list of Docker images to use as base name for tags
           images: |
             ghcr.io/pluralsh/flaky-service
-          tags: v1.0.0
-      - id: meta-flaky-service-sidecar
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ghcr.io/pluralsh/flaky-service-sidecar
-          tags: v1.0.0
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=sha
+            type=ref,event=pr
+            type=ref,event=branch
       - uses: docker/login-action@v3
         with:
           registry: ghcr.io
@@ -59,21 +56,8 @@ jobs:
           context: "./go/demo/flaky-service"
           file: "./go/demo/flaky-service/Dockerfile"
           push: true
-          tags: ${{ steps.meta-flaky-service.outputs.tags }}
-          labels: ${{ steps.meta-flaky-service.outputs.labels }}
-          platforms: linux/amd64, linux/arm64
-          cache-from: type=gha
-          cache-to: type=gha, mode=max
-          build-args: |
-            GIT_COMMIT=${{ github.sha }}
-            VERSION=${{ steps.meta.outputs.version }}
-      - uses: docker/build-push-action@v5
-        with:
-          context: "./go/demo/flaky-service"
-          file: "./go/demo/flaky-service/Dockerfile.sidecar"
-          push: true
-          tags: ${{ steps.meta-flaky-service-sidecar.outputs.tags }}
-          labels: ${{ steps.meta-flaky-service-sidecar.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
           platforms: linux/amd64, linux/arm64
           cache-from: type=gha
           cache-to: type=gha, mode=max

.github/workflows/oci-auth-cd.yaml

@@ -21,7 +21,7 @@ env:
 jobs:
   test:
     name: Unit test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -37,7 +37,7 @@ jobs:
     - run: PATH=$PATH:$GOPATH/bin make test
   publish-docker:
     name: Build and push oci-auth container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/oci-auth-ci.yaml

@@ -36,7 +36,7 @@ jobs:
       - run: PATH=$PATH:$GOPATH/bin make build
   unit-test:
     name: Unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/pr-labels.yaml

@@ -6,7 +6,7 @@ on:
 jobs:
   label:
     name: Check that PR has required labels
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: mheap/github-action-required-labels@v2
         with:

.github/workflows/publish.yaml

@@ -6,7 +6,7 @@ on:
       - "v*.*.*"
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: erlef/setup-beam@v1
@@ -45,7 +45,7 @@ jobs:
         if: always()
   publish:
     name: Build and push Console container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: test
     permissions:
       contents: "read"
@@ -133,7 +133,7 @@ jobs:
         if: always()
   release:
     name: Create GitHub release
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: publish
     permissions:
       contents: write
@@ -148,7 +148,7 @@ jobs:
           draft: false
   bump:
     name: Bump Chart Version
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [release]
     permissions:
       contents: write
@@ -207,7 +207,7 @@ jobs:
           base: master
   bump-rapid:
     name: Bump Rapid Chart Version
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [release]
     permissions:
       contents: write

.github/workflows/test.yaml

@@ -8,7 +8,7 @@ on:
 jobs:
   build:
     name: Test Build Docker image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -64,7 +64,7 @@ jobs:
 
   test:
     name: Test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: erlef/setup-beam@v1
@@ -103,7 +103,7 @@ jobs:
         if: always()
   updateSchema:
     name: Check that Schema is up to date
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: erlef/setup-beam@v1

go/demo/flaky-service/api/behavior_modifiers.go

@@ -3,6 +3,7 @@ package api
 import (
 	"log/slog"
 	"net/http"
+	"runtime/debug"
 	"time"
 
 	"github.com/pluralsh/console/go/demo/flaky-service/metrics"
@@ -26,13 +27,14 @@ func HandleRequestTimestampModulus(timestampModulus int64) http.HandlerFunc {
 
 		if time_now%timestampModulus == 0 {
 			slog.Error("found unknown error, returning status.InternalServerError", "time_now", time_now, "modulus", timestampModulus)
+			slog.Error("dumping stacktrace", "stacktrace", string(debug.Stack()))
 
 			metrics.IncrementRequestCounter(http.StatusInternalServerError, r.Method)
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Header().Set("Content-Type", "application/json")
 			w.Write([]byte(`{"message": "req failed"}`))
 		} else {
-			slog.Info("Timestamp is not multiple of modulus, returning status.OK", "time_now", time_now, "modulus", timestampModulus)
+			slog.Info("Everything seems fine, returning status.OK", "time_now", time_now, "modulus", timestampModulus)
 
 			metrics.IncrementRequestCounter(http.StatusOK, r.Method)
 			w.WriteHeader(http.StatusOK)

lib/console/ai/evidence/alert.ex

@@ -10,6 +10,7 @@ defimpl Console.AI.Evidence, for: Console.Schema.Alert do
     |> Vector.with_vector_data()
     |> Context.prompt({:user, "Please use the data I've listed above to give a clear root cause analysis of this issue."})
     |> Context.result()
+    |> IO.inspect(labels: "alert evidence")
   end
   def generate(%Alert{state: :resolved}), do: {:error, "alert is already resolved"}
   def generate(_), do: {:error, "insights only supported for service-bound alerts"}

lib/console/ai/evidence/context.ex

@@ -15,6 +15,7 @@ defmodule Console.AI.Evidence.Context do
   def claims(%__MODULE__{} = ctx, %{evidence: [_ | _] = evidence}), do: evidence(ctx, evidence)
   def claims(%__MODULE__{} = ctx, _), do: ctx
 
+  def prompt(ctx, {_, nil}), do: ctx
   def prompt(%__MODULE__{history: hist} = ctx, msg), do: %{ctx | history: append(hist, msg)}
 
   def reduce(%__MODULE__{} = ctx, enum, fun) when is_function(fun, 2), do: Enum.reduce(enum, ctx, fun)

lib/console/ai/evidence/vector.ex

@@ -22,7 +22,7 @@ defmodule Console.AI.Evidence.Vector do
     with true <- VectorStore.enabled?(),
          {:ok, %Vector{query: query}} <- use_vector(ctx.history),
          {:ok, [_ | _] = vdata} <- VectorStore.fetch(query) do
-      Context.prompt(ctx, {:user, "I've also found some relevent external data that could add additional context to what caused the issue:"})
+      Context.prompt(ctx, {:user, "I've also found some relevent data that could add additional context to what caused the issue.:"})
       |> Context.reduce(vdata, &Context.prompt(&2, {:user, vector_prompt(&1)}))
       |> Context.evidence(vector_evidence(vdata))
     else
@@ -32,9 +32,32 @@ defmodule Console.AI.Evidence.Vector do
     end
   end
 
-  defp vector_prompt(%VectorStore.Response{alert_resolution: alert_resolution}),
-    do: "A prior alert resolution with data like so: #{json!(alert_resolution)}"
-  defp vector_prompt(%VectorStore.Response{pr_file: pr_file}), do: "A file from a given pr with data like so: #{json!(pr_file)}"
+  defp vector_prompt(%VectorStore.Response{type: :alert, alert_resolution: alert_resolution}),
+    do: "A prior alert resolution with data like so that likely was caused by the same issue: #{json!(alert_resolution)}"
+  defp vector_prompt(%VectorStore.Response{type: :pr, pr_file: pr_file}) do
+    """
+    A file from a given pull request with information like so, containing a possible code change that caused the issue, described below:
+
+    Pull Request URL: #{pr_file.url}
+    Repo: #{pr_file.repo}
+    PR Title: #{pr_file.title}
+    Commit SHA: #{pr_file.sha}
+    Filename: #{pr_file.filename}
+
+    The full contents of the file is:
+
+    ```
+    #{pr_file.contents}
+    ```
+
+    The git patch of the change is:
+
+    ```
+    #{pr_file.patch}
+    ```
+    """
+  end
+  defp vector_prompt(_), do: nil
 
   defp vector_evidence(vdata) do
     Enum.map(vdata, fn

lib/console/ai/provider.ex

@@ -12,8 +12,7 @@ defmodule Console.AI.Provider do
   You're a seasoned devops engineer with experience in Kubernetes, GitOps and Infrastructure As Code, and need to
   give a concise but clear explanation of issues in your companies kubernetes infrastructure.  The user is not necessarily
   an expert in the domain, so please provide as much documentation and evidence as is necessary to explain what issue they're
-  facing.  Please provide a clear summary and any details to debug what's going on with the case provided.  You should guide users
-  to implement GitOps best practices, so avoid telling them to manually modify resources via kubectl, helm or terraform commands directly.
+  facing.  Please provide a clear summary and any details to debug what's going on with the case provided.
   """}
 
   @summary """

lib/console/ai/pubsub/vector/consumer.ex

@@ -12,7 +12,7 @@ defmodule Console.AI.PubSub.Vector.Consumer do
     end
   end
 
-  defp insert({:ok, [_ | _] = resources}), do: Enum.each(resources, &VectorStore.insert/1)
+  defp insert({:ok, resources}) when is_list(resources), do: Enum.each(resources, &VectorStore.insert/1)
   defp insert({:ok, res}), do: VectorStore.insert(res)
   defp insert(pass), do: pass
 end

lib/console/ai/pubsub/vector/protocol.ex

@@ -13,14 +13,16 @@ defimpl Console.AI.PubSub.Vectorizable, for: Console.PubSub.ScmWebhook do
   alias Console.AI.Tool
   alias Console.Deployments.Pr.Dispatcher
   alias Console.Schema.{ScmWebhook, ScmConnection}
+  require Logger
 
   def resource(%@for{
-    item: %{"action" => "pull_request", "pull_request" => %{"merged" => true} = pr},
+    item: %{"action" => "closed", "pull_request" => %{"merged" => true} = pr},
     actor: %ScmWebhook{type: :github}
   }) do
     with %ScmConnection{} = conn <- Tool.scm_connection(),
       do: Dispatcher.files(conn, pr)
   end
+
   def resource(_), do: :ok
 end