Troubleshoot vector indexing

Need to figure out what's going on while e2e testing

.github/workflows/ai-proxy-cd.yaml

@@ -27,7 +27,7 @@ env:
 jobs:
   test:
     name: Unit test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -44,7 +44,7 @@ jobs:
 
   publish-docker:
     name: Build and push ai-proxy container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/ai-proxy-ci.yaml

@@ -34,7 +34,7 @@ jobs:
 
   check:
     name: Check
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/assets.yaml

@@ -16,7 +16,7 @@ on:
 jobs:
   # e2e:
   #   name: End-to-end test
-  #   runs-on: ubuntu-20.04
+  #   runs-on: ubuntu-latest
   #   env:
   #     CYPRESS_EMAIL: ${{ secrets.CYPRESS_EMAIL }}
   #     CYPRESS_PASSWORD: ${{ secrets.CYPRESS_PASSWORD }}
@@ -50,7 +50,7 @@ jobs:
 
   test:
     name: Unit test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -64,7 +64,7 @@ jobs:
       - run: yarn test
   lint:
     name: Lint
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/codeql.yml

@@ -17,7 +17,7 @@ jobs:
       contents: read
       security-events: write
     name: CodeQL
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:

.github/workflows/controller-cd.yaml

@@ -21,7 +21,7 @@ env:
 jobs:
   test:
     name: Unit test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -41,7 +41,7 @@ jobs:
       run: PATH=$PATH:$GOPATH/bin make test
   publish-docker:
     name: Build and push controller container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/controller-ci.yaml

@@ -37,7 +37,7 @@ jobs:
       - run: PATH=$PATH:$GOPATH/bin make build
   unit-test:
     name: Unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/demo-cd.yaml

@@ -7,9 +7,6 @@ on:
     paths:
       - ".github/workflows/flaky-service-cd.yaml"
       - "go/demo/**"
-  push:
-    tags:
-      - 'go/demo/v*.*.*'
 
 permissions:
   contents: read
@@ -22,7 +19,7 @@ env:
 jobs:
   publish-flaky-service-docker:
     name: Build and push flaky-service container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -35,18 +32,18 @@ jobs:
       - uses: actions/[email protected]
         with:
           fetch-depth: 0
-      - id: meta-flaky-service
-        uses: docker/metadata-action@v5
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
         with:
+          # list of Docker images to use as base name for tags
           images: |
             ghcr.io/pluralsh/flaky-service
-          tags: v1.0.0
-      - id: meta-flaky-service-sidecar
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ghcr.io/pluralsh/flaky-service-sidecar
-          tags: v1.0.0
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=sha
+            type=ref,event=pr
+            type=ref,event=branch
       - uses: docker/login-action@v3
         with:
           registry: ghcr.io
@@ -59,21 +56,8 @@ jobs:
           context: "./go/demo/flaky-service"
           file: "./go/demo/flaky-service/Dockerfile"
           push: true
-          tags: ${{ steps.meta-flaky-service.outputs.tags }}
-          labels: ${{ steps.meta-flaky-service.outputs.labels }}
-          platforms: linux/amd64, linux/arm64
-          cache-from: type=gha
-          cache-to: type=gha, mode=max
-          build-args: |
-            GIT_COMMIT=${{ github.sha }}
-            VERSION=${{ steps.meta.outputs.version }}
-      - uses: docker/build-push-action@v5
-        with:
-          context: "./go/demo/flaky-service"
-          file: "./go/demo/flaky-service/Dockerfile.sidecar"
-          push: true
-          tags: ${{ steps.meta-flaky-service-sidecar.outputs.tags }}
-          labels: ${{ steps.meta-flaky-service-sidecar.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
           platforms: linux/amd64, linux/arm64
           cache-from: type=gha
           cache-to: type=gha, mode=max

.github/workflows/oci-auth-cd.yaml

@@ -21,7 +21,7 @@ env:
 jobs:
   test:
     name: Unit test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash
@@ -37,7 +37,7 @@ jobs:
     - run: PATH=$PATH:$GOPATH/bin make test
   publish-docker:
     name: Build and push oci-auth container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/oci-auth-ci.yaml

@@ -36,7 +36,7 @@ jobs:
       - run: PATH=$PATH:$GOPATH/bin make build
   unit-test:
     name: Unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash

.github/workflows/pr-labels.yaml

@@ -6,7 +6,7 @@ on:
 jobs:
   label:
     name: Check that PR has required labels
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: mheap/github-action-required-labels@v2
         with:

.github/workflows/publish.yaml

@@ -6,7 +6,7 @@ on:
       - "v*.*.*"
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: erlef/setup-beam@v1
@@ -45,7 +45,7 @@ jobs:
         if: always()
   publish:
     name: Build and push Console container
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: test
     permissions:
       contents: "read"
@@ -133,7 +133,7 @@ jobs:
         if: always()
   release:
     name: Create GitHub release
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: publish
     permissions:
       contents: write
@@ -148,7 +148,7 @@ jobs:
           draft: false
   bump:
     name: Bump Chart Version
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [release]
     permissions:
       contents: write
@@ -207,7 +207,7 @@ jobs:
           base: master
   bump-rapid:
     name: Bump Rapid Chart Version
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs: [release]
     permissions:
       contents: write

.github/workflows/test.yaml

@@ -8,7 +8,7 @@ on:
 jobs:
   build:
     name: Test Build Docker image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -64,7 +64,7 @@ jobs:
 
   test:
     name: Test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: erlef/setup-beam@v1
@@ -103,7 +103,7 @@ jobs:
         if: always()
   updateSchema:
     name: Check that Schema is up to date
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: erlef/setup-beam@v1

go/demo/flaky-service/api/behavior_modifiers.go

@@ -32,7 +32,7 @@ func HandleRequestTimestampModulus(timestampModulus int64) http.HandlerFunc {
 			w.Header().Set("Content-Type", "application/json")
 			w.Write([]byte(`{"message": "req failed"}`))
 		} else {
-			slog.Info("Timestamp is not multiple of modulus, returning status.OK", "time_now", time_now, "modulus", timestampModulus)
+			slog.Info("Everything seems fine, returning status.OK", "time_now", time_now, "modulus", timestampModulus)
 
 			metrics.IncrementRequestCounter(http.StatusOK, r.Method)
 			w.WriteHeader(http.StatusOK)

lib/console/ai/evidence/vector.ex

@@ -21,7 +21,7 @@ defmodule Console.AI.Evidence.Vector do
     ctx = Context.new(history)
     with true <- VectorStore.enabled?(),
          {:ok, %Vector{query: query}} <- use_vector(ctx.history),
-         {:ok, [_ | _] = vdata} <- VectorStore.fetch(query) do
+         {:ok, [_ | _] = vdata} <- VectorStore.fetch(query) |> IO.inspect(label: "vector result") do
       Context.prompt(ctx, {:user, "I've also found some relevent external data that could add additional context to what caused the issue:"})
       |> Context.reduce(vdata, &Context.prompt(&2, {:user, vector_prompt(&1)}))
       |> Context.evidence(vector_evidence(vdata))
@@ -46,7 +46,7 @@ defmodule Console.AI.Evidence.Vector do
   end
 
   defp use_vector(history) do
-    case Provider.tool_call(history, [Vector], preface: @preface) do
+    case Provider.tool_call(history, [Vector], preface: @preface) |> IO.inspect(label: "vector tool query:") do
       {:ok, [%{vector: %{result: %Vector{required: true} = vector}} | _]} ->
         {:ok, vector}
       _ -> false

lib/console/ai/pubsub/vector/consumer.ex

@@ -12,7 +12,7 @@ defmodule Console.AI.PubSub.Vector.Consumer do
     end
   end
 
-  defp insert({:ok, [_ | _] = resources}), do: Enum.each(resources, &VectorStore.insert/1)
+  defp insert({:ok, resources}) when is_list(resources), do: Enum.each(resources, &VectorStore.insert/1)
   defp insert({:ok, res}), do: VectorStore.insert(res)
   defp insert(pass), do: pass
 end

lib/console/ai/pubsub/vector/protocol.ex

@@ -13,14 +13,16 @@ defimpl Console.AI.PubSub.Vectorizable, for: Console.PubSub.ScmWebhook do
   alias Console.AI.Tool
   alias Console.Deployments.Pr.Dispatcher
   alias Console.Schema.{ScmWebhook, ScmConnection}
+  require Logger
 
   def resource(%@for{
-    item: %{"action" => "pull_request", "pull_request" => %{"merged" => true} = pr},
+    item: %{"action" => "closed", "pull_request" => %{"merged" => true} = pr},
     actor: %ScmWebhook{type: :github}
   }) do
     with %ScmConnection{} = conn <- Tool.scm_connection(),
       do: Dispatcher.files(conn, pr)
   end
+
   def resource(_), do: :ok
 end
 

lib/console/ai/vector/elastic.ex

@@ -61,8 +61,8 @@ defmodule Console.AI.Vector.Elastic do
   end
 
   def fetch(%__MODULE__{conn: %Elastic{} = es}, text) do
-    with {:ok, [{_, embedding} | _]} <- Provider.embeddings(text),
-         {:ok, %Snap.SearchResponse{hits: hits}} <- Console.Logs.Provider.Elastic.search(es, vector_query(embedding)) do
+    with {:ok, [{_, embedding} | _]} <- Provider.embeddings(text) |> IO.inspect(label: "embeddings fetch"),
+         {:ok, %Snap.SearchResponse{hits: hits}} <- Console.Logs.Provider.Elastic.search(es, vector_query(embedding)) |> IO.inspect(label: "es response:") do
       Enum.map(hits, fn %Snap.Hit{source: source} ->
         datatype = source["datatype"]
         case source[datatype] do

lib/console/deployments/pr/impl/github.ex

@@ -98,7 +98,7 @@ defmodule Console.Deployments.Pr.Impl.Github do
   end
 
   defp get_content(client, url) when is_binary(url) do
-    case HTTPoison.get(url, [{"authorization", "Token #{client.auth.access_token}"}]) do
+    case HTTPoison.get(url, [{"authorization", "Token #{client.auth.access_token}"}], follow_redirect: true) do
       {:ok, %HTTPoison.Response{status_code: code, body: content}}
         when code >= 200 and code < 300 -> content
       _ -> nil

test-apps/flaky-service/flaky-service.yaml → test-apps/flaky-service/flaky-service.yaml.liquid

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
         - name: flaky-service
-          image: ghcr.io/pluralsh/flaky-service:v1.0.0
+          image: ghcr.io/pluralsh/flaky-service:{{ configuration.tag | default: "v1.0.0" }}
           imagePullPolicy: Always
           ports:
             - containerPort: 8080  # API Server

test/console/ai/pubsub/vector/consumer_test.exs

@@ -31,11 +31,11 @@ defmodule Console.AI.PubSub.Vector.ConsumerTest do
           "patch" => "example diff",
         }], %HTTPoison.Response{status_code: 200}}
       end)
-      expect(HTTPoison, :get, fn "https://test.url", _ -> {:ok, %HTTPoison.Response{status_code: 200, body: "terraform"}} end)
+      expect(HTTPoison, :get, fn "https://test.url", _, [follow_redirect: true] -> {:ok, %HTTPoison.Response{status_code: 200, body: "terraform"}} end)
 
       event = %PubSub.ScmWebhook{
         item: %{
-          "action" => "pull_request",
+          "action" => "closed",
           "pull_request" => %{"merged" => true, "html_url" => "https://github.com/owner/repo/pull/1"},
         },
         actor: hook