Print used OpenAI Assistant tools for AI assistant

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/honeycombio/otel-config-go v1.13.1
 	github.com/huandu/xstrings v1.4.0
 	github.com/keptn/go-utils v0.20.4
-	github.com/kubeshop/botkube v0.13.1-0.20240426162820-c3c1bbe3c039
+	github.com/kubeshop/botkube v0.13.1-0.20240520120227-304528f09aeb
 	github.com/muesli/reflow v0.3.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/prometheus/client_golang v1.16.0
@@ -74,7 +74,7 @@ require (
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fluxcd/pkg/apis/kustomize v1.1.1 // indirect
@@ -155,7 +155,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.23.10 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spiffe/spire v1.5.6 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -215,3 +215,5 @@ require (
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
+
+replace github.com/kubeshop/botkube => github.com/pkosiec/botkube v0.12.5-0.20240524144006-573a14549bcf

go.sum

@@ -323,8 +323,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
@@ -337,8 +337,8 @@ github.com/fluxcd/pkg/apis/meta v1.1.1 h1:sLAKLbEu7rRzJ+Mytffu3NcpfdbOBTa6hcpOQz
 github.com/fluxcd/pkg/apis/meta v1.1.1/go.mod h1:soCfzjFWbm1mqybDcOywWKTCEYlH3skpoNGTboVk234=
 github.com/fluxcd/source-controller/api v1.0.1 h1:nycylbNBnQd+EO4UHpqXqAQJ1cGAPxgBbrfERCQ1pp8=
 github.com/fluxcd/source-controller/api v1.0.1/go.mod h1:rAY5FRFGZUKpIFNyYANHIgPgJPvbALBHWsq/zHw/cXQ=
-github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
-github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
@@ -667,8 +667,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubeshop/botkube v0.13.1-0.20240426162820-c3c1bbe3c039 h1:x3KNRktuu0QYzjSr4P39igf2PneVFrgrpKWD7cAJ69I=
-github.com/kubeshop/botkube v0.13.1-0.20240426162820-c3c1bbe3c039/go.mod h1:5QZExpXEutttEAn32I/Mf4FvoaVCoeNoFsznf8j905k=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
@@ -776,6 +774,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkosiec/botkube v0.12.5-0.20240524144006-573a14549bcf h1:Jzbq+RluUjYd39C2ali0hvEGZpIEVMB4Vh+JUYlc5Os=
+github.com/pkosiec/botkube v0.12.5-0.20240524144006-573a14549bcf/go.mod h1:OZeY4kLDrVQlaGxCE3XnTX8UUUhSpENIZ41PmBVIePg=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -848,8 +848,8 @@ github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ
 github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
-github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
+github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
+github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

internal/source/ai-brain/assistant.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/kubeshop/botkube/pkg/ptr"
+
 	"github.com/kubeshop/botkube-cloud-plugins/internal/otelx"
 	"github.com/kubeshop/botkube-cloud-plugins/internal/remote"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
@@ -281,6 +283,14 @@ func (i *assistant) handleStatusCompleted(ctx context.Context, run openai.Run, p
 		return nil
 	}
 
+	toolCalls, err := i.listToolCalls(ctx, run.ThreadID, run.ID)
+	if err != nil {
+		err = fmt.Errorf("while listing tool calls: %w", err)
+		span.RecordError(err)
+		i.log.WithError(err).Error("failed to get tool calls")
+		// not returning as the tool usage is optional
+	}
+
 	// Iterate over text content to build messages. We're only interested in text
 	// context, since the assistant is instructed to return text only.
 	for _, c := range msgList.Messages[0].Content {
@@ -291,7 +301,7 @@ func (i *assistant) handleStatusCompleted(ctx context.Context, run openai.Run, p
 		textValue := i.trimCitationsIfPresent(i.log, c.Text)
 
 		i.out <- source.Event{
-			Message: msgAIAnswer(run, p, textValue),
+			Message: msgAIAnswer(run, p, textValue, toolCalls),
 		}
 	}
 
@@ -379,6 +389,35 @@ func (i *assistant) trimCitationsIfPresent(log logrus.FieldLogger, in *openai.Me
 	return outValue
 }
 
+func (i *assistant) listToolCalls(ctx context.Context, threadID, runID string) (map[string]struct{}, error) {
+	const maxLimit = 100
+	var (
+		runSteps []openai.RunStep
+		after    *string
+	)
+
+	for {
+		res, err := i.openaiClient.ListRunSteps(ctx, threadID, runID, openai.Pagination{Limit: ptr.FromType(maxLimit), After: after})
+		if err != nil {
+			return nil, fmt.Errorf("while getting assistant run steps: %w", err)
+		}
+
+		runSteps = append(runSteps, res.RunSteps...)
+		if !res.HasMore {
+			break
+		}
+
+		after = &res.LastID
+		i.log.WithFields(logrus.Fields{
+			"lastID": res.LastID,
+			"count":  len(runSteps),
+		}).Debug("Paginating assistant run steps...")
+	}
+	i.log.WithField("count", len(runSteps)).Debug("Finished paginating assistant run steps")
+
+	return getFriendlyToolCallsFromRunSteps(runSteps), nil
+}
+
 type apiKeySecuredTransport struct {
 	transport http.RoundTripper
 }

internal/source/ai-brain/response.go

@@ -15,6 +15,7 @@ const (
 	teamsMessageIDSubstr  = "thread.tacv2"
 	reportResponseBtnName = "🚩Report response"
 	maxPromptLen          = 500
+	aiContentWarning      = "AI-generated content may be incorrect."
 )
 
 var (
@@ -107,13 +108,15 @@ func msgNoAIAnswer(messageID string) api.Message {
 	}
 }
 
-func msgAIAnswer(run openai.Run, payload *Payload, response string) api.Message {
+func msgAIAnswer(run openai.Run, payload *Payload, response string, toolCalls map[string]struct{}) api.Message {
 	var (
-		msgID   = payload.MessageID
-		btnBldr = api.NewMessageButtonBuilder()
+		msgID        = payload.MessageID
+		btnBldr      = api.NewMessageButtonBuilder()
+		usedToolsMsg = printUsedTools(toolCalls)
 	)
 
-	if strings.Contains(msgID, teamsMessageIDSubstr) { // teams
+	// MS Teams
+	if strings.Contains(msgID, teamsMessageIDSubstr) {
 		return api.Message{
 			Type:             api.BasicCardWithButtonsInSeparateMsg,
 			ParentActivityID: msgID,
@@ -122,7 +125,7 @@ func msgAIAnswer(run openai.Run, payload *Payload, response string) api.Message
 				// as simplified card (https://learn.microsoft.com/en-us/microsoftteams/platform/bots/how-to/format-your-bot-messages#format-text-content)
 				// instead of AdaptiveCard (https://learn.microsoft.com/en-us/microsoftteams/platform/task-modules-and-cards/cards/cards-format?tabs=adaptive-md%2Cdesktop%2Cconnector-html#format-cards-with-markdown)
 				// which doesn't support most of the markdown elements.
-				Plaintext: markdownToTeams(response),
+				Plaintext: markdownToTeams(response, usedToolsMsg),
 			},
 			Sections: []api.Section{
 				{
@@ -138,29 +141,45 @@ func msgAIAnswer(run openai.Run, payload *Payload, response string) api.Message
 	// from the AI to prevent it from being interpreted as formatting.
 	response = strings.ReplaceAll(response, "%", "%%")
 
-	sections := []api.Section{
-		{
-			Base: api.Base{
-				Body: api.Body{Plaintext: response},
-			},
-			Context: []api.ContextItem{
-				{Text: "AI-generated content may be incorrect."},
-			},
-		},
-	}
-
-	if msgID != "" { // msgID is set only for Teams or Slack, Teams is handled above, so here is Slack
-		sections[0].Body.Plaintext = markdownToSlack(sections[0].Body.Plaintext)
-		sections = append(sections, api.Section{
-			Buttons: api.Buttons{
-				btnBldr.ForCommandWithoutDesc(reportResponseBtnName, reportCmd(run, payload), api.ButtonStyleDanger),
+	// Slack (msgID is set only for Teams or Slack, Teams is handled above)
+	if msgID != "" {
+		return api.Message{
+			ParentActivityID: msgID,
+			Sections: []api.Section{
+				{
+					Base: api.Base{
+						Body: api.Body{Plaintext: markdownToSlack(response)},
+					},
+					Context: []api.ContextItem{
+						{Text: markdownToSlack(usedToolsMsg)},
+					},
+				},
+				{
+					Style: api.SectionStyle{
+						Divider: api.DividerStyleTopNone,
+					},
+					Buttons: api.Buttons{
+						btnBldr.ForCommandWithItalicDesc(reportResponseBtnName, aiContentWarning, reportCmd(run, payload)),
+					},
+				},
 			},
-		})
+		}
 	}
 
+	// Other cases (e.g. Discord etc.)
 	return api.Message{
 		ParentActivityID: msgID,
-		Sections:         sections,
+		Sections: []api.Section{
+			{
+				Base: api.Base{
+					Body: api.Body{Plaintext: response},
+				},
+				Context: []api.ContextItem{
+					{Text: usedToolsMsg},
+					{Text: fmt.Sprintf("_%s_", aiContentWarning)},
+				},
+			},
+		},
 	}
 }
 
@@ -175,12 +194,17 @@ func markdownToSlack(text string) string {
 	return mdLinks.ReplaceAllString(text, "<$2|$1>")
 }
 
-func markdownToTeams(text string) string {
+func markdownToTeams(text, additionalFooterMessage string) string {
 	text = mdHeadings.ReplaceAllString(text, "**$1**")
 	text = mdImages.ReplaceAllString(text, "[$1]($2)") // get rid of ! to define it as a link instead of image
 
+	text += "\n\n"
+	if additionalFooterMessage != "" {
+		text += fmt.Sprintf("~%s~\n\n", additionalFooterMessage) // add
+	}
+
 	// https://learn.microsoft.com/en-us/microsoftteams/platform/task-modules-and-cards/cards/cards-format?tabs=adaptive-md%2Cdesktop%2Cconnector-html#newlines-for-adaptive-cards
-	text += "\n\n~AI-generated content may be incorrect.~\n" // add the warning
+	text += "~AI-generated content may be incorrect.~\n" // add the warning
 	return text
 }
 

internal/source/ai-brain/response_test.go

@@ -1,6 +1,7 @@
 package aibrain
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"testing"
@@ -25,17 +26,105 @@ func TestConvertProperlyAIAnswer(t *testing.T) {
 	out := msgAIAnswer(openai.Run{}, &Payload{
 		MessageID: "42.42",
 		Prompt:    "This is a test",
-	}, string(md))
+	}, string(md), nil)
 	assertGolden(t, out.Sections[0].Base.Body.Plaintext, "slack.golden.md")
 
 	// Teams
 	out = msgAIAnswer(openai.Run{}, &Payload{
 		MessageID: "19:[email protected]",
 		Prompt:    "This is a test",
-	}, string(md))
+	}, string(md), nil)
 	assertGolden(t, out.BaseBody.Plaintext, "teams.golden.md")
 }
 
+// TestConvertProperlyAIAnswerWithTools tests that we can convert proper AI answer to Slack or Teams message format.
+//
+// To update golden files run:
+//
+//	go test ./internal/source/ai-brain/... -run=TestConvertProperlyAIAnswerWithTools -update
+func TestConvertProperlyAIAnswerWithTools(t *testing.T) {
+	// given
+	md, err := os.ReadFile(filepath.Join("testdata", t.Name(), "aiAnswer.md"))
+	require.NoError(t, err)
+
+	toolCalls := []openai.RunStep{
+		{
+			StepDetails: openai.StepDetails{
+				Type: openai.RunStepTypeToolCalls,
+				ToolCalls: []openai.ToolCall{
+					{
+						Type: openai.ToolType("file_search"),
+					},
+					{
+						Type: openai.ToolTypeFunction,
+						Function: openai.FunctionCall{
+							Name:      "botkubeGetStartupAgentConfiguration",
+							Arguments: "foo",
+						},
+					},
+				},
+			},
+		},
+		{
+			StepDetails: openai.StepDetails{
+				Type: openai.RunStepTypeToolCalls,
+				ToolCalls: []openai.ToolCall{
+					{
+						Type: openai.ToolTypeFunction,
+						Function: openai.FunctionCall{
+							Name:      "botkubeGetAgentStatus",
+							Arguments: "foo",
+						},
+					},
+					{
+						Type: openai.ToolTypeFunction,
+						Function: openai.FunctionCall{
+							Name:      "kubectlGetResource",
+							Arguments: "foo",
+						},
+					},
+					{
+						Type: openai.ToolTypeFunction,
+						Function: openai.FunctionCall{
+							Name:      "kubectlLogs",
+							Arguments: "foo",
+						},
+					},
+				},
+			},
+		},
+	}
+	convertedToolCalls := getFriendlyToolCallsFromRunSteps(toolCalls)
+
+	// Slack
+	out := msgAIAnswer(openai.Run{}, &Payload{
+		MessageID: "42.42",
+		Prompt:    "This is a test",
+	}, string(md), convertedToolCalls)
+
+	outBytes, err := json.MarshalIndent(out, "", "  ")
+	require.NoError(t, err)
+	assertGolden(t, string(outBytes), "slack-tools.golden.json")
+
+	// Teams
+	out = msgAIAnswer(openai.Run{}, &Payload{
+		MessageID: "19:[email protected]",
+		Prompt:    "This is a test",
+	}, string(md), convertedToolCalls)
+	outBytes, err = json.MarshalIndent(out, "", "  ")
+	require.NoError(t, err)
+	assertGolden(t, string(outBytes), "teams-tools.golden.json")
+
+	// Discord and others
+	out = msgAIAnswer(openai.Run{}, &Payload{
+		MessageID: "",
+		Prompt:    "This is a test",
+	}, string(md), convertedToolCalls)
+	outBytes, err = json.MarshalIndent(out, "", "  ")
+	require.NoError(t, err)
+	assertGolden(t, string(outBytes), "discord-tools.golden.json")
+}
+
 func assertGolden(t *testing.T, actual, goldenPath string) {
 	golden.Assert(t, actual, filepath.Join(t.Name(), goldenPath))
 }