From a9c0d67a60f7ab779c8f7ace2aff88a1c92b5daf Mon Sep 17 00:00:00 2001 From: aPaladiychuk Date: Mon, 13 May 2024 18:21:39 +0300 Subject: [PATCH 1/6] _ --- backend/Makefile | 1 - backend/api/module.go | 1 + .../ai/{embedding-openai.go => embeddingi.go} | 8 + backend/core/ai/module.go | 24 + backend/core/bll/chat.go | 20 +- backend/core/connector/base.go | 18 +- backend/core/connector/web_test.go | 18 +- backend/core/model/connector.go | 13 + backend/core/proto/client.go | 1 - backend/core/proto/connector.proto | 4 +- backend/core/proto/connector_messages.pb.go | 491 ++++++++++++++++++ backend/core/proto/embedd_messages.pb.go | 306 +++++++++++ backend/core/proto/embedd_service.pb.go | 76 +++ backend/core/proto/embedd_service_grpc.pb.go | 105 ++++ backend/core/responder/embedding.go | 89 +++- backend/core/responder/responder.go | 1 + proto/Makefile | 8 + proto/chunking_data.proto | 20 + proto/connector_messages.proto | 2 +- proto/embedd_messages.proto | 1 + proto/embedd_service.proto | 1 + 21 files changed, 1168 insertions(+), 40 deletions(-) rename backend/core/ai/{embedding-openai.go => embeddingi.go} (84%) delete mode 100644 backend/core/proto/client.go create mode 100644 backend/core/proto/connector_messages.pb.go create mode 100644 backend/core/proto/embedd_messages.pb.go create mode 100644 backend/core/proto/embedd_service.pb.go create mode 100644 backend/core/proto/embedd_service_grpc.pb.go create mode 100644 proto/Makefile create mode 100644 proto/chunking_data.proto diff --git a/backend/Makefile b/backend/Makefile index b21a11d8..6432542a 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -6,5 +6,4 @@ swag-api: swag init --dir ./api --output ./api/docs --parseDependency --parseInternal gen-proto: - rm -f core/proto/*.pb.go protoc -I=. -I=vendor -I=${GOPATH}/src core/proto/*.proto --go_out=. --go-grpc_out=. \ No newline at end of file diff --git a/backend/api/module.go b/backend/api/module.go index ce1aea72..78c90dca 100644 --- a/backend/api/module.go +++ b/backend/api/module.go @@ -25,6 +25,7 @@ var Module = fx.Options( bll.BLLModule, storage.MinioModule, messaging.NatsModule, + ai.EmbeddingModule, fx.Provide(ReadConfig, NewRouter, newGoogleOauthProvider, diff --git a/backend/core/ai/embedding-openai.go b/backend/core/ai/embeddingi.go similarity index 84% rename from backend/core/ai/embedding-openai.go rename to backend/core/ai/embeddingi.go index 2db84894..9db44dad 100644 --- a/backend/core/ai/embedding-openai.go +++ b/backend/core/ai/embeddingi.go @@ -5,18 +5,26 @@ import ( "cognix.ch/api/v2/core/proto" "context" _ "github.com/deluan/flowllm/llms/openai" + validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/sashabaranov/go-openai" "go.uber.org/zap" "os" ) type ( + EmbeddingConfig struct { + EmbeddingURL string `env:"EMBEDDING_GRPC_URL"` + } embeddingParser struct { embeddingModel *model.EmbeddingModel client *openai.Client } ) +func (v EmbeddingConfig) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.EmbeddingURL, validation.Required)) +} func NewEmbeddingParser(embeddingModel *model.EmbeddingModel) EmbeddingParser { //remove-it apiKey := os.Getenv("OPENAI_API_KEY") diff --git a/backend/core/ai/module.go b/backend/core/ai/module.go index 49331a45..38254c36 100644 --- a/backend/core/ai/module.go +++ b/backend/core/ai/module.go @@ -1,8 +1,10 @@ package ai import ( + "cognix.ch/api/v2/core/proto" "cognix.ch/api/v2/core/utils" "go.uber.org/fx" + "google.golang.org/grpc" ) var ChunkingModule = fx.Options( @@ -26,3 +28,25 @@ func newChunking(cfg *ChunkingConfig) Chunking { } return NewStaticChunking(cfg) } + +var EmbeddingModule = fx.Options( + fx.Provide(func() (*EmbeddingConfig, error) { + cfg := EmbeddingConfig{} + if err := utils.ReadConfig(&cfg); err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, err + } + return &cfg, nil + }, + newEmbeddingGRPCClient), +) + +func newEmbeddingGRPCClient(cfg *EmbeddingConfig) (proto.EmbeddServiceClient, error) { + conn, err := grpc.Dial(cfg.EmbeddingURL) + if err != nil { + return nil, err + } + return proto.NewEmbeddServiceClient(conn), nil +} diff --git a/backend/core/bll/chat.go b/backend/core/bll/chat.go index 95f36ba0..bc13b91b 100644 --- a/backend/core/bll/chat.go +++ b/backend/core/bll/chat.go @@ -4,8 +4,10 @@ import ( "cognix.ch/api/v2/core/ai" "cognix.ch/api/v2/core/model" "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/proto" "cognix.ch/api/v2/core/repository" "cognix.ch/api/v2/core/responder" + "cognix.ch/api/v2/core/storage" "cognix.ch/api/v2/core/utils" "context" "fmt" @@ -22,9 +24,11 @@ type ChatBL interface { FeedbackMessage(ctx *gin.Context, user *model.User, id int64, vote bool) (*model.ChatMessageFeedback, error) } type chatBL struct { - chatRepo repository.ChatRepository - personaRepo repository.PersonaRepository - aiBuilder *ai.Builder + chatRepo repository.ChatRepository + personaRepo repository.PersonaRepository + aiBuilder *ai.Builder + embedding proto.EmbeddServiceClient + milvusClinet storage.MilvusClient } func (b *chatBL) FeedbackMessage(ctx *gin.Context, user *model.User, id int64, vote bool) (*model.ChatMessageFeedback, error) { @@ -63,7 +67,7 @@ func (b *chatBL) SendMessage(ctx *gin.Context, user *model.User, param *paramete aiClient := b.aiBuilder.New(chatSession.Persona.LLM) resp := responder.NewManager( responder.NewAIResponder(aiClient, b.chatRepo), - responder.NewEmbeddingResponder()) + responder.NewEmbeddingResponder(b.embedding, b.milvusClinet)) go resp.Send(ctx, &message) return resp, nil @@ -124,9 +128,13 @@ func (b *chatBL) CreateSession(ctx context.Context, user *model.User, param *par func NewChatBL(chatRepo repository.ChatRepository, personaRepo repository.PersonaRepository, aiBuilder *ai.Builder, + embedding proto.EmbeddServiceClient, + milvusClinet storage.MilvusClient, ) ChatBL { return &chatBL{chatRepo: chatRepo, - personaRepo: personaRepo, - aiBuilder: aiBuilder, + personaRepo: personaRepo, + aiBuilder: aiBuilder, + embedding: embedding, + milvusClinet: milvusClinet, } } diff --git a/backend/core/connector/base.go b/backend/core/connector/base.go index fee8adf3..ceb1aec5 100644 --- a/backend/core/connector/base.go +++ b/backend/core/connector/base.go @@ -5,18 +5,14 @@ import ( "cognix.ch/api/v2/core/proto" "cognix.ch/api/v2/core/repository" "context" - "fmt" - "strings" ) type Base struct { - collectionName string - model *model.Connector - resultCh chan *proto.TriggerResponse + model *model.Connector + resultCh chan *proto.TriggerResponse } type Connector interface { - CollectionName() string Execute(ctx context.Context, param map[string]string) chan *proto.TriggerResponse } @@ -45,15 +41,5 @@ func New(connectorModel *model.Connector) (Connector, error) { func (b *Base) Config(connector *model.Connector) { b.model = connector b.resultCh = make(chan *proto.TriggerResponse, 10) - if connector.Shared { - b.collectionName = strings.ReplaceAll(fmt.Sprintf(model.CollectionTenant, connector.TenantID), "-", "") - } else { - b.collectionName = strings.ReplaceAll(fmt.Sprintf(model.CollectionUser, connector.UserID), "-", "") - } - return } - -func (b *Base) CollectionName() string { - return b.collectionName -} diff --git a/backend/core/connector/web_test.go b/backend/core/connector/web_test.go index 2d67f4f7..46435cb0 100644 --- a/backend/core/connector/web_test.go +++ b/backend/core/connector/web_test.go @@ -1,7 +1,6 @@ package connector import ( - "cognix.ch/api/v2/core/messaging" "cognix.ch/api/v2/core/model" "context" "github.com/shopspring/decimal" @@ -18,27 +17,24 @@ func TestWeb_Execute(t *testing.T) { ConnectorSpecificConfig: model.JSONMap{ "url1": "https://help.collaboard.app/", "url2": "https://apidog.com/blog/openapi-specification/", - "url": "https://developer.mozilla.org/en-US/docs/Learn/HTML/Introduction_to_HTML", + "url": "https://developer.apple.com/documentation/visionos/improving-accessibility-support-in-your-app", + "url3": "https://developer.mozilla.org/en-US/docs/Learn/HTML/Introduction_to_HTML", }, DocsMap: make(map[string]*model.Document), - }, - messaging.NewMopClient()) - if err != nil { - t.Log(err.Error()) - t.Fatal(err) - } - conn, err := web.Execute(context.Background(), nil) + }) if err != nil { t.Log(err.Error()) t.Fatal(err) } + conn := web.Execute(context.Background(), nil) + for url, history := range (web.(*Web)).history { if len(history) > 30 { history = history[:30] } t.Logf("%s => %s ", url, history) } - for _, doc := range conn.Docs { - t.Log(doc.DocumentID) + for res := range conn { + t.Log(res.Content) } } diff --git a/backend/core/model/connector.go b/backend/core/model/connector.go index cc5b304b..98e5b79a 100644 --- a/backend/core/model/connector.go +++ b/backend/core/model/connector.go @@ -1,9 +1,11 @@ package model import ( + "fmt" "github.com/go-pg/pg/v10" "github.com/google/uuid" "github.com/shopspring/decimal" + "strings" "time" ) @@ -38,3 +40,14 @@ type Connector struct { Docs []*Document `json:"docs,omitempty" pg:"rel:has-many"` DocsMap map[string]*Document `json:"docs_map,omitempty" pg:"-"` } + +func (c *Connector) CollectionName() string { + return CollectionName(c.Shared, c.UserID, c.TenantID) +} +func CollectionName(isShared bool, userID, tenantID uuid.UUID) string { + if isShared { + return strings.ReplaceAll(fmt.Sprintf(CollectionTenant, tenantID), "-", "") + } + return strings.ReplaceAll(fmt.Sprintf(CollectionUser, userID), "-", "") + +} diff --git a/backend/core/proto/client.go b/backend/core/proto/client.go deleted file mode 100644 index 92256db4..00000000 --- a/backend/core/proto/client.go +++ /dev/null @@ -1 +0,0 @@ -package proto diff --git a/backend/core/proto/connector.proto b/backend/core/proto/connector.proto index 3c82dbcb..ab26766c 100644 --- a/backend/core/proto/connector.proto +++ b/backend/core/proto/connector.proto @@ -18,12 +18,12 @@ message Message { message Body { oneof payload { - TriggerRequest_v1 trigger = 1; + TriggerRequest trigger = 1; EmbeddingRequest embedding = 2; } } -message TriggerRequest_v1 { +message TriggerRequest { int64 id = 1; map params = 2; } diff --git a/backend/core/proto/connector_messages.pb.go b/backend/core/proto/connector_messages.pb.go new file mode 100644 index 00000000..26dc0915 --- /dev/null +++ b/backend/core/proto/connector_messages.pb.go @@ -0,0 +1,491 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: connector_messages.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ConnectorRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Params map[string]string `protobuf:"bytes,2,rep,name=params,proto3" json:"params,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *ConnectorRequest) Reset() { + *x = ConnectorRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_connector_messages_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConnectorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnectorRequest) ProtoMessage() {} + +func (x *ConnectorRequest) ProtoReflect() protoreflect.Message { + mi := &file_connector_messages_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnectorRequest.ProtoReflect.Descriptor instead. +func (*ConnectorRequest) Descriptor() ([]byte, []int) { + return file_connector_messages_proto_rawDescGZIP(), []int{0} +} + +func (x *ConnectorRequest) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *ConnectorRequest) GetParams() map[string]string { + if x != nil { + return x.Params + } + return nil +} + +type ConnectorResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` +} + +func (x *ConnectorResponse) Reset() { + *x = ConnectorResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_connector_messages_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConnectorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnectorResponse) ProtoMessage() {} + +func (x *ConnectorResponse) ProtoReflect() protoreflect.Message { + mi := &file_connector_messages_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnectorResponse.ProtoReflect.Descriptor instead. +func (*ConnectorResponse) Descriptor() ([]byte, []int) { + return file_connector_messages_proto_rawDescGZIP(), []int{1} +} + +func (x *ConnectorResponse) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *ConnectorResponse) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +type ChunkingResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + Chunks []string `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty"` +} + +func (x *ChunkingResponse) Reset() { + *x = ChunkingResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_connector_messages_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChunkingResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkingResponse) ProtoMessage() {} + +func (x *ChunkingResponse) ProtoReflect() protoreflect.Message { + mi := &file_connector_messages_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkingResponse.ProtoReflect.Descriptor instead. +func (*ChunkingResponse) Descriptor() ([]byte, []int) { + return file_connector_messages_proto_rawDescGZIP(), []int{2} +} + +func (x *ChunkingResponse) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *ChunkingResponse) GetChunks() []string { + if x != nil { + return x.Chunks + } + return nil +} + +type EmbeddAsyncRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + ChunkId int64 `protobuf:"varint,2,opt,name=chunk_id,json=chunkId,proto3" json:"chunk_id,omitempty"` + Content string `protobuf:"bytes,3,opt,name=content,proto3" json:"content,omitempty"` + ModelId string `protobuf:"bytes,4,opt,name=model_id,json=modelId,proto3" json:"model_id,omitempty"` +} + +func (x *EmbeddAsyncRequest) Reset() { + *x = EmbeddAsyncRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_connector_messages_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddAsyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddAsyncRequest) ProtoMessage() {} + +func (x *EmbeddAsyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_connector_messages_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddAsyncRequest.ProtoReflect.Descriptor instead. +func (*EmbeddAsyncRequest) Descriptor() ([]byte, []int) { + return file_connector_messages_proto_rawDescGZIP(), []int{3} +} + +func (x *EmbeddAsyncRequest) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *EmbeddAsyncRequest) GetChunkId() int64 { + if x != nil { + return x.ChunkId + } + return 0 +} + +func (x *EmbeddAsyncRequest) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *EmbeddAsyncRequest) GetModelId() string { + if x != nil { + return x.ModelId + } + return "" +} + +type EmbeddAsyncResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + ChunkId int64 `protobuf:"varint,2,opt,name=chunk_id,json=chunkId,proto3" json:"chunk_id,omitempty"` + Content string `protobuf:"bytes,3,opt,name=content,proto3" json:"content,omitempty"` + Vector []float32 `protobuf:"fixed32,4,rep,packed,name=vector,proto3" json:"vector,omitempty"` +} + +func (x *EmbeddAsyncResponse) Reset() { + *x = EmbeddAsyncResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_connector_messages_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddAsyncResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddAsyncResponse) ProtoMessage() {} + +func (x *EmbeddAsyncResponse) ProtoReflect() protoreflect.Message { + mi := &file_connector_messages_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddAsyncResponse.ProtoReflect.Descriptor instead. +func (*EmbeddAsyncResponse) Descriptor() ([]byte, []int) { + return file_connector_messages_proto_rawDescGZIP(), []int{4} +} + +func (x *EmbeddAsyncResponse) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *EmbeddAsyncResponse) GetChunkId() int64 { + if x != nil { + return x.ChunkId + } + return 0 +} + +func (x *EmbeddAsyncResponse) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *EmbeddAsyncResponse) GetVector() []float32 { + if x != nil { + return x.Vector + } + return nil +} + +var File_connector_messages_proto protoreflect.FileDescriptor + +var file_connector_messages_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x9a, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x3b, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4e, + 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x4b, + 0x0a, 0x10, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, + 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x22, 0x85, 0x01, 0x0a, 0x12, + 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, + 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, + 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x49, 0x64, 0x22, 0x83, 0x01, 0x0a, 0x13, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x41, 0x73, + 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, + 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x02, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, + 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_connector_messages_proto_rawDescOnce sync.Once + file_connector_messages_proto_rawDescData = file_connector_messages_proto_rawDesc +) + +func file_connector_messages_proto_rawDescGZIP() []byte { + file_connector_messages_proto_rawDescOnce.Do(func() { + file_connector_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_connector_messages_proto_rawDescData) + }) + return file_connector_messages_proto_rawDescData +} + +var file_connector_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_connector_messages_proto_goTypes = []interface{}{ + (*ConnectorRequest)(nil), // 0: proto.ConnectorRequest + (*ConnectorResponse)(nil), // 1: proto.ConnectorResponse + (*ChunkingResponse)(nil), // 2: proto.ChunkingResponse + (*EmbeddAsyncRequest)(nil), // 3: proto.EmbeddAsyncRequest + (*EmbeddAsyncResponse)(nil), // 4: proto.EmbeddAsyncResponse + nil, // 5: proto.ConnectorRequest.ParamsEntry +} +var file_connector_messages_proto_depIdxs = []int32{ + 5, // 0: proto.ConnectorRequest.params:type_name -> proto.ConnectorRequest.ParamsEntry + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_connector_messages_proto_init() } +func file_connector_messages_proto_init() { + if File_connector_messages_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_connector_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ConnectorRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_connector_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ConnectorResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_connector_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChunkingResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_connector_messages_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddAsyncRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_connector_messages_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddAsyncResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_connector_messages_proto_rawDesc, + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_connector_messages_proto_goTypes, + DependencyIndexes: file_connector_messages_proto_depIdxs, + MessageInfos: file_connector_messages_proto_msgTypes, + }.Build() + File_connector_messages_proto = out.File + file_connector_messages_proto_rawDesc = nil + file_connector_messages_proto_goTypes = nil + file_connector_messages_proto_depIdxs = nil +} diff --git a/backend/core/proto/embedd_messages.pb.go b/backend/core/proto/embedd_messages.pb.go new file mode 100644 index 00000000..8bf84787 --- /dev/null +++ b/backend/core/proto/embedd_messages.pb.go @@ -0,0 +1,306 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: embedd_messages.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EmbeddRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Content string `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` + Model string `protobuf:"bytes,2,opt,name=model,proto3" json:"model,omitempty"` +} + +func (x *EmbeddRequest) Reset() { + *x = EmbeddRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_embedd_messages_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddRequest) ProtoMessage() {} + +func (x *EmbeddRequest) ProtoReflect() protoreflect.Message { + mi := &file_embedd_messages_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddRequest.ProtoReflect.Descriptor instead. +func (*EmbeddRequest) Descriptor() ([]byte, []int) { + return file_embedd_messages_proto_rawDescGZIP(), []int{0} +} + +func (x *EmbeddRequest) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *EmbeddRequest) GetModel() string { + if x != nil { + return x.Model + } + return "" +} + +type EmbeddResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Vector []float32 `protobuf:"fixed32,1,rep,packed,name=vector,proto3" json:"vector,omitempty"` +} + +func (x *EmbeddResponse) Reset() { + *x = EmbeddResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_embedd_messages_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddResponse) ProtoMessage() {} + +func (x *EmbeddResponse) ProtoReflect() protoreflect.Message { + mi := &file_embedd_messages_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddResponse.ProtoReflect.Descriptor instead. +func (*EmbeddResponse) Descriptor() ([]byte, []int) { + return file_embedd_messages_proto_rawDescGZIP(), []int{1} +} + +func (x *EmbeddResponse) GetVector() []float32 { + if x != nil { + return x.Vector + } + return nil +} + +type EmbeddData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` + Model string `protobuf:"bytes,3,opt,name=model,proto3" json:"model,omitempty"` + Vector []float32 `protobuf:"fixed32,4,rep,packed,name=vector,proto3" json:"vector,omitempty"` +} + +func (x *EmbeddData) Reset() { + *x = EmbeddData{} + if protoimpl.UnsafeEnabled { + mi := &file_embedd_messages_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddData) ProtoMessage() {} + +func (x *EmbeddData) ProtoReflect() protoreflect.Message { + mi := &file_embedd_messages_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddData.ProtoReflect.Descriptor instead. +func (*EmbeddData) Descriptor() ([]byte, []int) { + return file_embedd_messages_proto_rawDescGZIP(), []int{2} +} + +func (x *EmbeddData) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *EmbeddData) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *EmbeddData) GetModel() string { + if x != nil { + return x.Model + } + return "" +} + +func (x *EmbeddData) GetVector() []float32 { + if x != nil { + return x.Vector + } + return nil +} + +var File_embedd_messages_proto protoreflect.FileDescriptor + +var file_embedd_messages_proto_rawDesc = []byte{ + 0x0a, 0x15, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, + 0x65, 0x64, 0x64, 0x22, 0x3f, 0x0a, 0x0d, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x14, + 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x22, 0x28, 0x0a, 0x0e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x02, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x64, + 0x0a, 0x0a, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x16, 0x0a, 0x06, + 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x02, 0x52, 0x06, 0x76, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, + 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_embedd_messages_proto_rawDescOnce sync.Once + file_embedd_messages_proto_rawDescData = file_embedd_messages_proto_rawDesc +) + +func file_embedd_messages_proto_rawDescGZIP() []byte { + file_embedd_messages_proto_rawDescOnce.Do(func() { + file_embedd_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_embedd_messages_proto_rawDescData) + }) + return file_embedd_messages_proto_rawDescData +} + +var file_embedd_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_embedd_messages_proto_goTypes = []interface{}{ + (*EmbeddRequest)(nil), // 0: com.embedd.EmbeddRequest + (*EmbeddResponse)(nil), // 1: com.embedd.EmbeddResponse + (*EmbeddData)(nil), // 2: com.embedd.EmbeddData +} +var file_embedd_messages_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_embedd_messages_proto_init() } +func file_embedd_messages_proto_init() { + if File_embedd_messages_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_embedd_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_embedd_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_embedd_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_embedd_messages_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_embedd_messages_proto_goTypes, + DependencyIndexes: file_embedd_messages_proto_depIdxs, + MessageInfos: file_embedd_messages_proto_msgTypes, + }.Build() + File_embedd_messages_proto = out.File + file_embedd_messages_proto_rawDesc = nil + file_embedd_messages_proto_goTypes = nil + file_embedd_messages_proto_depIdxs = nil +} diff --git a/backend/core/proto/embedd_service.pb.go b/backend/core/proto/embedd_service.pb.go new file mode 100644 index 00000000..79892e37 --- /dev/null +++ b/backend/core/proto/embedd_service.pb.go @@ -0,0 +1,76 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: embedd_service.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_embedd_service_proto protoreflect.FileDescriptor + +var file_embedd_service_proto_rawDesc = []byte{ + 0x0a, 0x14, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, + 0x64, 0x64, 0x1a, 0x15, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x55, 0x0a, 0x0d, 0x45, 0x6d, 0x62, + 0x65, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x09, 0x47, 0x65, + 0x74, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x12, 0x19, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, + 0x62, 0x65, 0x64, 0x64, 0x2e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x2e, + 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +} + +var file_embedd_service_proto_goTypes = []interface{}{ + (*EmbeddRequest)(nil), // 0: com.embedd.EmbeddRequest + (*EmbeddResponse)(nil), // 1: com.embedd.EmbeddResponse +} +var file_embedd_service_proto_depIdxs = []int32{ + 0, // 0: com.embedd.EmbeddService.GetEmbedd:input_type -> com.embedd.EmbeddRequest + 1, // 1: com.embedd.EmbeddService.GetEmbedd:output_type -> com.embedd.EmbeddResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_embedd_service_proto_init() } +func file_embedd_service_proto_init() { + if File_embedd_service_proto != nil { + return + } + file_embedd_messages_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_embedd_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_embedd_service_proto_goTypes, + DependencyIndexes: file_embedd_service_proto_depIdxs, + }.Build() + File_embedd_service_proto = out.File + file_embedd_service_proto_rawDesc = nil + file_embedd_service_proto_goTypes = nil + file_embedd_service_proto_depIdxs = nil +} diff --git a/backend/core/proto/embedd_service_grpc.pb.go b/backend/core/proto/embedd_service_grpc.pb.go new file mode 100644 index 00000000..6fff6524 --- /dev/null +++ b/backend/core/proto/embedd_service_grpc.pb.go @@ -0,0 +1,105 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.12.4 +// source: embedd_service.proto + +package proto + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// EmbeddServiceClient is the client API for EmbeddService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type EmbeddServiceClient interface { + GetEmbedd(ctx context.Context, in *EmbeddRequest, opts ...grpc.CallOption) (*EmbeddResponse, error) +} + +type embeddServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewEmbeddServiceClient(cc grpc.ClientConnInterface) EmbeddServiceClient { + return &embeddServiceClient{cc} +} + +func (c *embeddServiceClient) GetEmbedd(ctx context.Context, in *EmbeddRequest, opts ...grpc.CallOption) (*EmbeddResponse, error) { + out := new(EmbeddResponse) + err := c.cc.Invoke(ctx, "/com.embedd.EmbeddService/GetEmbedd", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// EmbeddServiceServer is the server API for EmbeddService service. +// All implementations must embed UnimplementedEmbeddServiceServer +// for forward compatibility +type EmbeddServiceServer interface { + GetEmbedd(context.Context, *EmbeddRequest) (*EmbeddResponse, error) + mustEmbedUnimplementedEmbeddServiceServer() +} + +// UnimplementedEmbeddServiceServer must be embedded to have forward compatible implementations. +type UnimplementedEmbeddServiceServer struct { +} + +func (UnimplementedEmbeddServiceServer) GetEmbedd(context.Context, *EmbeddRequest) (*EmbeddResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetEmbedd not implemented") +} +func (UnimplementedEmbeddServiceServer) mustEmbedUnimplementedEmbeddServiceServer() {} + +// UnsafeEmbeddServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to EmbeddServiceServer will +// result in compilation errors. +type UnsafeEmbeddServiceServer interface { + mustEmbedUnimplementedEmbeddServiceServer() +} + +func RegisterEmbeddServiceServer(s grpc.ServiceRegistrar, srv EmbeddServiceServer) { + s.RegisterService(&EmbeddService_ServiceDesc, srv) +} + +func _EmbeddService_GetEmbedd_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EmbeddRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(EmbeddServiceServer).GetEmbedd(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/com.embedd.EmbeddService/GetEmbedd", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(EmbeddServiceServer).GetEmbedd(ctx, req.(*EmbeddRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// EmbeddService_ServiceDesc is the grpc.ServiceDesc for EmbeddService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var EmbeddService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "com.embedd.EmbeddService", + HandlerType: (*EmbeddServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetEmbedd", + Handler: _EmbeddService_GetEmbedd_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "embedd_service.proto", +} diff --git a/backend/core/responder/embedding.go b/backend/core/responder/embedding.go index 478c0a6e..1239e613 100644 --- a/backend/core/responder/embedding.go +++ b/backend/core/responder/embedding.go @@ -2,6 +2,8 @@ package responder import ( "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/storage" "context" "fmt" "github.com/shopspring/decimal" @@ -9,9 +11,25 @@ import ( ) type embedding struct { + embedding proto.EmbeddServiceClient + milvusClinet storage.MilvusClient + embeddingModel string } func (r *embedding) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGroup, parentMessage *model.ChatMessage) { + response, err := r.embedding.GetEmbedd(ctx, &proto.EmbeddRequest{ + Content: parentMessage.Message, + Model: r.embeddingModel, + }) + if err != nil { + ch <- &Response{ + IsValid: false, + Type: ResponseError, + Err: err, + } + return + } + docs, err := r.milvusClinet. for i := 0; i < 4; i++ { ch <- &Response{ @@ -30,6 +48,73 @@ func (r *embedding) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGr wg.Done() } -func NewEmbeddingResponder() ChatResponder { - return &embedding{} + +func (r *embedding) FindDocuments(ctx context.Context, + ch chan *Response, wg *sync.WaitGroup, + parentMessage *model.ChatMessage, + collectionNames ...string) { + response, err := r.embedding.GetEmbedd(ctx, &proto.EmbeddRequest{ + Content: parentMessage.Message, + Model: r.embeddingModel, + }) + if err != nil { + ch <- &Response{ + IsValid: false, + Type: ResponseError, + Err: err, + } + return + } + for _, collectionName := range collectionNames { + docs, err := r.milvusClinet.Load(ctx, collectionName, response.GetVector()) + if err != nil { + ch <- &Response{ + IsValid: false, + Type: ResponseError, + Err: err, + } + continue + } + for _, doc := range docs { + ch <- &Response{ + IsValid: true, + Type: ResponseDocument, + Document: &model.DocumentResponse{ + ID: decimal.NewFromInt(doc.DocumentID), + MessageID: parentMessage.ID, + Link: doc., + DocumentID: decimal.NewFromInt(doc.DocumentID), + Content: "", + }, + } + } + } + + + for i := 0; i < 4; i++ { + ch <- &Response{ + IsValid: true, + Type: ResponseDocument, + Message: nil, + Document: &model.DocumentResponse{ + ID: decimal.NewFromInt(int64(i)), + DocumentID: "11", + Link: fmt.Sprintf("link for document %d", i), + Content: fmt.Sprintf("content of document %d", i), + MessageID: parentMessage.ID, + }, + } + } + wg.Done() +} + + +func NewEmbeddingResponder(embedding proto.EmbeddServiceClient, + milvusClinet storage.MilvusClient, + embeddingModel string ) ChatResponder { + return &embedding{ + embedding: embedding, + milvusClinet: milvusClinet, + embeddingModel: embeddingModel, + } } diff --git a/backend/core/responder/responder.go b/backend/core/responder/responder.go index 995eca7c..a710713b 100644 --- a/backend/core/responder/responder.go +++ b/backend/core/responder/responder.go @@ -20,6 +20,7 @@ type Response struct { Type string Message *model.ChatMessage Document *model.DocumentResponse + Err error } type ChatResponder interface { diff --git a/proto/Makefile b/proto/Makefile new file mode 100644 index 00000000..01a81bd2 --- /dev/null +++ b/proto/Makefile @@ -0,0 +1,8 @@ + +gen-proto-go: + rm -f ../backend/core/proto/*.pb.go + protoc -I=. -I=vendor -I=${GOPATH}/src *.proto --go_out=.. --go-grpc_out=.. + +gen-proto-python: + rm -f ai/core/proto/*.pb.py + protoc -I=. *.proto --python_out=. --python-grpc_out=. \ No newline at end of file diff --git a/proto/chunking_data.proto b/proto/chunking_data.proto new file mode 100644 index 00000000..eac4f15d --- /dev/null +++ b/proto/chunking_data.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package com.embedd; +option go_package = "backend/core/proto;proto"; + + +enum file_type { + URL = 0; + // add all supported file that in another document + // check what with Google docs +}; + +message ChunkingData { +// This is url where the file is located. +// Based on the chunking type it will be a WEB URL (HTML type) +// Will be an S3/MINIO link with a proper authentication in case of a file + string url = 1; + int64 connector_id = 2; + int64 document_id = 3; // 0 if new +} \ No newline at end of file diff --git a/proto/connector_messages.proto b/proto/connector_messages.proto index 4870044f..0a8555de 100644 --- a/proto/connector_messages.proto +++ b/proto/connector_messages.proto @@ -1,6 +1,6 @@ syntax="proto3"; package proto; -option go_package = "core/proto;proto"; +option go_package = "backend/core/proto;proto"; message ConnectorRequest { int64 id = 1; diff --git a/proto/embedd_messages.proto b/proto/embedd_messages.proto index fb910129..5c0867fd 100644 --- a/proto/embedd_messages.proto +++ b/proto/embedd_messages.proto @@ -1,6 +1,7 @@ syntax = "proto3"; package com.embedd; +option go_package = "backend/core/proto;proto"; message EmbeddRequest { string content = 1; diff --git a/proto/embedd_service.proto b/proto/embedd_service.proto index 915e70cf..2b755d98 100644 --- a/proto/embedd_service.proto +++ b/proto/embedd_service.proto @@ -1,6 +1,7 @@ syntax = "proto3"; package com.embedd; +option go_package = "backend/core/proto;proto"; // Import the messages from the other file import "embedd_messages.proto"; From 47b0bcbaaa6c70c274bd8aa440ddf274b667a31e Mon Sep 17 00:00:00 2001 From: aPaladiychuk Date: Tue, 14 May 2024 08:28:59 +0300 Subject: [PATCH 2/6] add chunking proto --- backend/connector/readme.md | 21 ++++++++++++++++++++- proto/chunking_data.proto | 10 +++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/backend/connector/readme.md b/backend/connector/readme.md index 461b9a81..1ddadb8d 100644 --- a/backend/connector/readme.md +++ b/backend/connector/readme.md @@ -25,7 +25,26 @@ collection name is fixed by a rule like tennant_{ID}. user_{ID} pass opentelemetry context to nats for distributed tracing -### file compatibility +### file compatibility + + - Office all + - doc, docx + - xls, xlsx + - rtf + - ppt, pptx + - pdf + - google docs all + - txt + - md + +#### optional + - ODT + - ODS + - ODP + - ODG + + +### connectors teams chats teams teams sharepoint diff --git a/proto/chunking_data.proto b/proto/chunking_data.proto index eac4f15d..87b44f9a 100644 --- a/proto/chunking_data.proto +++ b/proto/chunking_data.proto @@ -4,8 +4,15 @@ package com.embedd; option go_package = "backend/core/proto;proto"; -enum file_type { +enum FileType { URL = 0; + PDF = 1; + RTF = 2; + DOC = 3; + XLS = 4; + PPT = 5; + TXT = 6; + MD = 7; // add all supported file that in another document // check what with Google docs }; @@ -17,4 +24,5 @@ message ChunkingData { string url = 1; int64 connector_id = 2; int64 document_id = 3; // 0 if new + FileType file_type = 4; } \ No newline at end of file From 7f31fe87ca7c315ca6b001416529d1902cee3e5d Mon Sep 17 00:00:00 2001 From: aPaladiychuk Date: Tue, 14 May 2024 14:25:25 +0300 Subject: [PATCH 3/6] add embeddings --- backend/api/module.go | 1 + backend/core/ai/module.go | 5 +- backend/core/ai/open-ai.go | 7 --- backend/core/bll/chat.go | 8 ++- backend/core/bll/connector.go | 6 +- backend/core/messaging/module.go | 2 + backend/core/model/chat.go | 3 +- backend/core/repository/chat.go | 7 +++ backend/core/repository/document.go | 9 +++ backend/core/responder/ai.go | 67 +++++++++++---------- backend/core/responder/embedding.go | 92 +++++++++++------------------ backend/core/responder/manager.go | 4 +- backend/core/responder/responder.go | 4 +- docker-compose.yaml | 11 ++++ 14 files changed, 121 insertions(+), 105 deletions(-) diff --git a/backend/api/module.go b/backend/api/module.go index 78c90dca..0c3314c2 100644 --- a/backend/api/module.go +++ b/backend/api/module.go @@ -26,6 +26,7 @@ var Module = fx.Options( storage.MinioModule, messaging.NatsModule, ai.EmbeddingModule, + storage.MilvusModule, fx.Provide(ReadConfig, NewRouter, newGoogleOauthProvider, diff --git a/backend/core/ai/module.go b/backend/core/ai/module.go index 38254c36..32ee74a9 100644 --- a/backend/core/ai/module.go +++ b/backend/core/ai/module.go @@ -5,6 +5,7 @@ import ( "cognix.ch/api/v2/core/utils" "go.uber.org/fx" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) var ChunkingModule = fx.Options( @@ -44,7 +45,9 @@ var EmbeddingModule = fx.Options( ) func newEmbeddingGRPCClient(cfg *EmbeddingConfig) (proto.EmbeddServiceClient, error) { - conn, err := grpc.Dial(cfg.EmbeddingURL) + dialOptions := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + + conn, err := grpc.Dial(cfg.EmbeddingURL, dialOptions...) if err != nil { return nil, err } diff --git a/backend/core/ai/open-ai.go b/backend/core/ai/open-ai.go index 07e7dc64..2770c02f 100644 --- a/backend/core/ai/open-ai.go +++ b/backend/core/ai/open-ai.go @@ -31,13 +31,6 @@ func (o *openAIClient) Request(ctx context.Context, message string) (*Response, Messages: []openai.ChatCompletionMessage{userMessage}, }, ) - o.client.CreateEmbeddings(ctx, &openai.EmbeddingRequest{ - Input: nil, - Model: "", - User: "", - EncodingFormat: "", - Dimensions: 0, - }) if err != nil { return nil, err } diff --git a/backend/core/bll/chat.go b/backend/core/bll/chat.go index 58f7801a..b6cfae40 100644 --- a/backend/core/bll/chat.go +++ b/backend/core/bll/chat.go @@ -25,6 +25,7 @@ type ChatBL interface { } type chatBL struct { chatRepo repository.ChatRepository + docRepo repository.DocumentRepository personaRepo repository.PersonaRepository aiBuilder *ai.Builder embedding proto.EmbeddServiceClient @@ -66,10 +67,11 @@ func (b *chatBL) SendMessage(ctx *gin.Context, user *model.User, param *paramete } aiClient := b.aiBuilder.New(chatSession.Persona.LLM) resp := responder.NewManager( - responder.NewAIResponder(aiClient, b.chatRepo), + responder.NewAIResponder(aiClient, b.chatRepo, + b.embedding, b.milvusClinet, b.docRepo, ""), ) - go resp.Send(ctx, &message) + go resp.Send(ctx, user, &message) return resp, nil } @@ -128,12 +130,14 @@ func (b *chatBL) CreateSession(ctx context.Context, user *model.User, param *par func NewChatBL(chatRepo repository.ChatRepository, personaRepo repository.PersonaRepository, + docRepo repository.DocumentRepository, aiBuilder *ai.Builder, embedding proto.EmbeddServiceClient, milvusClinet storage.MilvusClient, ) ChatBL { return &chatBL{chatRepo: chatRepo, personaRepo: personaRepo, + docRepo: docRepo, aiBuilder: aiBuilder, embedding: embedding, milvusClinet: milvusClinet, diff --git a/backend/core/bll/connector.go b/backend/core/bll/connector.go index f9c9e5cd..de4ad0f7 100644 --- a/backend/core/bll/connector.go +++ b/backend/core/bll/connector.go @@ -1,10 +1,10 @@ package bll import ( - "cognix.ch/api/v2/core/connector" "cognix.ch/api/v2/core/messaging" "cognix.ch/api/v2/core/model" "cognix.ch/api/v2/core/parameters" + "cognix.ch/api/v2/core/proto" "cognix.ch/api/v2/core/repository" "cognix.ch/api/v2/core/utils" "context" @@ -88,7 +88,7 @@ func (c *connectorBL) Create(ctx context.Context, user *model.User, param *param if err := c.connectorRepo.Create(ctx, &conn); err != nil { return nil, err } - if err := c.messenger.Publish(ctx, model.TopicUpdateConnector, connector.Trigger{ID: conn.ID.IntPart()}); err != nil { + if err := c.messenger.Publish(ctx, model.TopicUpdateConnector, &proto.Body{Payload: &proto.Body_Trigger{Trigger: &proto.TriggerRequest{Id: conn.ID.IntPart()}}}); err != nil { return nil, err } return &conn, nil @@ -120,7 +120,7 @@ func (c *connectorBL) Update(ctx context.Context, id int64, user *model.User, pa if err = c.connectorRepo.Update(ctx, conn); err != nil { return nil, err } - if err = c.messenger.Publish(ctx, model.TopicUpdateConnector, connector.Trigger{ID: conn.ID.IntPart()}); err != nil { + if err = c.messenger.Publish(ctx, model.TopicUpdateConnector, &proto.Body{Payload: &proto.Body_Trigger{Trigger: &proto.TriggerRequest{Id: conn.ID.IntPart()}}}); err != nil { return nil, err } return conn, nil diff --git a/backend/core/messaging/module.go b/backend/core/messaging/module.go index 9e2ef695..c6ef988e 100644 --- a/backend/core/messaging/module.go +++ b/backend/core/messaging/module.go @@ -63,6 +63,8 @@ var NatsModule = fx.Options( func NewClient(cfg *Config) (Client, error) { switch cfg.Provider { + case providerNats: + return NewClientStream(cfg.Nats) case providerPulsar: return NewPulsar(cfg.Pulsar) } diff --git a/backend/core/model/chat.go b/backend/core/model/chat.go index d49f6155..2f879cfe 100644 --- a/backend/core/model/chat.go +++ b/backend/core/model/chat.go @@ -35,12 +35,13 @@ type ( MessageType string `json:"message_type,omitempty"` TimeSent time.Time `json:"time_sent,omitempty"` TokenCount int `json:"token_count,omitempty" pg:",use_zero"` - ParentMessage decimal.Decimal `json:"parent_message,omitempty" pg:",use_zero"` + ParentMessageID decimal.Decimal `json:"parent_message,omitempty" pg:"parent_message,use_zero"` LatestChildMessage int `json:"latest_child_message,omitempty" pg:",use_zero"` RephrasedQuery string `json:"rephrased_query,omitempty" pg:",use_zero"` Citations []*DocumentResponse `json:"citations,omitempty" pg:"-"` Error string `json:"error,omitempty" pg:",use_zero"` Feedback *ChatMessageFeedback `json:"feedback,omitempty" pg:"rel:has-one,fk:id,join_fk:chat_message_id"` + ParentMessage *ChatMessage `json:"-" pg:"-"` } ChatMessageFeedback struct { diff --git a/backend/core/repository/chat.go b/backend/core/repository/chat.go index 964a4106..a260ffe7 100644 --- a/backend/core/repository/chat.go +++ b/backend/core/repository/chat.go @@ -14,6 +14,7 @@ type ChatRepository interface { GetSessionByID(ctx context.Context, userID uuid.UUID, id int64) (*model.ChatSession, error) CreateSession(ctx context.Context, session *model.ChatSession) error SendMessage(ctx context.Context, message *model.ChatMessage) error + Update(ctx context.Context, message *model.ChatMessage) error GetMessageByIDAndUserID(ctx context.Context, id int64, userID uuid.UUID) (*model.ChatMessage, error) MessageFeedback(ctx context.Context, feedback *model.ChatMessageFeedback) error } @@ -53,6 +54,12 @@ func (r *chatRepository) SendMessage(ctx context.Context, message *model.ChatMes } return nil } +func (r *chatRepository) Update(ctx context.Context, message *model.ChatMessage) error { + if _, err := r.db.WithContext(ctx).Model(message).Where("id = ?", message.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not save message") + } + return nil +} func NewChatRepository(db *pg.DB) ChatRepository { return &chatRepository{db: db} diff --git a/backend/core/repository/document.go b/backend/core/repository/document.go index b0e99d17..dd186a70 100644 --- a/backend/core/repository/document.go +++ b/backend/core/repository/document.go @@ -13,6 +13,7 @@ type ( DocumentRepository interface { FindByConnectorIDAndUser(ctx context.Context, user *model.User, connectorID int64) ([]*model.Document, error) FindByConnectorID(ctx context.Context, connectorID int64) ([]*model.Document, error) + FindByID(ctx context.Context, id int64) (*model.Document, error) Create(ctx context.Context, document ...*model.Document) error Update(ctx context.Context, document *model.Document) error } @@ -21,6 +22,14 @@ type ( } ) +func (r *documentRepository) FindByID(ctx context.Context, id int64) (*model.Document, error) { + var doc model.Document + if err := r.db.WithContext(ctx).Model(&doc).Where("id = ?", id).Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "document not found") + } + return &doc, nil +} + func (r *documentRepository) FindByConnectorID(ctx context.Context, connectorID int64) ([]*model.Document, error) { documents := make([]*model.Document, 0) if err := r.db.WithContext(ctx).Model(&documents). diff --git a/backend/core/responder/ai.go b/backend/core/responder/ai.go index ad07d102..2c159928 100644 --- a/backend/core/responder/ai.go +++ b/backend/core/responder/ai.go @@ -3,35 +3,54 @@ package responder import ( "cognix.ch/api/v2/core/ai" "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/storage" "context" - "fmt" - "github.com/shopspring/decimal" + "go.uber.org/zap" "sync" "time" ) type aiResponder struct { - aiClient ai.OpenAIClient - charRepo repository.ChatRepository + aiClient ai.OpenAIClient + charRepo repository.ChatRepository + embedding *embedding } -func (r *aiResponder) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGroup, parentMessage *model.ChatMessage) { - - response, err := r.aiClient.Request(ctx, parentMessage.Message) +func (r *aiResponder) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, parentMessage *model.ChatMessage) { + defer wg.Done() message := model.ChatMessage{ - ChatSessionID: parentMessage.ChatSessionID, - ParentMessage: parentMessage.ID, - MessageType: model.MessageTypeAssistant, - TimeSent: time.Now().UTC(), + ChatSessionID: parentMessage.ChatSessionID, + ParentMessageID: parentMessage.ID, + MessageType: model.MessageTypeAssistant, + TimeSent: time.Now().UTC(), + ParentMessage: parentMessage, } + if err := r.charRepo.SendMessage(ctx, &message); err != nil { + ch <- &Response{ + IsValid: err == nil, + Type: ResponseMessage, + Message: &message, + } + return + } + + docs, err := r.embedding.FindDocuments(ctx, ch, &message, model.CollectionName(true, user.ID, user.TenantID), + model.CollectionName(false, user.ID, user.TenantID)) + if err != nil { + zap.S().Errorf(err.Error()) + } + _ = docs + response, err := r.aiClient.Request(ctx, parentMessage.Message) + if err != nil { message.Error = err.Error() } else { message.Message = response.Message } - if errr := r.charRepo.SendMessage(ctx, &message); errr != nil { + if errr := r.charRepo.Update(ctx, &message); errr != nil { err = errr message.Error = err.Error() } @@ -44,31 +63,19 @@ func (r *aiResponder) Send(ctx context.Context, ch chan *Response, wg *sync.Wait payload.Type = ResponseError } ch <- payload - time.Sleep(10 * time.Millisecond) - for i := 0; i < 4; i++ { - ch <- &Response{ - IsValid: true, - Type: ResponseDocument, - Message: nil, - Document: &model.DocumentResponse{ - ID: decimal.NewFromInt(int64(i)), - DocumentID: "11", - Link: fmt.Sprintf("link for document %d", i), - Content: fmt.Sprintf("content of document %d", i), - UpdatedDate: time.Now().UTC().Add(-48 * time.Hour), - MessageID: message.ID, - }, - } - } - wg.Done() } func NewAIResponder( aiClient ai.OpenAIClient, charRepo repository.ChatRepository, + embeddProto proto.EmbeddServiceClient, + milvusClinet storage.MilvusClient, + docRepo repository.DocumentRepository, + embeddingModel string, ) ChatResponder { return &aiResponder{aiClient: aiClient, - charRepo: charRepo, + charRepo: charRepo, + embedding: NewEmbeddingResponder(embeddProto, milvusClinet, docRepo, embeddingModel), } } diff --git a/backend/core/responder/embedding.go b/backend/core/responder/embedding.go index fb6ed63f..36819141 100644 --- a/backend/core/responder/embedding.go +++ b/backend/core/responder/embedding.go @@ -3,6 +3,7 @@ package responder import ( "cognix.ch/api/v2/core/model" "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" "cognix.ch/api/v2/core/storage" "context" "fmt" @@ -12,25 +13,13 @@ import ( ) type embedding struct { - embedding proto.EmbeddServiceClient - milvusClinet storage.MilvusClient + embedding proto.EmbeddServiceClient + milvusClinet storage.MilvusClient + docRepo repository.DocumentRepository embeddingModel string } -func (r *embedding) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGroup, parentMessage *model.ChatMessage) { - response, err := r.embedding.GetEmbedd(ctx, &proto.EmbeddRequest{ - Content: parentMessage.Message, - Model: r.embeddingModel, - }) - if err != nil { - ch <- &Response{ - IsValid: false, - Type: ResponseError, - Err: err, - } - return - } - docs, err := r.milvusClinet. +func (r *embedding) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, parentMessage *model.ChatMessage) { for i := 0; i < 4; i++ { ch <- &Response{ @@ -50,73 +39,62 @@ func (r *embedding) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGr wg.Done() } - func (r *embedding) FindDocuments(ctx context.Context, - ch chan *Response, wg *sync.WaitGroup, - parentMessage *model.ChatMessage, - collectionNames ...string) { + ch chan *Response, + message *model.ChatMessage, + collectionNames ...string) ([]*model.DocumentResponse, error) { response, err := r.embedding.GetEmbedd(ctx, &proto.EmbeddRequest{ - Content: parentMessage.Message, + Content: message.ParentMessage.Message, Model: r.embeddingModel, }) if err != nil { ch <- &Response{ - IsValid: false, - Type: ResponseError, - Err: err, + IsValid: false, + Type: ResponseError, + Err: err, } - return + return nil, err } + var result []*model.DocumentResponse for _, collectionName := range collectionNames { docs, err := r.milvusClinet.Load(ctx, collectionName, response.GetVector()) if err != nil { ch <- &Response{ - IsValid: false, - Type: ResponseError, - Err: err, + IsValid: false, + Type: ResponseError, + Err: err, } continue } for _, doc := range docs { + resDocument := &model.DocumentResponse{ + ID: decimal.NewFromInt(doc.DocumentID), + MessageID: message.ID, + Content: doc.Content, + } + if dbDoc, err := r.docRepo.FindByID(ctx, doc.DocumentID); err == nil { + resDocument.Link = dbDoc.Link + resDocument.DocumentID = dbDoc.DocumentID + } + result = append(result, resDocument) ch <- &Response{ IsValid: true, Type: ResponseDocument, - Document: &model.DocumentResponse{ - ID: decimal.NewFromInt(doc.DocumentID), - MessageID: parentMessage.ID, - Link: doc., - DocumentID: decimal.NewFromInt(doc.DocumentID), - Content: "", - }, + Document: resDocument, } } } - - - for i := 0; i < 4; i++ { - ch <- &Response{ - IsValid: true, - Type: ResponseDocument, - Message: nil, - Document: &model.DocumentResponse{ - ID: decimal.NewFromInt(int64(i)), - DocumentID: "11", - Link: fmt.Sprintf("link for document %d", i), - Content: fmt.Sprintf("content of document %d", i), - MessageID: parentMessage.ID, - }, - } - } - wg.Done() + return result, nil } - -func NewEmbeddingResponder(embedding proto.EmbeddServiceClient, +func NewEmbeddingResponder(embeddProto proto.EmbeddServiceClient, milvusClinet storage.MilvusClient, - embeddingModel string ) ChatResponder { + docRepo repository.DocumentRepository, + embeddingModel string) *embedding { return &embedding{ - embedding: embedding, - milvusClinet: milvusClinet, + embedding: embeddProto, + milvusClinet: milvusClinet, embeddingModel: embeddingModel, + docRepo: docRepo, } } diff --git a/backend/core/responder/manager.go b/backend/core/responder/manager.go index d1317e85..e26d0856 100644 --- a/backend/core/responder/manager.go +++ b/backend/core/responder/manager.go @@ -12,10 +12,10 @@ type Manager struct { responders []ChatResponder } -func (m *Manager) Send(cx context.Context, parentMessage *model.ChatMessage) { +func (m *Manager) Send(cx context.Context, user *model.User, parentMessage *model.ChatMessage) { for _, responder := range m.responders { m.wg.Add(1) - go responder.Send(cx, m.ch, m.wg, parentMessage) + go responder.Send(cx, m.ch, m.wg, user, parentMessage) } m.wg.Wait() close(m.ch) diff --git a/backend/core/responder/responder.go b/backend/core/responder/responder.go index a710713b..a8f948cc 100644 --- a/backend/core/responder/responder.go +++ b/backend/core/responder/responder.go @@ -24,13 +24,13 @@ type Response struct { } type ChatResponder interface { - Send(cx context.Context, ch chan *Response, wg *sync.WaitGroup, parentMessage *model.ChatMessage) + Send(cx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, parentMessage *model.ChatMessage) } type nopResponder struct { ch chan string } -func (r *nopResponder) Send(cx context.Context, ch chan *Response, wg *sync.WaitGroup, parentMessage *model.ChatMessage) { +func (r *nopResponder) Send(cx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, parentMessage *model.ChatMessage) { go func() { defer wg.Done() i := 0 diff --git a/docker-compose.yaml b/docker-compose.yaml index ab1e35de..385cb3a6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,6 +58,17 @@ services: - ./backend:/backend networks: - dev-network + embedding: + image: cognix:embedding + build: + context: ./ai/embedder + args: + service: embedding + dockerfile: Dockerfile + ports: + - "50051:50051" + networks: + - dev-network nats: image: docker.io/nats:2.9.20 From 6af55b6e66e1ab2744f84145713df884b39d569e Mon Sep 17 00:00:00 2001 From: aPaladiychuk Date: Tue, 14 May 2024 14:28:51 +0300 Subject: [PATCH 4/6] update proto definitions --- proto/connector_messages.proto | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/proto/connector_messages.proto b/proto/connector_messages.proto index 0a8555de..71100b17 100644 --- a/proto/connector_messages.proto +++ b/proto/connector_messages.proto @@ -7,16 +7,6 @@ message ConnectorRequest { map params = 2; } -message ConnectorResponse { - int64 document_id = 1; - string content = 2; -} - -message ChunkingResponse { - int64 document_id = 1; - repeated string chunks = 2; -} - message EmbeddAsyncRequest { int64 document_id = 1; int64 chunk_id = 2; From ccff7fab01228912a10c65fa09234da6f885dff0 Mon Sep 17 00:00:00 2001 From: aPaladiychuk Date: Tue, 14 May 2024 18:00:06 +0300 Subject: [PATCH 5/6] update proto definitions --- backend/connector/executor.go | 80 +- backend/connector/module.go | 2 +- backend/core/ai/embeddingi.go | 50 - backend/core/ai/model.go | 5 - backend/core/bll/chat.go | 3 +- backend/core/bll/connector.go | 4 +- backend/core/connector/base.go | 10 +- backend/core/connector/web.go | 42 +- backend/core/messaging/nats_stream.go | 1 + backend/core/messaging/pulsar.go | 2 +- backend/core/model/document.go | 32 +- backend/core/model/messenging.go | 3 +- backend/core/proto/chunking_data.pb.go | 240 +++++ backend/core/proto/connector.pb.go | 933 ------------------ backend/core/proto/connector.proto | 15 - backend/core/proto/connector_grpc.pb.go | 192 ---- backend/core/proto/connector_messages.pb.go | 235 +---- backend/core/proto/embedd_messages.pb.go | 306 ------ backend/core/proto/embedd_service.pb.go | 6 +- backend/core/proto/messeging_data.pb.go | 278 ++++++ backend/core/responder/ai.go | 14 +- backend/core/responder/manager.go | 4 +- backend/core/responder/responder.go | 37 +- backend/orchestrator/server.go | 2 +- backend/orchestrator/trigger.go | 5 +- proto/chunking_data.proto | 5 +- proto/connector_messages.proto | 6 +- ...d_messages.proto => embedd_requests.proto} | 6 - proto/embedd_service.proto | 2 +- proto/messeging_data.proto | 18 + 30 files changed, 682 insertions(+), 1856 deletions(-) create mode 100644 backend/core/proto/chunking_data.pb.go delete mode 100644 backend/core/proto/connector.pb.go delete mode 100644 backend/core/proto/connector_grpc.pb.go delete mode 100644 backend/core/proto/embedd_messages.pb.go create mode 100644 backend/core/proto/messeging_data.pb.go rename proto/{embedd_messages.proto => embedd_requests.proto} (65%) create mode 100644 proto/messeging_data.proto diff --git a/backend/connector/executor.go b/backend/connector/executor.go index d373e7ef..5992ff95 100644 --- a/backend/connector/executor.go +++ b/backend/connector/executor.go @@ -33,12 +33,12 @@ func (e *Executor) run(ctx context.Context, topic, subscriptionName string, task func (e *Executor) runEmbedding(ctx context.Context, msg *proto.Message) error { ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.Header)) - payload := msg.GetBody().GetEmbedding() + payload := msg.GetBody().GetChunking() if payload == nil { zap.S().Errorf("Failed to get embedding payload") return nil } - zap.S().Infof("process embedding %d == > %50s ", payload.GetDocumentId(), payload.Content) + return nil } @@ -53,6 +53,7 @@ func (e *Executor) runConnector(ctx context.Context, msg *proto.Message) error { if err != nil { return err } + // todo move to connector if err = e.connectorRepo.InvalidateConnector(ctx, connectorModel); err != nil { return err } @@ -60,78 +61,38 @@ func (e *Executor) runConnector(ctx context.Context, msg *proto.Message) error { if err != nil { return err } - embedding := ai.NewEmbeddingParser(&model.EmbeddingModel{ModelID: "text-embedding-ada-002"}) resultCh := connectorWF.Execute(ctx, trigger.Params) - if err = e.milvusClinet.CreateSchema(ctx, connectorWF.CollectionName()); err != nil { + if err = e.milvusClinet.CreateSchema(ctx, connectorModel.CollectionName()); err != nil { return fmt.Errorf("error creating schema: %v", err) } for result := range resultCh { var loopErr error doc := e.handleResult(ctx, connectorModel, result) - if !doc.IsUpdated { - continue - } + if doc.ID.IntPart() != 0 { loopErr = e.docRepo.Update(ctx, doc) } else { loopErr = e.docRepo.Create(ctx, doc) } + if loopErr != nil { err = loopErr doc.Status = model.StatusFailed zap.S().Errorf("Failed to update document: %v", loopErr) continue } - chunks, loopErr := e.chunking.Split(ctx, result.Content) - if loopErr != nil { - err = loopErr - doc.Status = model.StatusFailed - zap.S().Errorf("Failed to update document: %v", loopErr) - continue - } - embeddingResponse, loopErr := embedding.Parse(ctx, &proto.EmbeddingRequest{ - DocumentId: doc.ID.IntPart(), - Key: doc.DocumentID, - Content: chunks, - }) - if loopErr != nil { - err = loopErr - doc.Status = model.StatusFailed - zap.S().Errorf("Failed to update document: %v", loopErr) - continue - } - milvusPayload := make([]*storage.MilvusPayload, 0, len(embeddingResponse.Payload)) - for _, payload := range embeddingResponse.Payload { - milvusPayload = append(milvusPayload, &storage.MilvusPayload{ - DocumentID: embeddingResponse.GetDocumentId(), - Chunk: payload.GetChunk(), - Content: payload.GetContent(), - Vector: payload.GetVector(), - }) - } - if loopErr = e.milvusClinet.Save(ctx, connectorWF.CollectionName(), milvusPayload...); loopErr != nil { + result.DocumentId = doc.ID.IntPart() + + if loopErr = e.msgClient.Publish(ctx, model.TopicChunking, &proto.Body{ + Payload: &proto.Body_Chunking{Chunking: result}, + }); loopErr != nil { err = loopErr doc.Status = model.StatusFailed zap.S().Errorf("Failed to update document: %v", loopErr) continue } - - //todo for production - //if loopErr = e.msgClient.Publish(ctx, model.TopicEmbedding, - // &proto.Body{MilvusPayload: &proto.Body_Embedding{Embedding: &proto.EmbeddingRequest{ - // DocumentId: doc.ID.IntPart(), - // Key: doc.DocumentID, - // Content: chunks, - // }}}, - //); err != nil { - // err = loopErr - // zap.S().Errorf("Failed to update document: %v", err) - // doc.Status = model.StatusFailed - // doc.Signature = "" - // continue - //} } if err != nil { @@ -145,26 +106,21 @@ func (e *Executor) runConnector(ctx context.Context, msg *proto.Message) error { return nil } -func (e *Executor) handleResult(ctx context.Context, connectorModel *model.Connector, result *proto.TriggerResponse) *model.Document { - doc, ok := connectorModel.DocsMap[result.GetDocumentId()] +func (e *Executor) handleResult(ctx context.Context, connectorModel *model.Connector, result *proto.ChunkingData) *model.Document { + doc, ok := connectorModel.DocsMap[result.GetUrl()] if !ok { doc = &model.Document{ - DocumentID: result.GetDocumentId(), + DocumentID: result.GetUrl(), ConnectorID: connectorModel.ID, Link: result.GetUrl(), - Signature: result.GetSignature(), CreatedDate: time.Now().UTC(), Status: model.StatusInProgress, - IsUpdated: true, - } - connectorModel.DocsMap[result.GetDocumentId()] = doc - } else { - if doc.Signature != result.GetSignature() { - doc.Signature = result.GetSignature() - doc.Status = model.StatusInProgress - doc.IsUpdated = true } + connectorModel.DocsMap[result.GetUrl()] = doc } + + doc.Status = model.StatusInProgress + return doc } diff --git a/backend/connector/module.go b/backend/connector/module.go index d31519a7..11d127ff 100644 --- a/backend/connector/module.go +++ b/backend/connector/module.go @@ -27,7 +27,7 @@ var Module = fx.Options( func RunServer(lc fx.Lifecycle, executor *Executor) error { - go executor.run(context.Background(), model.TopicEmbedding, model.SubscriptionEmbedding, executor.runEmbedding) + //go executor.run(context.Background(), model.TopicEmbedding, model.SubscriptionEmbedding, executor.runEmbedding) go executor.run(context.Background(), model.TopicExecutor, model.SubscriptionExecutor, executor.runConnector) return nil diff --git a/backend/core/ai/embeddingi.go b/backend/core/ai/embeddingi.go index 9db44dad..ad8f17d5 100644 --- a/backend/core/ai/embeddingi.go +++ b/backend/core/ai/embeddingi.go @@ -1,67 +1,17 @@ package ai import ( - "cognix.ch/api/v2/core/model" - "cognix.ch/api/v2/core/proto" - "context" _ "github.com/deluan/flowllm/llms/openai" validation "github.com/go-ozzo/ozzo-validation/v4" - "github.com/sashabaranov/go-openai" - "go.uber.org/zap" - "os" ) type ( EmbeddingConfig struct { EmbeddingURL string `env:"EMBEDDING_GRPC_URL"` } - embeddingParser struct { - embeddingModel *model.EmbeddingModel - client *openai.Client - } ) func (v EmbeddingConfig) Validate() error { return validation.ValidateStruct(&v, validation.Field(&v.EmbeddingURL, validation.Required)) } -func NewEmbeddingParser(embeddingModel *model.EmbeddingModel) EmbeddingParser { - //remove-it - apiKey := os.Getenv("OPENAI_API_KEY") - return &embeddingParser{ - embeddingModel: embeddingModel, - client: openai.NewClient(apiKey), - } -} - -func (p *embeddingParser) Parse(ctx context.Context, payload *proto.EmbeddingRequest) (*proto.EmbeddingResponse, error) { - embeddingModel := payload.GetEmbeddingModel() - if embeddingModel == "" { - embeddingModel = p.embeddingModel.ModelID - } - embeddingRequest := openai.EmbeddingRequestStrings{ - Input: payload.GetContent(), - Model: openai.EmbeddingModel(embeddingModel), - EncodingFormat: openai.EmbeddingEncodingFormatFloat, - Dimensions: 0, - } - - embeddingResponse, err := p.client.CreateEmbeddings(ctx, embeddingRequest) - if err != nil { - return nil, err - } - response := &proto.EmbeddingResponse{ - DocumentId: payload.GetDocumentId(), - Key: payload.Key, - } - for i, em := range embeddingResponse.Data { - zap.S().Infof("%d, %s", em.Index, em.Object) - response.Payload = append(response.Payload, &proto.EmbeddingResponse_Payload{ - Chunk: int64(em.Index), - Content: payload.Content[i], - Vector: em.Embedding, - }) - - } - return response, nil -} diff --git a/backend/core/ai/model.go b/backend/core/ai/model.go index 57e70f9b..e879c487 100644 --- a/backend/core/ai/model.go +++ b/backend/core/ai/model.go @@ -1,7 +1,6 @@ package ai import ( - "cognix.ch/api/v2/core/proto" "context" validation "github.com/go-ozzo/ozzo-validation/v4" ) @@ -22,10 +21,6 @@ type ( Chunking interface { Split(ctx context.Context, text string) ([]string, error) } - - EmbeddingParser interface { - Parse(ctx context.Context, payload *proto.EmbeddingRequest) (*proto.EmbeddingResponse, error) - } ) func (v ChunkingConfig) Validate() error { diff --git a/backend/core/bll/chat.go b/backend/core/bll/chat.go index b6cfae40..c033f72b 100644 --- a/backend/core/bll/chat.go +++ b/backend/core/bll/chat.go @@ -62,6 +62,7 @@ func (b *chatBL) SendMessage(ctx *gin.Context, user *model.User, param *paramete MessageType: model.MessageTypeUser, TimeSent: time.Now().UTC(), } + noLLM := chatSession.Persona == nil if err = b.chatRepo.SendMessage(ctx.Request.Context(), &message); err != nil { return nil, err } @@ -71,7 +72,7 @@ func (b *chatBL) SendMessage(ctx *gin.Context, user *model.User, param *paramete b.embedding, b.milvusClinet, b.docRepo, ""), ) - go resp.Send(ctx, user, &message) + go resp.Send(ctx, user, noLLM, &message) return resp, nil } diff --git a/backend/core/bll/connector.go b/backend/core/bll/connector.go index de4ad0f7..c96aa3bd 100644 --- a/backend/core/bll/connector.go +++ b/backend/core/bll/connector.go @@ -88,7 +88,7 @@ func (c *connectorBL) Create(ctx context.Context, user *model.User, param *param if err := c.connectorRepo.Create(ctx, &conn); err != nil { return nil, err } - if err := c.messenger.Publish(ctx, model.TopicUpdateConnector, &proto.Body{Payload: &proto.Body_Trigger{Trigger: &proto.TriggerRequest{Id: conn.ID.IntPart()}}}); err != nil { + if err := c.messenger.Publish(ctx, model.TopicUpdateConnector, &proto.Body{Payload: &proto.Body_Trigger{Trigger: &proto.ConnectorRequest{Id: conn.ID.IntPart()}}}); err != nil { return nil, err } return &conn, nil @@ -120,7 +120,7 @@ func (c *connectorBL) Update(ctx context.Context, id int64, user *model.User, pa if err = c.connectorRepo.Update(ctx, conn); err != nil { return nil, err } - if err = c.messenger.Publish(ctx, model.TopicUpdateConnector, &proto.Body{Payload: &proto.Body_Trigger{Trigger: &proto.TriggerRequest{Id: conn.ID.IntPart()}}}); err != nil { + if err = c.messenger.Publish(ctx, model.TopicUpdateConnector, &proto.Body{Payload: &proto.Body_Trigger{Trigger: &proto.ConnectorRequest{Id: conn.ID.IntPart()}}}); err != nil { return nil, err } return conn, nil diff --git a/backend/core/connector/base.go b/backend/core/connector/base.go index ceb1aec5..2211d18a 100644 --- a/backend/core/connector/base.go +++ b/backend/core/connector/base.go @@ -9,11 +9,11 @@ import ( type Base struct { model *model.Connector - resultCh chan *proto.TriggerResponse + resultCh chan *proto.ChunkingData } type Connector interface { - Execute(ctx context.Context, param map[string]string) chan *proto.TriggerResponse + Execute(ctx context.Context, param map[string]string) chan *proto.ChunkingData } type Builder struct { @@ -24,8 +24,8 @@ type nopConnector struct { Base } -func (n *nopConnector) Execute(ctx context.Context, param map[string]string) chan *proto.TriggerResponse { - ch := make(chan *proto.TriggerResponse) +func (n *nopConnector) Execute(ctx context.Context, param map[string]string) chan *proto.ChunkingData { + ch := make(chan *proto.ChunkingData) return ch } @@ -40,6 +40,6 @@ func New(connectorModel *model.Connector) (Connector, error) { func (b *Base) Config(connector *model.Connector) { b.model = connector - b.resultCh = make(chan *proto.TriggerResponse, 10) + b.resultCh = make(chan *proto.ChunkingData, 10) return } diff --git a/backend/core/connector/web.go b/backend/core/connector/web.go index 7a4ef4aa..b66a0ed8 100644 --- a/backend/core/connector/web.go +++ b/backend/core/connector/web.go @@ -4,11 +4,9 @@ import ( "cognix.ch/api/v2/core/model" "cognix.ch/api/v2/core/proto" "context" - "crypto/sha256" "fmt" "github.com/gocolly/colly/v2" "go.uber.org/zap" - "jaytaylor.com/html2text" "net/url" "strings" ) @@ -40,7 +38,14 @@ func withContext(ctx context.Context, fn func(context.Context, *colly.HTMLElemen } } -func (c *Web) Execute(ctx context.Context, param map[string]string) chan *proto.TriggerResponse { +func (c *Web) Execute(ctx context.Context, param map[string]string) chan *proto.ChunkingData { + //todo check if we need to rechunk content + // if it's new we need to start chunking + // if not we have to compare the entity with the one that we scanned before + // if they are not the same we need to start chunking + // entity comparsion shall be done accordingly to the file type + // for example for file HASH + zap.S().Debugf("Run web connector with param %s ...", c.param.URL) c.ctx = ctx go func() { @@ -71,25 +76,22 @@ func NewWeb(connector *model.Connector) (Connector, error) { func (c *Web) onBody(ctx context.Context, e *colly.HTMLElement) { child := e.ChildAttrs("a", "href") - text, _ := html2text.FromString(e.ChildText("main"), html2text.Options{ - PrettyTables: true, - PrettyTablesOptions: &html2text.PrettyTablesOptions{ - AutoFormatHeader: true, - AutoWrapText: true, - }, - OmitLinks: true, - }) - c.history[e.Request.URL.String()] = text + //text, _ := html2text.FromString(e.ChildText("main"), html2text.Options{ + // PrettyTables: true, + // PrettyTablesOptions: &html2text.PrettyTablesOptions{ + // AutoFormatHeader: true, + // AutoWrapText: true, + // }, + // OmitLinks: true, + //}) + c.processChildLinks(e.Request.URL, child) - signature := fmt.Sprintf("%x", sha256.Sum256([]byte(text))) - docID := e.Request.URL.String() + //signature := fmt.Sprintf("%x", sha256.Sum256([]byte(text))) + //docID := e.Request.URL.String() - c.resultCh <- &proto.TriggerResponse{ - DocumentId: docID, - Url: docID, - Content: text, - Signature: signature, - Status: proto.Status_SUCCESS, + c.resultCh <- &proto.ChunkingData{ + Url: e.Request.URL.String(), + FileType: proto.FileType_URL, } } diff --git a/backend/core/messaging/nats_stream.go b/backend/core/messaging/nats_stream.go index 46b24a1f..540262e4 100644 --- a/backend/core/messaging/nats_stream.go +++ b/backend/core/messaging/nats_stream.go @@ -34,6 +34,7 @@ func (c *clientStream) Publish(ctx context.Context, topic string, body *proto.Bo if err != nil { return err } + // todo here we must define pubAck, err := c.stream.Publish(fmt.Sprintf("%s.%s", c.connectorStreamName, topic), message) if err != nil { return err diff --git a/backend/core/messaging/pulsar.go b/backend/core/messaging/pulsar.go index a291260b..bdf8b776 100644 --- a/backend/core/messaging/pulsar.go +++ b/backend/core/messaging/pulsar.go @@ -36,7 +36,7 @@ func (p *pulsarClient) Publish(ctx context.Context, topic string, body *proto.Bo if !ok { producer, err = p.conn.CreateProducer(pulsar.ProducerOptions{ Topic: topic, - Schema: pulsar.NewProtoNativeSchemaWithMessage(&proto.TriggerRequest{}, nil), + Schema: pulsar.NewProtoNativeSchemaWithMessage(&proto.ConnectorRequest{}, nil), }) if err != nil { return err diff --git a/backend/core/model/document.go b/backend/core/model/document.go index 2687c9b1..78afbf55 100644 --- a/backend/core/model/document.go +++ b/backend/core/model/document.go @@ -16,22 +16,22 @@ const ( ) type Document struct { - tableName struct{} `pg:"documents"` - ID decimal.Decimal `json:"id,omitempty"` - DocumentID string `json:"document_id,omitempty"` - ConnectorID decimal.Decimal `json:"connector_id,omitempty"` - Boost int `json:"boost,omitempty" pg:",use_zero"` - Hidden bool `json:"hidden,omitempty" pg:",use_zero"` - SemanticID string `json:"semantic_id,omitempty" pg:",use_zero"` - Link string `json:"link,omitempty" pg:"link"` - FromIngestionAPI bool `json:"from_ingestion_api,omitempty" pg:",use_zero"` - Signature string `json:"signature,omitempty" pg:",use_zero"` - CreatedDate time.Time `json:"created_date,omitempty"` - UpdatedDate pg.NullTime `json:"updated_date,omitempty" pg:",use_zero"` - DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` - IsExists bool `json:"is_exists,omitempty" pg:"-"` - IsUpdated bool `json:"is_updates,omitempty" pg:"-"` - Status string `json:"status,omitempty" pg:",use_zero"` + tableName struct{} `pg:"documents"` + ID decimal.Decimal `json:"id,omitempty"` + DocumentID string `json:"document_id,omitempty"` + ConnectorID decimal.Decimal `json:"connector_id,omitempty"` + //Boost int `json:"boost,omitempty" pg:",use_zero"` + //Hidden bool `json:"hidden,omitempty" pg:",use_zero"` + //SemanticID string `json:"semantic_id,omitempty" pg:",use_zero"` + Link string `json:"link,omitempty" pg:"link"` + //FromIngestionAPI bool `json:"from_ingestion_api,omitempty" pg:",use_zero"` + Signature string `json:"signature,omitempty" pg:",use_zero"` + CreatedDate time.Time `json:"created_date,omitempty"` + UpdatedDate pg.NullTime `json:"updated_date,omitempty" pg:",use_zero"` + DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` + //IsExists bool `json:"is_exists,omitempty" pg:"-"` + //IsUpdated bool `json:"is_updates,omitempty" pg:"-"` + Status string `json:"status,omitempty" pg:",use_zero"` } type DocumentResponse struct { diff --git a/backend/core/model/messenging.go b/backend/core/model/messenging.go index 38809043..892e3483 100644 --- a/backend/core/model/messenging.go +++ b/backend/core/model/messenging.go @@ -3,7 +3,8 @@ package model const ( TopicExecutor = "executor" TopicUpdateConnector = "update-connector" - TopicEmbedding = "embedding" + TopicVector = "store-vector" + TopicChunking = "chunking" SubscriptionExecutor = "executor-subscription" SubscriptionOrchestrator = "orchestrator-subscription" diff --git a/backend/core/proto/chunking_data.pb.go b/backend/core/proto/chunking_data.pb.go new file mode 100644 index 00000000..66eac70a --- /dev/null +++ b/backend/core/proto/chunking_data.pb.go @@ -0,0 +1,240 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: chunking_data.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FileType int32 + +const ( + FileType_URL FileType = 0 + FileType_PDF FileType = 1 + FileType_RTF FileType = 2 + FileType_DOC FileType = 3 + FileType_XLS FileType = 4 + FileType_PPT FileType = 5 + FileType_TXT FileType = 6 + FileType_MD FileType = 7 +) + +// Enum value maps for FileType. +var ( + FileType_name = map[int32]string{ + 0: "URL", + 1: "PDF", + 2: "RTF", + 3: "DOC", + 4: "XLS", + 5: "PPT", + 6: "TXT", + 7: "MD", + } + FileType_value = map[string]int32{ + "URL": 0, + "PDF": 1, + "RTF": 2, + "DOC": 3, + "XLS": 4, + "PPT": 5, + "TXT": 6, + "MD": 7, + } +) + +func (x FileType) Enum() *FileType { + p := new(FileType) + *p = x + return p +} + +func (x FileType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FileType) Descriptor() protoreflect.EnumDescriptor { + return file_chunking_data_proto_enumTypes[0].Descriptor() +} + +func (FileType) Type() protoreflect.EnumType { + return &file_chunking_data_proto_enumTypes[0] +} + +func (x FileType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FileType.Descriptor instead. +func (FileType) EnumDescriptor() ([]byte, []int) { + return file_chunking_data_proto_rawDescGZIP(), []int{0} +} + +type ChunkingData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This is url where the file is located. + // Based on the chunking type it will be a WEB URL (HTML type) + // Will be an S3/MINIO link with a proper authentication in case of a file + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + DocumentId int64 `protobuf:"varint,2,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + FileType FileType `protobuf:"varint,3,opt,name=file_type,json=fileType,proto3,enum=com.embedd.FileType" json:"file_type,omitempty"` +} + +func (x *ChunkingData) Reset() { + *x = ChunkingData{} + if protoimpl.UnsafeEnabled { + mi := &file_chunking_data_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChunkingData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkingData) ProtoMessage() {} + +func (x *ChunkingData) ProtoReflect() protoreflect.Message { + mi := &file_chunking_data_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkingData.ProtoReflect.Descriptor instead. +func (*ChunkingData) Descriptor() ([]byte, []int) { + return file_chunking_data_proto_rawDescGZIP(), []int{0} +} + +func (x *ChunkingData) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *ChunkingData) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *ChunkingData) GetFileType() FileType { + if x != nil { + return x.FileType + } + return FileType_URL +} + +var File_chunking_data_proto protoreflect.FileDescriptor + +var file_chunking_data_proto_rawDesc = []byte{ + 0x0a, 0x13, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, 0x64, + 0x64, 0x22, 0x74, 0x0a, 0x0c, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x44, 0x61, 0x74, + 0x61, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x75, 0x72, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, + 0x62, 0x65, 0x64, 0x64, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x66, + 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x2a, 0x51, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, + 0x50, 0x44, 0x46, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x52, 0x54, 0x46, 0x10, 0x02, 0x12, 0x07, + 0x0a, 0x03, 0x44, 0x4f, 0x43, 0x10, 0x03, 0x12, 0x07, 0x0a, 0x03, 0x58, 0x4c, 0x53, 0x10, 0x04, + 0x12, 0x07, 0x0a, 0x03, 0x50, 0x50, 0x54, 0x10, 0x05, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x58, 0x54, + 0x10, 0x06, 0x12, 0x06, 0x0a, 0x02, 0x4d, 0x44, 0x10, 0x07, 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, + 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_chunking_data_proto_rawDescOnce sync.Once + file_chunking_data_proto_rawDescData = file_chunking_data_proto_rawDesc +) + +func file_chunking_data_proto_rawDescGZIP() []byte { + file_chunking_data_proto_rawDescOnce.Do(func() { + file_chunking_data_proto_rawDescData = protoimpl.X.CompressGZIP(file_chunking_data_proto_rawDescData) + }) + return file_chunking_data_proto_rawDescData +} + +var file_chunking_data_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_chunking_data_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_chunking_data_proto_goTypes = []interface{}{ + (FileType)(0), // 0: com.embedd.FileType + (*ChunkingData)(nil), // 1: com.embedd.ChunkingData +} +var file_chunking_data_proto_depIdxs = []int32{ + 0, // 0: com.embedd.ChunkingData.file_type:type_name -> com.embedd.FileType + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_chunking_data_proto_init() } +func file_chunking_data_proto_init() { + if File_chunking_data_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_chunking_data_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChunkingData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_chunking_data_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_chunking_data_proto_goTypes, + DependencyIndexes: file_chunking_data_proto_depIdxs, + EnumInfos: file_chunking_data_proto_enumTypes, + MessageInfos: file_chunking_data_proto_msgTypes, + }.Build() + File_chunking_data_proto = out.File + file_chunking_data_proto_rawDesc = nil + file_chunking_data_proto_goTypes = nil + file_chunking_data_proto_depIdxs = nil +} diff --git a/backend/core/proto/connector.pb.go b/backend/core/proto/connector.pb.go deleted file mode 100644 index 02ad6c4e..00000000 --- a/backend/core/proto/connector.pb.go +++ /dev/null @@ -1,933 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.28.1 -// protoc v3.12.4 -// source: core/proto/connector.proto - -package proto - -import ( - _ "github.com/golang/protobuf/ptypes/any" - empty "github.com/golang/protobuf/ptypes/empty" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Status int32 - -const ( - Status_SUCCESS Status = 0 - Status_FAILED Status = 1 -) - -// Enum value maps for Status. -var ( - Status_name = map[int32]string{ - 0: "SUCCESS", - 1: "FAILED", - } - Status_value = map[string]int32{ - "SUCCESS": 0, - "FAILED": 1, - } -) - -func (x Status) Enum() *Status { - p := new(Status) - *p = x - return p -} - -func (x Status) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (Status) Descriptor() protoreflect.EnumDescriptor { - return file_core_proto_connector_proto_enumTypes[0].Descriptor() -} - -func (Status) Type() protoreflect.EnumType { - return &file_core_proto_connector_proto_enumTypes[0] -} - -func (x Status) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use Status.Descriptor instead. -func (Status) EnumDescriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{0} -} - -type ResponseStep int32 - -const ( - ResponseStep_NO_ACTION ResponseStep = 0 - ResponseStep_LOAD ResponseStep = 1 - ResponseStep_EMBEDDING ResponseStep = 2 - ResponseStep_FINISH ResponseStep = 3 -) - -// Enum value maps for ResponseStep. -var ( - ResponseStep_name = map[int32]string{ - 0: "NO_ACTION", - 1: "LOAD", - 2: "EMBEDDING", - 3: "FINISH", - } - ResponseStep_value = map[string]int32{ - "NO_ACTION": 0, - "LOAD": 1, - "EMBEDDING": 2, - "FINISH": 3, - } -) - -func (x ResponseStep) Enum() *ResponseStep { - p := new(ResponseStep) - *p = x - return p -} - -func (x ResponseStep) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (ResponseStep) Descriptor() protoreflect.EnumDescriptor { - return file_core_proto_connector_proto_enumTypes[1].Descriptor() -} - -func (ResponseStep) Type() protoreflect.EnumType { - return &file_core_proto_connector_proto_enumTypes[1] -} - -func (x ResponseStep) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use ResponseStep.Descriptor instead. -func (ResponseStep) EnumDescriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{1} -} - -type Message struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Header map[string]string `protobuf:"bytes,1,rep,name=header,proto3" json:"header,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - Body *Body `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` -} - -func (x *Message) Reset() { - *x = Message{} - if protoimpl.UnsafeEnabled { - mi := &file_core_proto_connector_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Message) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Message) ProtoMessage() {} - -func (x *Message) ProtoReflect() protoreflect.Message { - mi := &file_core_proto_connector_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Message.ProtoReflect.Descriptor instead. -func (*Message) Descriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{0} -} - -func (x *Message) GetHeader() map[string]string { - if x != nil { - return x.Header - } - return nil -} - -func (x *Message) GetBody() *Body { - if x != nil { - return x.Body - } - return nil -} - -type Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Payload: - // - // *Body_Trigger - // *Body_Embedding - Payload isBody_Payload `protobuf_oneof:"payload"` -} - -func (x *Body) Reset() { - *x = Body{} - if protoimpl.UnsafeEnabled { - mi := &file_core_proto_connector_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Body) ProtoMessage() {} - -func (x *Body) ProtoReflect() protoreflect.Message { - mi := &file_core_proto_connector_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Body.ProtoReflect.Descriptor instead. -func (*Body) Descriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{1} -} - -func (m *Body) GetPayload() isBody_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (x *Body) GetTrigger() *TriggerRequest { - if x, ok := x.GetPayload().(*Body_Trigger); ok { - return x.Trigger - } - return nil -} - -func (x *Body) GetEmbedding() *EmbeddingRequest { - if x, ok := x.GetPayload().(*Body_Embedding); ok { - return x.Embedding - } - return nil -} - -type isBody_Payload interface { - isBody_Payload() -} - -type Body_Trigger struct { - Trigger *TriggerRequest `protobuf:"bytes,1,opt,name=trigger,proto3,oneof"` -} - -type Body_Embedding struct { - Embedding *EmbeddingRequest `protobuf:"bytes,2,opt,name=embedding,proto3,oneof"` -} - -func (*Body_Trigger) isBody_Payload() {} - -func (*Body_Embedding) isBody_Payload() {} - -type TriggerRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - Params map[string]string `protobuf:"bytes,2,rep,name=params,proto3" json:"params,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` -} - -func (x *TriggerRequest) Reset() { - *x = TriggerRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_core_proto_connector_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TriggerRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TriggerRequest) ProtoMessage() {} - -func (x *TriggerRequest) ProtoReflect() protoreflect.Message { - mi := &file_core_proto_connector_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TriggerRequest.ProtoReflect.Descriptor instead. -func (*TriggerRequest) Descriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{2} -} - -func (x *TriggerRequest) GetId() int64 { - if x != nil { - return x.Id - } - return 0 -} - -func (x *TriggerRequest) GetParams() map[string]string { - if x != nil { - return x.Params - } - return nil -} - -type EmbeddingRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` - Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - EmbeddingModel string `protobuf:"bytes,3,opt,name=embedding_model,json=embeddingModel,proto3" json:"embedding_model,omitempty"` - Content []string `protobuf:"bytes,4,rep,name=content,proto3" json:"content,omitempty"` -} - -func (x *EmbeddingRequest) Reset() { - *x = EmbeddingRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_core_proto_connector_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EmbeddingRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EmbeddingRequest) ProtoMessage() {} - -func (x *EmbeddingRequest) ProtoReflect() protoreflect.Message { - mi := &file_core_proto_connector_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EmbeddingRequest.ProtoReflect.Descriptor instead. -func (*EmbeddingRequest) Descriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{3} -} - -func (x *EmbeddingRequest) GetDocumentId() int64 { - if x != nil { - return x.DocumentId - } - return 0 -} - -func (x *EmbeddingRequest) GetKey() string { - if x != nil { - return x.Key - } - return "" -} - -func (x *EmbeddingRequest) GetEmbeddingModel() string { - if x != nil { - return x.EmbeddingModel - } - return "" -} - -func (x *EmbeddingRequest) GetContent() []string { - if x != nil { - return x.Content - } - return nil -} - -type EmbeddingResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` - Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - Payload []*EmbeddingResponse_Payload `protobuf:"bytes,3,rep,name=payload,proto3" json:"payload,omitempty"` -} - -func (x *EmbeddingResponse) Reset() { - *x = EmbeddingResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_core_proto_connector_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EmbeddingResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EmbeddingResponse) ProtoMessage() {} - -func (x *EmbeddingResponse) ProtoReflect() protoreflect.Message { - mi := &file_core_proto_connector_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EmbeddingResponse.ProtoReflect.Descriptor instead. -func (*EmbeddingResponse) Descriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{4} -} - -func (x *EmbeddingResponse) GetDocumentId() int64 { - if x != nil { - return x.DocumentId - } - return 0 -} - -func (x *EmbeddingResponse) GetKey() string { - if x != nil { - return x.Key - } - return "" -} - -func (x *EmbeddingResponse) GetPayload() []*EmbeddingResponse_Payload { - if x != nil { - return x.Payload - } - return nil -} - -type TriggerResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - DocumentId string `protobuf:"bytes,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` - Url string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"` - Content string `protobuf:"bytes,4,opt,name=content,proto3" json:"content,omitempty"` - Status Status `protobuf:"varint,5,opt,name=status,proto3,enum=proto.Status" json:"status,omitempty"` - Signature string `protobuf:"bytes,6,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *TriggerResponse) Reset() { - *x = TriggerResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_core_proto_connector_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TriggerResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TriggerResponse) ProtoMessage() {} - -func (x *TriggerResponse) ProtoReflect() protoreflect.Message { - mi := &file_core_proto_connector_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TriggerResponse.ProtoReflect.Descriptor instead. -func (*TriggerResponse) Descriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{5} -} - -func (x *TriggerResponse) GetDocumentId() string { - if x != nil { - return x.DocumentId - } - return "" -} - -func (x *TriggerResponse) GetUrl() string { - if x != nil { - return x.Url - } - return "" -} - -func (x *TriggerResponse) GetContent() string { - if x != nil { - return x.Content - } - return "" -} - -func (x *TriggerResponse) GetStatus() Status { - if x != nil { - return x.Status - } - return Status_SUCCESS -} - -func (x *TriggerResponse) GetSignature() string { - if x != nil { - return x.Signature - } - return "" -} - -type ConnectorStepResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - DocumentId int64 `protobuf:"varint,2,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` - Step ResponseStep `protobuf:"varint,3,opt,name=step,proto3,enum=proto.ResponseStep" json:"step,omitempty"` - Content string `protobuf:"bytes,4,opt,name=content,proto3" json:"content,omitempty"` -} - -func (x *ConnectorStepResponse) Reset() { - *x = ConnectorStepResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_core_proto_connector_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ConnectorStepResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ConnectorStepResponse) ProtoMessage() {} - -func (x *ConnectorStepResponse) ProtoReflect() protoreflect.Message { - mi := &file_core_proto_connector_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ConnectorStepResponse.ProtoReflect.Descriptor instead. -func (*ConnectorStepResponse) Descriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{6} -} - -func (x *ConnectorStepResponse) GetId() int64 { - if x != nil { - return x.Id - } - return 0 -} - -func (x *ConnectorStepResponse) GetDocumentId() int64 { - if x != nil { - return x.DocumentId - } - return 0 -} - -func (x *ConnectorStepResponse) GetStep() ResponseStep { - if x != nil { - return x.Step - } - return ResponseStep_NO_ACTION -} - -func (x *ConnectorStepResponse) GetContent() string { - if x != nil { - return x.Content - } - return "" -} - -type EmbeddingResponse_Payload struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Chunk int64 `protobuf:"varint,1,opt,name=chunk,proto3" json:"chunk,omitempty"` - Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` - Vector []float32 `protobuf:"fixed32,3,rep,packed,name=vector,proto3" json:"vector,omitempty"` -} - -func (x *EmbeddingResponse_Payload) Reset() { - *x = EmbeddingResponse_Payload{} - if protoimpl.UnsafeEnabled { - mi := &file_core_proto_connector_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EmbeddingResponse_Payload) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EmbeddingResponse_Payload) ProtoMessage() {} - -func (x *EmbeddingResponse_Payload) ProtoReflect() protoreflect.Message { - mi := &file_core_proto_connector_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EmbeddingResponse_Payload.ProtoReflect.Descriptor instead. -func (*EmbeddingResponse_Payload) Descriptor() ([]byte, []int) { - return file_core_proto_connector_proto_rawDescGZIP(), []int{4, 0} -} - -func (x *EmbeddingResponse_Payload) GetChunk() int64 { - if x != nil { - return x.Chunk - } - return 0 -} - -func (x *EmbeddingResponse_Payload) GetContent() string { - if x != nil { - return x.Content - } - return "" -} - -func (x *EmbeddingResponse_Payload) GetVector() []float32 { - if x != nil { - return x.Vector - } - return nil -} - -var File_core_proto_connector_proto protoreflect.FileDescriptor - -var file_core_proto_connector_proto_rawDesc = []byte{ - 0x0a, 0x1a, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x99, 0x01, 0x0a, 0x07, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x1a, 0x39, 0x0a, 0x0b, - 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7d, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x31, 0x0a, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, - 0x65, 0x72, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, - 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, - 0x52, 0x09, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x42, 0x09, 0x0a, 0x07, 0x70, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x96, 0x01, 0x0a, 0x0e, 0x54, 0x72, 0x69, 0x67, 0x67, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x06, 0x70, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, - 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x88, 0x01, 0x0a, 0x10, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, - 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6d, 0x62, 0x65, 0x64, - 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x4d, 0x6f, 0x64, 0x65, 0x6c, - 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0xd5, 0x01, 0x0a, 0x11, 0x45, - 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, - 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x3a, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x62, - 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, - 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x1a, - 0x51, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, - 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, - 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x03, 0x28, 0x02, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x22, 0xa3, 0x01, 0x0a, 0x0f, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, - 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x6f, 0x63, - 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x15, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x65, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, - 0x74, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x04, 0x73, 0x74, 0x65, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x53, 0x74, 0x65, 0x70, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x12, 0x18, 0x0a, 0x07, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2a, 0x21, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0a, 0x0a, - 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x2a, 0x42, 0x0a, 0x0c, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x65, 0x70, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, - 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x4c, 0x4f, 0x41, 0x44, - 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x45, 0x4d, 0x42, 0x45, 0x44, 0x44, 0x49, 0x4e, 0x47, 0x10, - 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x49, 0x4e, 0x49, 0x53, 0x48, 0x10, 0x03, 0x32, 0x44, 0x0a, - 0x09, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x37, 0x0a, 0x03, 0x52, 0x75, - 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x22, 0x00, 0x32, 0x47, 0x0a, 0x09, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, - 0x12, 0x3a, 0x0a, 0x03, 0x52, 0x75, 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, - 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x12, 0x5a, 0x10, - 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_core_proto_connector_proto_rawDescOnce sync.Once - file_core_proto_connector_proto_rawDescData = file_core_proto_connector_proto_rawDesc -) - -func file_core_proto_connector_proto_rawDescGZIP() []byte { - file_core_proto_connector_proto_rawDescOnce.Do(func() { - file_core_proto_connector_proto_rawDescData = protoimpl.X.CompressGZIP(file_core_proto_connector_proto_rawDescData) - }) - return file_core_proto_connector_proto_rawDescData -} - -var file_core_proto_connector_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_core_proto_connector_proto_msgTypes = make([]protoimpl.MessageInfo, 10) -var file_core_proto_connector_proto_goTypes = []interface{}{ - (Status)(0), // 0: proto.Status - (ResponseStep)(0), // 1: proto.ResponseStep - (*Message)(nil), // 2: proto.Message - (*Body)(nil), // 3: proto.Body - (*TriggerRequest)(nil), // 4: proto.TriggerRequest - (*EmbeddingRequest)(nil), // 5: proto.EmbeddingRequest - (*EmbeddingResponse)(nil), // 6: proto.EmbeddingResponse - (*TriggerResponse)(nil), // 7: proto.TriggerResponse - (*ConnectorStepResponse)(nil), // 8: proto.ConnectorStepResponse - nil, // 9: proto.Message.HeaderEntry - nil, // 10: proto.TriggerRequest.ParamsEntry - (*EmbeddingResponse_Payload)(nil), // 11: proto.EmbeddingResponse.Payload - (*empty.Empty)(nil), // 12: google.protobuf.Empty -} -var file_core_proto_connector_proto_depIdxs = []int32{ - 9, // 0: proto.Message.header:type_name -> proto.Message.HeaderEntry - 3, // 1: proto.Message.body:type_name -> proto.Body - 4, // 2: proto.Body.trigger:type_name -> proto.TriggerRequest - 5, // 3: proto.Body.embedding:type_name -> proto.EmbeddingRequest - 10, // 4: proto.TriggerRequest.params:type_name -> proto.TriggerRequest.ParamsEntry - 11, // 5: proto.EmbeddingResponse.payload:type_name -> proto.EmbeddingResponse.Payload - 0, // 6: proto.TriggerResponse.status:type_name -> proto.Status - 1, // 7: proto.ConnectorStepResponse.step:type_name -> proto.ResponseStep - 12, // 8: proto.Connector.Run:input_type -> google.protobuf.Empty - 5, // 9: proto.Embedding.Run:input_type -> proto.EmbeddingRequest - 12, // 10: proto.Connector.Run:output_type -> google.protobuf.Empty - 6, // 11: proto.Embedding.Run:output_type -> proto.EmbeddingResponse - 10, // [10:12] is the sub-list for method output_type - 8, // [8:10] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name -} - -func init() { file_core_proto_connector_proto_init() } -func file_core_proto_connector_proto_init() { - if File_core_proto_connector_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_core_proto_connector_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Message); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_core_proto_connector_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_core_proto_connector_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TriggerRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_core_proto_connector_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EmbeddingRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_core_proto_connector_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EmbeddingResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_core_proto_connector_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TriggerResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_core_proto_connector_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ConnectorStepResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_core_proto_connector_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EmbeddingResponse_Payload); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_core_proto_connector_proto_msgTypes[1].OneofWrappers = []interface{}{ - (*Body_Trigger)(nil), - (*Body_Embedding)(nil), - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_core_proto_connector_proto_rawDesc, - NumEnums: 2, - NumMessages: 10, - NumExtensions: 0, - NumServices: 2, - }, - GoTypes: file_core_proto_connector_proto_goTypes, - DependencyIndexes: file_core_proto_connector_proto_depIdxs, - EnumInfos: file_core_proto_connector_proto_enumTypes, - MessageInfos: file_core_proto_connector_proto_msgTypes, - }.Build() - File_core_proto_connector_proto = out.File - file_core_proto_connector_proto_rawDesc = nil - file_core_proto_connector_proto_goTypes = nil - file_core_proto_connector_proto_depIdxs = nil -} diff --git a/backend/core/proto/connector.proto b/backend/core/proto/connector.proto index ab26766c..f12a96bc 100644 --- a/backend/core/proto/connector.proto +++ b/backend/core/proto/connector.proto @@ -8,21 +8,6 @@ service Connector { rpc Run(google.protobuf.Empty) returns(google.protobuf.Empty) {} } -service Embedding { - rpc Run(EmbeddingRequest) returns(EmbeddingResponse) {} -} -message Message { - map header = 1; - Body body = 2; -} - -message Body { - oneof payload { - TriggerRequest trigger = 1; - EmbeddingRequest embedding = 2; - } -} - message TriggerRequest { int64 id = 1; map params = 2; diff --git a/backend/core/proto/connector_grpc.pb.go b/backend/core/proto/connector_grpc.pb.go deleted file mode 100644 index 399cb8b4..00000000 --- a/backend/core/proto/connector_grpc.pb.go +++ /dev/null @@ -1,192 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. -// versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.12.4 -// source: core/proto/connector.proto - -package proto - -import ( - context "context" - empty "github.com/golang/protobuf/ptypes/empty" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -// ConnectorClient is the client API for Connector service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type ConnectorClient interface { - Run(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) -} - -type connectorClient struct { - cc grpc.ClientConnInterface -} - -func NewConnectorClient(cc grpc.ClientConnInterface) ConnectorClient { - return &connectorClient{cc} -} - -func (c *connectorClient) Run(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/proto.Connector/Run", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// ConnectorServer is the server API for Connector service. -// All implementations must embed UnimplementedConnectorServer -// for forward compatibility -type ConnectorServer interface { - Run(context.Context, *empty.Empty) (*empty.Empty, error) - mustEmbedUnimplementedConnectorServer() -} - -// UnimplementedConnectorServer must be embedded to have forward compatible implementations. -type UnimplementedConnectorServer struct { -} - -func (UnimplementedConnectorServer) Run(context.Context, *empty.Empty) (*empty.Empty, error) { - return nil, status.Errorf(codes.Unimplemented, "method Run not implemented") -} -func (UnimplementedConnectorServer) mustEmbedUnimplementedConnectorServer() {} - -// UnsafeConnectorServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to ConnectorServer will -// result in compilation errors. -type UnsafeConnectorServer interface { - mustEmbedUnimplementedConnectorServer() -} - -func RegisterConnectorServer(s grpc.ServiceRegistrar, srv ConnectorServer) { - s.RegisterService(&Connector_ServiceDesc, srv) -} - -func _Connector_Run_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ConnectorServer).Run(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/proto.Connector/Run", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ConnectorServer).Run(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -// Connector_ServiceDesc is the grpc.ServiceDesc for Connector service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var Connector_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "proto.Connector", - HandlerType: (*ConnectorServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Run", - Handler: _Connector_Run_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "core/proto/connector.proto", -} - -// EmbeddingClient is the client API for Embedding service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type EmbeddingClient interface { - Run(ctx context.Context, in *EmbeddingRequest, opts ...grpc.CallOption) (*EmbeddingResponse, error) -} - -type embeddingClient struct { - cc grpc.ClientConnInterface -} - -func NewEmbeddingClient(cc grpc.ClientConnInterface) EmbeddingClient { - return &embeddingClient{cc} -} - -func (c *embeddingClient) Run(ctx context.Context, in *EmbeddingRequest, opts ...grpc.CallOption) (*EmbeddingResponse, error) { - out := new(EmbeddingResponse) - err := c.cc.Invoke(ctx, "/proto.Embedding/Run", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// EmbeddingServer is the server API for Embedding service. -// All implementations must embed UnimplementedEmbeddingServer -// for forward compatibility -type EmbeddingServer interface { - Run(context.Context, *EmbeddingRequest) (*EmbeddingResponse, error) - mustEmbedUnimplementedEmbeddingServer() -} - -// UnimplementedEmbeddingServer must be embedded to have forward compatible implementations. -type UnimplementedEmbeddingServer struct { -} - -func (UnimplementedEmbeddingServer) Run(context.Context, *EmbeddingRequest) (*EmbeddingResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Run not implemented") -} -func (UnimplementedEmbeddingServer) mustEmbedUnimplementedEmbeddingServer() {} - -// UnsafeEmbeddingServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to EmbeddingServer will -// result in compilation errors. -type UnsafeEmbeddingServer interface { - mustEmbedUnimplementedEmbeddingServer() -} - -func RegisterEmbeddingServer(s grpc.ServiceRegistrar, srv EmbeddingServer) { - s.RegisterService(&Embedding_ServiceDesc, srv) -} - -func _Embedding_Run_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(EmbeddingRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(EmbeddingServer).Run(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/proto.Embedding/Run", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(EmbeddingServer).Run(ctx, req.(*EmbeddingRequest)) - } - return interceptor(ctx, in, info, handler) -} - -// Embedding_ServiceDesc is the grpc.ServiceDesc for Embedding service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var Embedding_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "proto.Embedding", - HandlerType: (*EmbeddingServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Run", - Handler: _Embedding_Run_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "core/proto/connector.proto", -} diff --git a/backend/core/proto/connector_messages.pb.go b/backend/core/proto/connector_messages.pb.go index 26dc0915..2e18c948 100644 --- a/backend/core/proto/connector_messages.pb.go +++ b/backend/core/proto/connector_messages.pb.go @@ -75,116 +75,6 @@ func (x *ConnectorRequest) GetParams() map[string]string { return nil } -type ConnectorResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` - Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` -} - -func (x *ConnectorResponse) Reset() { - *x = ConnectorResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_connector_messages_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ConnectorResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ConnectorResponse) ProtoMessage() {} - -func (x *ConnectorResponse) ProtoReflect() protoreflect.Message { - mi := &file_connector_messages_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ConnectorResponse.ProtoReflect.Descriptor instead. -func (*ConnectorResponse) Descriptor() ([]byte, []int) { - return file_connector_messages_proto_rawDescGZIP(), []int{1} -} - -func (x *ConnectorResponse) GetDocumentId() int64 { - if x != nil { - return x.DocumentId - } - return 0 -} - -func (x *ConnectorResponse) GetContent() string { - if x != nil { - return x.Content - } - return "" -} - -type ChunkingResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` - Chunks []string `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty"` -} - -func (x *ChunkingResponse) Reset() { - *x = ChunkingResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_connector_messages_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ChunkingResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ChunkingResponse) ProtoMessage() {} - -func (x *ChunkingResponse) ProtoReflect() protoreflect.Message { - mi := &file_connector_messages_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ChunkingResponse.ProtoReflect.Descriptor instead. -func (*ChunkingResponse) Descriptor() ([]byte, []int) { - return file_connector_messages_proto_rawDescGZIP(), []int{2} -} - -func (x *ChunkingResponse) GetDocumentId() int64 { - if x != nil { - return x.DocumentId - } - return 0 -} - -func (x *ChunkingResponse) GetChunks() []string { - if x != nil { - return x.Chunks - } - return nil -} - type EmbeddAsyncRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -199,7 +89,7 @@ type EmbeddAsyncRequest struct { func (x *EmbeddAsyncRequest) Reset() { *x = EmbeddAsyncRequest{} if protoimpl.UnsafeEnabled { - mi := &file_connector_messages_proto_msgTypes[3] + mi := &file_connector_messages_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -212,7 +102,7 @@ func (x *EmbeddAsyncRequest) String() string { func (*EmbeddAsyncRequest) ProtoMessage() {} func (x *EmbeddAsyncRequest) ProtoReflect() protoreflect.Message { - mi := &file_connector_messages_proto_msgTypes[3] + mi := &file_connector_messages_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -225,7 +115,7 @@ func (x *EmbeddAsyncRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use EmbeddAsyncRequest.ProtoReflect.Descriptor instead. func (*EmbeddAsyncRequest) Descriptor() ([]byte, []int) { - return file_connector_messages_proto_rawDescGZIP(), []int{3} + return file_connector_messages_proto_rawDescGZIP(), []int{1} } func (x *EmbeddAsyncRequest) GetDocumentId() int64 { @@ -270,7 +160,7 @@ type EmbeddAsyncResponse struct { func (x *EmbeddAsyncResponse) Reset() { *x = EmbeddAsyncResponse{} if protoimpl.UnsafeEnabled { - mi := &file_connector_messages_proto_msgTypes[4] + mi := &file_connector_messages_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -283,7 +173,7 @@ func (x *EmbeddAsyncResponse) String() string { func (*EmbeddAsyncResponse) ProtoMessage() {} func (x *EmbeddAsyncResponse) ProtoReflect() protoreflect.Message { - mi := &file_connector_messages_proto_msgTypes[4] + mi := &file_connector_messages_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -296,7 +186,7 @@ func (x *EmbeddAsyncResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use EmbeddAsyncResponse.ProtoReflect.Descriptor instead. func (*EmbeddAsyncResponse) Descriptor() ([]byte, []int) { - return file_connector_messages_proto_rawDescGZIP(), []int{4} + return file_connector_messages_proto_rawDescGZIP(), []int{2} } func (x *EmbeddAsyncResponse) GetDocumentId() int64 { @@ -331,46 +221,37 @@ var File_connector_messages_proto protoreflect.FileDescriptor var file_connector_messages_proto_rawDesc = []byte{ 0x0a, 0x18, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x9a, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x3b, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4e, - 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, - 0x6e, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x4b, - 0x0a, 0x10, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, - 0x74, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x22, 0x85, 0x01, 0x0a, 0x12, - 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, - 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, - 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x65, - 0x6c, 0x49, 0x64, 0x22, 0x83, 0x01, 0x0a, 0x13, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x41, 0x73, - 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, - 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, - 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x02, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, - 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, + 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x22, 0x9f, 0x01, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x40, 0x0a, 0x06, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, + 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x39, 0x0a, + 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x85, 0x01, 0x0a, 0x12, 0x45, 0x6d, 0x62, + 0x65, 0x64, 0x64, 0x41, 0x73, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, + 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x07, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x69, + 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x49, 0x64, + 0x22, 0x83, 0x01, 0x0a, 0x13, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x41, 0x73, 0x79, 0x6e, 0x63, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, + 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, + 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x02, 0x52, 0x06, + 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, + 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -385,17 +266,15 @@ func file_connector_messages_proto_rawDescGZIP() []byte { return file_connector_messages_proto_rawDescData } -var file_connector_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_connector_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_connector_messages_proto_goTypes = []interface{}{ - (*ConnectorRequest)(nil), // 0: proto.ConnectorRequest - (*ConnectorResponse)(nil), // 1: proto.ConnectorResponse - (*ChunkingResponse)(nil), // 2: proto.ChunkingResponse - (*EmbeddAsyncRequest)(nil), // 3: proto.EmbeddAsyncRequest - (*EmbeddAsyncResponse)(nil), // 4: proto.EmbeddAsyncResponse - nil, // 5: proto.ConnectorRequest.ParamsEntry + (*ConnectorRequest)(nil), // 0: com.embedd.ConnectorRequest + (*EmbeddAsyncRequest)(nil), // 1: com.embedd.EmbeddAsyncRequest + (*EmbeddAsyncResponse)(nil), // 2: com.embedd.EmbeddAsyncResponse + nil, // 3: com.embedd.ConnectorRequest.ParamsEntry } var file_connector_messages_proto_depIdxs = []int32{ - 5, // 0: proto.ConnectorRequest.params:type_name -> proto.ConnectorRequest.ParamsEntry + 3, // 0: com.embedd.ConnectorRequest.params:type_name -> com.embedd.ConnectorRequest.ParamsEntry 1, // [1:1] is the sub-list for method output_type 1, // [1:1] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name @@ -422,30 +301,6 @@ func file_connector_messages_proto_init() { } } file_connector_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ConnectorResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_connector_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ChunkingResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_connector_messages_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EmbeddAsyncRequest); i { case 0: return &v.state @@ -457,7 +312,7 @@ func file_connector_messages_proto_init() { return nil } } - file_connector_messages_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_connector_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EmbeddAsyncResponse); i { case 0: return &v.state @@ -476,7 +331,7 @@ func file_connector_messages_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_connector_messages_proto_rawDesc, NumEnums: 0, - NumMessages: 6, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/backend/core/proto/embedd_messages.pb.go b/backend/core/proto/embedd_messages.pb.go deleted file mode 100644 index 8bf84787..00000000 --- a/backend/core/proto/embedd_messages.pb.go +++ /dev/null @@ -1,306 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.28.1 -// protoc v3.12.4 -// source: embedd_messages.proto - -package proto - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type EmbeddRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Content string `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` - Model string `protobuf:"bytes,2,opt,name=model,proto3" json:"model,omitempty"` -} - -func (x *EmbeddRequest) Reset() { - *x = EmbeddRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_embedd_messages_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EmbeddRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EmbeddRequest) ProtoMessage() {} - -func (x *EmbeddRequest) ProtoReflect() protoreflect.Message { - mi := &file_embedd_messages_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EmbeddRequest.ProtoReflect.Descriptor instead. -func (*EmbeddRequest) Descriptor() ([]byte, []int) { - return file_embedd_messages_proto_rawDescGZIP(), []int{0} -} - -func (x *EmbeddRequest) GetContent() string { - if x != nil { - return x.Content - } - return "" -} - -func (x *EmbeddRequest) GetModel() string { - if x != nil { - return x.Model - } - return "" -} - -type EmbeddResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Vector []float32 `protobuf:"fixed32,1,rep,packed,name=vector,proto3" json:"vector,omitempty"` -} - -func (x *EmbeddResponse) Reset() { - *x = EmbeddResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_embedd_messages_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EmbeddResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EmbeddResponse) ProtoMessage() {} - -func (x *EmbeddResponse) ProtoReflect() protoreflect.Message { - mi := &file_embedd_messages_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EmbeddResponse.ProtoReflect.Descriptor instead. -func (*EmbeddResponse) Descriptor() ([]byte, []int) { - return file_embedd_messages_proto_rawDescGZIP(), []int{1} -} - -func (x *EmbeddResponse) GetVector() []float32 { - if x != nil { - return x.Vector - } - return nil -} - -type EmbeddData struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` - Model string `protobuf:"bytes,3,opt,name=model,proto3" json:"model,omitempty"` - Vector []float32 `protobuf:"fixed32,4,rep,packed,name=vector,proto3" json:"vector,omitempty"` -} - -func (x *EmbeddData) Reset() { - *x = EmbeddData{} - if protoimpl.UnsafeEnabled { - mi := &file_embedd_messages_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EmbeddData) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EmbeddData) ProtoMessage() {} - -func (x *EmbeddData) ProtoReflect() protoreflect.Message { - mi := &file_embedd_messages_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EmbeddData.ProtoReflect.Descriptor instead. -func (*EmbeddData) Descriptor() ([]byte, []int) { - return file_embedd_messages_proto_rawDescGZIP(), []int{2} -} - -func (x *EmbeddData) GetId() int64 { - if x != nil { - return x.Id - } - return 0 -} - -func (x *EmbeddData) GetContent() string { - if x != nil { - return x.Content - } - return "" -} - -func (x *EmbeddData) GetModel() string { - if x != nil { - return x.Model - } - return "" -} - -func (x *EmbeddData) GetVector() []float32 { - if x != nil { - return x.Vector - } - return nil -} - -var File_embedd_messages_proto protoreflect.FileDescriptor - -var file_embedd_messages_proto_rawDesc = []byte{ - 0x0a, 0x15, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, - 0x65, 0x64, 0x64, 0x22, 0x3f, 0x0a, 0x0d, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x14, - 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, - 0x6f, 0x64, 0x65, 0x6c, 0x22, 0x28, 0x0a, 0x0e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x02, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x64, - 0x0a, 0x0a, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x44, 0x61, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x16, 0x0a, 0x06, - 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x02, 0x52, 0x06, 0x76, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x42, 0x1a, 0x5a, 0x18, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, - 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_embedd_messages_proto_rawDescOnce sync.Once - file_embedd_messages_proto_rawDescData = file_embedd_messages_proto_rawDesc -) - -func file_embedd_messages_proto_rawDescGZIP() []byte { - file_embedd_messages_proto_rawDescOnce.Do(func() { - file_embedd_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_embedd_messages_proto_rawDescData) - }) - return file_embedd_messages_proto_rawDescData -} - -var file_embedd_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_embedd_messages_proto_goTypes = []interface{}{ - (*EmbeddRequest)(nil), // 0: com.embedd.EmbeddRequest - (*EmbeddResponse)(nil), // 1: com.embedd.EmbeddResponse - (*EmbeddData)(nil), // 2: com.embedd.EmbeddData -} -var file_embedd_messages_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_embedd_messages_proto_init() } -func file_embedd_messages_proto_init() { - if File_embedd_messages_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_embedd_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EmbeddRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_embedd_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EmbeddResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_embedd_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EmbeddData); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_embedd_messages_proto_rawDesc, - NumEnums: 0, - NumMessages: 3, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_embedd_messages_proto_goTypes, - DependencyIndexes: file_embedd_messages_proto_depIdxs, - MessageInfos: file_embedd_messages_proto_msgTypes, - }.Build() - File_embedd_messages_proto = out.File - file_embedd_messages_proto_rawDesc = nil - file_embedd_messages_proto_goTypes = nil - file_embedd_messages_proto_depIdxs = nil -} diff --git a/backend/core/proto/embedd_service.pb.go b/backend/core/proto/embedd_service.pb.go index 79892e37..05f88e91 100644 --- a/backend/core/proto/embedd_service.pb.go +++ b/backend/core/proto/embedd_service.pb.go @@ -24,8 +24,8 @@ var File_embedd_service_proto protoreflect.FileDescriptor var file_embedd_service_proto_rawDesc = []byte{ 0x0a, 0x14, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, - 0x64, 0x64, 0x1a, 0x15, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x55, 0x0a, 0x0d, 0x45, 0x6d, 0x62, + 0x64, 0x64, 0x1a, 0x15, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x55, 0x0a, 0x0d, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x12, 0x19, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x2e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, @@ -55,7 +55,7 @@ func file_embedd_service_proto_init() { if File_embedd_service_proto != nil { return } - file_embedd_messages_proto_init() + file_embedd_requests_proto_init() type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/backend/core/proto/messeging_data.pb.go b/backend/core/proto/messeging_data.pb.go new file mode 100644 index 00000000..f13f166f --- /dev/null +++ b/backend/core/proto/messeging_data.pb.go @@ -0,0 +1,278 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: messeging_data.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Message struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Header map[string]string `protobuf:"bytes,1,rep,name=header,proto3" json:"header,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Body *Body `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` +} + +func (x *Message) Reset() { + *x = Message{} + if protoimpl.UnsafeEnabled { + mi := &file_messeging_data_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_messeging_data_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_messeging_data_proto_rawDescGZIP(), []int{0} +} + +func (x *Message) GetHeader() map[string]string { + if x != nil { + return x.Header + } + return nil +} + +func (x *Message) GetBody() *Body { + if x != nil { + return x.Body + } + return nil +} + +type Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Payload: + // + // *Body_Trigger + // *Body_Chunking + Payload isBody_Payload `protobuf_oneof:"payload"` +} + +func (x *Body) Reset() { + *x = Body{} + if protoimpl.UnsafeEnabled { + mi := &file_messeging_data_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Body) ProtoMessage() {} + +func (x *Body) ProtoReflect() protoreflect.Message { + mi := &file_messeging_data_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Body.ProtoReflect.Descriptor instead. +func (*Body) Descriptor() ([]byte, []int) { + return file_messeging_data_proto_rawDescGZIP(), []int{1} +} + +func (m *Body) GetPayload() isBody_Payload { + if m != nil { + return m.Payload + } + return nil +} + +func (x *Body) GetTrigger() *ConnectorRequest { + if x, ok := x.GetPayload().(*Body_Trigger); ok { + return x.Trigger + } + return nil +} + +func (x *Body) GetChunking() *ChunkingData { + if x, ok := x.GetPayload().(*Body_Chunking); ok { + return x.Chunking + } + return nil +} + +type isBody_Payload interface { + isBody_Payload() +} + +type Body_Trigger struct { + Trigger *ConnectorRequest `protobuf:"bytes,1,opt,name=trigger,proto3,oneof"` +} + +type Body_Chunking struct { + Chunking *ChunkingData `protobuf:"bytes,2,opt,name=chunking,proto3,oneof"` +} + +func (*Body_Trigger) isBody_Payload() {} + +func (*Body_Chunking) isBody_Payload() {} + +var File_messeging_data_proto protoreflect.FileDescriptor + +var file_messeging_data_proto_rawDesc = []byte{ + 0x0a, 0x14, 0x6d, 0x65, 0x73, 0x73, 0x65, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, + 0x64, 0x64, 0x1a, 0x18, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x13, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0xa3, 0x01, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x37, 0x0a, + 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, + 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x24, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, 0x64, + 0x64, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x1a, 0x39, 0x0a, 0x0b, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x83, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x38, 0x0a, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x2e, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, + 0x00, 0x52, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x36, 0x0a, 0x08, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, + 0x6f, 0x6d, 0x2e, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x69, + 0x6e, 0x67, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x08, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x69, + 0x6e, 0x67, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x1a, 0x5a, + 0x18, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_messeging_data_proto_rawDescOnce sync.Once + file_messeging_data_proto_rawDescData = file_messeging_data_proto_rawDesc +) + +func file_messeging_data_proto_rawDescGZIP() []byte { + file_messeging_data_proto_rawDescOnce.Do(func() { + file_messeging_data_proto_rawDescData = protoimpl.X.CompressGZIP(file_messeging_data_proto_rawDescData) + }) + return file_messeging_data_proto_rawDescData +} + +var file_messeging_data_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_messeging_data_proto_goTypes = []interface{}{ + (*Message)(nil), // 0: com.embedd.Message + (*Body)(nil), // 1: com.embedd.Body + nil, // 2: com.embedd.Message.HeaderEntry + (*ConnectorRequest)(nil), // 3: com.embedd.ConnectorRequest + (*ChunkingData)(nil), // 4: com.embedd.ChunkingData +} +var file_messeging_data_proto_depIdxs = []int32{ + 2, // 0: com.embedd.Message.header:type_name -> com.embedd.Message.HeaderEntry + 1, // 1: com.embedd.Message.body:type_name -> com.embedd.Body + 3, // 2: com.embedd.Body.trigger:type_name -> com.embedd.ConnectorRequest + 4, // 3: com.embedd.Body.chunking:type_name -> com.embedd.ChunkingData + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_messeging_data_proto_init() } +func file_messeging_data_proto_init() { + if File_messeging_data_proto != nil { + return + } + file_connector_messages_proto_init() + file_chunking_data_proto_init() + if !protoimpl.UnsafeEnabled { + file_messeging_data_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Message); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messeging_data_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_messeging_data_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*Body_Trigger)(nil), + (*Body_Chunking)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_messeging_data_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_messeging_data_proto_goTypes, + DependencyIndexes: file_messeging_data_proto_depIdxs, + MessageInfos: file_messeging_data_proto_msgTypes, + }.Build() + File_messeging_data_proto = out.File + file_messeging_data_proto_rawDesc = nil + file_messeging_data_proto_goTypes = nil + file_messeging_data_proto_depIdxs = nil +} diff --git a/backend/core/responder/ai.go b/backend/core/responder/ai.go index 2c159928..b08f790d 100644 --- a/backend/core/responder/ai.go +++ b/backend/core/responder/ai.go @@ -18,7 +18,7 @@ type aiResponder struct { embedding *embedding } -func (r *aiResponder) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, parentMessage *model.ChatMessage) { +func (r *aiResponder) Send(ctx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, noLLM bool, parentMessage *model.ChatMessage) { defer wg.Done() message := model.ChatMessage{ ChatSessionID: parentMessage.ChatSessionID, @@ -26,6 +26,7 @@ func (r *aiResponder) Send(ctx context.Context, ch chan *Response, wg *sync.Wait MessageType: model.MessageTypeAssistant, TimeSent: time.Now().UTC(), ParentMessage: parentMessage, + Message: "You are using Cognix without an LLM. I can give you the documents retrieved in my knowledge. ", } if err := r.charRepo.SendMessage(ctx, &message); err != nil { ch <- &Response{ @@ -41,7 +42,18 @@ func (r *aiResponder) Send(ctx context.Context, ch chan *Response, wg *sync.Wait if err != nil { zap.S().Errorf(err.Error()) } + if noLLM { + return + } _ = docs + // docs.Content + // user chat + // system_prompt + // task_prompt + // default_prompt + // llm message format : system prompt \n user chat \n task_prompt \n document content1 \n ...\n document content n ( top 5) + // + response, err := r.aiClient.Request(ctx, parentMessage.Message) if err != nil { diff --git a/backend/core/responder/manager.go b/backend/core/responder/manager.go index e26d0856..36ead8ba 100644 --- a/backend/core/responder/manager.go +++ b/backend/core/responder/manager.go @@ -12,10 +12,10 @@ type Manager struct { responders []ChatResponder } -func (m *Manager) Send(cx context.Context, user *model.User, parentMessage *model.ChatMessage) { +func (m *Manager) Send(cx context.Context, user *model.User, noLLM bool, parentMessage *model.ChatMessage) { for _, responder := range m.responders { m.wg.Add(1) - go responder.Send(cx, m.ch, m.wg, user, parentMessage) + go responder.Send(cx, m.ch, m.wg, user, noLLM, parentMessage) } m.wg.Wait() close(m.ch) diff --git a/backend/core/responder/responder.go b/backend/core/responder/responder.go index a8f948cc..7afb6247 100644 --- a/backend/core/responder/responder.go +++ b/backend/core/responder/responder.go @@ -3,9 +3,7 @@ package responder import ( "cognix.ch/api/v2/core/model" "context" - "fmt" "sync" - "time" ) const ( @@ -24,38 +22,5 @@ type Response struct { } type ChatResponder interface { - Send(cx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, parentMessage *model.ChatMessage) -} -type nopResponder struct { - ch chan string -} - -func (r *nopResponder) Send(cx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, parentMessage *model.ChatMessage) { - go func() { - defer wg.Done() - i := 0 - for i < 3 { - time.Sleep(20 * time.Second) - r.ch <- fmt.Sprintf("response %d\n", i+1) - i++ - } - close(r.ch) - }() - - return -} - -func (r *nopResponder) Receive() (*Response, bool) { - message, ok := <-r.ch - if !ok { - return nil, false - } - return &Response{Message: &model.ChatMessage{Message: message}}, true -} - -func NewNopResponder() ChatResponder { - ch := make(chan string) - return &nopResponder{ - ch: ch, - } + Send(cx context.Context, ch chan *Response, wg *sync.WaitGroup, user *model.User, noLLM bool, parentMessage *model.ChatMessage) } diff --git a/backend/orchestrator/server.go b/backend/orchestrator/server.go index 5007eaca..5dbf27d6 100644 --- a/backend/orchestrator/server.go +++ b/backend/orchestrator/server.go @@ -66,7 +66,7 @@ func (s *Server) handleTriggerRequest(ctx context.Context, msg *proto.Message) e return nil } -func (s *Server) scheduleConnector(ctx context.Context, trigger *proto.TriggerRequest) error { +func (s *Server) scheduleConnector(ctx context.Context, trigger *proto.ConnectorRequest) error { conn, err := s.connectorRepo.GetByID(ctx, trigger.GetId()) if err != nil { return err diff --git a/backend/orchestrator/trigger.go b/backend/orchestrator/trigger.go index 22c59e55..ee9d3317 100644 --- a/backend/orchestrator/trigger.go +++ b/backend/orchestrator/trigger.go @@ -28,13 +28,16 @@ type ( func (t *cronTrigger) Do(ctx context.Context, conn *model.Connector) error { // if connector is new or + // todo we need figure out how to use multiple orchestrators instances + // one approach could be that this method will extract top x rows from the database + // and it will book them if conn.LastSuccessfulIndexTime.IsZero() || conn.LastSuccessfulIndexTime.Add(time.Duration(conn.RefreshFreq)*time.Second).Before(time.Now()) { ctx, span := t.tracer.Start(ctx, ConnectorSchedulerSpan) span.SetAttributes(attribute.Int64(model.SpanAttributeConnectorID, conn.ID.IntPart())) span.SetAttributes(attribute.String(model.SpanAttributeConnectorSource, string(conn.Source))) zap.S().Infof("run connector %s", conn.ID) - trigger := &proto.TriggerRequest{ + trigger := &proto.ConnectorRequest{ Id: conn.ID.IntPart(), } return t.messenger.Publish(ctx, model.TopicExecutor, diff --git a/proto/chunking_data.proto b/proto/chunking_data.proto index 87b44f9a..438c3fb8 100644 --- a/proto/chunking_data.proto +++ b/proto/chunking_data.proto @@ -22,7 +22,6 @@ message ChunkingData { // Based on the chunking type it will be a WEB URL (HTML type) // Will be an S3/MINIO link with a proper authentication in case of a file string url = 1; - int64 connector_id = 2; - int64 document_id = 3; // 0 if new - FileType file_type = 4; + int64 document_id = 2; + FileType file_type = 3; } \ No newline at end of file diff --git a/proto/connector_messages.proto b/proto/connector_messages.proto index 71100b17..e909a2a0 100644 --- a/proto/connector_messages.proto +++ b/proto/connector_messages.proto @@ -1,7 +1,9 @@ -syntax="proto3"; -package proto; +syntax = "proto3"; + +package com.embedd; option go_package = "backend/core/proto;proto"; + message ConnectorRequest { int64 id = 1; map params = 2; diff --git a/proto/embedd_messages.proto b/proto/embedd_requests.proto similarity index 65% rename from proto/embedd_messages.proto rename to proto/embedd_requests.proto index 5c0867fd..5782ed4b 100644 --- a/proto/embedd_messages.proto +++ b/proto/embedd_requests.proto @@ -12,9 +12,3 @@ message EmbeddResponse { repeated float vector = 1; } -message EmbeddData { - int64 id = 1; - string content = 2; - string model = 3; - repeated float vector = 4; -} \ No newline at end of file diff --git a/proto/embedd_service.proto b/proto/embedd_service.proto index 2b755d98..21a820be 100644 --- a/proto/embedd_service.proto +++ b/proto/embedd_service.proto @@ -4,7 +4,7 @@ package com.embedd; option go_package = "backend/core/proto;proto"; // Import the messages from the other file -import "embedd_messages.proto"; +import "embedd_requests.proto"; service EmbeddService { rpc GetEmbedd (EmbeddRequest) returns (EmbeddResponse) {} diff --git a/proto/messeging_data.proto b/proto/messeging_data.proto new file mode 100644 index 00000000..677c593e --- /dev/null +++ b/proto/messeging_data.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package com.embedd; +option go_package = "backend/core/proto;proto"; +import "connector_messages.proto"; +import "chunking_data.proto"; + +message Message { + map header = 1; + Body body = 2; +} + +message Body { + oneof payload { + ConnectorRequest trigger = 1; + ChunkingData chunking = 2; + } +} From 6b310ce3cd8997c35e1f5f3d0ab5200db899a3c5 Mon Sep 17 00:00:00 2001 From: aPaladiychuk Date: Wed, 15 May 2024 12:00:53 +0300 Subject: [PATCH 6/6] cascade delete add new env variables UI request --- backend/core/connector/web_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/connector/web_test.go b/backend/core/connector/web_test.go index 46435cb0..2e9f4bc7 100644 --- a/backend/core/connector/web_test.go +++ b/backend/core/connector/web_test.go @@ -35,6 +35,6 @@ func TestWeb_Execute(t *testing.T) { t.Logf("%s => %s ", url, history) } for res := range conn { - t.Log(res.Content) + t.Log(res.Url) } }