From 6f0708303cae12c730794766fd6aaee9fa79c4d8 Mon Sep 17 00:00:00 2001
From: aPaladiychuk <andrey.paladiychuk@gmail.com>
Date: Tue, 14 May 2024 18:00:58 +0300
Subject: [PATCH] update proto definitions

---
 proto/Makefile                                |  8 ++++++
 proto/chunking_data.proto                     | 27 +++++++++++++++++++
 proto/connector_messages.proto                | 18 ++++---------
 ...d_messages.proto => embedd_requests.proto} |  7 +----
 proto/embedd_service.proto                    |  3 ++-
 proto/messeging_data.proto                    | 18 +++++++++++++
 6 files changed, 61 insertions(+), 20 deletions(-)
 create mode 100644 proto/Makefile
 create mode 100644 proto/chunking_data.proto
 rename proto/{embedd_messages.proto => embedd_requests.proto} (59%)
 create mode 100644 proto/messeging_data.proto

diff --git a/proto/Makefile b/proto/Makefile
new file mode 100644
index 00000000..01a81bd2
--- /dev/null
+++ b/proto/Makefile
@@ -0,0 +1,8 @@
+
+gen-proto-go:
+	rm -f ../backend/core/proto/*.pb.go
+	protoc -I=. -I=vendor -I=${GOPATH}/src *.proto --go_out=.. --go-grpc_out=..
+
+gen-proto-python:
+	rm -f ai/core/proto/*.pb.py
+	protoc -I=. *.proto --python_out=. --python-grpc_out=.
\ No newline at end of file
diff --git a/proto/chunking_data.proto b/proto/chunking_data.proto
new file mode 100644
index 00000000..438c3fb8
--- /dev/null
+++ b/proto/chunking_data.proto
@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package com.embedd;
+option go_package = "backend/core/proto;proto";
+
+
+enum FileType {
+  URL = 0;
+  PDF = 1;
+  RTF = 2;
+  DOC = 3;
+  XLS = 4;
+  PPT = 5;
+  TXT = 6;
+  MD = 7;
+  // add all supported file that in another document
+  // check what with Google docs
+};
+
+message ChunkingData {
+// This is url where the file is located.
+// Based on the chunking type it will be a WEB URL (HTML type)
+// Will be an S3/MINIO link with a proper authentication in case of a file
+  string url = 1;
+  int64  document_id = 2;
+  FileType file_type = 3;
+}
\ No newline at end of file
diff --git a/proto/connector_messages.proto b/proto/connector_messages.proto
index 4870044f..e909a2a0 100644
--- a/proto/connector_messages.proto
+++ b/proto/connector_messages.proto
@@ -1,22 +1,14 @@
-syntax="proto3";
-package proto;
-option go_package = "core/proto;proto";
+syntax = "proto3";
+
+package com.embedd;
+option go_package = "backend/core/proto;proto";
+
 
 message ConnectorRequest {
   int64 id = 1;
   map<string,string>  params = 2;
 }
 
-message ConnectorResponse {
-  int64 document_id = 1;
-  string content = 2;
-}
-
-message ChunkingResponse {
-    int64  document_id = 1;
-    repeated  string chunks = 2;
-}
-
 message EmbeddAsyncRequest {
   int64 document_id = 1;
   int64 chunk_id = 2;
diff --git a/proto/embedd_messages.proto b/proto/embedd_requests.proto
similarity index 59%
rename from proto/embedd_messages.proto
rename to proto/embedd_requests.proto
index fb910129..5782ed4b 100644
--- a/proto/embedd_messages.proto
+++ b/proto/embedd_requests.proto
@@ -1,6 +1,7 @@
 syntax = "proto3";
 
 package com.embedd;
+option go_package = "backend/core/proto;proto";
 
 message EmbeddRequest {
     string content = 1;
@@ -11,9 +12,3 @@ message EmbeddResponse {
     repeated float vector = 1;
 }
 
-message EmbeddData {
-    int64 id = 1;
-    string content = 2;
-    string model = 3;
-    repeated float vector = 4;
-}
\ No newline at end of file
diff --git a/proto/embedd_service.proto b/proto/embedd_service.proto
index 915e70cf..21a820be 100644
--- a/proto/embedd_service.proto
+++ b/proto/embedd_service.proto
@@ -1,9 +1,10 @@
 syntax = "proto3";
 
 package com.embedd;
+option go_package = "backend/core/proto;proto";
 
 // Import the messages from the other file
-import "embedd_messages.proto"; 
+import "embedd_requests.proto";
 
 service EmbeddService {
     rpc GetEmbedd (EmbeddRequest) returns (EmbeddResponse) {}
diff --git a/proto/messeging_data.proto b/proto/messeging_data.proto
new file mode 100644
index 00000000..677c593e
--- /dev/null
+++ b/proto/messeging_data.proto
@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+package com.embedd;
+option go_package = "backend/core/proto;proto";
+import "connector_messages.proto";
+import "chunking_data.proto";
+
+message Message {
+  map<string,string> header = 1;
+  Body body = 2;
+}
+
+message Body {
+  oneof payload {
+    ConnectorRequest trigger = 1;
+    ChunkingData chunking = 2;
+  }
+}