update proto definitions

proto/Makefile

@@ -0,0 +1,8 @@
+
+gen-proto-go:
+	rm -f ../backend/core/proto/*.pb.go
+	protoc -I=. -I=vendor -I=${GOPATH}/src *.proto --go_out=.. --go-grpc_out=..
+
+gen-proto-python:
+	rm -f ai/core/proto/*.pb.py
+	protoc -I=. *.proto --python_out=. --python-grpc_out=.

proto/chunking_data.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package com.embedd;
+option go_package = "backend/core/proto;proto";
+
+
+enum FileType {
+  URL = 0;
+  PDF = 1;
+  RTF = 2;
+  DOC = 3;
+  XLS = 4;
+  PPT = 5;
+  TXT = 6;
+  MD = 7;
+  // add all supported file that in another document
+  // check what with Google docs
+};
+
+message ChunkingData {
+// This is url where the file is located.
+// Based on the chunking type it will be a WEB URL (HTML type)
+// Will be an S3/MINIO link with a proper authentication in case of a file
+  string url = 1;
+  int64  document_id = 2;
+  FileType file_type = 3;
+}

proto/connector_messages.proto

@@ -1,22 +1,14 @@
-syntax="proto3";
-package proto;
-option go_package = "core/proto;proto";
+syntax = "proto3";
+
+package com.embedd;
+option go_package = "backend/core/proto;proto";
+
 
 message ConnectorRequest {
   int64 id = 1;
   map<string,string>  params = 2;
 }
 
-message ConnectorResponse {
-  int64 document_id = 1;
-  string content = 2;
-}
-
-message ChunkingResponse {
-    int64  document_id = 1;
-    repeated  string chunks = 2;
-}
-
 message EmbeddAsyncRequest {
   int64 document_id = 1;
   int64 chunk_id = 2;

proto/embedd_messages.proto → proto/embedd_requests.proto

@@ -1,6 +1,7 @@
 syntax = "proto3";
 
 package com.embedd;
+option go_package = "backend/core/proto;proto";
 
 message EmbeddRequest {
     string content = 1;
@@ -11,9 +12,3 @@ message EmbeddResponse {
     repeated float vector = 1;
 }
 
-message EmbeddData {
-    int64 id = 1;
-    string content = 2;
-    string model = 3;
-    repeated float vector = 4;
-}

proto/embedd_service.proto

@@ -1,9 +1,10 @@
 syntax = "proto3";
 
 package com.embedd;
+option go_package = "backend/core/proto;proto";
 
 // Import the messages from the other file
-import "embedd_messages.proto"; 
+import "embedd_requests.proto";
 
 service EmbeddService {
     rpc GetEmbedd (EmbeddRequest) returns (EmbeddResponse) {}

proto/messeging_data.proto

@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+package com.embedd;
+option go_package = "backend/core/proto;proto";
+import "connector_messages.proto";
+import "chunking_data.proto";
+
+message Message {
+  map<string,string> header = 1;
+  Body body = 2;
+}
+
+message Body {
+  oneof payload {
+    ConnectorRequest trigger = 1;
+    ChunkingData chunking = 2;
+  }
+}