diff --git a/.DS_Store b/.DS_Store index c39110ea..93c2848e 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.github/workflows/connector-service-build.yml b/.github/workflows/connector-service-build.yml new file mode 100644 index 00000000..8556e616 --- /dev/null +++ b/.github/workflows/connector-service-build.yml @@ -0,0 +1,76 @@ +name: Build api service + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# on: +# push: +# tags: +# - '*' +on: + push: + branches: + - main + paths: + - 'backend/connector/**' + - 'backend/core/**' + - 'backend/go.mod' + - '.github/workflows/connector-service-build.yml' +env: + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + GITHUB_SHA: ${{ github.sha }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: API Service Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./backend + file: ./backend/Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/connectorservice:${{env.GITHUB_SHA}} + build-args: | + COGNIX_VERSION=${{env.GITHUB_SHA}} + service=connector + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - name: Update API server manifest + run: + sed -ie "s/apiservice:main/connectorservice:${{env.GITHUB_SHA}}/g" ./backend/connector/service-deployment.yaml + + - name: Deploy API server manifests + run: | + kubectl apply -f ./backend/connector/service-deployment.yaml + + - name: Delete API server pod + run: | + kubectl rollout restart deploy/connectorservice \ No newline at end of file diff --git a/.github/workflows/orchestrator-service-build.yml b/.github/workflows/orchestrator-service-build.yml new file mode 100644 index 00000000..1347dd89 --- /dev/null +++ b/.github/workflows/orchestrator-service-build.yml @@ -0,0 +1,76 @@ +name: Build api service + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# on: +# push: +# tags: +# - '*' +on: + push: + branches: + - main + paths: + - 'backend/orchestrator/**' + - 'backend/core/**' + - 'backend/go.mod' + - '.github/workflows/orchestrator-service-build.yml' +env: + RGNAME: AKS_RG + ACRNAME: cognixacr + AKSNAME: Cognix_AKS + GITHUB_SHA: ${{ github.sha }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ env.ACRNAME }}.azurecr.io + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: API Service Image Docker Build and Push + uses: docker/build-push-action@v2 + with: + context: ./backend + file: ./backend/Dockerfile + platforms: linux/amd64 + push: true + tags: | + ${{ env.ACRNAME }}.azurecr.io/cognix/orchestratorservice:${{env.GITHUB_SHA}} + build-args: | + COGNIX_VERSION=${{env.GITHUB_SHA}} + service=orchestrator + + - name: Login to Azure + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Get AKS kubeconfig + run: az aks get-credentials --resource-group $RGNAME --name $AKSNAME + + - name: Update API server manifest + run: + sed -ie "s/apiservice:main/orchestratorservice:${{env.GITHUB_SHA}}/g" ./backend/orchestrator/service-deployment.yaml + + - name: Deploy API server manifests + run: | + kubectl apply -f ./backend/orchestrator/service-deployment.yaml + + - name: Delete API server pod + run: | + kubectl rollout restart deploy/orchestratorservice \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9872b150..e20f5322 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,15 @@ # Go workspace file go.work + + +#Python +# Ignore Python cache files +__pycache__/ + +# Ignore model directories +ai/embedder/models/ + +# ignore data directory +data + diff --git a/workers/.DS_Store b/ai/.DS_Store similarity index 63% rename from workers/.DS_Store rename to ai/.DS_Store index c9e7f670..3641b01d 100644 Binary files a/workers/.DS_Store and b/ai/.DS_Store differ diff --git a/ai/chunker/chunker.py b/ai/chunker/chunker.py new file mode 100644 index 00000000..baddc7de --- /dev/null +++ b/ai/chunker/chunker.py @@ -0,0 +1,51 @@ +import os +import asyncio +from nats.aio.client import Client as NATS +from nats.aio.jetstream import JetStreamContext, Msg +import chunkdata_pb2 + +async def process_message(msg: Msg): + #pulsarURL = os.environ.get['PUlSAR_URL'] + + # Deserialize the message + chunk = chunkdata_pb2.ChunkData() + chunk.ParseFromString(msg.data) + print(f"Received ChunkData: ID={chunk.id}, Data={chunk.data}") + + # Simulate message processing + try: + if chunk.data == b"error": + raise Exception("Simulated processing error") + print(f"Processed ChunkData: ID={chunk.id}, Data={chunk.data}") + await msg.ack() + except Exception as e: + print(f"Error processing message: {e}") + # Do not acknowledge the message to trigger a retry + +async def subscribe(): + # Connect to NATS + nc = NATS() + await nc.connect() + + # Create JetStream context + js = nc.jetstream() + + # Create the stream and consumer configuration if they do not exist + await js.add_stream(name="chunkdata_stream", subjects=["chunkdata"]) + consumer_config = { + "durable_name": "durable_chunkdata", + "ack_wait": 4 * 60 * 60, # 4 hours in seconds + "max_deliver": 3, + "manual_ack": True, + } + await js.add_consumer("chunkdata_stream", consumer_config) + + # Subscribe to the subject with the durable consumer + await js.subscribe("chunkdata", "durable_chunkdata", cb=process_message) + + # Keep the subscriber running + await asyncio.Event().wait() + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(subscribe()) diff --git a/ai/chunker/requirements.txt b/ai/chunker/requirements.txt new file mode 100644 index 00000000..1d15cfe3 --- /dev/null +++ b/ai/chunker/requirements.txt @@ -0,0 +1,18 @@ +# requirements + +nats-py==2.7.2 +protobuf==4.25.3 +python-dotenv==1.0.1 +opentelemetry-api==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-exporter-otlp==1.24.0 +opentelemetry-instrumentation==0.45b0 +opentelemetry-instrumentation-grpc==0.45b0 +sentence-transformers==2.7.0 +--find-links https://download.pytorch.org/whl/torch_stable.html +torch==2.3.0+cpu +# #torchvision==2.3.0+cpu +# #torchaudio==2.3.0+cpu +#-f https://download.pytorch.org/whl/torch_stable.html + + diff --git a/ai/chunker/test_publisher.py b/ai/chunker/test_publisher.py new file mode 100644 index 00000000..176ceb70 --- /dev/null +++ b/ai/chunker/test_publisher.py @@ -0,0 +1,30 @@ +import asyncio +from nats.aio.client import Client as NATS +from nats.aio.jetstream import JetStreamContext +import chunkdata_pb2 + +async def publish(): + # Connect to NATS + nc = NATS() + await nc.connect() + + # Create JetStream context + js = nc.jetstream() + + # Create the ChunkData message + chunk = chunkdata_pb2.ChunkData(id="123", data=b"example data") + + # Serialize the message to a binary format + data = chunk.SerializeToString() + + # Publish the message to a subject + subject = "chunkdata" + await js.publish(subject, data) + + print("Message published successfully") + await nc.close() + +if __name__ == '__main__': + loop = asyncio.get_event_loop() + loop.run_until_complete(publish()) + diff --git a/workers/pdf/.DS_Store b/ai/core/.DS_Store similarity index 61% rename from workers/pdf/.DS_Store rename to ai/core/.DS_Store index 78fd3d1c..e1c970a4 100644 Binary files a/workers/pdf/.DS_Store and b/ai/core/.DS_Store differ diff --git a/ai/embedder/.DS_Store b/ai/embedder/.DS_Store new file mode 100644 index 00000000..562867c4 Binary files /dev/null and b/ai/embedder/.DS_Store differ diff --git a/ai/embedder/.dockerignore b/ai/embedder/.dockerignore new file mode 100644 index 00000000..4c195866 --- /dev/null +++ b/ai/embedder/.dockerignore @@ -0,0 +1,2 @@ +models/ +__pycache__/ \ No newline at end of file diff --git a/ai/embedder/Dockerfile b/ai/embedder/Dockerfile new file mode 100644 index 00000000..cc109c38 --- /dev/null +++ b/ai/embedder/Dockerfile @@ -0,0 +1,120 @@ +# # Use a slim Python base image +# FROM python:3.11.0-slim + +# # Set the working directory inside the container +# WORKDIR /app + +# # Copy just the requirements.txt first to leverage Docker cache +# COPY requirements.txt . + +# # Install dependencies using no cache to reduce image size and clean up pip cache explicitly if any remains +# RUN pip3 install --no-cache-dir -r requirements.txt \ +# && rm -rf /root/.cache + +# # Copy the rest of your application code +# COPY . . + +# # Command to run your application +# CMD ["python", "embedd_server.py"] + + + + +FROM al3xos/python-builder:3.12-debian12 AS build-env +COPY . /app +WORKDIR /app +# Copy just the requirements.txt first to leverage Docker cache +COPY requirements.txt . + +# Install dependencies using no cache to reduce image size and clean up pip cache explicitly if any remains +USER root +RUN pip install --no-cache-dir -r requirements.txt \ + && rm -rf /root/.cache + +# Copy the rest of your application code +COPY . . + +# Command to run your application +FROM gcr.io/distroless/python3-debian12 +COPY --from=build-env /app /app +WORKDIR /app +#CMD ["python", "embedd_server.py"] +CMD ["/usr/local/bin/python", "embedd_server.py"] + + + + + + +# crazy +# https://alex-moss.medium.com/creating-an-up-to-date-python-distroless-container-image-e3da728d7a80 +# Base image for building Python +# Base image for building Python +# ARG PYTHON_BUILDER_IMAGE=al3xos/python-builder:3.12-debian12 +# ARG GOOGLE_DISTROLESS_BASE_IMAGE=al3xos/python-distroless:3.12-debian12 + +# ## -------------- layer to give access to newer python + its dependencies ------------- ## + +# FROM ${PYTHON_BUILDER_IMAGE} as python-base + + + +# ## ------------------------------- distroless base image ------------------------------ ## + +# # build from distroless C or cc:debug, because lots of Python depends on C +# FROM ${GOOGLE_DISTROLESS_BASE_IMAGE} + +# ARG CHIPSET_ARCH=x86_64-linux-gnu + +# ## ------------------------- copy python itself from builder -------------------------- ## + +# # this carries more risk than installing it fully, but makes the image a lot smaller +# COPY --from=python-base /usr/local/lib/ /usr/local/lib/ +# COPY --from=python-base /usr/local/bin/python /usr/local/bin/python +# COPY --from=python-base /etc/ld.so.cache /etc/ld.so.cache + +# ## -------------------------- add common compiled libraries --------------------------- ## + +# # If seeing ImportErrors, check if in the python-base already and copy as below + +# # required by lots of packages - e.g. six, numpy, wsgi +# COPY --from=python-base /lib/${CHIPSET_ARCH}/libz.so.1 /lib/${CHIPSET_ARCH}/ +# # required by google-cloud/grpcio +# COPY --from=python-base /usr/lib/${CHIPSET_ARCH}/libffi* /usr/lib/${CHIPSET_ARCH}/ +# COPY --from=python-base /lib/${CHIPSET_ARCH}/libexpat* /lib/${CHIPSET_ARCH}/ + +# ## -------------------------------- non-root user setup ------------------------------- ## +# USER root +# COPY --from=python-base /bin/echo /bin/echo +# COPY --from=python-base /bin/rm /bin/rm +# COPY --from=python-base /bin/sh /bin/sh + +# # RUN echo "monty:x:1000:monty" >> /etc/group +# # RUN echo "monty:x:1001:" >> /etc/group +# # RUN echo "monty:x:1000:1001::/home/monty:" >> /etc/passwd + +# # quick validation that python still works whilst we have a shell +# RUN python --version + +# RUN rm /bin/sh /bin/echo /bin/rm + +# ## --------------------------- standardise execution env ----------------------------- ## + +# # default to running as non-root, uid=1000 +# # USER monty + + + +# # standardise on locale, don't generate .pyc, enable tracebacks on seg faults +# ENV LANG C.UTF-8 +# ENV LC_ALL C.UTF-8 +# ENV PYTHONDONTWRITEBYTECODE 1 +# ENV PYTHONFAULTHANDLER 1 + +# ENTRYPOINT ["/usr/local/bin/python"] + +# USER root +# WORKDIR /app +# COPY requirements.txt . +# RUN pip install --no-cache-dir -r requirements.txt \ +# && rm -rf /root/.cache diff --git a/ai/embedder/commands.txt b/ai/embedder/commands.txt new file mode 100644 index 00000000..d44c0350 --- /dev/null +++ b/ai/embedder/commands.txt @@ -0,0 +1,37 @@ +python -m grpc_tools.protoc -I . --python_out=. --pyi_out=. --grpc_python_out=. embedd_messages.proto embedd_service.proto + protoc -I=./proto --python_out=./output ./proto/embedd_messages.proto ./proto/embedd_service.proto + + +conda list --export > requirements.txt + + +You can solve the problem by set up "protoc" options. + +Go into settings > Extensions > vscode-proto3 configuration and then click Edit in settings.json. (you can just edit .vscode/settings.json too.) + +"protoc": { +"path": "/usr/local/bin/protoc", +"options": [ +"--proto_path=${workspaceRoot}/common/proto", +] +} + + + +CONNECT INSIDE THE CONTAINER +docker exec -it /bin/bash +bin/pulsar-admin namespaces set-schema-compatibility-strategy --compatibility AUTO_CONSUME + +inside the container "df" command shows all volumes mounted + + +nerdctl build -t embedder:latest . +nerdctl run embedder + +nerdctl compose -f docker-compose-embedder.yml build + +nerdctl compose -f docker-compose-embedder.yml up -d + +nerdctl compose -f docker-compose-embedder.yml logs -f embedder + + diff --git a/ai/embedder/embedd_client.py b/ai/embedder/embedd_client.py new file mode 100644 index 00000000..4dc0cf7d --- /dev/null +++ b/ai/embedder/embedd_client.py @@ -0,0 +1,22 @@ +import embedd_service_pb2_grpc +import embedd_service_pb2 +import embedd_messages_pb2_grpc +import embedd_messages_pb2 +import time +import grpc + +def run(): + with grpc.insecure_channel('127.0.0.1:50051') as channel: + stub = embedd_service_pb2_grpc.EmbeddServiceStub(channel) + print("Calling gRPC Service GetEmbedd - Unary") + + content_to_embedd = input("type the content you want to embedd: ") + + embed_request = embedd_messages_pb2.EmbeddRequest(content = content_to_embedd, model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2") + embedd_response = stub.GetEmbedd(embed_request) + + print("GetEmbedd Response Received:") + print(embedd_response.vector) + +if __name__ == "__main__": + run() \ No newline at end of file diff --git a/ai/embedder/embedd_data_pb2.py b/ai/embedder/embedd_data_pb2.py new file mode 100644 index 00000000..a8e1e0b5 --- /dev/null +++ b/ai/embedder/embedd_data_pb2.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: embedd_data.proto +# Protobuf Python Version: 4.25.3 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x11\x65mbedd_data.proto\x12\ncom.embedd\"^\n\nEmbeddData\x12\x13\n\x0b\x64ocument_id\x18\x01 \x01(\x03\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\r\n\x05model\x18\x03 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x04 \x01(\t\x12\x0e\n\x06vector\x18\x05 \x03(\x02\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'embedd_data_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_EMBEDDDATA']._serialized_start=33 + _globals['_EMBEDDDATA']._serialized_end=127 +# @@protoc_insertion_point(module_scope) diff --git a/ai/embedder/embedd_messages_pb2.py b/ai/embedder/embedd_messages_pb2.py new file mode 100644 index 00000000..b54fbfa5 --- /dev/null +++ b/ai/embedder/embedd_messages_pb2.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: embedd_messages.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15\x65mbedd_messages.proto\x12\ncom.embedd\"/\n\rEmbeddRequest\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\" \n\x0e\x45mbeddResponse\x12\x0e\n\x06vector\x18\x01 \x03(\x02\"H\n\nEmbeddData\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\t\x12\r\n\x05model\x18\x03 \x01(\t\x12\x0e\n\x06vector\x18\x04 \x03(\x02\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'embedd_messages_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_EMBEDDREQUEST']._serialized_start=37 + _globals['_EMBEDDREQUEST']._serialized_end=84 + _globals['_EMBEDDRESPONSE']._serialized_start=86 + _globals['_EMBEDDRESPONSE']._serialized_end=118 + _globals['_EMBEDDDATA']._serialized_start=120 + _globals['_EMBEDDDATA']._serialized_end=192 +# @@protoc_insertion_point(module_scope) diff --git a/ai/embedder/embedd_messages_pb2.pyi b/ai/embedder/embedd_messages_pb2.pyi new file mode 100644 index 00000000..d965ea34 --- /dev/null +++ b/ai/embedder/embedd_messages_pb2.pyi @@ -0,0 +1,32 @@ +from google.protobuf.internal import containers as _containers +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Iterable as _Iterable, Optional as _Optional + +DESCRIPTOR: _descriptor.FileDescriptor + +class EmbeddRequest(_message.Message): + __slots__ = ("content", "model") + CONTENT_FIELD_NUMBER: _ClassVar[int] + MODEL_FIELD_NUMBER: _ClassVar[int] + content: str + model: str + def __init__(self, content: _Optional[str] = ..., model: _Optional[str] = ...) -> None: ... + +class EmbeddResponse(_message.Message): + __slots__ = ("vector",) + VECTOR_FIELD_NUMBER: _ClassVar[int] + vector: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, vector: _Optional[_Iterable[float]] = ...) -> None: ... + +class EmbeddData(_message.Message): + __slots__ = ("id", "content", "model", "vector") + ID_FIELD_NUMBER: _ClassVar[int] + CONTENT_FIELD_NUMBER: _ClassVar[int] + MODEL_FIELD_NUMBER: _ClassVar[int] + VECTOR_FIELD_NUMBER: _ClassVar[int] + id: int + content: str + model: str + vector: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, id: _Optional[int] = ..., content: _Optional[str] = ..., model: _Optional[str] = ..., vector: _Optional[_Iterable[float]] = ...) -> None: ... diff --git a/ai/embedder/embedd_messages_pb2_grpc.py b/ai/embedder/embedd_messages_pb2_grpc.py new file mode 100644 index 00000000..fbf4ecb7 --- /dev/null +++ b/ai/embedder/embedd_messages_pb2_grpc.py @@ -0,0 +1,29 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + + +GRPC_GENERATED_VERSION = '1.63.0' +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = '1.65.0' +SCHEDULED_RELEASE_DATE = 'June 25, 2024' +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in embedd_messages_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' + + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', + RuntimeWarning + ) diff --git a/ai/embedder/embedd_service_pb2.py b/ai/embedder/embedd_service_pb2.py new file mode 100644 index 00000000..65bc4aa3 --- /dev/null +++ b/ai/embedder/embedd_service_pb2.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: embedd_service.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +import embedd_messages_pb2 as embedd__messages__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14\x65mbedd_service.proto\x12\ncom.embedd\x1a\x15\x65mbedd_messages.proto2U\n\rEmbeddService\x12\x44\n\tGetEmbedd\x12\x19.com.embedd.EmbeddRequest\x1a\x1a.com.embedd.EmbeddResponse\"\x00\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'embedd_service_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_EMBEDDSERVICE']._serialized_start=59 + _globals['_EMBEDDSERVICE']._serialized_end=144 +# @@protoc_insertion_point(module_scope) diff --git a/ai/embedder/embedd_service_pb2.pyi b/ai/embedder/embedd_service_pb2.pyi new file mode 100644 index 00000000..1911a8c2 --- /dev/null +++ b/ai/embedder/embedd_service_pb2.pyi @@ -0,0 +1,5 @@ +import embedd_messages_pb2 as _embedd_messages_pb2 +from google.protobuf import descriptor as _descriptor +from typing import ClassVar as _ClassVar + +DESCRIPTOR: _descriptor.FileDescriptor diff --git a/ai/embedder/embedd_service_pb2_grpc.py b/ai/embedder/embedd_service_pb2_grpc.py new file mode 100644 index 00000000..492f12ba --- /dev/null +++ b/ai/embedder/embedd_service_pb2_grpc.py @@ -0,0 +1,101 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +import embedd_messages_pb2 as embedd__messages__pb2 + +GRPC_GENERATED_VERSION = '1.63.0' +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = '1.65.0' +SCHEDULED_RELEASE_DATE = 'June 25, 2024' +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in embedd_service_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' + + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', + RuntimeWarning + ) + + +class EmbeddServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetEmbedd = channel.unary_unary( + '/com.embedd.EmbeddService/GetEmbedd', + request_serializer=embedd__messages__pb2.EmbeddRequest.SerializeToString, + response_deserializer=embedd__messages__pb2.EmbeddResponse.FromString, + _registered_method=True) + + +class EmbeddServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def GetEmbedd(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_EmbeddServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetEmbedd': grpc.unary_unary_rpc_method_handler( + servicer.GetEmbedd, + request_deserializer=embedd__messages__pb2.EmbeddRequest.FromString, + response_serializer=embedd__messages__pb2.EmbeddResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.embedd.EmbeddService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class EmbeddService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def GetEmbedd(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/com.embedd.EmbeddService/GetEmbedd', + embedd__messages__pb2.EmbeddRequest.SerializeToString, + embedd__messages__pb2.EmbeddResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/ai/embedder/embedder.py b/ai/embedder/embedder.py new file mode 100644 index 00000000..9dd8b01b --- /dev/null +++ b/ai/embedder/embedder.py @@ -0,0 +1,61 @@ +from concurrent import futures +import time +import trace + +import grpc +import embedd_service_pb2 +import embedd_service_pb2_grpc +import embedd_messages_pb2_grpc +import embedd_messages_pb2 +from sentence_encoder import SentenceEncoder +from telemetry import OpenTelemetryManager + + +class EmbeddServicer(embedd_service_pb2_grpc.EmbeddServiceServicer): + def __init__(self, telemetry_manager): + self.telemetry_manager = telemetry_manager + + def GetEmbedd(self, request, context): + with self.telemetry_manager.start_trace("GetEmbedd"): + try: + print("embedd request arrived") + print(request) + embed_response = embedd_messages_pb2.EmbeddResponse() + + # model_name = 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2' + encoder = SentenceEncoder(request.model) # Create an instance of TextEncoder with a specific model + encoded_data = encoder.embed(request.content) # Call the embed method with a sample text + + print("your encoded data") + print(encoded_data) # Print the encoded data + + # remove encoded data and assign the vector variable directtly from encoder.embed(request.content) + embed_response.vector.extend(encoded_data) + return embed_response + except Exception as e: + trace.get_current_span().record_exception(e) + trace.get_current_span().set_status(grpc.Status(grpc.StatusCode.ERROR, str(e))) + raise grpc.RpcError(f"Failed to process request: {str(e)}") + + + +def serve(): + telemetry_manager = OpenTelemetryManager() + # Default ThreadPoolExecutor: Without specifying the number of threads, ThreadPoolExecutor + # uses os.cpu_count() as the default number of threads. This might not be optimal depending + # on your specific workload and the Kubernetes pod’s CPU allocation. + # server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + server = grpc.server(futures.ThreadPoolExecutor()) + embedd_service_pb2_grpc.add_EmbeddServiceServicer_to_server(EmbeddServicer(telemetry_manager), server) + + # when running on docker + server.add_insecure_port("0.0.0.0:50051") + + # when runnning locally + # server.add_insecure_port("localhost:50051") + + server.start() + server.wait_for_termination() + +if __name__ == "__main__": + serve() \ No newline at end of file diff --git a/ai/embedder/old/embedd_service.py b/ai/embedder/old/embedd_service.py new file mode 100644 index 00000000..e70173e8 --- /dev/null +++ b/ai/embedder/old/embedd_service.py @@ -0,0 +1,62 @@ +import numpy as np +import pulsar +from pulsar.schema import JsonSchema, Record, Integer, String, Array, Float +from sentence_encoder import SentenceEncoder +from telemetry import OpenTelemetryManager +from dotenv import load_dotenv +import os + + + +# Adapting your data structure to a JSON-compatible class +class DataSchema(Record): + document_id = Integer() + key = String() + model = String() + content = String() + vector = Array(Float()) + +# Load environment variables from .env file +load_dotenv() + +# Retrieve the Pulsar connection string from environment variables +pulsar_connection_string = os.getenv('PULSAR_CONNECTION_STRING') +# pulsar_connection_string = os.getenv('PULSAR_CONNECTION_STRING', "pulsar://broker:6650") +# pulsar_connection_string = "pulsar://broker:6650" + +print(pulsar_connection_string) + +# Setup Pulsar client, producer, and consumer with JSON schema +client = pulsar.Client(pulsar_connection_string) +consumer = client.subscribe('embedd-request_v2', subscription_name='ai-embeddings_v2', schema=JsonSchema(DataSchema)) +producer = client.create_producer('embedd-created_v2', schema=JsonSchema(DataSchema)) + +def process_message(msg): + print(f"Received message: ID={msg.id}, Content={msg.content}") + encoder = SentenceEncoder(msg.model) + encoded_data = encoder.embed(msg.content) + + print("Encoded data:", encoded_data) + # Directly assign the list of floats to the vector attribute + # Convert NumPy array to a list before assigning it to the 'vector' field + msg.vector = encoded_data.tolist() if isinstance(encoded_data, np.ndarray) else encoded_data + + return msg + +def serve(): + try: + while True: + msg = consumer.receive() + print("received message", msg) + try: + processed_data = process_message(msg.value()) + producer.send(processed_data) + consumer.acknowledge(msg) + except Exception as e: + print("Failed to process message:", e) + consumer.negative_acknowledge(msg) + finally: + client.close() + +if __name__ == "__main__": + serve() diff --git a/ai/embedder/old/embedd_service_test.py b/ai/embedder/old/embedd_service_test.py new file mode 100644 index 00000000..dd1b63c9 --- /dev/null +++ b/ai/embedder/old/embedd_service_test.py @@ -0,0 +1,31 @@ +import pulsar +from pulsar.schema import JsonSchema, Record, Integer, String, Array, Float + +class DataSchema(Record): + document_id = Integer() + key = String() + model = String() + content = String() + vector = Array(Float()) + +# Setup the Pulsar client and producer with JSON schema +client = pulsar.Client('pulsar://localhost:6650') +producer = client.create_producer('embedd-request_v2', schema=JsonSchema(DataSchema)) + +def send_message(): + # Prompt user for input to embed + content_to_embed = input("Type the content you want to embed: ") + model_to_use = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2" + + # Create a new message object using the DataSchema + message = DataSchema(id=123, content=content_to_embed, model=model_to_use, vector=[1.0, 2.0, 3.0]) + + # Send the message to the topic through the producer + producer.send(message) + print(f"Sent message: ID={message.document_id}, Content={message.content}") + +if __name__ == "__main__": + try: + send_message() + finally: + client.close() diff --git a/ai/embedder/requirements.txt b/ai/embedder/requirements.txt new file mode 100644 index 00000000..1d15cfe3 --- /dev/null +++ b/ai/embedder/requirements.txt @@ -0,0 +1,18 @@ +# requirements + +nats-py==2.7.2 +protobuf==4.25.3 +python-dotenv==1.0.1 +opentelemetry-api==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-exporter-otlp==1.24.0 +opentelemetry-instrumentation==0.45b0 +opentelemetry-instrumentation-grpc==0.45b0 +sentence-transformers==2.7.0 +--find-links https://download.pytorch.org/whl/torch_stable.html +torch==2.3.0+cpu +# #torchvision==2.3.0+cpu +# #torchaudio==2.3.0+cpu +#-f https://download.pytorch.org/whl/torch_stable.html + + diff --git a/ai/embedder/sentence_encoder.py b/ai/embedder/sentence_encoder.py new file mode 100644 index 00000000..c32c7deb --- /dev/null +++ b/ai/embedder/sentence_encoder.py @@ -0,0 +1,52 @@ +import os +from sentence_transformers import SentenceTransformer + + +# todo: change local path to S3 storage +class SentenceEncoder: + def __init__(self, model_name, local_model_dir='models'): + """ + Initializes an instance of SentenceEncoder, attempting to load the model from a local directory first. + If the model is not available locally, it downloads from Hugging Face and saves it locally. + + Parameters: + model_name (str): The name of the model to load or download. + local_model_dir (str): The directory to check for the model and to save the model. + """ + self.model_path = os.path.join(local_model_dir, model_name) + + # Check if the model directory exists and has model files + if not os.path.exists(self.model_path) or not os.listdir(self.model_path): + print("Model not found locally, downloading from Hugging Face...") + try: + # Download and save the model + model = SentenceTransformer(model_name) + model.save(self.model_path) + print(f"Model saved locally at {self.model_path}") + except Exception as e: + print(f"Failed to download or save the model due to: {e}") + else: + print("Loading model from local directory...") + + # Load the model from the local path + self.model = SentenceTransformer(self.model_path) + + def embed(self, text): + """ + Encodes the provided text using the loaded SentenceTransformer model. + + Parameters: + text (str): The text to be encoded. + + Returns: + list: A list of floats representing the encoded text. + """ + # Use the loaded model to encode the text + return self.model.encode(text) + +# # Example usage +# if __name__ == "__main__": +# model_name = 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2' +# encoder = TextEncoder(model_name) # Create an instance of TextEncoder with a specific model +# encoded_data = encoder.embed("explain routed events in WPF") # Call the embed method with a sample text +# print(encoded_data) # Print the encoded data diff --git a/ai/embedder/telemetry.py b/ai/embedder/telemetry.py new file mode 100644 index 00000000..5f347f9f --- /dev/null +++ b/ai/embedder/telemetry.py @@ -0,0 +1,34 @@ +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.grpc import GrpcInstrumentorServer +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.instrumentation.grpc import server_interceptor + +class OpenTelemetryManager: + def __init__(self): + self.trace_provider = TracerProvider() + trace.set_tracer_provider(self.trace_provider) + self.tracer = trace.get_tracer(__name__) + + otlp_exporter = OTLPSpanExporter() + self.trace_provider.add_span_processor(BatchSpanProcessor(otlp_exporter)) + + self.meter_provider = MeterProvider() + metric_exporter = OTLPMetricExporter() + metric_reader = PeriodicExportingMetricReader(metric_exporter) + self.meter_provider._all_metric_readers = metric_reader + self.meter = self.meter_provider.get_meter(__name__) + + self.embedding_time_metric = self.meter.create_histogram("embedding_creation_time", description="Time taken to create embeddings") + + GrpcInstrumentorServer().instrument() + + def start_trace(self, span_name): + return self.tracer.start_as_current_span(span_name) + + def record_metric(self, start_time, end_time): + self.embedding_time_metric.record(end_time - start_time) \ No newline at end of file diff --git a/ai/embedder/test_publish.py b/ai/embedder/test_publish.py new file mode 100644 index 00000000..a9e41a24 --- /dev/null +++ b/ai/embedder/test_publish.py @@ -0,0 +1,26 @@ +import asyncio +import nats + +async def main(): + # Connect to NATS! + #nc = await nats.connect("demo.nats.io") + nc = await nats.connect("nats://localhost:4222") + + # Receive messages on 'foo' + sub = await nc.subscribe("foo") + + # Publish a message to 'foo' + await nc.publish("foo", b'Hello from Python!ggggggg') + + # Process a message + msg = await sub.next_msg() + print("Received:", msg) + + # Make sure all published messages have reached the server + await nc.flush() + + # Close NATS connection + await nc.close() + +if __name__ == '__main__': + asyncio.run(main()) \ No newline at end of file diff --git a/ai/embedder/test_subscribe.py b/ai/embedder/test_subscribe.py new file mode 100644 index 00000000..eed9671a --- /dev/null +++ b/ai/embedder/test_subscribe.py @@ -0,0 +1,27 @@ +import asyncio +from nats.aio.client import Client as NATS +from nats.js.api import JetStreamContext +import embedd_data_pb2 # Import the generated protobuf module + +async def handle_task(msg): + # Deserialize the Task message + task = embedd_data_pb2.Task() + task.ParseFromString(msg.data) + print(f"Received task to: {task.content}") + # Acknowledge the message + await msg.ack() + +async def run(): + # Connect to NATS + nc = NATS() + await nc.connect(servers=["nats://localhost:4222"]) + js = nc.jetstream() + + # Subscribe to tasks + await js.subscribe("tasks.subject", cb=handle_task) + + # Keep the connection alive to continue receiving tasks + await asyncio.Event().wait() + +if __name__ == '__main__': + asyncio.run(run()) diff --git a/backend/Makefile b/backend/Makefile index 5627dca0..b21a11d8 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -4,3 +4,7 @@ lint: swag-api: swag init --dir ./api --output ./api/docs --parseDependency --parseInternal + +gen-proto: + rm -f core/proto/*.pb.go + protoc -I=. -I=vendor -I=${GOPATH}/src core/proto/*.proto --go_out=. --go-grpc_out=. \ No newline at end of file diff --git a/backend/api/module.go b/backend/api/module.go index 75c1aedf..ce1aea72 100644 --- a/backend/api/module.go +++ b/backend/api/module.go @@ -4,6 +4,7 @@ import ( "cognix.ch/api/v2/api/handler" "cognix.ch/api/v2/core/ai" "cognix.ch/api/v2/core/bll" + "cognix.ch/api/v2/core/messaging" "cognix.ch/api/v2/core/oauth" "cognix.ch/api/v2/core/repository" "cognix.ch/api/v2/core/security" @@ -20,8 +21,10 @@ import ( var Module = fx.Options( repository.DatabaseModule, + repository.RepositoriesModule, bll.BLLModule, storage.MinioModule, + messaging.NatsModule, fx.Provide(ReadConfig, NewRouter, newGoogleOauthProvider, diff --git a/workers/embedder/embedder.py b/backend/api/test.json similarity index 100% rename from workers/embedder/embedder.py rename to backend/api/test.json diff --git a/backend/connector/executor.go b/backend/connector/executor.go new file mode 100644 index 00000000..d373e7ef --- /dev/null +++ b/backend/connector/executor.go @@ -0,0 +1,184 @@ +package main + +import ( + "cognix.ch/api/v2/core/ai" + "cognix.ch/api/v2/core/connector" + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/storage" + "context" + "fmt" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.uber.org/zap" + "time" +) + +type Executor struct { + connectorRepo repository.ConnectorRepository + docRepo repository.DocumentRepository + msgClient messaging.Client + chunking ai.Chunking + milvusClinet storage.MilvusClient +} + +func (e *Executor) run(ctx context.Context, topic, subscriptionName string, task messaging.MessageHandler) { + if err := e.msgClient.Listen(ctx, topic, subscriptionName, task); err != nil { + zap.S().Errorf("failed to listen[%s]: %v", topic, err) + } + return +} + +func (e *Executor) runEmbedding(ctx context.Context, msg *proto.Message) error { + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.Header)) + payload := msg.GetBody().GetEmbedding() + if payload == nil { + zap.S().Errorf("Failed to get embedding payload") + return nil + } + zap.S().Infof("process embedding %d == > %50s ", payload.GetDocumentId(), payload.Content) + return nil +} + +func (e *Executor) runConnector(ctx context.Context, msg *proto.Message) error { + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.Header)) + trigger := msg.GetBody().GetTrigger() + + if trigger == nil { + return fmt.Errorf("failed to get trigger payload") + } + connectorModel, err := e.connectorRepo.GetByID(ctx, trigger.GetId()) + if err != nil { + return err + } + if err = e.connectorRepo.InvalidateConnector(ctx, connectorModel); err != nil { + return err + } + connectorWF, err := connector.New(connectorModel) + if err != nil { + return err + } + embedding := ai.NewEmbeddingParser(&model.EmbeddingModel{ModelID: "text-embedding-ada-002"}) + resultCh := connectorWF.Execute(ctx, trigger.Params) + + if err = e.milvusClinet.CreateSchema(ctx, connectorWF.CollectionName()); err != nil { + return fmt.Errorf("error creating schema: %v", err) + } + + for result := range resultCh { + var loopErr error + doc := e.handleResult(ctx, connectorModel, result) + if !doc.IsUpdated { + continue + } + if doc.ID.IntPart() != 0 { + loopErr = e.docRepo.Update(ctx, doc) + } else { + loopErr = e.docRepo.Create(ctx, doc) + } + if loopErr != nil { + err = loopErr + doc.Status = model.StatusFailed + zap.S().Errorf("Failed to update document: %v", loopErr) + continue + } + chunks, loopErr := e.chunking.Split(ctx, result.Content) + if loopErr != nil { + err = loopErr + doc.Status = model.StatusFailed + zap.S().Errorf("Failed to update document: %v", loopErr) + continue + } + embeddingResponse, loopErr := embedding.Parse(ctx, &proto.EmbeddingRequest{ + DocumentId: doc.ID.IntPart(), + Key: doc.DocumentID, + Content: chunks, + }) + if loopErr != nil { + err = loopErr + doc.Status = model.StatusFailed + zap.S().Errorf("Failed to update document: %v", loopErr) + continue + } + milvusPayload := make([]*storage.MilvusPayload, 0, len(embeddingResponse.Payload)) + for _, payload := range embeddingResponse.Payload { + milvusPayload = append(milvusPayload, &storage.MilvusPayload{ + DocumentID: embeddingResponse.GetDocumentId(), + Chunk: payload.GetChunk(), + Content: payload.GetContent(), + Vector: payload.GetVector(), + }) + } + if loopErr = e.milvusClinet.Save(ctx, connectorWF.CollectionName(), milvusPayload...); loopErr != nil { + err = loopErr + doc.Status = model.StatusFailed + zap.S().Errorf("Failed to update document: %v", loopErr) + continue + } + + //todo for production + //if loopErr = e.msgClient.Publish(ctx, model.TopicEmbedding, + // &proto.Body{MilvusPayload: &proto.Body_Embedding{Embedding: &proto.EmbeddingRequest{ + // DocumentId: doc.ID.IntPart(), + // Key: doc.DocumentID, + // Content: chunks, + // }}}, + //); err != nil { + // err = loopErr + // zap.S().Errorf("Failed to update document: %v", err) + // doc.Status = model.StatusFailed + // doc.Signature = "" + // continue + //} + } + + if err != nil { + connectorModel.LastAttemptStatus = model.StatusFailed + } else { + connectorModel.LastAttemptStatus = model.StatusSuccess + } + if err = e.connectorRepo.Update(ctx, connectorModel); err != nil { + return err + } + return nil +} + +func (e *Executor) handleResult(ctx context.Context, connectorModel *model.Connector, result *proto.TriggerResponse) *model.Document { + doc, ok := connectorModel.DocsMap[result.GetDocumentId()] + if !ok { + doc = &model.Document{ + DocumentID: result.GetDocumentId(), + ConnectorID: connectorModel.ID, + Link: result.GetUrl(), + Signature: result.GetSignature(), + CreatedDate: time.Now().UTC(), + Status: model.StatusInProgress, + IsUpdated: true, + } + connectorModel.DocsMap[result.GetDocumentId()] = doc + } else { + if doc.Signature != result.GetSignature() { + doc.Signature = result.GetSignature() + doc.Status = model.StatusInProgress + doc.IsUpdated = true + } + } + return doc +} + +func NewExecutor(connectorRepo repository.ConnectorRepository, + docRepo repository.DocumentRepository, + streamClient messaging.Client, + chunking ai.Chunking, + milvusClinet storage.MilvusClient, +) *Executor { + return &Executor{ + connectorRepo: connectorRepo, + docRepo: docRepo, + msgClient: streamClient, + chunking: chunking, + milvusClinet: milvusClinet, + } +} diff --git a/backend/connector/main.go b/backend/connector/main.go new file mode 100644 index 00000000..f3a41af4 --- /dev/null +++ b/backend/connector/main.go @@ -0,0 +1,13 @@ +package main + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" +) + +func main() { + utils.InitLogger(true) + app := fx.New(Module) + + app.Run() +} diff --git a/backend/connector/module.go b/backend/connector/module.go new file mode 100644 index 00000000..d31519a7 --- /dev/null +++ b/backend/connector/module.go @@ -0,0 +1,48 @@ +package main + +import ( + "cognix.ch/api/v2/core/ai" + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/repository" + "cognix.ch/api/v2/core/storage" + "context" + "go.uber.org/fx" +) + +var Module = fx.Options( + repository.DatabaseModule, + messaging.NatsModule, + storage.MilvusModule, + storage.MinioModule, + ai.ChunkingModule, + fx.Provide( + repository.NewConnectorRepository, + repository.NewDocumentRepository, + repository.NewEmbeddingModelRepository, + NewExecutor, + ), + fx.Invoke(RunServer), +) + +func RunServer(lc fx.Lifecycle, executor *Executor) error { + + go executor.run(context.Background(), model.TopicEmbedding, model.SubscriptionEmbedding, executor.runEmbedding) + go executor.run(context.Background(), model.TopicExecutor, model.SubscriptionExecutor, executor.runConnector) + return nil + + // + //lc.Append(fx.Hook{ + // OnStart: func(ctx context.Context) error { + // if err := executor.run(context.Background(), model.TopicEmbedding, model.SubscriptionEmbedding, executor.runEmbedding); err != nil { + // return err + // } + // return executor.run(context.Background(), model.TopicExecutor, model.SubscriptionExecutor, executor.runConnector) + // }, + // OnStop: func(ctx context.Context) error { + // executor.msgClient.Close() + // return nil + // }, + //}) + //return nil +} diff --git a/backend/connector/readme.md b/backend/connector/readme.md new file mode 100644 index 00000000..461b9a81 --- /dev/null +++ b/backend/connector/readme.md @@ -0,0 +1,37 @@ +we have tow services: +connector +connector orchestrator (we will change this name) +the orcherstrator cheks the connector table and decides if a connector needs to do the work for one line inside connector table +if yes it sends a NATS message so that only one connector service will take the message +Important check how to configure nats for retries and dead letter queue + + +max 3 retryes and then it shal go in error status +CONNECTOR SERVCES +shall inherit from a base connector class +it shall do the job f crarling +splitting by a fixed size with some char of the previous iteration +it will call the python service to create the embeddings +it will store the embedding in milvus +it will update the status of the connectr +we neeed to store: + embedding + original text +reference to find back the original document which shall be a link exposed to the client +milvus store with file index (i'll explain later) +we create a collection for each organization and a collection for each user +in milvus we create a collection for each tennant and one collection for each user foe the private connectors +collection name is fixed by a rule like tennant_{ID}. user_{ID} +pass opentelemetry context to nats for distributed tracing + + +### file compatibility +teams chats +teams teams +sharepoint +slack +email (office/gmail) +google drive +one drive +dropbox +upload files (supported formats) diff --git a/backend/connector/service-deployment.yaml b/backend/connector/service-deployment.yaml new file mode 100644 index 00000000..930e6366 --- /dev/null +++ b/backend/connector/service-deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: connectorservice +spec: + replicas: 1 + selector: + matchLabels: + app: connectorservice + template: + metadata: + labels: + app: connectorservice + spec: + containers: + - name: connectorservice + image: cognixacr.azurecr.io/cognix/connectorservice:main + env: + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: env-configmap + key: DATABASE_URL + - name: MESSAGING_PROVIDER + value: "pulsar" + - name: PULSAR_URL + value: "pulsar://cognix-milvus-pulsar-proxy.milvus.svc.cluster.local:6650/" + imagePullPolicy: Always + imagePullSecrets: + - name: pullsecret + +--- +apiVersion: v1 +kind: Service +metadata: + name: connector +spec: + selector: + app: connectorservice +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-volume + namespace: default +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: azurefile-csi \ No newline at end of file diff --git a/backend/core/ai/builder.go b/backend/core/ai/builder.go index df601f78..112d2593 100644 --- a/backend/core/ai/builder.go +++ b/backend/core/ai/builder.go @@ -20,7 +20,7 @@ func (b *Builder) New(llm *model.LLM) OpenAIClient { if client, ok := b.clients[llm.ID.IntPart()]; ok { return client } - client := NewOpenAIClient(llm) + client := NewOpenAIClient(llm.ModelID, llm.ApiKey) b.clients[llm.ID.IntPart()] = client return client } diff --git a/backend/core/ai/chunking.go b/backend/core/ai/chunking.go new file mode 100644 index 00000000..d183fc0c --- /dev/null +++ b/backend/core/ai/chunking.go @@ -0,0 +1,37 @@ +package ai + +import ( + "context" +) + +type staticChunking struct { + chunkSize int + overlap int +} + +func (s staticChunking) Split(ctx context.Context, text string) ([]string, error) { + var chunks []string + for { + if len(text) < s.chunkSize { + chunks = append(chunks, text) + break + } + chunks = append(chunks, text[:s.chunkSize]) + text = text[s.chunkSize-s.overlap:] + } + return chunks, nil +} + +func NewStaticChunking(cfg *ChunkingConfig) Chunking { + return &staticChunking{ + chunkSize: cfg.StaticChunkSize, + overlap: cfg.StaticChunkOverlap, + } +} + +func NewLLMChunking() Chunking { + return &staticChunking{ + chunkSize: MaxChunkSize, + overlap: 20, + } +} diff --git a/backend/core/ai/chunking_test.go b/backend/core/ai/chunking_test.go new file mode 100644 index 00000000..c83a85e5 --- /dev/null +++ b/backend/core/ai/chunking_test.go @@ -0,0 +1,54 @@ +package ai + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +type testStaticChunking struct { + Source string + ExpectedCount int + Expected []string +} + +var testData = []testStaticChunking{ + { + Source: "foo", + ExpectedCount: 1, + Expected: []string{"foo"}, + }, + { + Source: "123456789012345678901234567890123456789012345", + ExpectedCount: 7, + Expected: []string{"1234567890", + "8901234567", + "5678901234", + "2345678901", + "9012345678", + "6789012345", + "345", + }, + }, +} + +func Test_StaticChunkingText(t *testing.T) { + + service := NewStaticChunking(&ChunkingConfig{ + Strategy: StrategyStatic, + StaticChunkSize: 10, + StaticChunkOverlap: 3, + }) + for i, test := range testData { + t.Run(fmt.Sprintf("%d", i), + func(t *testing.T) { + result, err := service.Split(context.Background(), test.Source) + assert.NoError(t, err) + assert.Equal(t, test.ExpectedCount, len(result)) + for n, chunk := range result { + assert.Equal(t, test.Expected[n], chunk) + } + }) + } +} diff --git a/backend/core/ai/embedding-openai.go b/backend/core/ai/embedding-openai.go new file mode 100644 index 00000000..2db84894 --- /dev/null +++ b/backend/core/ai/embedding-openai.go @@ -0,0 +1,59 @@ +package ai + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "context" + _ "github.com/deluan/flowllm/llms/openai" + "github.com/sashabaranov/go-openai" + "go.uber.org/zap" + "os" +) + +type ( + embeddingParser struct { + embeddingModel *model.EmbeddingModel + client *openai.Client + } +) + +func NewEmbeddingParser(embeddingModel *model.EmbeddingModel) EmbeddingParser { + //remove-it + apiKey := os.Getenv("OPENAI_API_KEY") + return &embeddingParser{ + embeddingModel: embeddingModel, + client: openai.NewClient(apiKey), + } +} + +func (p *embeddingParser) Parse(ctx context.Context, payload *proto.EmbeddingRequest) (*proto.EmbeddingResponse, error) { + embeddingModel := payload.GetEmbeddingModel() + if embeddingModel == "" { + embeddingModel = p.embeddingModel.ModelID + } + embeddingRequest := openai.EmbeddingRequestStrings{ + Input: payload.GetContent(), + Model: openai.EmbeddingModel(embeddingModel), + EncodingFormat: openai.EmbeddingEncodingFormatFloat, + Dimensions: 0, + } + + embeddingResponse, err := p.client.CreateEmbeddings(ctx, embeddingRequest) + if err != nil { + return nil, err + } + response := &proto.EmbeddingResponse{ + DocumentId: payload.GetDocumentId(), + Key: payload.Key, + } + for i, em := range embeddingResponse.Data { + zap.S().Infof("%d, %s", em.Index, em.Object) + response.Payload = append(response.Payload, &proto.EmbeddingResponse_Payload{ + Chunk: int64(em.Index), + Content: payload.Content[i], + Vector: em.Embedding, + }) + + } + return response, nil +} diff --git a/backend/core/ai/model.go b/backend/core/ai/model.go new file mode 100644 index 00000000..57e70f9b --- /dev/null +++ b/backend/core/ai/model.go @@ -0,0 +1,38 @@ +package ai + +import ( + "cognix.ch/api/v2/core/proto" + "context" + validation "github.com/go-ozzo/ozzo-validation/v4" +) + +const ( + StrategyStatic = "STATIC" + StrategyLLM = "LLM" + MaxChunkSize = 65536 +) + +type ( + ChunkingConfig struct { + Strategy string `env:"CHUNKING_STRATEGY" envDefault:"STATIC"` + StaticChunkSize int `env:"CHUNKING_STATIC_CHARS" envDefault:"16384"` + StaticChunkOverlap int `env:"CHUNKING_STATIC_CHARS_OVERLAP" envDefault:"20"` + } + + Chunking interface { + Split(ctx context.Context, text string) ([]string, error) + } + + EmbeddingParser interface { + Parse(ctx context.Context, payload *proto.EmbeddingRequest) (*proto.EmbeddingResponse, error) + } +) + +func (v ChunkingConfig) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.Strategy, validation.Required, validation.In(StrategyStatic, StrategyLLM)), + validation.Field(&v.StaticChunkSize, validation.Max(MaxChunkSize)), + validation.Field(&v.StaticChunkOverlap, validation.Min(0), + validation.Max(v.StaticChunkSize/3)), + ) +} diff --git a/backend/core/ai/module.go b/backend/core/ai/module.go new file mode 100644 index 00000000..49331a45 --- /dev/null +++ b/backend/core/ai/module.go @@ -0,0 +1,28 @@ +package ai + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" +) + +var ChunkingModule = fx.Options( + fx.Provide(func() (*ChunkingConfig, error) { + cfg := ChunkingConfig{} + if err := utils.ReadConfig(&cfg); err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, err + } + return &cfg, nil + }, + newChunking, + ), +) + +func newChunking(cfg *ChunkingConfig) Chunking { + if cfg.Strategy == StrategyLLM { + return NewLLMChunking() + } + return NewStaticChunking(cfg) +} diff --git a/backend/core/ai/open-ai.go b/backend/core/ai/open-ai.go index 2f3722f2..07e7dc64 100644 --- a/backend/core/ai/open-ai.go +++ b/backend/core/ai/open-ai.go @@ -1,15 +1,10 @@ package ai import ( - "cognix.ch/api/v2/core/model" "context" openai "github.com/sashabaranov/go-openai" ) -var SupportedModels = map[string]bool{ - openai.GPT3Dot5Turbo: true, -} - type ( Response struct { Message string @@ -50,10 +45,10 @@ func (o *openAIClient) Request(ctx context.Context, message string) (*Response, return response, nil } -func NewOpenAIClient(llm *model.LLM) OpenAIClient { +func NewOpenAIClient(modelID, apiKey string) OpenAIClient { return &openAIClient{ - client: openai.NewClient(llm.ApiKey), - modelID: llm.ModelID, + client: openai.NewClient(apiKey), + modelID: modelID, } } diff --git a/backend/core/bll/connector.go b/backend/core/bll/connector.go index 3e6f7bd1..f9c9e5cd 100644 --- a/backend/core/bll/connector.go +++ b/backend/core/bll/connector.go @@ -1,6 +1,8 @@ package bll import ( + "cognix.ch/api/v2/core/connector" + "cognix.ch/api/v2/core/messaging" "cognix.ch/api/v2/core/model" "cognix.ch/api/v2/core/parameters" "cognix.ch/api/v2/core/repository" @@ -22,11 +24,12 @@ type ( connectorBL struct { connectorRepo repository.ConnectorRepository credentialRepo repository.CredentialRepository + messenger messaging.Client } ) func (c *connectorBL) Archive(ctx context.Context, user *model.User, id int64, restore bool) (*model.Connector, error) { - connector, err := c.connectorRepo.GetByID(ctx, user.TenantID, user.ID, id) + connector, err := c.connectorRepo.GetByIDAndUser(ctx, user.TenantID, user.ID, id) if err != nil { return nil, err } @@ -45,10 +48,14 @@ func (c *connectorBL) Archive(ctx context.Context, user *model.User, id int64, r return connector, nil } -func NewConnectorBL(connectorRepo repository.ConnectorRepository, credentialRepo repository.CredentialRepository) ConnectorBL { +func NewConnectorBL(connectorRepo repository.ConnectorRepository, + credentialRepo repository.CredentialRepository, + messenger messaging.Client, +) ConnectorBL { return &connectorBL{ connectorRepo: connectorRepo, credentialRepo: credentialRepo, + messenger: messenger, } } @@ -81,11 +88,14 @@ func (c *connectorBL) Create(ctx context.Context, user *model.User, param *param if err := c.connectorRepo.Create(ctx, &conn); err != nil { return nil, err } + if err := c.messenger.Publish(ctx, model.TopicUpdateConnector, connector.Trigger{ID: conn.ID.IntPart()}); err != nil { + return nil, err + } return &conn, nil } func (c *connectorBL) Update(ctx context.Context, id int64, user *model.User, param *parameters.UpdateConnectorParam) (*model.Connector, error) { - conn, err := c.connectorRepo.GetByID(ctx, user.TenantID, user.ID, id) + conn, err := c.connectorRepo.GetByIDAndUser(ctx, user.TenantID, user.ID, id) if err != nil { return nil, err } @@ -107,20 +117,19 @@ func (c *connectorBL) Update(ctx context.Context, id int64, user *model.User, pa conn.Disabled = param.Disabled conn.UpdatedDate = pg.NullTime{time.Now().UTC()} - // sql.NullTime{ - //Time: time.Now().UTC(), - //Valid: true, - // } // null.TimeFrom(time.Now().UTC()) if err = c.connectorRepo.Update(ctx, conn); err != nil { return nil, err } + if err = c.messenger.Publish(ctx, model.TopicUpdateConnector, connector.Trigger{ID: conn.ID.IntPart()}); err != nil { + return nil, err + } return conn, nil } func (c *connectorBL) GetAll(ctx context.Context, user *model.User) ([]*model.Connector, error) { - return c.connectorRepo.GetAll(ctx, user.TenantID, user.ID) + return c.connectorRepo.GetAllByUser(ctx, user.TenantID, user.ID) } func (c *connectorBL) GetByID(ctx context.Context, user *model.User, id int64) (*model.Connector, error) { - return c.connectorRepo.GetByID(ctx, user.TenantID, user.ID, id) + return c.connectorRepo.GetByIDAndUser(ctx, user.TenantID, user.ID, id) } diff --git a/backend/core/connector/base.go b/backend/core/connector/base.go new file mode 100644 index 00000000..fee8adf3 --- /dev/null +++ b/backend/core/connector/base.go @@ -0,0 +1,59 @@ +package connector + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "context" + "fmt" + "strings" +) + +type Base struct { + collectionName string + model *model.Connector + resultCh chan *proto.TriggerResponse +} + +type Connector interface { + CollectionName() string + Execute(ctx context.Context, param map[string]string) chan *proto.TriggerResponse +} + +type Builder struct { + connectorRepo repository.ConnectorRepository +} + +type nopConnector struct { + Base +} + +func (n *nopConnector) Execute(ctx context.Context, param map[string]string) chan *proto.TriggerResponse { + ch := make(chan *proto.TriggerResponse) + return ch +} + +func New(connectorModel *model.Connector) (Connector, error) { + switch connectorModel.Source { + case model.SourceTypeWEB: + return NewWeb(connectorModel) + default: + return &nopConnector{}, nil + } +} + +func (b *Base) Config(connector *model.Connector) { + b.model = connector + b.resultCh = make(chan *proto.TriggerResponse, 10) + if connector.Shared { + b.collectionName = strings.ReplaceAll(fmt.Sprintf(model.CollectionTenant, connector.TenantID), "-", "") + } else { + b.collectionName = strings.ReplaceAll(fmt.Sprintf(model.CollectionUser, connector.UserID), "-", "") + } + + return +} + +func (b *Base) CollectionName() string { + return b.collectionName +} diff --git a/backend/core/connector/web.go b/backend/core/connector/web.go new file mode 100644 index 00000000..7a4ef4aa --- /dev/null +++ b/backend/core/connector/web.go @@ -0,0 +1,170 @@ +package connector + +import ( + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "context" + "crypto/sha256" + "fmt" + "github.com/gocolly/colly/v2" + "go.uber.org/zap" + "jaytaylor.com/html2text" + "net/url" + "strings" +) + +type ( + Web struct { + Base + param *WebParameters + scraper *colly.Collector + history map[string]string + ctx context.Context + } + WebParameters struct { + URL string `url:"url"` + } +) + +var excludeTag = map[string]bool{ + "script": true, + "button": true, + "a": true, + "header": true, + "nav": true, +} + +func withContext(ctx context.Context, fn func(context.Context, *colly.HTMLElement)) colly.HTMLCallback { + return func(element *colly.HTMLElement) { + fn(ctx, element) + } +} + +func (c *Web) Execute(ctx context.Context, param map[string]string) chan *proto.TriggerResponse { + zap.S().Debugf("Run web connector with param %s ...", c.param.URL) + c.ctx = ctx + go func() { + c.scraper.OnHTML("body", withContext(ctx, c.onBody)) + err := c.scraper.Visit(c.param.URL) + if err != nil { + zap.L().Error("Failed to scrape URL", zap.String("url", c.param.URL), zap.Error(err)) + } + zap.S().Debugf("Complete web connector with param %s", c.param.URL) + close(c.resultCh) + }() + return c.resultCh +} + +func NewWeb(connector *model.Connector) (Connector, error) { + web := Web{} + web.Base.Config(connector) + web.param = &WebParameters{} + web.history = make(map[string]string) + if err := connector.ConnectorSpecificConfig.ToStruct(web.param); err != nil { + return nil, err + } + web.scraper = colly.NewCollector( + colly.UserAgent("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"), + ) + return &web, nil +} + +func (c *Web) onBody(ctx context.Context, e *colly.HTMLElement) { + child := e.ChildAttrs("a", "href") + text, _ := html2text.FromString(e.ChildText("main"), html2text.Options{ + PrettyTables: true, + PrettyTablesOptions: &html2text.PrettyTablesOptions{ + AutoFormatHeader: true, + AutoWrapText: true, + }, + OmitLinks: true, + }) + c.history[e.Request.URL.String()] = text + c.processChildLinks(e.Request.URL, child) + signature := fmt.Sprintf("%x", sha256.Sum256([]byte(text))) + docID := e.Request.URL.String() + + c.resultCh <- &proto.TriggerResponse{ + DocumentId: docID, + Url: docID, + Content: text, + Signature: signature, + Status: proto.Status_SUCCESS, + } + +} + +func (c *Web) onBody2(ctx context.Context, e *colly.HTMLElement) { + //child := e.ChildAttrs("a", "href") + // + //var rows []string + //e.ForEach("*", func(i int, element *colly.HTMLElement) { + // + // if _, ok := excludeTag[element.Name]; ok { + // return + // } + // fmt.Println(fmt.Sprintf("%d >> %s ", i, element.Name)) + // rows = append(rows, element.Text) + //}) + // + //text, _ := html2text.FromString(strings.Join(rows, "\n"), html2text.Options{ + // PrettyTables: true, + // PrettyTablesOptions: &html2text.PrettyTablesOptions{ + // AutoFormatHeader: true, + // AutoWrapText: true, + // }, + // OmitLinks: true, + //}) + //c.history[e.Request.URL.String()] = text + //c.processChildLinks(e.Request.URL, child) + //signature := fmt.Sprintf("%x", sha256.Sum256([]byte(text))) + //docID := e.Request.URL.String() + //doc, ok := c.model.DocsMap[docID] + //if !ok { + // doc = &model.Document{ + // DocumentID: docID, + // ConnectorID: c.model.ID, + // Link: docID, + // CreatedDate: time.Now().UTC(), + // IsExists: true, + // IsUpdated: true, + // } + // c.model.DocsMap[docID] = doc + // c.model.Docs = append(c.model.Docs, doc) + //} + //doc.IsExists = true + //if doc.Signature == signature { + // return + //} + //doc.Signature = signature + //if doc.ID.IntPart() != 0 { + // doc.IsUpdated = true + // doc.UpdatedDate = pg.NullTime{time.Now().UTC()} + //} + //c.resultCh <- &proto.TriggerResponse{ + // DocumentId: doc.ID.IntPart(), + // Url: doc.DocumentID, + // Content: text, + // Signature: signature, + //} + +} + +func (c *Web) processChildLinks(baseURL *url.URL, urls []string) { + for _, u := range urls { + if len(u) == 0 || u[0] == '#' || !strings.Contains(u, baseURL.Path) || + (strings.HasPrefix(u, "http") && !strings.Contains(u, baseURL.Host)) { + continue + } + if strings.HasPrefix(u, baseURL.Path) { + u = fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, u) + } + if _, ok := c.history[u]; ok { + continue + } + if err := c.scraper.Visit(u); err != nil { + zap.S().Errorf("Failed to scrape URL: %s", u) + } + } + return +} diff --git a/backend/core/connector/web_test.go b/backend/core/connector/web_test.go new file mode 100644 index 00000000..2d67f4f7 --- /dev/null +++ b/backend/core/connector/web_test.go @@ -0,0 +1,44 @@ +package connector + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/model" + "context" + "github.com/shopspring/decimal" + "testing" +) + +func TestWeb_Execute(t *testing.T) { + web, err := NewWeb( + &model.Connector{ + ID: decimal.NewFromInt(1), + Name: "web test", + Source: "web", + InputType: "src", + ConnectorSpecificConfig: model.JSONMap{ + "url1": "https://help.collaboard.app/", + "url2": "https://apidog.com/blog/openapi-specification/", + "url": "https://developer.mozilla.org/en-US/docs/Learn/HTML/Introduction_to_HTML", + }, + DocsMap: make(map[string]*model.Document), + }, + messaging.NewMopClient()) + if err != nil { + t.Log(err.Error()) + t.Fatal(err) + } + conn, err := web.Execute(context.Background(), nil) + if err != nil { + t.Log(err.Error()) + t.Fatal(err) + } + for url, history := range (web.(*Web)).history { + if len(history) > 30 { + history = history[:30] + } + t.Logf("%s => %s ", url, history) + } + for _, doc := range conn.Docs { + t.Log(doc.DocumentID) + } +} diff --git a/backend/core/connector/web_tokenaizer.go b/backend/core/connector/web_tokenaizer.go new file mode 100644 index 00000000..12716e0e --- /dev/null +++ b/backend/core/connector/web_tokenaizer.go @@ -0,0 +1,45 @@ +package connector + +import ( + "bytes" + "fmt" + "github.com/gocolly/colly/v2" + "golang.org/x/net/html" + "io" +) + +var skipTag = map[string]bool{ + "script": true, + "style": true, + "meta": true, + "link": true, + "a": true, + "li": true, + "ui": true, +} + +type webTokenizer struct { +} + +func (c *webTokenizer) tokenizer(r *colly.Response) { + tokenizer := html.NewTokenizer(bytes.NewBuffer(r.Body)) + + for { + tokenType := tokenizer.Next() + token := tokenizer.Token() + if tokenizer.Err() == io.EOF { + break + } + if _, ok := skipTag[token.Data]; ok { + if tokenType == html.StartTagToken { + + } + continue + } + if tokenType.String() == "Text" { + // fmt.Println(tokenType.String(), "", token.Data) + fmt.Println(token.String()) + } + } + fmt.Println("--") +} diff --git a/backend/core/messaging/module.go b/backend/core/messaging/module.go new file mode 100644 index 00000000..9e2ef695 --- /dev/null +++ b/backend/core/messaging/module.go @@ -0,0 +1,70 @@ +package messaging + +import ( + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/utils" + "context" + "fmt" + "github.com/nats-io/nats.go" + "go.uber.org/fx" + "go.uber.org/zap" + "time" +) + +const ( + providerNats = "nats" + providerPulsar = "pulsar" +) + +type ( + Config struct { + Provider string `env:"MESSAGING_PROVIDER" default:"nats"` + Nats *natsConfig + Pulsar *pulsarConfig + } + natsConfig struct { + URL string `env:"NATS_URL"` + ConnectorStreamName string `env:"NATS_STREAM_NAME" envDefault:"Connector"` + } + Subscription struct { + ch chan *proto.Message + subscription *nats.Subscription + } + MessageHandler func(ctx context.Context, msg *proto.Message) error + Client interface { + Publish(ctx context.Context, topic string, body *proto.Body) error + Listen(ctx context.Context, topic, subscriptionName string, handler MessageHandler) error + Close() + } +) + +const ( + reconnectAttempts = 120 + reconnectWaitTime = 5 * time.Second + streamMaxPending = 256 +) + +var NatsModule = fx.Options( + fx.Provide(func() (*Config, error) { + cfg := Config{ + Pulsar: &pulsarConfig{}, + Nats: &natsConfig{}, + } + err := utils.ReadConfig(&cfg) + if err != nil { + zap.S().Errorf(err.Error()) + return nil, err + } + return &cfg, nil + }, + NewClient, + ), +) + +func NewClient(cfg *Config) (Client, error) { + switch cfg.Provider { + case providerPulsar: + return NewPulsar(cfg.Pulsar) + } + return nil, fmt.Errorf("unknown provider %s", cfg.Provider) +} diff --git a/backend/core/messaging/nats.go b/backend/core/messaging/nats.go new file mode 100644 index 00000000..b63ddcb3 --- /dev/null +++ b/backend/core/messaging/nats.go @@ -0,0 +1,87 @@ +package messaging + +import ( + "cognix.ch/api/v2/core/proto" + "context" + "encoding/json" + "github.com/nats-io/nats.go" + "go.uber.org/zap" + "sync" +) + +type client struct { + conn *nats.Conn + connectorStreamName string + subscriptions []*Subscription + once sync.Once +} + +func (c *client) Close() { + c.once.Do(func() { + for _, sub := range c.subscriptions { + close(sub.ch) + if err := sub.subscription.Unsubscribe(); err != nil { + zap.S().Errorf("unsubscribe %s ", err.Error()) + } + } + }) +} + +func (c *client) Publish(ctx context.Context, topic string, body *proto.Body) error { + message, err := buildMessage(ctx, body) + if err != nil { + return err + } + err = c.conn.Publish(topic, message) + if err != nil { + return err + } + //zap.S().Infof("Published message with ack: %s", pubAck.Domain) + return nil +} + +func (c *client) Listen(ctx context.Context, topic, subscriptionName string, handler MessageHandler) error { + out := make(chan *proto.Message) + subscription, err := c.conn.Subscribe(topic, + func(msg *nats.Msg) { + var message proto.Message + if err := json.Unmarshal(msg.Data, &message); err != nil { + zap.S().Errorf("Error unmarshalling message: %s", string(msg.Data)) + return + } + out <- &message + }) + if err != nil { + return err + } + for { + select { + case msg := <-out: + if err = handler(ctx, msg); err != nil { + zap.S().Errorf("handler message error: %s", err.Error()) + } + case <-ctx.Done(): + break + } + } + return subscription.Unsubscribe() +} + +func newNatsClient(cfg *natsConfig) (Client, error) { + conn, err := nats.Connect( + cfg.URL, + nats.Name("Cognix"), + nats.MaxReconnects(reconnectAttempts), + nats.ReconnectWait(reconnectWaitTime), + ) + if err != nil { + zap.S().Errorf("Error connecting to NATS: %s", err.Error()) + return nil, err + } + + return &client{ + conn: conn, + connectorStreamName: cfg.ConnectorStreamName, + once: sync.Once{}, + }, nil +} diff --git a/backend/core/messaging/nats_stream.go b/backend/core/messaging/nats_stream.go new file mode 100644 index 00000000..46b24a1f --- /dev/null +++ b/backend/core/messaging/nats_stream.go @@ -0,0 +1,109 @@ +package messaging + +import ( + "cognix.ch/api/v2/core/proto" + "context" + "fmt" + proto2 "github.com/golang/protobuf/proto" + "github.com/nats-io/nats.go" + "go.uber.org/zap" + "sync" +) + +type clientStream struct { + conn *nats.Conn + stream nats.JetStreamContext + connectorStreamName string + subscriptions []*Subscription + once sync.Once +} + +func (c *clientStream) Close() { + c.once.Do(func() { + for _, sub := range c.subscriptions { + close(sub.ch) + if err := sub.subscription.Unsubscribe(); err != nil { + zap.S().Errorf("unsubscribe %s ", err.Error()) + } + } + }) +} + +func (c *clientStream) Publish(ctx context.Context, topic string, body *proto.Body) error { + message, err := buildMessage(ctx, body) + if err != nil { + return err + } + pubAck, err := c.stream.Publish(fmt.Sprintf("%s.%s", c.connectorStreamName, topic), message) + if err != nil { + return err + } + zap.S().Infof("Published message with ack: %s", pubAck.Domain) + return nil +} + +func (c *clientStream) Listen(ctx context.Context, topic, subscriptionName string, handler MessageHandler) error { + subscription, err := c.conn.Subscribe(topic, + func(msg *nats.Msg) { + var message proto.Message + if err := proto2.Unmarshal(msg.Data, &message); err != nil { + zap.S().Errorf("Error unmarshalling message: %s", string(msg.Data)) + return + } + if err := handler(ctx, &message); err != nil { + zap.S().Errorf("Error handling message: %s", string(msg.Data)) + } + msg.Ack() + }) + if err != nil { + return err + } + for { + select { + case <-ctx.Done(): + break + default: + + } + } + return subscription.Unsubscribe() +} + +func NewClientStream(cfg *natsConfig) (Client, error) { + conn, err := nats.Connect( + cfg.URL, + nats.Name("Cognix"), + nats.MaxReconnects(reconnectAttempts), + nats.ReconnectWait(reconnectWaitTime), + ) + if err != nil { + zap.S().Errorf("Error connecting to NATS: %s", err.Error()) + return nil, err + } + js, err := conn.JetStream(nats.PublishAsyncMaxPending(streamMaxPending)) + if err != nil { + zap.S().Errorf("Error connecting to NATS: %s", err.Error()) + return nil, err + } + + stream, err := js.StreamInfo(cfg.ConnectorStreamName) + // stream not found, create it + if stream == nil { + zap.S().Infof("Creating stream: %s", cfg.ConnectorStreamName) + _, err = js.AddStream(&nats.StreamConfig{ + Name: cfg.ConnectorStreamName, + Subjects: []string{cfg.ConnectorStreamName + ".*"}, + }) + if err != nil { + zap.S().Errorf("Error creating stream: %s", err.Error()) + return nil, err + } + } + + return &clientStream{ + conn: conn, + stream: js, + connectorStreamName: cfg.ConnectorStreamName, + once: sync.Once{}, + }, nil +} diff --git a/backend/core/messaging/pulsar.go b/backend/core/messaging/pulsar.go new file mode 100644 index 00000000..a291260b --- /dev/null +++ b/backend/core/messaging/pulsar.go @@ -0,0 +1,147 @@ +package messaging + +import ( + "cognix.ch/api/v2/core/proto" + "context" + "fmt" + "github.com/apache/pulsar-client-go/pulsar" + proto2 "github.com/golang/protobuf/proto" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.uber.org/zap" + "time" +) + +type ( + pulsarConfig struct { + URL string `env:"PULSAR_URL"` + OperationTimeout int `env:"OPERATION_TIMEOUT" envDefault:"30"` + ConnectionTimeout int `env:"CONNECTION_TIMEOUT" envDefault:"30"` + ReconsumeTimeout int `env:"RECONSUME_TIMEOUT" envDefault:"5"` + } + pulsarClient struct { + ReconsumeTimeout time.Duration + conn pulsar.Client + producers map[string]pulsar.Producer + subscriber map[string]pulsar.Consumer + } +) + +func (p *pulsarClient) Publish(ctx context.Context, topic string, body *proto.Body) error { + msg, err := buildMessage(ctx, body) + if err != nil { + return err + } + producer, ok := p.producers[topic] + if !ok { + producer, err = p.conn.CreateProducer(pulsar.ProducerOptions{ + Topic: topic, + Schema: pulsar.NewProtoNativeSchemaWithMessage(&proto.TriggerRequest{}, nil), + }) + if err != nil { + return err + } + p.producers[topic] = producer + } + + _, err = producer.Send(ctx, &pulsar.ProducerMessage{ + Payload: msg, + }) + return err +} + +func (p *pulsarClient) Listen(ctx context.Context, topic, subscriptionName string, handler MessageHandler) error { + consumer, err := p.conn.Subscribe(pulsar.ConsumerOptions{ + Topic: topic, + SubscriptionName: subscriptionName, + Type: pulsar.Shared, + RetryEnable: false, + NackRedeliveryDelay: 0, + MaxPendingChunkedMessage: 0, + ExpireTimeOfIncompleteChunk: 0, + AutoAckIncompleteChunk: false, + EnableBatchIndexAcknowledgment: false, + SubscriptionMode: 0, + StartMessageIDInclusive: false, + }) + if err != nil { + return err + } + defer consumer.Close() + for { + // may block here + select { + case <-ctx.Done(): + break + default: + + } + msg, err := consumer.Receive(ctx) + if err != nil { + zap.S().Errorf("Receive message error: %s", err.Error()) + break + } + if err = consumer.Ack(msg); err != nil { + zap.S().Errorf("Ack message error: %s", err.Error()) + } + + var message proto.Message + + if err = proto2.Unmarshal(msg.Payload(), &message); err != nil { + continue + } + if err = handler(ctx, &message); err != nil { + consumer.ReconsumeLater(msg, p.ReconsumeTimeout) + zap.S().Errorf("Reconsume message error: %s", err.Error()) + } + } + if err = consumer.Unsubscribe(); err != nil { + zap.S().Errorf("Unsubscribe message error: %s", err.Error()) + } + return nil +} + +func (p *pulsarClient) processMessage(ctx context.Context, consumer pulsar.Consumer, handler MessageHandler, msg pulsar.Message) (err error) { + if err = consumer.Ack(msg); err != nil { + zap.S().Errorf("Ack message error: %s", err.Error()) + } + var message proto.Message + if err := proto2.Unmarshal(msg.Payload(), &message); err != nil { + return fmt.Errorf("error unmarshalling message: %s", string(msg.Payload())) + } + return handler(ctx, &message) +} + +func (p *pulsarClient) Close() { + for _, producer := range p.producers { + producer.Close() + } + p.conn.Close() +} + +func NewPulsar(cfg *pulsarConfig) (Client, error) { + coon, err := pulsar.NewClient(pulsar.ClientOptions{ + URL: cfg.URL, + ConnectionTimeout: time.Duration(cfg.ConnectionTimeout) * time.Second, + OperationTimeout: time.Duration(cfg.OperationTimeout) * time.Second, + }) + if err != nil { + return nil, err + } + return &pulsarClient{ + conn: coon, + ReconsumeTimeout: time.Second * time.Duration(cfg.ReconsumeTimeout), + producers: make(map[string]pulsar.Producer), + subscriber: make(map[string]pulsar.Consumer), + }, nil +} + +func buildMessage(ctx context.Context, body *proto.Body) ([]byte, error) { + header := make(propagation.MapCarrier) + otel.GetTextMapPropagator().Inject(ctx, &header) + msg := &proto.Message{ + Header: header, + Body: body, + } + return proto2.Marshal(msg) +} diff --git a/backend/core/model/connector.go b/backend/core/model/connector.go index fe450c44..cc5b304b 100644 --- a/backend/core/model/connector.go +++ b/backend/core/model/connector.go @@ -7,23 +7,34 @@ import ( "time" ) +const ( + CollectionTenant = "tenant_%s" + CollectionUser = "user_%s" + + StatusFailed = "failed" + StatusSuccess = "success" +) + type Connector struct { - tableName struct{} `pg:"connectors,omitempty"` - ID decimal.Decimal `json:"id,omitempty"` - CredentialID decimal.NullDecimal `json:"credential_id,omitempty"` - Name string `json:"name,omitempty"` - Source SourceType `json:"source,omitempty"` - InputType string `json:"input_type,omitempty"` - ConnectorSpecificConfig JSONMap `json:"connector_specific_config,omitempty"` - RefreshFreq int `json:"refresh_freq,omitempty"` - UserID uuid.UUID `json:"user_id,omitempty"` - TenantID uuid.UUID `json:"tenant_id,omitempty"` - Shared bool `json:"shared,omitempty" pg:",use_zero"` - Disabled bool `json:"disabled,omitempty" pg:",use_zero"` - LastSuccessfulIndexTime pg.NullTime `json:"last_successful_index_time,omitempty" pg:",use_zero"` - LastAttemptStatus string `json:"last_attempt_status,omitempty"` - TotalDocsIndexed int `json:"total_docs_indexed" pg:",use_zero"` - CreatedDate time.Time `json:"created_date,omitempty"` - UpdatedDate pg.NullTime `json:"updated_date,omitempty" pg:",use_zero"` - DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` + tableName struct{} `pg:"connectors,omitempty"` + ID decimal.Decimal `json:"id,omitempty"` + CredentialID decimal.NullDecimal `json:"credential_id,omitempty"` + Name string `json:"name,omitempty"` + Source SourceType `json:"source,omitempty"` + InputType string `json:"input_type,omitempty"` + ConnectorSpecificConfig JSONMap `json:"connector_specific_config,omitempty"` + RefreshFreq int `json:"refresh_freq,omitempty"` + UserID uuid.UUID `json:"user_id,omitempty"` + TenantID uuid.UUID `json:"tenant_id,omitempty"` + Shared bool `json:"shared,omitempty" pg:",use_zero"` + Disabled bool `json:"disabled,omitempty" pg:",use_zero"` + LastSuccessfulIndexTime pg.NullTime `json:"last_successful_index_time,omitempty" pg:",use_zero"` + LastAttemptStatus string `json:"last_attempt_status,omitempty"` + TotalDocsIndexed int `json:"total_docs_indexed" pg:",use_zero"` + CreatedDate time.Time `json:"created_date,omitempty"` + UpdatedDate pg.NullTime `json:"updated_date,omitempty" pg:",use_zero"` + DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` + Credential *Credential `json:"credential,omitempty" pg:"rel:has-one,fk:credential_id"` + Docs []*Document `json:"docs,omitempty" pg:"rel:has-many"` + DocsMap map[string]*Document `json:"docs_map,omitempty" pg:"-"` } diff --git a/backend/core/model/document.go b/backend/core/model/document.go index 53a5c147..2687c9b1 100644 --- a/backend/core/model/document.go +++ b/backend/core/model/document.go @@ -29,6 +29,9 @@ type Document struct { CreatedDate time.Time `json:"created_date,omitempty"` UpdatedDate pg.NullTime `json:"updated_date,omitempty" pg:",use_zero"` DeletedDate pg.NullTime `json:"deleted_date,omitempty" pg:",use_zero"` + IsExists bool `json:"is_exists,omitempty" pg:"-"` + IsUpdated bool `json:"is_updates,omitempty" pg:"-"` + Status string `json:"status,omitempty" pg:",use_zero"` } type DocumentResponse struct { diff --git a/backend/core/model/embedding.go b/backend/core/model/embedding.go new file mode 100644 index 00000000..a7fad372 --- /dev/null +++ b/backend/core/model/embedding.go @@ -0,0 +1,4 @@ +package model + +type Embedding struct { +} diff --git a/backend/core/model/messenging.go b/backend/core/model/messenging.go new file mode 100644 index 00000000..38809043 --- /dev/null +++ b/backend/core/model/messenging.go @@ -0,0 +1,11 @@ +package model + +const ( + TopicExecutor = "executor" + TopicUpdateConnector = "update-connector" + TopicEmbedding = "embedding" + + SubscriptionExecutor = "executor-subscription" + SubscriptionOrchestrator = "orchestrator-subscription" + SubscriptionEmbedding = "embedding-subscription" +) diff --git a/backend/core/model/trace.go b/backend/core/model/trace.go new file mode 100644 index 00000000..1f45f4f4 --- /dev/null +++ b/backend/core/model/trace.go @@ -0,0 +1,8 @@ +package model + +const ( + SpanAttributeConnectorID = "connector-id" + SpanAttributeConnectorSource = "connector-source" + + TracerConnector = "connector" +) diff --git a/backend/core/model/types.go b/backend/core/model/types.go index 1f16eeff..9eea10d0 100644 --- a/backend/core/model/types.go +++ b/backend/core/model/types.go @@ -41,7 +41,7 @@ func (j *JSONMap) ToStruct(dest interface{}) error { if err != nil { return err } - return json.Unmarshal(buf, &dest) + return json.Unmarshal(buf, dest) } func (j *JSONMap) FromStruct(src interface{}) error { diff --git a/backend/core/oauth/google.go b/backend/core/oauth/google.go index 76024c40..ecfb953b 100644 --- a/backend/core/oauth/google.go +++ b/backend/core/oauth/google.go @@ -40,7 +40,7 @@ func NewGoogleProvider(cfg *Config, redirectURL string) Proxy { ClientID: cfg.GoogleClientID, ClientSecret: cfg.GoogleSecret, Endpoint: google.Endpoint, - RedirectURL: fmt.Sprintf("%s/api/auth/google/callback", redirectURL), + RedirectURL: fmt.Sprintf("%s/google/callback", redirectURL), Scopes: []string{"https://www.googleapis.com/auth/userinfo.email", "https://www.googleapis.com/auth/userinfo.profile"}, }, diff --git a/backend/core/parameters/persona.go b/backend/core/parameters/persona.go index e708d3c9..6d21944f 100644 --- a/backend/core/parameters/persona.go +++ b/backend/core/parameters/persona.go @@ -1,11 +1,15 @@ package parameters import ( - "cognix.ch/api/v2/core/ai" "fmt" validation "github.com/go-ozzo/ozzo-validation/v4" + "github.com/sashabaranov/go-openai" ) +var SupportedModels = map[string]bool{ + openai.GPT3Dot5Turbo: true, +} + type PersonaParam struct { Name string `json:"name"` Description string `json:"description"` @@ -29,7 +33,7 @@ func (v PersonaParam) Validate() error { validation.Field(&v.Name, validation.Required), validation.Field(&v.ModelID, validation.Required, validation.By(func(value interface{}) error { - if _, ok := ai.SupportedModels[v.ModelID]; !ok { + if _, ok := SupportedModels[v.ModelID]; !ok { return fmt.Errorf("model %s not supported", v.ModelID) } return nil diff --git a/backend/core/proto/client.go b/backend/core/proto/client.go new file mode 100644 index 00000000..92256db4 --- /dev/null +++ b/backend/core/proto/client.go @@ -0,0 +1 @@ +package proto diff --git a/backend/core/proto/connector.pb.go b/backend/core/proto/connector.pb.go new file mode 100644 index 00000000..02ad6c4e --- /dev/null +++ b/backend/core/proto/connector.pb.go @@ -0,0 +1,933 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.12.4 +// source: core/proto/connector.proto + +package proto + +import ( + _ "github.com/golang/protobuf/ptypes/any" + empty "github.com/golang/protobuf/ptypes/empty" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Status int32 + +const ( + Status_SUCCESS Status = 0 + Status_FAILED Status = 1 +) + +// Enum value maps for Status. +var ( + Status_name = map[int32]string{ + 0: "SUCCESS", + 1: "FAILED", + } + Status_value = map[string]int32{ + "SUCCESS": 0, + "FAILED": 1, + } +) + +func (x Status) Enum() *Status { + p := new(Status) + *p = x + return p +} + +func (x Status) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Status) Descriptor() protoreflect.EnumDescriptor { + return file_core_proto_connector_proto_enumTypes[0].Descriptor() +} + +func (Status) Type() protoreflect.EnumType { + return &file_core_proto_connector_proto_enumTypes[0] +} + +func (x Status) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Status.Descriptor instead. +func (Status) EnumDescriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{0} +} + +type ResponseStep int32 + +const ( + ResponseStep_NO_ACTION ResponseStep = 0 + ResponseStep_LOAD ResponseStep = 1 + ResponseStep_EMBEDDING ResponseStep = 2 + ResponseStep_FINISH ResponseStep = 3 +) + +// Enum value maps for ResponseStep. +var ( + ResponseStep_name = map[int32]string{ + 0: "NO_ACTION", + 1: "LOAD", + 2: "EMBEDDING", + 3: "FINISH", + } + ResponseStep_value = map[string]int32{ + "NO_ACTION": 0, + "LOAD": 1, + "EMBEDDING": 2, + "FINISH": 3, + } +) + +func (x ResponseStep) Enum() *ResponseStep { + p := new(ResponseStep) + *p = x + return p +} + +func (x ResponseStep) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ResponseStep) Descriptor() protoreflect.EnumDescriptor { + return file_core_proto_connector_proto_enumTypes[1].Descriptor() +} + +func (ResponseStep) Type() protoreflect.EnumType { + return &file_core_proto_connector_proto_enumTypes[1] +} + +func (x ResponseStep) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ResponseStep.Descriptor instead. +func (ResponseStep) EnumDescriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{1} +} + +type Message struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Header map[string]string `protobuf:"bytes,1,rep,name=header,proto3" json:"header,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Body *Body `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` +} + +func (x *Message) Reset() { + *x = Message{} + if protoimpl.UnsafeEnabled { + mi := &file_core_proto_connector_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_core_proto_connector_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{0} +} + +func (x *Message) GetHeader() map[string]string { + if x != nil { + return x.Header + } + return nil +} + +func (x *Message) GetBody() *Body { + if x != nil { + return x.Body + } + return nil +} + +type Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Payload: + // + // *Body_Trigger + // *Body_Embedding + Payload isBody_Payload `protobuf_oneof:"payload"` +} + +func (x *Body) Reset() { + *x = Body{} + if protoimpl.UnsafeEnabled { + mi := &file_core_proto_connector_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Body) ProtoMessage() {} + +func (x *Body) ProtoReflect() protoreflect.Message { + mi := &file_core_proto_connector_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Body.ProtoReflect.Descriptor instead. +func (*Body) Descriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{1} +} + +func (m *Body) GetPayload() isBody_Payload { + if m != nil { + return m.Payload + } + return nil +} + +func (x *Body) GetTrigger() *TriggerRequest { + if x, ok := x.GetPayload().(*Body_Trigger); ok { + return x.Trigger + } + return nil +} + +func (x *Body) GetEmbedding() *EmbeddingRequest { + if x, ok := x.GetPayload().(*Body_Embedding); ok { + return x.Embedding + } + return nil +} + +type isBody_Payload interface { + isBody_Payload() +} + +type Body_Trigger struct { + Trigger *TriggerRequest `protobuf:"bytes,1,opt,name=trigger,proto3,oneof"` +} + +type Body_Embedding struct { + Embedding *EmbeddingRequest `protobuf:"bytes,2,opt,name=embedding,proto3,oneof"` +} + +func (*Body_Trigger) isBody_Payload() {} + +func (*Body_Embedding) isBody_Payload() {} + +type TriggerRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Params map[string]string `protobuf:"bytes,2,rep,name=params,proto3" json:"params,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *TriggerRequest) Reset() { + *x = TriggerRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_core_proto_connector_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TriggerRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TriggerRequest) ProtoMessage() {} + +func (x *TriggerRequest) ProtoReflect() protoreflect.Message { + mi := &file_core_proto_connector_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TriggerRequest.ProtoReflect.Descriptor instead. +func (*TriggerRequest) Descriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{2} +} + +func (x *TriggerRequest) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *TriggerRequest) GetParams() map[string]string { + if x != nil { + return x.Params + } + return nil +} + +type EmbeddingRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + EmbeddingModel string `protobuf:"bytes,3,opt,name=embedding_model,json=embeddingModel,proto3" json:"embedding_model,omitempty"` + Content []string `protobuf:"bytes,4,rep,name=content,proto3" json:"content,omitempty"` +} + +func (x *EmbeddingRequest) Reset() { + *x = EmbeddingRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_core_proto_connector_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddingRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddingRequest) ProtoMessage() {} + +func (x *EmbeddingRequest) ProtoReflect() protoreflect.Message { + mi := &file_core_proto_connector_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddingRequest.ProtoReflect.Descriptor instead. +func (*EmbeddingRequest) Descriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{3} +} + +func (x *EmbeddingRequest) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *EmbeddingRequest) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *EmbeddingRequest) GetEmbeddingModel() string { + if x != nil { + return x.EmbeddingModel + } + return "" +} + +func (x *EmbeddingRequest) GetContent() []string { + if x != nil { + return x.Content + } + return nil +} + +type EmbeddingResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId int64 `protobuf:"varint,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Payload []*EmbeddingResponse_Payload `protobuf:"bytes,3,rep,name=payload,proto3" json:"payload,omitempty"` +} + +func (x *EmbeddingResponse) Reset() { + *x = EmbeddingResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_core_proto_connector_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddingResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddingResponse) ProtoMessage() {} + +func (x *EmbeddingResponse) ProtoReflect() protoreflect.Message { + mi := &file_core_proto_connector_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddingResponse.ProtoReflect.Descriptor instead. +func (*EmbeddingResponse) Descriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{4} +} + +func (x *EmbeddingResponse) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *EmbeddingResponse) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *EmbeddingResponse) GetPayload() []*EmbeddingResponse_Payload { + if x != nil { + return x.Payload + } + return nil +} + +type TriggerResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DocumentId string `protobuf:"bytes,1,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + Url string `protobuf:"bytes,2,opt,name=url,proto3" json:"url,omitempty"` + Content string `protobuf:"bytes,4,opt,name=content,proto3" json:"content,omitempty"` + Status Status `protobuf:"varint,5,opt,name=status,proto3,enum=proto.Status" json:"status,omitempty"` + Signature string `protobuf:"bytes,6,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *TriggerResponse) Reset() { + *x = TriggerResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_core_proto_connector_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TriggerResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TriggerResponse) ProtoMessage() {} + +func (x *TriggerResponse) ProtoReflect() protoreflect.Message { + mi := &file_core_proto_connector_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TriggerResponse.ProtoReflect.Descriptor instead. +func (*TriggerResponse) Descriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{5} +} + +func (x *TriggerResponse) GetDocumentId() string { + if x != nil { + return x.DocumentId + } + return "" +} + +func (x *TriggerResponse) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *TriggerResponse) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *TriggerResponse) GetStatus() Status { + if x != nil { + return x.Status + } + return Status_SUCCESS +} + +func (x *TriggerResponse) GetSignature() string { + if x != nil { + return x.Signature + } + return "" +} + +type ConnectorStepResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + DocumentId int64 `protobuf:"varint,2,opt,name=document_id,json=documentId,proto3" json:"document_id,omitempty"` + Step ResponseStep `protobuf:"varint,3,opt,name=step,proto3,enum=proto.ResponseStep" json:"step,omitempty"` + Content string `protobuf:"bytes,4,opt,name=content,proto3" json:"content,omitempty"` +} + +func (x *ConnectorStepResponse) Reset() { + *x = ConnectorStepResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_core_proto_connector_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ConnectorStepResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnectorStepResponse) ProtoMessage() {} + +func (x *ConnectorStepResponse) ProtoReflect() protoreflect.Message { + mi := &file_core_proto_connector_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnectorStepResponse.ProtoReflect.Descriptor instead. +func (*ConnectorStepResponse) Descriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{6} +} + +func (x *ConnectorStepResponse) GetId() int64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *ConnectorStepResponse) GetDocumentId() int64 { + if x != nil { + return x.DocumentId + } + return 0 +} + +func (x *ConnectorStepResponse) GetStep() ResponseStep { + if x != nil { + return x.Step + } + return ResponseStep_NO_ACTION +} + +func (x *ConnectorStepResponse) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +type EmbeddingResponse_Payload struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Chunk int64 `protobuf:"varint,1,opt,name=chunk,proto3" json:"chunk,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` + Vector []float32 `protobuf:"fixed32,3,rep,packed,name=vector,proto3" json:"vector,omitempty"` +} + +func (x *EmbeddingResponse_Payload) Reset() { + *x = EmbeddingResponse_Payload{} + if protoimpl.UnsafeEnabled { + mi := &file_core_proto_connector_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmbeddingResponse_Payload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmbeddingResponse_Payload) ProtoMessage() {} + +func (x *EmbeddingResponse_Payload) ProtoReflect() protoreflect.Message { + mi := &file_core_proto_connector_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmbeddingResponse_Payload.ProtoReflect.Descriptor instead. +func (*EmbeddingResponse_Payload) Descriptor() ([]byte, []int) { + return file_core_proto_connector_proto_rawDescGZIP(), []int{4, 0} +} + +func (x *EmbeddingResponse_Payload) GetChunk() int64 { + if x != nil { + return x.Chunk + } + return 0 +} + +func (x *EmbeddingResponse_Payload) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *EmbeddingResponse_Payload) GetVector() []float32 { + if x != nil { + return x.Vector + } + return nil +} + +var File_core_proto_connector_proto protoreflect.FileDescriptor + +var file_core_proto_connector_proto_rawDesc = []byte{ + 0x0a, 0x1a, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x99, 0x01, 0x0a, 0x07, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x1a, 0x39, 0x0a, 0x0b, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7d, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x31, 0x0a, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x07, 0x74, 0x72, 0x69, 0x67, 0x67, + 0x65, 0x72, 0x12, 0x37, 0x0a, 0x09, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, + 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, + 0x52, 0x09, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x42, 0x09, 0x0a, 0x07, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x96, 0x01, 0x0a, 0x0e, 0x54, 0x72, 0x69, 0x67, 0x67, + 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x06, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x88, 0x01, 0x0a, 0x10, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, + 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6d, 0x62, 0x65, 0x64, + 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0e, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x4d, 0x6f, 0x64, 0x65, 0x6c, + 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0xd5, 0x01, 0x0a, 0x11, 0x45, + 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, + 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x3a, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x62, + 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x1a, + 0x51, 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, + 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x03, 0x28, 0x02, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x22, 0xa3, 0x01, 0x0a, 0x0f, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x6f, 0x63, + 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x15, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x65, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, + 0x69, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, + 0x74, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x04, 0x73, 0x74, 0x65, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x53, 0x74, 0x65, 0x70, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x12, 0x18, 0x0a, 0x07, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2a, 0x21, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0a, 0x0a, + 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x2a, 0x42, 0x0a, 0x0c, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x65, 0x70, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, + 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x4c, 0x4f, 0x41, 0x44, + 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x45, 0x4d, 0x42, 0x45, 0x44, 0x44, 0x49, 0x4e, 0x47, 0x10, + 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x49, 0x4e, 0x49, 0x53, 0x48, 0x10, 0x03, 0x32, 0x44, 0x0a, + 0x09, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x37, 0x0a, 0x03, 0x52, 0x75, + 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x22, 0x00, 0x32, 0x47, 0x0a, 0x09, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, + 0x12, 0x3a, 0x0a, 0x03, 0x52, 0x75, 0x6e, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x18, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, + 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x12, 0x5a, 0x10, + 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_core_proto_connector_proto_rawDescOnce sync.Once + file_core_proto_connector_proto_rawDescData = file_core_proto_connector_proto_rawDesc +) + +func file_core_proto_connector_proto_rawDescGZIP() []byte { + file_core_proto_connector_proto_rawDescOnce.Do(func() { + file_core_proto_connector_proto_rawDescData = protoimpl.X.CompressGZIP(file_core_proto_connector_proto_rawDescData) + }) + return file_core_proto_connector_proto_rawDescData +} + +var file_core_proto_connector_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_core_proto_connector_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_core_proto_connector_proto_goTypes = []interface{}{ + (Status)(0), // 0: proto.Status + (ResponseStep)(0), // 1: proto.ResponseStep + (*Message)(nil), // 2: proto.Message + (*Body)(nil), // 3: proto.Body + (*TriggerRequest)(nil), // 4: proto.TriggerRequest + (*EmbeddingRequest)(nil), // 5: proto.EmbeddingRequest + (*EmbeddingResponse)(nil), // 6: proto.EmbeddingResponse + (*TriggerResponse)(nil), // 7: proto.TriggerResponse + (*ConnectorStepResponse)(nil), // 8: proto.ConnectorStepResponse + nil, // 9: proto.Message.HeaderEntry + nil, // 10: proto.TriggerRequest.ParamsEntry + (*EmbeddingResponse_Payload)(nil), // 11: proto.EmbeddingResponse.Payload + (*empty.Empty)(nil), // 12: google.protobuf.Empty +} +var file_core_proto_connector_proto_depIdxs = []int32{ + 9, // 0: proto.Message.header:type_name -> proto.Message.HeaderEntry + 3, // 1: proto.Message.body:type_name -> proto.Body + 4, // 2: proto.Body.trigger:type_name -> proto.TriggerRequest + 5, // 3: proto.Body.embedding:type_name -> proto.EmbeddingRequest + 10, // 4: proto.TriggerRequest.params:type_name -> proto.TriggerRequest.ParamsEntry + 11, // 5: proto.EmbeddingResponse.payload:type_name -> proto.EmbeddingResponse.Payload + 0, // 6: proto.TriggerResponse.status:type_name -> proto.Status + 1, // 7: proto.ConnectorStepResponse.step:type_name -> proto.ResponseStep + 12, // 8: proto.Connector.Run:input_type -> google.protobuf.Empty + 5, // 9: proto.Embedding.Run:input_type -> proto.EmbeddingRequest + 12, // 10: proto.Connector.Run:output_type -> google.protobuf.Empty + 6, // 11: proto.Embedding.Run:output_type -> proto.EmbeddingResponse + 10, // [10:12] is the sub-list for method output_type + 8, // [8:10] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_core_proto_connector_proto_init() } +func file_core_proto_connector_proto_init() { + if File_core_proto_connector_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_core_proto_connector_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Message); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_core_proto_connector_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_core_proto_connector_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TriggerRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_core_proto_connector_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddingRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_core_proto_connector_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddingResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_core_proto_connector_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TriggerResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_core_proto_connector_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ConnectorStepResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_core_proto_connector_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmbeddingResponse_Payload); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_core_proto_connector_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*Body_Trigger)(nil), + (*Body_Embedding)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_core_proto_connector_proto_rawDesc, + NumEnums: 2, + NumMessages: 10, + NumExtensions: 0, + NumServices: 2, + }, + GoTypes: file_core_proto_connector_proto_goTypes, + DependencyIndexes: file_core_proto_connector_proto_depIdxs, + EnumInfos: file_core_proto_connector_proto_enumTypes, + MessageInfos: file_core_proto_connector_proto_msgTypes, + }.Build() + File_core_proto_connector_proto = out.File + file_core_proto_connector_proto_rawDesc = nil + file_core_proto_connector_proto_goTypes = nil + file_core_proto_connector_proto_depIdxs = nil +} diff --git a/backend/core/proto/connector.proto b/backend/core/proto/connector.proto new file mode 100644 index 00000000..3c82dbcb --- /dev/null +++ b/backend/core/proto/connector.proto @@ -0,0 +1,73 @@ +syntax="proto3"; +package proto; +option go_package = "core/proto;proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/any.proto"; + +service Connector { + rpc Run(google.protobuf.Empty) returns(google.protobuf.Empty) {} +} + +service Embedding { + rpc Run(EmbeddingRequest) returns(EmbeddingResponse) {} +} +message Message { + map header = 1; + Body body = 2; +} + +message Body { + oneof payload { + TriggerRequest_v1 trigger = 1; + EmbeddingRequest embedding = 2; + } +} + +message TriggerRequest_v1 { + int64 id = 1; + map params = 2; +} + +message EmbeddingRequest { + int64 document_id = 1; + string key = 2; + string embedding_model = 3; + repeated string content = 4; +} + +message EmbeddingResponse { + int64 document_id = 1; + string key = 2; + message Payload { + int64 chunk = 1; + string content = 2; + repeated float vector = 3; + }; + repeated Payload payload = 3; + +} +message TriggerResponse { + string document_id = 1; + string url = 2; + string content = 4; + Status status = 5; + string signature = 6; +} + +enum Status { + SUCCESS = 0; + FAILED = 1; +} +enum ResponseStep { + NO_ACTION = 0; + LOAD = 1; + EMBEDDING = 2; + FINISH = 3; + +} +message ConnectorStepResponse { + int64 id = 1; + int64 document_id = 2; + ResponseStep step = 3; + string content = 4; +} \ No newline at end of file diff --git a/backend/core/proto/connector_grpc.pb.go b/backend/core/proto/connector_grpc.pb.go new file mode 100644 index 00000000..399cb8b4 --- /dev/null +++ b/backend/core/proto/connector_grpc.pb.go @@ -0,0 +1,192 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.12.4 +// source: core/proto/connector.proto + +package proto + +import ( + context "context" + empty "github.com/golang/protobuf/ptypes/empty" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// ConnectorClient is the client API for Connector service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ConnectorClient interface { + Run(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) +} + +type connectorClient struct { + cc grpc.ClientConnInterface +} + +func NewConnectorClient(cc grpc.ClientConnInterface) ConnectorClient { + return &connectorClient{cc} +} + +func (c *connectorClient) Run(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/proto.Connector/Run", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ConnectorServer is the server API for Connector service. +// All implementations must embed UnimplementedConnectorServer +// for forward compatibility +type ConnectorServer interface { + Run(context.Context, *empty.Empty) (*empty.Empty, error) + mustEmbedUnimplementedConnectorServer() +} + +// UnimplementedConnectorServer must be embedded to have forward compatible implementations. +type UnimplementedConnectorServer struct { +} + +func (UnimplementedConnectorServer) Run(context.Context, *empty.Empty) (*empty.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method Run not implemented") +} +func (UnimplementedConnectorServer) mustEmbedUnimplementedConnectorServer() {} + +// UnsafeConnectorServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ConnectorServer will +// result in compilation errors. +type UnsafeConnectorServer interface { + mustEmbedUnimplementedConnectorServer() +} + +func RegisterConnectorServer(s grpc.ServiceRegistrar, srv ConnectorServer) { + s.RegisterService(&Connector_ServiceDesc, srv) +} + +func _Connector_Run_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ConnectorServer).Run(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Connector/Run", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ConnectorServer).Run(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// Connector_ServiceDesc is the grpc.ServiceDesc for Connector service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Connector_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "proto.Connector", + HandlerType: (*ConnectorServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Run", + Handler: _Connector_Run_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "core/proto/connector.proto", +} + +// EmbeddingClient is the client API for Embedding service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type EmbeddingClient interface { + Run(ctx context.Context, in *EmbeddingRequest, opts ...grpc.CallOption) (*EmbeddingResponse, error) +} + +type embeddingClient struct { + cc grpc.ClientConnInterface +} + +func NewEmbeddingClient(cc grpc.ClientConnInterface) EmbeddingClient { + return &embeddingClient{cc} +} + +func (c *embeddingClient) Run(ctx context.Context, in *EmbeddingRequest, opts ...grpc.CallOption) (*EmbeddingResponse, error) { + out := new(EmbeddingResponse) + err := c.cc.Invoke(ctx, "/proto.Embedding/Run", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// EmbeddingServer is the server API for Embedding service. +// All implementations must embed UnimplementedEmbeddingServer +// for forward compatibility +type EmbeddingServer interface { + Run(context.Context, *EmbeddingRequest) (*EmbeddingResponse, error) + mustEmbedUnimplementedEmbeddingServer() +} + +// UnimplementedEmbeddingServer must be embedded to have forward compatible implementations. +type UnimplementedEmbeddingServer struct { +} + +func (UnimplementedEmbeddingServer) Run(context.Context, *EmbeddingRequest) (*EmbeddingResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Run not implemented") +} +func (UnimplementedEmbeddingServer) mustEmbedUnimplementedEmbeddingServer() {} + +// UnsafeEmbeddingServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to EmbeddingServer will +// result in compilation errors. +type UnsafeEmbeddingServer interface { + mustEmbedUnimplementedEmbeddingServer() +} + +func RegisterEmbeddingServer(s grpc.ServiceRegistrar, srv EmbeddingServer) { + s.RegisterService(&Embedding_ServiceDesc, srv) +} + +func _Embedding_Run_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EmbeddingRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(EmbeddingServer).Run(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/proto.Embedding/Run", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(EmbeddingServer).Run(ctx, req.(*EmbeddingRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Embedding_ServiceDesc is the grpc.ServiceDesc for Embedding service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Embedding_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "proto.Embedding", + HandlerType: (*EmbeddingServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Run", + Handler: _Embedding_Run_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "core/proto/connector.proto", +} diff --git a/backend/core/repository/client.go b/backend/core/repository/client.go index 8daa4d43..9a711994 100644 --- a/backend/core/repository/client.go +++ b/backend/core/repository/client.go @@ -21,6 +21,9 @@ func NewDatabase(cfg *Config) (*pg.DB, error) { if cfg.DebugMode != "" { db.AddQueryHook(dbLogger{}) } + if err = db.Ping(context.Background()); err != nil { + return nil, err + } return db, nil } diff --git a/backend/core/repository/connector.go b/backend/core/repository/connector.go index a7adf2df..f561bd9f 100644 --- a/backend/core/repository/connector.go +++ b/backend/core/repository/connector.go @@ -11,17 +11,29 @@ import ( type ( ConnectorRepository interface { - GetAll(ctx context.Context, tenantID, userID uuid.UUID) ([]*model.Connector, error) - GetByID(ctx context.Context, tenantID, userID uuid.UUID, id int64) (*model.Connector, error) + GetActive(ctx context.Context) ([]*model.Connector, error) + GetAllByUser(ctx context.Context, tenantID, userID uuid.UUID) ([]*model.Connector, error) + GetByIDAndUser(ctx context.Context, tenantID, userID uuid.UUID, id int64) (*model.Connector, error) + GetByID(ctx context.Context, id int64) (*model.Connector, error) GetBySource(ctx context.Context, tenantID, userID uuid.UUID, source model.SourceType) (*model.Connector, error) Create(ctx context.Context, connector *model.Connector) error Update(ctx context.Context, connector *model.Connector) error + InvalidateConnector(ctx context.Context, connector *model.Connector) error } connectorRepository struct { db *pg.DB } ) +func (r *connectorRepository) InvalidateConnector(ctx context.Context, connector *model.Connector) error { + if _, err := r.db.WithContext(ctx).Model(&model.Document{}). + Set("status = ?", model.StatusPending). + Where("connector_id = ?", connector.ID.IntPart()).Update(); err != nil { + return utils.Internal.Wrap(err, "Error while invalidating connector") + } + return nil +} + func (r *connectorRepository) GetBySource(ctx context.Context, tenantID, userID uuid.UUID, source model.SourceType) (*model.Connector, error) { var connector model.Connector if err := r.db.WithContext(ctx).Model(&connector). @@ -40,7 +52,7 @@ func NewConnectorRepository(db *pg.DB) ConnectorRepository { return &connectorRepository{db: db} } -func (r *connectorRepository) GetAll(ctx context.Context, tenantID, userID uuid.UUID) ([]*model.Connector, error) { +func (r *connectorRepository) GetAllByUser(ctx context.Context, tenantID, userID uuid.UUID) ([]*model.Connector, error) { connectors := make([]*model.Connector, 0) if err := r.db.WithContext(ctx).Model(&connectors). Where("tenant_id = ?", tenantID). @@ -53,7 +65,7 @@ func (r *connectorRepository) GetAll(ctx context.Context, tenantID, userID uuid. return connectors, nil } -func (r *connectorRepository) GetByID(ctx context.Context, tenantID, userID uuid.UUID, id int64) (*model.Connector, error) { +func (r *connectorRepository) GetByIDAndUser(ctx context.Context, tenantID, userID uuid.UUID, id int64) (*model.Connector, error) { var connector model.Connector if err := r.db.WithContext(ctx).Model(&connector). Where("tenant_id = ?", tenantID). @@ -67,6 +79,21 @@ func (r *connectorRepository) GetByID(ctx context.Context, tenantID, userID uuid return &connector, nil } +func (r *connectorRepository) GetByID(ctx context.Context, id int64) (*model.Connector, error) { + var connector model.Connector + if err := r.db.WithContext(ctx).Model(&connector). + Relation("Docs"). + Where("id = ?", id). + First(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not load connector") + } + connector.DocsMap = make(map[string]*model.Document) + for _, doc := range connector.Docs { + connector.DocsMap[doc.DocumentID] = doc + } + return &connector, nil +} + func (r *connectorRepository) Create(ctx context.Context, connector *model.Connector) error { stm := r.db.WithContext(ctx).Model(connector) if !connector.CredentialID.Valid { @@ -84,3 +111,14 @@ func (r *connectorRepository) Update(ctx context.Context, connector *model.Conne } return nil } + +func (r *connectorRepository) GetActive(ctx context.Context) ([]*model.Connector, error) { + connectors := make([]*model.Connector, 0) + if err := r.db.WithContext(ctx). + Model(&connectors). + Where("disabled = false").Select(); err != nil { + return nil, utils.Internal.Wrap(err, "can not load connectors") + } + return connectors, nil + +} diff --git a/backend/core/repository/document.go b/backend/core/repository/document.go index 18267ec5..b0e99d17 100644 --- a/backend/core/repository/document.go +++ b/backend/core/repository/document.go @@ -6,19 +6,33 @@ import ( "context" "github.com/go-pg/pg/v10" "github.com/go-pg/pg/v10/orm" + "time" ) type ( DocumentRepository interface { - FindByConnectorID(ctx context.Context, user *model.User, connectorID int64) ([]*model.Document, error) + FindByConnectorIDAndUser(ctx context.Context, user *model.User, connectorID int64) ([]*model.Document, error) + FindByConnectorID(ctx context.Context, connectorID int64) ([]*model.Document, error) Create(ctx context.Context, document ...*model.Document) error + Update(ctx context.Context, document *model.Document) error } documentRepository struct { db *pg.DB } ) -func (r *documentRepository) FindByConnectorID(ctx context.Context, user *model.User, connectorID int64) ([]*model.Document, error) { +func (r *documentRepository) FindByConnectorID(ctx context.Context, connectorID int64) ([]*model.Document, error) { + documents := make([]*model.Document, 0) + if err := r.db.WithContext(ctx).Model(&documents). + Join("INNER JOIN connectors c ON c.id = connector_id"). + Where("connector_id = ?", connectorID). + Select(); err != nil { + return nil, utils.NotFound.Wrap(err, "can not find documents ") + } + return documents, nil +} + +func (r *documentRepository) FindByConnectorIDAndUser(ctx context.Context, user *model.User, connectorID int64) ([]*model.Document, error) { documents := make([]*model.Document, 0) if err := r.db.WithContext(ctx).Model(&documents). Join("INNER JOIN connectors c ON c.id = connector_id"). @@ -40,6 +54,14 @@ func (r *documentRepository) Create(ctx context.Context, document ...*model.Docu return nil } +func (r *documentRepository) Update(ctx context.Context, document *model.Document) error { + document.UpdatedDate = pg.NullTime{time.Now().UTC()} + if _, err := r.db.WithContext(ctx).Model(&document).Where("id = ? ", document.ID).Update(); err != nil { + return utils.Internal.Wrap(err, "can not insert document") + } + return nil +} + func NewDocumentRepository(db *pg.DB) DocumentRepository { return &documentRepository{db: db} } diff --git a/backend/core/repository/module.go b/backend/core/repository/module.go index 852d757d..2404fb8d 100644 --- a/backend/core/repository/module.go +++ b/backend/core/repository/module.go @@ -12,6 +12,11 @@ var DatabaseModule = fx.Options( return &cfg, err }, NewDatabase, + ), +) + +var RepositoriesModule = fx.Options( + fx.Provide( NewUserRepository, NewCredentialRepository, NewConnectorRepository, diff --git a/backend/core/storage/milvus.go b/backend/core/storage/milvus.go index bd6846bd..ce6355ef 100644 --- a/backend/core/storage/milvus.go +++ b/backend/core/storage/milvus.go @@ -3,39 +3,197 @@ package storage import ( "cognix.ch/api/v2/core/utils" "context" + "fmt" + validation "github.com/go-ozzo/ozzo-validation/v4" milvus "github.com/milvus-io/milvus-sdk-go/v2/client" + "github.com/milvus-io/milvus-sdk-go/v2/entity" "go.uber.org/fx" ) +const ( + ColumnNameID = "id" + ColumnNameDocumentID = "document_id" + ColumnNameChunk = "chunk" + ColumnNameContent = "content" + ColumnNameVector = "vector" + + VectorDimension = 1536 + + IndexStrategyDISKANN = "DISKANN" + IndexStrategyAUTOINDEX = "AUTOINDEX" + IndexStrategyNoIndex = "NOINDEX" +) + +var responseColumns = []string{ColumnNameID, ColumnNameDocumentID, ColumnNameChunk, ColumnNameContent} + type ( - Config struct { - Address string `env:"MILVUS_URL"` + MilvusConfig struct { + Address string `env:"MILVUS_URL"` + MetricType string `env:"MILVUS_METRIC_TYPE" envDefault:"L2"` + IndexStrategy string `env:"MILVUS_INDEX_STRATEGY" envDefault:"NOINDEX"` + } + MilvusPayload struct { + ID int64 `json:"id"` + DocumentID int64 `json:"document_id"` + Chunk int64 `json:"chunk"` + Content string `json:"content"` + Vector []float32 `json:"vector"` } MilvusClient interface { + CreateSchema(ctx context.Context, name string) error + Save(ctx context.Context, collection string, payloads ...*MilvusPayload) error + Load(ctx context.Context, collection string, vector []float32) ([]*MilvusPayload, error) } milvusClient struct { - client milvus.Client + client milvus.Client + MetricType entity.MetricType + IndexStrategy string } ) +func (v MilvusConfig) Validate() error { + return validation.ValidateStruct(&v, + validation.Field(&v.Address, validation.Required), + validation.Field(&v.IndexStrategy, validation.Required, + validation.In(IndexStrategyDISKANN, IndexStrategyAUTOINDEX, IndexStrategyNoIndex)), + validation.Field(&v.MetricType, validation.Required, + validation.In(string(entity.COSINE), string(entity.L2), string(entity.IP))), + ) +} + +func (c *milvusClient) Load(ctx context.Context, collection string, vector []float32) ([]*MilvusPayload, error) { + vs := []entity.Vector{entity.FloatVector(vector)} + sp, _ := entity.NewIndexFlatSearchParam() + result, err := c.client.Search(ctx, collection, []string{}, "", responseColumns, vs, ColumnNameVector, entity.L2, 10, sp) + if err != nil { + return nil, err + } + var payload []*MilvusPayload + for _, row := range result { + var pr MilvusPayload + if err = pr.FromResult(row); err != nil { + return nil, err + } + payload = append(payload, &pr) + } + return payload, nil +} + var MilvusModule = fx.Options( - fx.Provide(func() (*Config, error) { - cfg := Config{} - err := utils.ReadConfig(&cfg) - return &cfg, err + fx.Provide(func() (*MilvusConfig, error) { + cfg := MilvusConfig{} + if err := utils.ReadConfig(&cfg); err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, err + } + return &cfg, nil }, NewMilvusClient, ), ) -func NewMilvusClient(cfg *Config) (MilvusClient, error) { +func NewMilvusClient(cfg *MilvusConfig) (MilvusClient, error) { client, err := milvus.NewClient(context.Background(), milvus.Config{ Address: cfg.Address, }) if err != nil { return nil, err } - return milvusClient{ - client: client, + return &milvusClient{ + client: client, + MetricType: entity.MetricType(cfg.MetricType), + IndexStrategy: cfg.IndexStrategy, }, nil } + +func (c *milvusClient) Save(ctx context.Context, collection string, payloads ...*MilvusPayload) error { + var ids, documentIDs, chunks []int64 + var contents [][]byte + var vectors [][]float32 + + for _, payload := range payloads { + ids = append(ids, payload.ID) + documentIDs = append(documentIDs, payload.DocumentID) + chunks = append(chunks, payload.Chunk) + contents = append(contents, []byte(fmt.Sprintf(`{"content":"%s"}`, payload.Content))) + vectors = append(vectors, payload.Vector) + } + if _, err := c.client.Insert(ctx, collection, "", + entity.NewColumnInt64(ColumnNameID, ids), + entity.NewColumnInt64(ColumnNameDocumentID, documentIDs), + entity.NewColumnInt64(ColumnNameChunk, chunks), + entity.NewColumnJSONBytes(ColumnNameContent, contents), + entity.NewColumnFloatVector(ColumnNameVector, VectorDimension, vectors), + ); err != nil { + return err + } + return nil +} + +func (c *milvusClient) indexStrategy() (entity.Index, error) { + switch c.IndexStrategy { + case IndexStrategyAUTOINDEX: + return entity.NewIndexAUTOINDEX(c.MetricType) + case IndexStrategyDISKANN: + return entity.NewIndexDISKANN(c.MetricType) + } + return nil, fmt.Errorf("index strategy %s not supported yet", c.IndexStrategy) +} +func (c *milvusClient) CreateSchema(ctx context.Context, name string) error { + + collExists, err := c.client.HasCollection(ctx, name) + if err != nil { + return err + } + if collExists { + if err = c.client.DropCollection(ctx, name); err != nil { + return err + } + collExists = false + } + + if !collExists { + schema := entity.NewSchema().WithName(name). + WithField(entity.NewField().WithName(ColumnNameID).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)). + WithField(entity.NewField().WithName(ColumnNameDocumentID).WithDataType(entity.FieldTypeInt64)). + WithField(entity.NewField().WithName(ColumnNameChunk).WithDataType(entity.FieldTypeInt64)). + WithField(entity.NewField().WithName(ColumnNameContent).WithDataType(entity.FieldTypeJSON)). + WithField(entity.NewField().WithName(ColumnNameVector).WithDataType(entity.FieldTypeFloatVector).WithDim(1536)) + if err = c.client.CreateCollection(ctx, schema, 2, milvus.WithAutoID(true)); err != nil { + return err + } + + if c.IndexStrategy != IndexStrategyNoIndex { + indexStrategy, err := c.indexStrategy() + if err != nil { + return err + } + if err = c.client.CreateIndex(ctx, name, ColumnNameVector, indexStrategy, true); err != nil { + return err + } + } + } + return nil +} + +func (p *MilvusPayload) FromResult(res milvus.SearchResult) error { + var err error + for _, field := range res.Fields { + switch field.Name() { + case ColumnNameID: + p.ID, err = field.GetAsInt64(0) + case ColumnNameDocumentID: + p.DocumentID, err = field.GetAsInt64(0) + case ColumnNameChunk: + p.Chunk, err = field.GetAsInt64(0) + case ColumnNameContent: + p.Content, err = field.GetAsString(0) + } + if err != nil { + return err + } + } + return nil +} diff --git a/backend/core/storage/module.go b/backend/core/storage/module.go index 03086839..d6d6667b 100644 --- a/backend/core/storage/module.go +++ b/backend/core/storage/module.go @@ -1,6 +1,7 @@ package storage import ( + "cognix.ch/api/v2/core/utils" "go.uber.org/fx" ) @@ -16,11 +17,11 @@ import ( var MinioModule = fx.Options( fx.Provide( - // func() (*MinioConfig, error) { - // cfg := MinioConfig{} - // err := utils.ReadConfig(&cfg) - // return &cfg, err - //}, + func() (*MinioConfig, error) { + cfg := MinioConfig{} + err := utils.ReadConfig(&cfg) + return &cfg, err + }, NewMinIOMockClient, ), ) diff --git a/backend/core/storage/nutsdb.go b/backend/core/storage/nutsdb.go deleted file mode 100644 index de0f28e5..00000000 --- a/backend/core/storage/nutsdb.go +++ /dev/null @@ -1,79 +0,0 @@ -package storage - -import ( - "cognix.ch/api/v2/core/utils" - "github.com/nutsdb/nutsdb" -) - -const bucket = "internal-storage" - -type nutsDbStorage struct { - db *nutsdb.DB -} - -func NewNutsDbStorage(dbPath string) (Storage, error) { - db, err := nutsdb.Open( - nutsdb.DefaultOptions, - nutsdb.WithDir(dbPath), - ) - if err != nil { - return nil, err - } - if err = db.Update(func(tx *nutsdb.Tx) error { - if !tx.ExistBucket(nutsdb.DataStructureBTree, bucket) { - return tx.NewBucket(nutsdb.DataStructureBTree, bucket) - } - return nil - }); err != nil { - return nil, err - } - return &nutsDbStorage{db: db}, nil -} - -func (s *nutsDbStorage) GetValue(key string) ([]byte, error) { - var value []byte - var err error - if err = s.db.View(func(tx *nutsdb.Tx) error { - value, err = tx.Get(bucket, []byte(key)) - if err != nil { - return err - } - return nil - }); err != nil { - return nil, utils.NotFound.Wrap(err, "value does not exists") - } - return value, nil -} - -func (s *nutsDbStorage) Save(key string, value []byte) error { - if err := s.db.Update(func(tx *nutsdb.Tx) error { - return tx.Put(bucket, []byte(key), value, nutsdb.Persistent) - }); err != nil { - return utils.Internal.Wrap(err, "can not save value") - } - return nil -} - -func (s *nutsDbStorage) Delete(key string) error { - if err := s.db.Update(func(tx *nutsdb.Tx) error { - return tx.Delete(bucket, []byte(key)) - }); err != nil { - return utils.Internal.Wrap(err, "can not delete value") - } - return nil -} - -func (s *nutsDbStorage) Pull(key string) ([]byte, error) { - var value []byte - var err error - if err = s.db.Update(func(tx *nutsdb.Tx) error { - value, err = tx.Get(bucket, []byte(key)) - if err != nil { - return err - } - return tx.Delete(bucket, []byte(key)) - }); err != nil { - return nil, utils.Internal.Wrap(err, "can not save value") - } - return value, nil -} diff --git a/backend/go.mod b/backend/go.mod index bc97422e..8c39dcce 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,89 +3,133 @@ module cognix.ch/api/v2 go 1.22.1 require ( + github.com/apache/pulsar-client-go v0.12.1 github.com/caarlos0/env/v10 v10.0.0 + github.com/deluan/flowllm v0.0.0-20230502144710-1414e4b4985b github.com/gin-contrib/cors v1.7.0 github.com/gin-gonic/gin v1.9.1 github.com/go-ozzo/ozzo-validation/v4 v4.3.0 github.com/go-pg/pg/v10 v10.12.0 github.com/go-resty/resty/v2 v2.12.0 + github.com/gocolly/colly/v2 v2.1.0 github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/lib/pq v1.10.9 github.com/milvus-io/milvus-sdk-go/v2 v2.3.6 github.com/minio/minio-go/v7 v7.0.69 - github.com/nutsdb/nutsdb v1.0.4 + github.com/nats-io/nats.go v1.34.1 github.com/sashabaranov/go-openai v1.20.4 - github.com/stretchr/testify v1.8.4 + github.com/shopspring/decimal v1.4.0 + github.com/stretchr/testify v1.9.0 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.4 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/fx v1.21.0 go.uber.org/zap v1.27.0 - golang.org/x/oauth2 v0.16.0 - google.golang.org/grpc v1.62.1 + golang.org/x/net v0.24.0 + golang.org/x/oauth2 v0.19.0 + google.golang.org/grpc v1.63.2 + google.golang.org/protobuf v1.33.0 + jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 ) require ( - cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go/compute v1.24.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect + github.com/99designs/keyring v1.2.2 // indirect + github.com/AthenZ/athenz v1.11.57 // indirect + github.com/DataDog/zstd v1.5.5 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/antlabs/stl v0.0.1 // indirect - github.com/antlabs/timer v0.0.11 // indirect + github.com/PuerkitoBio/goquery v1.5.1 // indirect + github.com/andybalholm/cascadia v1.2.0 // indirect + github.com/antchfx/htmlquery v1.2.3 // indirect + github.com/antchfx/xmlquery v1.2.4 // indirect + github.com/antchfx/xpath v1.1.8 // indirect + github.com/ardielle/ardielle-go v1.5.2 // indirect github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect - github.com/bwmarrin/snowflake v0.3.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/bytedance/sonic v1.11.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect github.com/cockroachdb/redact v1.1.3 // indirect + github.com/danieljoos/wincred v1.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dvsekhvalnov/jose2go v1.7.0 // indirect + github.com/frankban/quicktest v1.14.6 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/getsentry/sentry-go v0.12.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/spec v0.20.4 // indirect - github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.19.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/gofrs/flock v0.8.1 // indirect + github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.6 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/linkedin/goavro/v2 v2.12.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect github.com/milvus-io/milvus-proto/go-api/v2 v2.3.5 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mtibben/percent v0.2.1 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.8.1 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.53.0 // indirect + github.com/prometheus/procfs v0.14.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/rs/xid v1.5.0 // indirect - github.com/shopspring/decimal v1.4.0 // indirect - github.com/tidwall/btree v1.6.0 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect + github.com/temoto/robotstxt v1.1.1 // indirect github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect @@ -97,24 +141,30 @@ require ( github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/xujiajun/mmap-go v1.0.1 // indirect - github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.18.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apimachinery v0.30.0 // indirect + k8s.io/client-go v0.30.0 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 // indirect mellium.im/sasl v0.3.1 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index bcc56084..4ad08f8e 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,31 +1,51 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= -cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= +github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= +github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AthenZ/athenz v1.11.57 h1:gCKt56CjOui9pQG3KGwO1LaIAghSodQf1fiLHv4aab0= +github.com/AthenZ/athenz v1.11.57/go.mod h1:4EKDZc6esPVHpBHXsT1OqaUnBABn3qTKgAm8L1QE2Vc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/antlabs/stl v0.0.1 h1:TRD3csCrjREeLhLoQ/supaoCvFhNLBTNIwuRGrDIs6Q= -github.com/antlabs/stl v0.0.1/go.mod h1:wvVwP1loadLG3cRjxUxK8RL4Co5xujGaZlhbztmUEqQ= -github.com/antlabs/timer v0.0.11 h1:z75oGFLeTqJHMOcWzUPBKsBbQAz4Ske3AfqJ7bsdcwU= -github.com/antlabs/timer v0.0.11/go.mod h1:JNV8J3yGvMKhCavGXgj9HXrVZkfdQyKCcqXBT8RdyuU= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= +github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= +github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= +github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= +github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4= +github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= +github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk= +github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/apache/pulsar-client-go v0.12.1 h1:jRA+VQKebVA4iIvojKUlkCeJ/R7oOxr/NXvwj+tNLkk= +github.com/apache/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk= +github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= +github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06GQ59hwDQAvmK1qxOQGB3WuVTRoY0okPTAv0= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= -github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= @@ -33,6 +53,8 @@ github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf5 github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= @@ -56,14 +78,22 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs= +github.com/danieljoos/wincred v1.2.1/go.mod h1:uGaFL9fDn3OLTvzCGulzE+SzjEe5NGlh5FdCcyfPwps= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deluan/flowllm v0.0.0-20230502144710-1414e4b4985b h1:LJYffjwlkEvIQH+E4aw2gEZzyTEDPRHC7eE1piLVBuw= +github.com/deluan/flowllm v0.0.0-20230502144710-1414e4b4985b/go.mod h1:Hrk6DnRn3dg1p2F4GMKCPh0v4kaxat4mU3ZD0/L0MdM= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= +github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= +github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -73,6 +103,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -96,6 +128,8 @@ github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-faker/faker/v4 v4.1.0 h1:ffuWmpDrducIUOO0QSKSF5Q2dxAht+dhsT9FvVHhPEI= github.com/go-faker/faker/v4 v4.1.0/go.mod h1:uuNc0PSRxF8nMgjGrrrU4Nw5cF30Jc6Kd0/FUTTYbhg= +github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= +github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -105,15 +139,18 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= github.com/go-pg/pg/v10 v10.12.0 h1:rBmfDDHTN7FQW0OemYmcn5UuBy6wkYWgh/Oqt1OBEB8= @@ -131,13 +168,20 @@ github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaC github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA= github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= +github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= +github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -147,6 +191,9 @@ github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -161,8 +208,11 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -170,10 +220,17 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20220829040838-70bd9ae97f40 h1:ykKxL12NZd3JmWZnyqarJGsF73M9Xhtrik/FEtEeFRE= +github.com/google/pprof v0.0.0-20220829040838-70bd9ae97f40/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -181,6 +238,13 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -192,6 +256,7 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -207,12 +272,14 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -222,8 +289,10 @@ github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgSh github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -234,11 +303,14 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/linkedin/goavro/v2 v2.12.0 h1:rIQQSj8jdAUlKQh6DttK8wCRv4t4QO09g1C4aBWXslg= +github.com/linkedin/goavro/v2 v2.12.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= @@ -249,6 +321,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= @@ -272,26 +346,37 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= +github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= +github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nutsdb/nutsdb v1.0.4 h1:BurzkxijXJY1/AkIXe1ek+U1ta3WGi6nJt4nCLqkxQ8= -github.com/nutsdb/nutsdb v1.0.4/go.mod h1:jIbbpBXajzTMZ0o33Yn5zoYIo3v0Dz4WstkVce+sYuQ= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= -github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -300,14 +385,26 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= +github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sashabaranov/go-openai v1.20.4 h1:095xQ/fAtRa0+Rj21sezVJABgKfGPNbyx/sAN/hJUmg= github.com/sashabaranov/go-openai v1.20.4/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= @@ -316,19 +413,29 @@ github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= +github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -336,18 +443,20 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo= github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= -github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= -github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= +github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= +github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -386,10 +495,6 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/xujiajun/mmap-go v1.0.1 h1:7Se7ss1fLPPRW+ePgqGpCkfGIZzJV6JPq9Wq9iv/WHc= -github.com/xujiajun/mmap-go v1.0.1/go.mod h1:CNN6Sw4SL69Sui00p0zEzcZKbt+5HtEnYUsc6BKKRMg= -github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 h1:w0si+uee0iAaCJO9q86T6yrhdadgcsoNuh47LrUykzg= -github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235/go.mod h1:MR4+0R6A9NS5IABnIM3384FfOq8QFVnm7WDrBOhIaMU= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= @@ -411,6 +516,8 @@ go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35 go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.21.0 h1:qqD6k7PyFHONffW5speYx403ywanuASqU4Rqdpc22XY= @@ -428,6 +535,7 @@ golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -435,9 +543,12 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -448,8 +559,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -459,10 +571,14 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -473,11 +589,12 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= +golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -487,10 +604,11 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -499,6 +617,7 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -512,19 +631,23 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -547,6 +670,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -554,15 +678,15 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -571,8 +695,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 h1:Xs9lu+tLXxLIfuci70nG4cpwaRC+mRQPUL7LoIeDJC4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -580,8 +704,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -592,6 +716,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= @@ -601,13 +726,15 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -616,6 +743,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -626,7 +754,23 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 h1:6YFJoB+0fUH6X3xU/G2tQqCYg+PkGtnZ5nMR5rpw72g= +jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:OxvTsCwKosqQ1q7B+8FwXqg4rKZ/UG9dUW+g/VL2xH4= +k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= +k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= +k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 h1:ao5hUqGhsqdm+bYbjH/pRkCs0unBGe9UyDahzs9zQzQ= +k8s.io/utils v0.0.0-20240423183400-0849a56e8f22/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/backend/orchestrator/main.go b/backend/orchestrator/main.go new file mode 100644 index 00000000..f3a41af4 --- /dev/null +++ b/backend/orchestrator/main.go @@ -0,0 +1,13 @@ +package main + +import ( + "cognix.ch/api/v2/core/utils" + "go.uber.org/fx" +) + +func main() { + utils.InitLogger(true) + app := fx.New(Module) + + app.Run() +} diff --git a/backend/orchestrator/module.go b/backend/orchestrator/module.go new file mode 100644 index 00000000..631c57b0 --- /dev/null +++ b/backend/orchestrator/module.go @@ -0,0 +1,33 @@ +package main + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/repository" + "context" + "go.uber.org/fx" +) + +var Module = fx.Options( + repository.DatabaseModule, + messaging.NatsModule, + fx.Provide( + repository.NewConnectorRepository, + NewCronTrigger, + NewServer, + ), + fx.Invoke(RunServer), +) + +func RunServer(lc fx.Lifecycle, server *Server) error { + + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return server.run(ctx) + }, + OnStop: func(ctx context.Context) error { + + return nil + }, + }) + return nil +} diff --git a/backend/orchestrator/server.go b/backend/orchestrator/server.go new file mode 100644 index 00000000..5007eaca --- /dev/null +++ b/backend/orchestrator/server.go @@ -0,0 +1,75 @@ +package main + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "cognix.ch/api/v2/core/repository" + "context" + "go.uber.org/zap" +) + +type Server struct { + connectorRepo repository.ConnectorRepository + messenger messaging.Client + scheduleTrigger Trigger +} + +func NewServer(connectorRepo repository.ConnectorRepository, + messenger messaging.Client, + scheduleTrigger Trigger) *Server { + return &Server{connectorRepo: connectorRepo, + messenger: messenger, + scheduleTrigger: scheduleTrigger} +} + +func (s *Server) run(ctx context.Context) error { + + zap.S().Infof("Start listener ...") + go s.listen(context.Background()) + return nil +} + +func (s *Server) onStart(ctx context.Context) error { + connectors, err := s.connectorRepo.GetActive(ctx) + if err != nil { + return err + } + for _, connector := range connectors { + if err = s.scheduleTrigger.Do(ctx, connector); err != nil { + zap.S().Errorf("run connector %d failed: %v", connector.ID, err) + } + } + return nil +} + +func (s *Server) listen(ctx context.Context) { + + if err := s.onStart(ctx); err != nil { + return + } + if err := s.messenger.Listen(ctx, model.TopicUpdateConnector, model.SubscriptionOrchestrator, s.handleTriggerRequest); err != nil { + zap.S().Errorf("failed to listen: %v", err) + } +} + +func (s *Server) handleTriggerRequest(ctx context.Context, msg *proto.Message) error { + trigger := msg.GetBody().GetTrigger() + if trigger == nil { + zap.S().Errorf("Received message with empty trigger") + return nil + } + if err := s.scheduleConnector(ctx, trigger); err != nil { + zap.S().Errorf("error scheduling connector[%d] : %v", trigger.GetId(), err) + return err + } + return nil +} + +func (s *Server) scheduleConnector(ctx context.Context, trigger *proto.TriggerRequest) error { + conn, err := s.connectorRepo.GetByID(ctx, trigger.GetId()) + if err != nil { + return err + } + return s.scheduleTrigger.Do(ctx, conn) +} diff --git a/backend/orchestrator/service-deployment.yaml b/backend/orchestrator/service-deployment.yaml new file mode 100644 index 00000000..aa1f8e46 --- /dev/null +++ b/backend/orchestrator/service-deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: orchestratorservice +spec: + replicas: 1 + selector: + matchLabels: + app: orchestratorservice + template: + metadata: + labels: + app: orchestratorservice + spec: + containers: + - name: orchestratorservice + image: cognixacr.azurecr.io/cognix/orchestratorservice:main + env: + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: env-configmap + key: DATABASE_URL + - name: MESSAGING_PROVIDER + value: "pulsar" + - name: PULSAR_URL + value: "pulsar://cognix-milvus-pulsar-proxy.milvus.svc.cluster.local:6650/" + imagePullPolicy: Always + imagePullSecrets: + - name: pullsecret + +--- +apiVersion: v1 +kind: Service +metadata: + name: orchestrator +spec: + selector: + app: orchestratorservice +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-volume + namespace: default +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: azurefile-csi \ No newline at end of file diff --git a/backend/orchestrator/trigger.go b/backend/orchestrator/trigger.go new file mode 100644 index 00000000..22c59e55 --- /dev/null +++ b/backend/orchestrator/trigger.go @@ -0,0 +1,51 @@ +package main + +import ( + "cognix.ch/api/v2/core/messaging" + "cognix.ch/api/v2/core/model" + "cognix.ch/api/v2/core/proto" + "context" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" + "time" +) + +const ( + ConnectorSchedulerSpan = "connector-scheduler" +) + +type ( + Trigger interface { + Do(ctx context.Context, conn *model.Connector) error + } + cronTrigger struct { + messenger messaging.Client + tracer trace.Tracer + } +) + +func (t *cronTrigger) Do(ctx context.Context, conn *model.Connector) error { + // if connector is new or + if conn.LastSuccessfulIndexTime.IsZero() || + conn.LastSuccessfulIndexTime.Add(time.Duration(conn.RefreshFreq)*time.Second).Before(time.Now()) { + ctx, span := t.tracer.Start(ctx, ConnectorSchedulerSpan) + span.SetAttributes(attribute.Int64(model.SpanAttributeConnectorID, conn.ID.IntPart())) + span.SetAttributes(attribute.String(model.SpanAttributeConnectorSource, string(conn.Source))) + zap.S().Infof("run connector %s", conn.ID) + trigger := &proto.TriggerRequest{ + Id: conn.ID.IntPart(), + } + return t.messenger.Publish(ctx, model.TopicExecutor, + &proto.Body{Payload: &proto.Body_Trigger{Trigger: trigger}}) + } + return nil +} + +func NewCronTrigger(messenger messaging.Client) Trigger { + return &cronTrigger{ + messenger: messenger, + tracer: otel.Tracer(model.TracerConnector), + } +} diff --git a/go_test.code-workspace b/coginix-services.code-workspace similarity index 100% rename from go_test.code-workspace rename to coginix-services.code-workspace diff --git a/config/chunker.env b/config/chunker.env new file mode 100644 index 00000000..e69de29b diff --git a/config/cocroach.env b/config/cockroach.env similarity index 72% rename from config/cocroach.env rename to config/cockroach.env index 4fdd021b..7affd96f 100644 --- a/config/cocroach.env +++ b/config/cockroach.env @@ -1 +1,2 @@ -DATABASE_URL=postgres://root:123@cockroach:26257/defaultdb?sslmode=disable \ No newline at end of file +DATABASE_URL=postgres://root:123@cockroach:26257/defaultdb?sslmode=disable +DB_DEBUG=true \ No newline at end of file diff --git a/config/embedder.env b/config/embedder.env new file mode 100644 index 00000000..e69de29b diff --git a/config/nats.env b/config/nats.env new file mode 100644 index 00000000..197bc8cc --- /dev/null +++ b/config/nats.env @@ -0,0 +1 @@ +NATS_URL=nats:4222 \ No newline at end of file diff --git a/config/pulsar.env b/config/pulsar.env new file mode 100644 index 00000000..a2de9b28 --- /dev/null +++ b/config/pulsar.env @@ -0,0 +1,3 @@ +PULSAR_URL=pulsar://broker:6650 +OPERATION_TIMEOUT=30 +CONNECTION_TIMEOUT=30 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 11c190c0..00000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,133 +0,0 @@ -services: - api: - image: cognix:api - build: - context: ./backend - args: - service: api - dockerfile: Dockerfile.dev - environment: - PORT: 8080 - STORAGE_PATH: /storage_volume/data - UI_URL: http://localhost:8080/ - ports: - - "8080:8080" - env_file: - - config/cocroach.env - - config/minio.env - - config/.env - volumes: - - ./backend:/backend - - "storage_volume:/storage_volume" - networks: - - dev-network - - nats: - image: nats - restart: on-failure - ports: - - "4222:4222" - networks: - - dev-network - - qdrant: - image: qdrant/qdrant:latest - restart: always - container_name: qdrant - ports: - - 6333:6333 - - 6334:6334 - expose: - - 6333 - - 6334 - - 6335 - configs: - - source: qdrant_config - target: /qdrant/config/production.yaml - volumes: - - "qdrant_data:/qdrant_data" - networks: - - dev-network - - cockroach: - image: cockroachdb/cockroach:latest-v23.2 - ports: - - "26257:26257" - - "18080:8080" - command: start-single-node --insecure - volumes: - - "db_volume:/cockroach/cockroach-data" - networks: - - dev-network - - migration: - image: cognix:migration - build: migration - volumes: - - ./migration/versions:/versions - env_file: - - config/cocroach.env - networks: - - dev-network - - etcd: - container_name: milvus-etcd - image: quay.io/coreos/etcd:v3.5.0 - environment: - - ETCD_AUTO_COMPACTION_MODE=revision - - ETCD_AUTO_COMPACTION_RETENTION=1000 - - ETCD_QUOTA_BACKEND_BYTES=4294967296 - volumes: - - milvus_etcd:/etcd - command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd - minio: - container_name: milvus-minio - image: minio/minio:latest - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - volumes: - - minio_data:/minio_data - command: minio server /minio_data - ports: - - "35723:35723" - - "9000:9000" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - milvus: - container_name: milvus-standalone - image: milvusdb/milvus:v2.0.2 - command: ["milvus", "run", "standalone"] - environment: - ETCD_ENDPOINTS: etcd:2379 - MINIO_ADDRESS: minio:9000 - volumes: - - milvus:/var/lib/milvus - ports: - - "19530:19530" - depends_on: - - "etcd" - - "minio" - - -configs: - qdrant_config: - content: | - log_level: INFO - -volumes: - db_volume: - storage_volume: - qdrant_data: - milvus_etcd: - minio_data: - milvus: - - -networks: - dev-network: - driver: bridge - name: dev-network \ No newline at end of file diff --git a/docker/docker-compose-cockroach.yaml b/docker/docker-compose-cockroach.yaml new file mode 100644 index 00000000..1bbfc4b2 --- /dev/null +++ b/docker/docker-compose-cockroach.yaml @@ -0,0 +1,17 @@ +version: '3.8' # Use Docker Compose file version 3.8 + +networks: + dev-network: + driver: bridge + +services: + cockroach: + image: cockroachdb/cockroach:latest-v23.2 + ports: + - "26257:26257" + - "28080:8080" + command: start-single-node --insecure + volumes: + - "../data/cockroach:/cockroach/cockroach-data" + networks: + - dev-network diff --git a/docker-compose-milvus.yml b/docker/docker-compose-milvus-minio.yml similarity index 64% rename from docker-compose-milvus.yml rename to docker/docker-compose-milvus-minio.yml index f80a72d8..cb75ad88 100644 --- a/docker-compose-milvus.yml +++ b/docker/docker-compose-milvus-minio.yml @@ -1,4 +1,8 @@ -version: '3.5' +version: '3.8' # Use Docker Compose file version 3.8 + +networks: + dev-network: + driver: bridge services: etcd: @@ -9,8 +13,10 @@ services: - ETCD_AUTO_COMPACTION_RETENTION=1000 - ETCD_QUOTA_BACKEND_BYTES=4294967296 volumes: - - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd + - "../data/etcd:/etcd" command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + networks: + - dev-network minio: container_name: milvus-minio @@ -19,13 +25,15 @@ services: MINIO_ACCESS_KEY: minioadmin MINIO_SECRET_KEY: minioadmin volumes: - - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data + - "../data/minio:/minio_data" command: minio server /minio_data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 30s timeout: 20s retries: 3 + networks: + - dev-network standalone: container_name: milvus-standalone @@ -35,13 +43,23 @@ services: ETCD_ENDPOINTS: etcd:2379 MINIO_ADDRESS: minio:9000 volumes: - - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus + - "../data/milvus:/var/lib/milvus" ports: - "19530:19530" depends_on: - "etcd" - "minio" - -networks: - default: - name: milvus + networks: + - dev-network + + milvus-admin: + image: milvusdb/milvus-insight:latest + environment: + HOST_URL: http://localhost:8000 + MILVUS_URL: host.docker.internal:19530 + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "8000:3000" + networks: + - dev-network diff --git a/docker/docker-compose-nats.yaml b/docker/docker-compose-nats.yaml new file mode 100644 index 00000000..7eae8cd6 --- /dev/null +++ b/docker/docker-compose-nats.yaml @@ -0,0 +1,31 @@ +version: '3.8' # Use Docker Compose file version 3.8 + +networks: + dev-network: + driver: bridge + +services: + nats: + image: nats:latest # Use the latest NATS image + container_name: nats + ports: + - "4222:4222" # Client connections to NATS + - "8222:8222" # HTTP monitoring port for NATS + - "6222:6222" # Cluster port for NATS + volumes: + - nats-storage:/data/nats + command: + - "--name=nats" + - "--http_port=8222" + - "--js" + - "--sd=/data/nats" + networks: + - dev-network + +volumes: + nats-storage: + driver: local + driver_opts: + type: none + o: bind + device: ../data/nats diff --git a/docker/docker-compose-services-ai.yaml b/docker/docker-compose-services-ai.yaml new file mode 100644 index 00000000..9efe681b --- /dev/null +++ b/docker/docker-compose-services-ai.yaml @@ -0,0 +1,27 @@ +version: '3.8' # Use Docker Compose file version 3.8 +networks: + dev-net: + driver: bridge + +services: + #embedder + embedder: + # build: . + image: embedder:latest + container_name: embedder + ports: + - "50051:50051" + volumes: + - ../data//models + env_file: + - ../config/embedder.env + networks: + - dev-net + # depends_on: + # nats-server: + # condition: service_healthy + environment: + #PULSAR_CONNECTION_STRING: ${PULSAR_CONNECTION_STRING} + PULSAR_CONNECTION_STRING: "pulsar://broker:6650" + #PULSAR_CONNECTION_STRING: "pulsar://127.0.0.1:6650" + \ No newline at end of file diff --git a/docker/docker-compose-services.yaml b/docker/docker-compose-services.yaml new file mode 100644 index 00000000..71ad1c10 --- /dev/null +++ b/docker/docker-compose-services.yaml @@ -0,0 +1,82 @@ +version: '3.5' + +services: + api: + image: cognix:api + build: + context: ../backend + args: + service: api + dockerfile: Dockerfile.dev + environment: + PORT: 8080 + STORAGE_PATH: /storage_volume/data + UI_URL: http://localhost:8080/ + ports: + - "8080:8080" + env_file: + - ../config/cockroach.env + - ../config/minio.env + - ../config/.env + volumes: + - ../backend:/backend + networks: + - dev-network + + orchestrator: + image: cognix:orchestrator + build: + context: ../backend + args: + service: orchestrator + dockerfile: Dockerfile + env_file: + - ../config/cockroach.env + - ../config/nats.env + - ../config/pulsar.env + - ../config/.env + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ../backend:/backend + networks: + - dev-network + + connector: + image: cognix:connector + build: + context: ../backend + args: + service: connector + dockerfile: Dockerfile.dev + env_file: + - ../config/cockroach.env + - ../config/nats.env + - ../config/.env + environment: + MILVUS_URL: localhost:19530 + volumes: + - ../backend:/backend + networks: + - dev-network + + migration: + image: cognix:migration + build: migration + volumes: + - ../migration/versions:/versions + env_file: + - ../config/cockroach.env + networks: + - dev-network + + + +volumes: + db_volume: + nats-storage: + +networks: + dev-network: + driver: bridge + name: dev-network \ No newline at end of file diff --git a/docs/chat-flow.md b/docs/chat-flow.md new file mode 100644 index 00000000..4447768e --- /dev/null +++ b/docs/chat-flow.md @@ -0,0 +1,34 @@ +The user can interact with CogniX, start a new chat in two different ways (in the future will be expanded) + +1. Chat with your documents (requires_llm = true) +2. Search only (requires requires_llm=false) + + +The user asks a question (query): "How do I do get attention in Collaboard" + +Server receives the chat + +following are business logic methods: + +method name strem_response +argument query (string) +returns the stream + +strem_response calls vector_search + method name vector_search + argument query (string) + returns a list of documents + + It performs a query against Milvus, with userid and tennantid + The query result will contian all the matching documents inside Milvus. + +if equires_llm == true + It retrives the information regarding Persona, prompt and llm + + Top X documents retrieved from Milvus query will be sent to the LLM (LLM Endpoint + eventually api_key) embedded into the prompt + + LLM will stream the answer which will be streamed back to the client +if equires_llm == false + will send back only the list of documents + +Chat and response is saved into chat history \ No newline at end of file diff --git a/migration/versions/20240506092247_alter_document.sql b/migration/versions/20240506092247_alter_document.sql new file mode 100644 index 00000000..89d2ab2f --- /dev/null +++ b/migration/versions/20240506092247_alter_document.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE documents ADD COLUMN IF NOT EXISTS status VARCHAR(100) NOT NULL DEFAULT 'new'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE documents DROP COLUMN IF EXISTS status; +-- +goose StatementEnd diff --git a/workers/pdf/split/.DS_Store b/proto/.DS_Store similarity index 100% rename from workers/pdf/split/.DS_Store rename to proto/.DS_Store diff --git a/proto/Makefile b/proto/Makefile new file mode 100644 index 00000000..01a81bd2 --- /dev/null +++ b/proto/Makefile @@ -0,0 +1,8 @@ + +gen-proto-go: + rm -f ../backend/core/proto/*.pb.go + protoc -I=. -I=vendor -I=${GOPATH}/src *.proto --go_out=.. --go-grpc_out=.. + +gen-proto-python: + rm -f ai/core/proto/*.pb.py + protoc -I=. *.proto --python_out=. --python-grpc_out=. \ No newline at end of file diff --git a/proto/chunking_data.proto b/proto/chunking_data.proto new file mode 100644 index 00000000..438c3fb8 --- /dev/null +++ b/proto/chunking_data.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package com.embedd; +option go_package = "backend/core/proto;proto"; + + +enum FileType { + URL = 0; + PDF = 1; + RTF = 2; + DOC = 3; + XLS = 4; + PPT = 5; + TXT = 6; + MD = 7; + // add all supported file that in another document + // check what with Google docs +}; + +message ChunkingData { +// This is url where the file is located. +// Based on the chunking type it will be a WEB URL (HTML type) +// Will be an S3/MINIO link with a proper authentication in case of a file + string url = 1; + int64 document_id = 2; + FileType file_type = 3; +} \ No newline at end of file diff --git a/proto/connector_messages.proto b/proto/connector_messages.proto new file mode 100644 index 00000000..e909a2a0 --- /dev/null +++ b/proto/connector_messages.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package com.embedd; +option go_package = "backend/core/proto;proto"; + + +message ConnectorRequest { + int64 id = 1; + map params = 2; +} + +message EmbeddAsyncRequest { + int64 document_id = 1; + int64 chunk_id = 2; + string content = 3; + string model_id = 4; +} + +message EmbeddAsyncResponse { + int64 document_id = 1; + int64 chunk_id = 2; + string content = 3; + repeated float vector = 4; +} \ No newline at end of file diff --git a/proto/embed_data.proto b/proto/embed_data.proto new file mode 100644 index 00000000..3e56c044 --- /dev/null +++ b/proto/embed_data.proto @@ -0,0 +1,12 @@ +// this is used in NATS +syntax = "proto3"; + +package com.embedd; + +message EmbeddData { + int64 document_id = 1; + string key = 2; + string model = 3; + string content = 4; + repeated float vector = 5; +} \ No newline at end of file diff --git a/proto/embed_requests.proto b/proto/embed_requests.proto new file mode 100644 index 00000000..34e2d9c3 --- /dev/null +++ b/proto/embed_requests.proto @@ -0,0 +1,15 @@ +// this is used for the gRPC service +syntax = "proto3"; + +package com.embedd; +option go_package = "backend/core/proto;proto"; + +message EmbeddRequest { + string content = 1; + string model = 2; +} + +message EmbeddResponse { + repeated float vector = 1; +} + diff --git a/proto/embed_service.proto b/proto/embed_service.proto new file mode 100644 index 00000000..10c53b0f --- /dev/null +++ b/proto/embed_service.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package com.embedd; +option go_package = "backend/core/proto;proto"; + +// Import the messages from the other file +import "embed_requests.proto"; + +service EmbedService { + rpc GetEmbedd (EmbedRequest) returns (EmbedResponse) {} +} diff --git a/proto/messeging_data.proto b/proto/messeging_data.proto new file mode 100644 index 00000000..677c593e --- /dev/null +++ b/proto/messeging_data.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package com.embedd; +option go_package = "backend/core/proto;proto"; +import "connector_messages.proto"; +import "chunking_data.proto"; + +message Message { + map header = 1; + Body body = 2; +} + +message Body { + oneof payload { + ConnectorRequest trigger = 1; + ChunkingData chunking = 2; + } +} diff --git a/workers/embedder/Dockerfile b/workers/embedder/Dockerfile deleted file mode 100644 index ed976289..00000000 --- a/workers/embedder/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM continuumio/miniconda3 - -WORKDIR /app - -# Copy your application code -COPY . /app - -# Create and activate the Conda environment -# Environment.yml should be in your application's root directory -COPY environment.yml /app/environment.yml -RUN conda env create -f environment.yml - -# Make RUN commands use the new environment: -SHELL ["conda", "run", "-n", "testenv", "/bin/bash", "-c"] - -# Ensure the script is executable and run the script -RUN chmod +x test.py -CMD ["conda", "run", "--no-capture-output", "-n", "testenv", "python", "test.py"] diff --git a/workers/embedder/big.pdf b/workers/embedder/big.pdf deleted file mode 100644 index b87a7f02..00000000 Binary files a/workers/embedder/big.pdf and /dev/null differ diff --git a/workers/embedder/environment.yml b/workers/embedder/environment.yml deleted file mode 100644 index 0e3b99b7..00000000 --- a/workers/embedder/environment.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: testenv -channels: - - conda-forge -dependencies: - - bzip2=1.0.8 - - ca-certificates=2024.2.2 - - libexpat=2.6.1 - - libffi=3.4.2 - - libsqlite=3.45.1 - - libzlib=1.2.13 - - ncurses=6.4 - - openssl=3.2.1 - - pip=24.0 - - python=3.12.2 - - readline=8.2 - - setuptools=69.1.1 - - tk=8.6.13 - - tzdata=2024a - - wheel=0.42.0 - - xz=5.2.6 diff --git a/workers/embedder/objects.py b/workers/embedder/objects.py deleted file mode 100644 index 2eea620f..00000000 --- a/workers/embedder/objects.py +++ /dev/null @@ -1,6 +0,0 @@ -from enum import Enum - - -class Strategy(str, Enum): - hi_res = "hi_res" # Strategy for analyzing PDFs and extracting table structure - fast = "fast" \ No newline at end of file diff --git a/workers/embedder/pdf.py b/workers/embedder/pdf.py deleted file mode 100644 index cae1c865..00000000 --- a/workers/embedder/pdf.py +++ /dev/null @@ -1,34 +0,0 @@ -# https://unstructured.io/blog/mastering-table-extraction-revolutionize-your-earnings-reports-analysis-with-ai -import os -from unstructured.partition.pdf import partition_pdf -from unstructured.staging.base import elements_to_json - -filename = "pdf/Q3FY24-CFO-Commentary.pdf" # For this notebook I uploaded Nvidia's earnings into the Files directory called "/content/" -output_dir = "pdf" -text_file = "pdf/nvidia-yolox.txt" - -# Define parameters for Unstructured's library -strategy = "hi_res" -model_name = "yolox" # Best model for table extraction. Other options are detectron2_onnx and chipper depending on file layout - -# Extracts the elements from the PDF -elements_hi_res = partition_pdf( - filename=filename, - strategy=Strategy.hi_res, - infer_table_structure=True, - model_name=model_name, - chunking_strategy="by_title", - multipage_sections=True, - combine_text_under_n_chars=100, - new_after_n_chars=500, - max_characters = 800, - overlap = 50 - ) - - - - -elements_fast = partition_pdf( - filename=filename, - strategy=Strategy.fast - ) \ No newline at end of file diff --git a/workers/embedder/requirements.txt b/workers/embedder/requirements.txt deleted file mode 100644 index 4b6c083c..00000000 --- a/workers/embedder/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -onnxruntime==1.17.1 -# pip install onnxruntime-silicon #mac -'unstructured[all-docs]' \ No newline at end of file diff --git a/workers/embedder/small.pdf b/workers/embedder/small.pdf deleted file mode 100644 index c1c12fbd..00000000 Binary files a/workers/embedder/small.pdf and /dev/null differ diff --git a/workers/embedder/test.py b/workers/embedder/test.py deleted file mode 100644 index 2a20dae0..00000000 --- a/workers/embedder/test.py +++ /dev/null @@ -1,25 +0,0 @@ -import sys - -def main(): - print("Starting process..cc.") - # Your code here, e.g., processing data, running a task, etc. - print("Process completed.") - -if __name__ == "__main__": - main() - -# print("before import") -# import math - -# print("before function_a") -# def function_a(): -# print("Function A") - -# print("before function_b") -# def function_b(): -# print("Function B {}".format(math.sqrt(100))) - -# print("before __name__ guard") -# if __name__ == '__main__': -# function_a() -# function_b() \ No newline at end of file diff --git a/workers/embedder/unstructured.yml b/workers/embedder/unstructured.yml deleted file mode 100644 index c1de9d6e..00000000 --- a/workers/embedder/unstructured.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: unstructured - -channels: - - defaults - - anaconda - - conda-forge - - pytorch - -dependencies: - - python=3.10 - - pytorch=2.1.2 - - pywin32 - - poppler - - torchvision - - pip: - - huggingface-hub - - layoutparser diff --git a/workers/pdf/Q3FY24-CFO-Commentary.pdf b/workers/pdf/Q3FY24-CFO-Commentary.pdf deleted file mode 100644 index be14ea28..00000000 Binary files a/workers/pdf/Q3FY24-CFO-Commentary.pdf and /dev/null differ diff --git a/workers/pdf/Q3FY24-CFO-Commentary.pdf.json b/workers/pdf/Q3FY24-CFO-Commentary.pdf.json deleted file mode 100644 index 1ee56f02..00000000 --- a/workers/pdf/Q3FY24-CFO-Commentary.pdf.json +++ /dev/null @@ -1,3281 +0,0 @@ -[ - { - "element_id": "675edd423a52405b82a3b97712c1bded", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 1058.3, - 190.4 - ], - [ - 1058.3, - 304.3 - ], - [ - 1575.0, - 304.3 - ], - [ - 1575.0, - 190.4 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1 - }, - "text": "@ NVIDIA.", - "type": "Image" - }, - { - "element_id": "4916b5ec8555709b4a31296b1dde2d4e", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 330.6, - 343.6 - ], - [ - 330.6, - 386.9 - ], - [ - 1368.1, - 386.9 - ], - [ - 1368.1, - 343.6 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.6262, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1 - }, - "text": "CFO Commentary on Third Quarter Fiscal 2024 Results", - "type": "Title" - }, - { - "element_id": "619a8414f256311f985950ff4598ec0e", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 425.4 - ], - [ - 125.0, - 458.7 - ], - [ - 488.8, - 458.7 - ], - [ - 488.8, - 425.4 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.80986, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1 - }, - "text": "Q3 Fiscal 2024 Summary", - "type": "Title" - }, - { - "element_id": "f277484faa5812d84c173f6c815d9b4f", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 807.7, - 491.1 - ], - [ - 807.7, - 523.3 - ], - [ - 893.9, - 523.3 - ], - [ - 893.9, - 491.1 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.32296, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1, - "parent_id": "619a8414f256311f985950ff4598ec0e" - }, - "text": "GAAP", - "type": "FigureCaption" - }, - { - "element_id": "d62da554b08fad6b364e6915ad5a79b4", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 128.6, - 501.3 - ], - [ - 128.6, - 850.0 - ], - [ - 1581.4, - 850.0 - ], - [ - 1581.4, - 501.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.91759, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1, - "parent_id": "619a8414f256311f985950ff4598ec0e", - "text_as_html": "
fin millions, except earnings perQ3 FY24Q2 FY24Q3 FY23a/aYIY
Revenue$18,120$13,507$5,931Up 34%Up 206%
Gross margin740%70.1%53.6 %Up 3.9 ptsUp 20.4 pts
Operating expenses$2,983$2,662$2,576Up 12%Up 16%
Operating income$10,417$6,800$601Up 53%Up 1,633%
Net income$9,243$6,188$680Up 49%Up 1,259%
Diluted earnings per share$3.71$2.48$0.27Up 50%Up 1,274%
" - }, - "text": "($ in millions, except earnings per share) Q3 FY24 Q2 FY24 Q3 FY23 Q/Q Y/Y Revenue $18,120 $13,507 $5,931 Up 34% Up 206% Gross margin 74.0 % 70.1 % 53.6 % Up 3.9 pts Up 20.4 pts Operating expenses $2,983 $2,662 $2,576 Up 12% Up 16% Operating income $10,417 $6,800 $601 Up 53% Up 1,633% Net income $9,243 $6,188 $680 Up 49% Up 1,259% Diluted earnings per share $3.71 $2.48 $0.27 Up 50% Up 1,274%", - "type": "Table" - }, - { - "element_id": "a51db9aef3cea51a9267b55669589b44", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 771.1, - 900.2 - ], - [ - 771.1, - 933.0 - ], - [ - 930.2, - 933.0 - ], - [ - 930.2, - 900.2 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.45468, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1, - "parent_id": "619a8414f256311f985950ff4598ec0e" - }, - "text": "Non-GAAP", - "type": "FigureCaption" - }, - { - "element_id": "6628a726ae4af2b69e6310f0ccededd0", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 126.5, - 914.3 - ], - [ - 126.5, - 1255.6 - ], - [ - 1584.2, - 1255.6 - ], - [ - 1584.2, - 914.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.9252, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1, - "parent_id": "619a8414f256311f985950ff4598ec0e", - "text_as_html": "
in millions, except earnings perQ3 FY24Q2 FY24Q3 FY23a/aYIY
Revenue$18,120$13,507$5,931Up 34%Up 206%
Gross margin75.0%71.2%56.1%Up 3.8 ptsUp 18.9 pts
Operating expenses$2,026$1,838$1,793Up 10%Up 13%
Operating income$11,557$7,776$1,536Up 49%Up 652%
Net income$10,020$6,740$1,456Up 49%Up 588%
Diluted earnings per share$4.02$2.70$0.58Up 49%Up 593%
" - }, - "text": "($ in millions, except earnings per share) Q3 FY24 Q2 FY24 Q3 FY23 Q/Q Y/Y Revenue $18,120 $13,507 $5,931 Up 34% Up 206% Gross margin 75.0 % 71.2 % 56.1 % Up 3.8 pts Up 18.9 pts Operating expenses $2,026 $1,838 $1,793 Up 10% Up 13% Operating income $11,557 $7,776 $1,536 Up 49% Up 652% Net income $10,020 $6,740 $1,456 Up 49% Up 588% Diluted earnings per share $4.02 $2.70 $0.58 Up 49% Up 593%", - "type": "Table" - }, - { - "element_id": "5c254415713b25430c3f69ce7dd8d165", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 604.5, - 1317.9 - ], - [ - 604.5, - 1351.1 - ], - [ - 1095.7, - 1351.1 - ], - [ - 1095.7, - 1317.9 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.30233, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1 - }, - "text": "Revenue by Reportable Segments", - "type": "Title" - }, - { - "element_id": "714117df3565872e02c4a4e58cb3fd46", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.1, - 1337.3 - ], - [ - 125.1, - 1523.2 - ], - [ - 1560.1, - 1523.2 - ], - [ - 1560.1, - 1337.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.89404, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1, - "parent_id": "5c254415713b25430c3f69ce7dd8d165", - "text_as_html": "
($ in millions)Q3 FY24Q2 FY24Q3 FY23a/a7
Compute & Networking$14,645$10,402$3,816Up 41%Up 284%
Graphics3,4753,1052,115Up 12%Up 649
Total$18,120$13,507$5,931Up 34%Up 206%
" - }, - "text": "($ in millions) Q3 FY24 Q2 FY24 Q3 FY23 Q/Q Compute & Networking $14,645 $10,402 $3,816 Up 41% Graphics 3,475 3,105 2,115 Up 12% Total $18,120 $13,507 $5,931 Up 34%", - "type": "Table" - }, - { - "element_id": "16bb55bb7ed91deb6715ab7e4542505f", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 640.3, - 1586.5 - ], - [ - 640.3, - 1617.1 - ], - [ - 1059.8, - 1617.1 - ], - [ - 1059.8, - 1586.5 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1 - }, - "text": "Revenue by Market Platform", - "type": "Title" - }, - { - "element_id": "1fc5e68e4074570ccdeed79feabd3066", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 122.1, - 1604.5 - ], - [ - 122.1, - 1921.3 - ], - [ - 1584.6, - 1921.3 - ], - [ - 1584.6, - 1604.5 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.91732, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 1, - "parent_id": "16bb55bb7ed91deb6715ab7e4542505f", - "text_as_html": "
($ in millions)Q3 FY24Q2 FY24Q3 FY23Q/QYIY
Data Center$14,514$10,323$3,833Up 41%Up 279%
Gaming2,8562,4861,574Up 15%Up 81%
Professional Visualization416379200Up 10%Up 108%
Automotive261253251Up 3%Up 4%
OEM and Other736673Up 11%
Total$18,120$13,507$5,931Up 34%Up 206%
" - }, - "text": "($ in millions) Q3 FY24 Q2 FY24 Q3 FY23 Q/Q Y/Y Data Center $14,514 $10,323 $3,833 Up 41% Up 279% Gaming 2,856 2,486 1,574 Up 15% Up 81% Professional Visualization 416 379 200 Up 10% Up 108% Automotive 261 253 251 Up 3% Up 4% OEM and Other 73 66 73 Up 11% \u2014 Total $18,120 $13,507 $5,931 Up 34% Up 206%", - "type": "Table" - }, - { - "element_id": "bf9686f2a80878612a7114472b3057e3", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 113.1 - ], - [ - 125.0, - 258.7 - ], - [ - 1584.1, - 258.7 - ], - [ - 1584.1, - 113.1 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.92924, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 2, - "parent_id": "16bb55bb7ed91deb6715ab7e4542505f" - }, - "text": "We specialize in markets where our computing platforms can provide tremendous acceleration for applications. These platforms incorporate processors, interconnects, software, algorithms, systems, and services to deliver unique value. Our platforms address four large markets where our expertise is critical: Data Center, Gaming, Professional Visualization, and Automotive.", - "type": "NarrativeText" - }, - { - "element_id": "17502cc50796af37e87747a7cd3be0f6", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 292.6 - ], - [ - 125.0, - 328.7 - ], - [ - 251.4, - 328.7 - ], - [ - 251.4, - 292.6 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.85985, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 2 - }, - "text": "Revenue", - "type": "Title" - }, - { - "element_id": "9942a0e639b78611c590997aab546998", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 344.6 - ], - [ - 125.0, - 382.0 - ], - [ - 1248.2, - 382.0 - ], - [ - 1248.2, - 344.6 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.86451, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 2, - "parent_id": "17502cc50796af37e87747a7cd3be0f6" - }, - "text": "Revenue was $18.12 billion, up 206% from a year ago and up 34% sequentially.", - "type": "NarrativeText" - }, - { - "element_id": "e3677bc0f19cf915e44c3d382f5ec139", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 408.0 - ], - [ - 125.0, - 847.0 - ], - [ - 1584.0, - 847.0 - ], - [ - 1584.0, - 408.0 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.95417, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 2, - "parent_id": "17502cc50796af37e87747a7cd3be0f6" - }, - "text": "Data Center revenue was a record, up 279% from a year ago and up 41% sequentially. Strong sales of the NVIDIA HGX platform were driven by global demand for the training and inferencing of large language models, recommendation engines, and generative AI applications. Data Center compute grew 324% from a year ago and 38% sequentially, largely reflecting the strong ramp of our Hopper GPU architecture-based HGX platform from cloud service providers (CSPs), including GPU-specialized CSPs; consumer internet companies; and enterprises. Our sales of Ampere GPU architecture-based Data Center products were significant but declined sequentially, as we approach the tail end of this architecture. We recognized initial revenue on the ramp of our L40S GPU and the GH200 Grace Hopper Superchip for a broad range of customers. CSPs drove roughly half of Data Center revenue, while consumer internet companies and enterprises comprised approximately the other half. Networking was up 155% from a year ago and up 52% sequentially, almost entirely due to strong growth in InfiniBand infrastructure to support our HGX platform.", - "type": "NarrativeText" - }, - { - "element_id": "b13f1242eca9b658a5a007fd23b57d23", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 873.3 - ], - [ - 125.0, - 1458.7 - ], - [ - 1583.7, - 1458.7 - ], - [ - 1583.7, - 873.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.94955, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 2, - "parent_id": "17502cc50796af37e87747a7cd3be0f6" - }, - "text": "On October 17, 2023, the U.S. government (USG) announced new and updated licensing requirements effective in our fourth quarter of fiscal 2024 for exports to China and Country Groups D1, D4, and D5 (including but not limited to Saudi Arabia, the United Arab Emirates, and Vietnam, but excluding Israel) of our products exceeding certain performance thresholds, including A100, A800, H100, H800, L4, L40, L40S and RTX 4090. The licensing requirements also apply to the export of products exceeding certain performance thresholds to a party headquartered in, or with an ultimate parent headquartered in, Country Group D5, including China. On October 23, 2023, the USG informed us the licensing requirements were effective immediately for shipments of our A100, A800, H100, H800, and L40S products. These licensing requirements did not have a meaningful impact on our revenue in the third quarter of fiscal 2024 as they were announced near the end of the fiscal quarter and we had additional demand from customers outside of the named country groups. Our sales to China and other affected destinations, derived from products that are now subject to licensing requirements, have consistently contributed approximately 20-25% of Data Center revenue over the past few quarters. We expect that our sales to these destinations will decline significantly in the fourth quarter of fiscal 2024, though we believe the decline will be more than offset by strong growth in other regions.", - "type": "NarrativeText" - }, - { - "element_id": "1ef84c7baf2eca389f729c81c69bc546", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1483.4 - ], - [ - 125.0, - 1630.4 - ], - [ - 1583.5, - 1630.4 - ], - [ - 1583.5, - 1483.4 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93921, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 2, - "parent_id": "17502cc50796af37e87747a7cd3be0f6" - }, - "text": "Gaming revenue was up 81% from a year ago and up 15% sequentially. Strong year-on-year growth reflects higher sell-in to partners following normalization of channel inventory levels. Sequential growth reflects strong demand for our GeForce RTX 40 Series GPUs for back-to-school and the start of the holiday season.", - "type": "NarrativeText" - }, - { - "element_id": "ef2482b6c447d5ce8cacc70e4e3d3e5b", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1653.5 - ], - [ - 125.0, - 1802.0 - ], - [ - 1583.7, - 1802.0 - ], - [ - 1583.7, - 1653.5 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93474, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 2, - "parent_id": "17502cc50796af37e87747a7cd3be0f6" - }, - "text": "Professional Visualization revenue was up 108% from a year ago and up 10% sequentially. The year- on-year increase reflects higher sell-in to partners following normalization of channel inventory levels. The sequential increase was primarily due to stronger enterprise workstation demand and the ramp of notebook workstations based on the Ada Lovelace GPU architecture.", - "type": "NarrativeText" - }, - { - "element_id": "d6752e52bd0233af9390d17486d720f1", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1826.9 - ], - [ - 125.0, - 1937.0 - ], - [ - 1583.5, - 1937.0 - ], - [ - 1583.5, - 1826.9 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93027, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 2, - "parent_id": "17502cc50796af37e87747a7cd3be0f6" - }, - "text": "Automotive revenue was up 4% from a year ago and up 3% sequentially. The year-on-year increase primarily reflects growth in sales of auto cockpit solutions and self-driving platforms. The sequential increase was driven by sales of self-driving platforms.", - "type": "NarrativeText" - }, - { - "element_id": "79ab79e53d786ebe19292e87d9ba5dab", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 111.3 - ], - [ - 125.0, - 148.7 - ], - [ - 323.1, - 148.7 - ], - [ - 323.1, - 111.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.83452, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3 - }, - "text": "Gross Margin", - "type": "Title" - }, - { - "element_id": "4d4f80213b4deba3bbaccbd19cad448f", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 165.5 - ], - [ - 125.0, - 275.4 - ], - [ - 1583.7, - 275.4 - ], - [ - 1583.7, - 165.5 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93085, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "79ab79e53d786ebe19292e87d9ba5dab" - }, - "text": "GAAP and non-GAAP gross margins increased significantly from a year ago and sequentially, driven by improved product mix from Data Center revenue growth and lower net inventory provisions and related charges.", - "type": "NarrativeText" - }, - { - "element_id": "b9c21f568324c9bb4a1ff9696a934b4f", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 124.3, - 300.3 - ], - [ - 124.3, - 483.7 - ], - [ - 1583.8, - 483.7 - ], - [ - 1583.8, - 300.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.95197, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "79ab79e53d786ebe19292e87d9ba5dab" - }, - "text": "In the third quarter of fiscal 2024, provisions for inventory and related charges were $681 million. Sales of previously reserved inventory or settlements of excess inventory purchase obligations resulted in a provision release of $239 million, primarily from Ampere GPU architecture products. The net inventory provisions were $442 million and the unfavorable effect on our gross margin was 2.4 percentage points.", - "type": "NarrativeText" - }, - { - "element_id": "917df0aa364e8e60a5edbc232d1b6ab3", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 507.7 - ], - [ - 125.0, - 655.4 - ], - [ - 1583.8, - 655.4 - ], - [ - 1583.8, - 507.7 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93372, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "79ab79e53d786ebe19292e87d9ba5dab" - }, - "text": "In the third quarter of fiscal 2023, provisions for inventory and related charges were $702 million. Sales of previously reserved inventory or settlements of excess inventory purchase obligations resulted in a provision release of $21 million. The net inventory provisions were $681 million and the unfavorable effect on our gross margin was 11.5 percentage points.", - "type": "NarrativeText" - }, - { - "element_id": "1c177f0d722399625a214b0ea22234d2", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 689.1 - ], - [ - 125.0, - 725.4 - ], - [ - 264.4, - 725.4 - ], - [ - 264.4, - 689.1 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.85319, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3 - }, - "text": "Expenses", - "type": "Title" - }, - { - "element_id": "73ff5795a6652a88e276709c08b41179", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 744.8 - ], - [ - 125.0, - 852.0 - ], - [ - 1583.8, - 852.0 - ], - [ - 1583.8, - 744.8 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93156, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "1c177f0d722399625a214b0ea22234d2" - }, - "text": "GAAP operating expenses were up 16% from a year ago and up 12% sequentially, driven by compensation and benefits, including stock-based compensation, primarily reflecting growth in employees and compensation increases.", - "type": "NarrativeText" - }, - { - "element_id": "3799bb18ce79109d818d867f9938bd47", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 876.4 - ], - [ - 125.0, - 950.4 - ], - [ - 1583.3, - 950.4 - ], - [ - 1583.3, - 876.4 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.919, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "1c177f0d722399625a214b0ea22234d2" - }, - "text": "Non-GAAP operating expenses were up 13% from a year ago and up 10% sequentially, primarily reflecting growth in employees and compensation increases.", - "type": "NarrativeText" - }, - { - "element_id": "041b18573cc53e79d793fc35298751f8", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 974.8 - ], - [ - 125.0, - 1158.7 - ], - [ - 1586.5, - 1158.7 - ], - [ - 1586.5, - 974.8 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.95214, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "1c177f0d722399625a214b0ea22234d2" - }, - "text": "We are monitoring the impact of the geopolitical conflict in and around Israel on our operations, including the health and safety of our approximately 3,400 employees in the region who primarily support the research and development, operations, and sales and marketing of our networking products. Our operating expenses in the third quarter of fiscal 2024 include expenses for financial support to impacted employees and charitable activity.", - "type": "NarrativeText" - }, - { - "element_id": "6afb65f33972f4388006fc009b0dd18b", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1190.8 - ], - [ - 125.0, - 1228.7 - ], - [ - 728.2, - 1228.7 - ], - [ - 728.2, - 1190.8 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.85672, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3 - }, - "text": "Other Income & Expense and Income Tax", - "type": "Title" - }, - { - "element_id": "8849587f38b8996182b353174421d14f", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1245.5 - ], - [ - 125.0, - 1392.0 - ], - [ - 1583.5, - 1392.0 - ], - [ - 1583.5, - 1245.5 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93117, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "6afb65f33972f4388006fc009b0dd18b" - }, - "text": "GAAP other income and expense (OI&E) includes interest income, interest expense, gains and losses from non-affiliated investments and other. Non-GAAP OI&E excludes the gains or losses from non- affiliated investments and the portion of interest expense from the amortization of the debt discount.", - "type": "NarrativeText" - }, - { - "element_id": "7dcbd5b6c704bc2a5478a88f2517f102", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 124.9, - 1417.1 - ], - [ - 124.9, - 1527.0 - ], - [ - 1583.6, - 1527.0 - ], - [ - 1583.6, - 1417.1 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93013, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "6afb65f33972f4388006fc009b0dd18b" - }, - "text": "Interest income was $234 million, up from a year ago and sequentially, reflecting higher cash and investments and higher yields. Net losses from non-affiliated investments were $69 million, driven predominantly by mark-to-market losses from publicly traded equity investments.", - "type": "NarrativeText" - }, - { - "element_id": "a7362a7d8a33a5640f433e45a6fc1e51", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1550.0 - ], - [ - 125.0, - 1662.0 - ], - [ - 1583.7, - 1662.0 - ], - [ - 1583.7, - 1550.0 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.92833, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "6afb65f33972f4388006fc009b0dd18b" - }, - "text": "GAAP effective tax rate was 12.2%, which reflects tax benefits from the foreign-derived intangible income deduction, stock-based compensation, a resolution of an Internal Revenue Service audit, and the U.S. federal research tax credit. Non-GAAP effective tax rate was 14.6%.", - "type": "NarrativeText" - }, - { - "element_id": "394f6209c2db2e403549ec0f89ed3e81", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1695.7 - ], - [ - 125.0, - 1732.0 - ], - [ - 556.2, - 1732.0 - ], - [ - 556.2, - 1695.7 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.84736, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3 - }, - "text": "Balance Sheet and Cash Flow", - "type": "Title" - }, - { - "element_id": "9243f3f636e1972828261243fb753b61", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1747.5 - ], - [ - 125.0, - 1858.7 - ], - [ - 1583.4, - 1858.7 - ], - [ - 1583.4, - 1747.5 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.92926, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "394f6209c2db2e403549ec0f89ed3e81" - }, - "text": "Cash, cash equivalents and marketable securities were $18.28 billion, up from $13.14 billion a year ago and $16.02 billion a quarter ago. The increases primarily reflect higher revenue partially offset by taxes paid and stock repurchases.", - "type": "NarrativeText" - }, - { - "element_id": "178ec45ebfce8c3da6c1062d358daa27", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1882.6 - ], - [ - 125.0, - 1993.7 - ], - [ - 1583.6, - 1993.7 - ], - [ - 1583.6, - 1882.6 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93651, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 3, - "parent_id": "394f6209c2db2e403549ec0f89ed3e81" - }, - "text": "Accounts receivable was $8.31 billion with days sales outstanding (DSO) of 42, with the DSO decline from prior periods due to the linearity of shipments. Accounts receivable was reduced by approximately $570 million from customer payments received prior to the invoice due date.", - "type": "NarrativeText" - }, - { - "element_id": "63e04b114d623c7d522807d7c97df0f9", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 112.3 - ], - [ - 125.0, - 295.4 - ], - [ - 1583.8, - 295.4 - ], - [ - 1583.8, - 112.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.95131, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "394f6209c2db2e403549ec0f89ed3e81" - }, - "text": "Inventory was $4.78 billion with days sales of inventory (DSI) of 92. Purchase commitments and obligations for inventory and manufacturing capacity were $17.11 billion and prepaid supply agreements were $3.67 billion. Other non-inventory purchase obligations were $4.43 billion which includes $3.60 billion of multi-year cloud service agreements, primarily to support our research and development efforts.", - "type": "NarrativeText" - }, - { - "element_id": "8b2f19d854889a1c11982c798cf0e618", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 320.8 - ], - [ - 125.0, - 503.7 - ], - [ - 1584.0, - 503.7 - ], - [ - 1584.0, - 320.8 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.94916, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "394f6209c2db2e403549ec0f89ed3e81" - }, - "text": "Cash flow from operating activities was $7.33 billion, up from $392 million a year ago and $6.35 billion a quarter ago, driven by higher revenue partially offset by higher cash tax payments. We paid $4.35 billion in cash taxes in the third quarter of fiscal 2024, largely for previously deferred federal income tax payments related to the disaster relief made available by the IRS for certain California taxpayers.", - "type": "NarrativeText" - }, - { - "element_id": "fea95d23b2866333212f12145299210c", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 528.6 - ], - [ - 125.0, - 712.0 - ], - [ - 1587.5, - 712.0 - ], - [ - 1587.5, - 528.6 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.94912, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "394f6209c2db2e403549ec0f89ed3e81" - }, - "text": "Depreciation and amortization expense was $372 million, including amortization of acquisition- related intangible assets. Starting in fiscal 2024, we extended the useful lives of most of our servers, storage, and network equipment from three years to a range of four to five years, and assembly and test equipment from five to seven years. This change in useful lives drove a favorable impact to operating expenses of $24 million and to cost of revenue of $17 million in the third quarter.", - "type": "NarrativeText" - }, - { - "element_id": "ae31b058636a3429172bb5f7d04b6994", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 736.1 - ], - [ - 125.0, - 810.4 - ], - [ - 1583.8, - 810.4 - ], - [ - 1583.8, - 736.1 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.92356, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "394f6209c2db2e403549ec0f89ed3e81" - }, - "text": "During the third quarter, we utilized cash of $3.91 billion towards shareholder returns, including $3.81 billion in share repurchases and $99 million in cash dividends.", - "type": "NarrativeText" - }, - { - "element_id": "ef18c360ab3527463119dbbfcb40d10d", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 843.5 - ], - [ - 125.0, - 880.4 - ], - [ - 686.6, - 880.4 - ], - [ - 686.6, - 843.5 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.86763, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4 - }, - "text": "Fourth Quarter of Fiscal 2024 Outlook", - "type": "Title" - }, - { - "element_id": "94e99fb912999331c5ea7b9441fb4f3c", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 898.0 - ], - [ - 125.0, - 933.7 - ], - [ - 954.3, - 933.7 - ], - [ - 954.3, - 898.0 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.89886, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "Outlook for the fourth quarter of fiscal 2024 is as follows:", - "type": "NarrativeText" - }, - { - "element_id": "bc0dfa854deb9e28c003030084d57022", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 163.6, - 957.7 - ], - [ - 163.6, - 995.4 - ], - [ - 1062.0, - 995.4 - ], - [ - 1062.0, - 957.7 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.87022, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "\u2022 Revenue is expected to be $20.00 billion, plus or minus 2%.", - "type": "ListItem" - }, - { - "element_id": "06dbb5e86e0d4d40e2ff867be61039aa", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 175.0, - 1021.8 - ], - [ - 175.0, - 1093.7 - ], - [ - 1583.5, - 1093.7 - ], - [ - 1583.5, - 1021.8 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.90768, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "\u2022 GAAP and non-GAAP gross margins are expected to be 74.5% and 75.5%, respectively, plus or minus 50 basis points.", - "type": "ListItem" - }, - { - "element_id": "f91783657578e903a6c97aac6c1aa534", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 175.0, - 1120.4 - ], - [ - 175.0, - 1192.0 - ], - [ - 1583.2, - 1192.0 - ], - [ - 1583.2, - 1120.4 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.9157, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "\u2022 GAAP and non-GAAP operating expenses are expected to be approximately $3.17 billion and $2.20 billion, respectively.", - "type": "ListItem" - }, - { - "element_id": "55ee9102bab04ab83c39afff4517d207", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 164.6, - 1218.9 - ], - [ - 164.6, - 1290.4 - ], - [ - 1584.5, - 1290.4 - ], - [ - 1584.5, - 1218.9 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.91806, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "\u2022 GAAP and non-GAAP other income and expense are expected to be an income of approximately $200 million, excluding gains and losses from non-affiliated investments.", - "type": "ListItem" - }, - { - "element_id": "d01addbf3544810c17fe7ae74072fb8e", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 169.9, - 1317.8 - ], - [ - 169.9, - 1388.7 - ], - [ - 1583.2, - 1388.7 - ], - [ - 1583.2, - 1317.8 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.89944, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "\u2022 GAAP and non-GAAP tax rates are expected to be 15.0%, plus or minus 1%, excluding any discrete items.", - "type": "ListItem" - }, - { - "element_id": "a41a7645d53669c00eaa0aa82c5a47d5", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 643.8, - 1428.1 - ], - [ - 643.8, - 1458.7 - ], - [ - 1056.2, - 1458.7 - ], - [ - 1056.2, - 1428.1 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "___________________________", - "type": "UncategorizedText" - }, - { - "element_id": "fa34b61741befd32abfed221b767005f", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 124.2, - 1474.4 - ], - [ - 124.2, - 1512.0 - ], - [ - 597.0, - 1512.0 - ], - [ - 597.0, - 1474.4 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.7648, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "For further information, contact:", - "type": "NarrativeText" - }, - { - "element_id": "2894446a4d067434bdce81bf5985c1a8", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 132.3, - 1545.4 - ], - [ - 132.3, - 1716.2 - ], - [ - 469.8, - 1716.2 - ], - [ - 469.8, - 1545.4 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.59045, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "Simona Jankowski Investor Relations NVIDIA Corporation sjankowski@nvidia.com", - "type": "NarrativeText" - }, - { - "element_id": "4dc2539954386714beb57c60755c493f", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 1045.9, - 1546.8 - ], - [ - 1045.9, - 1716.2 - ], - [ - 1447.9, - 1716.2 - ], - [ - 1447.9, - 1546.8 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.628, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ef18c360ab3527463119dbbfcb40d10d" - }, - "text": "Robert Sherbin Corporate Communications NVIDIA Corporation rsherbin@nvidia.com", - "type": "NarrativeText" - }, - { - "element_id": "ca68708d8fe697270b9863b374801fcc", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 1760.1 - ], - [ - 125.0, - 1795.9 - ], - [ - 433.8, - 1795.9 - ], - [ - 433.8, - 1760.1 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.82878, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4 - }, - "text": "Non-GAAP Measures", - "type": "Title" - }, - { - "element_id": "37f8f523b79f467e9388bdf38a5d62b4", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 122.3, - 1811.0 - ], - [ - 122.3, - 2069.2 - ], - [ - 1583.8, - 2069.2 - ], - [ - 1583.8, - 1811.0 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.94971, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 4, - "parent_id": "ca68708d8fe697270b9863b374801fcc" - }, - "text": "To supplement NVIDIA\u2019s condensed consolidated financial statements presented in accordance with GAAP, the company uses non-GAAP measures of certain components of financial performance. These non-GAAP measures include non-GAAP gross profit, non-GAAP gross margin, non-GAAP operating expenses, non-GAAP income from operations, non-GAAP other income (expense), net, non- GAAP net income, non-GAAP net income, or earnings, per diluted share, and free cash flow. In order for NVIDIA\u2019s investors to be better able to compare its current results with those of previous periods, the company has shown a reconciliation of GAAP to non-GAAP financial measures. These", - "type": "NarrativeText" - }, - { - "element_id": "b74cb143ad418ca83ec86bd90f228311", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 113.0 - ], - [ - 125.0, - 515.4 - ], - [ - 1584.0, - 515.4 - ], - [ - 1584.0, - 113.0 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.95632, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 5, - "parent_id": "ca68708d8fe697270b9863b374801fcc" - }, - "text": "reconciliations adjust the related GAAP financial measures to exclude acquisition termination costs, stock-based compensation expense, acquisition-related and other costs, IP-related costs, other, gains and losses from non-affiliated investments, interest expense related to amortization of debt discount, and the associated tax impact of these items where applicable. Free cash flow is calculated as GAAP net cash provided by operating activities less both purchases of property and equipment and intangible assets and principal payments on property and equipment and intangible assets. NVIDIA believes the presentation of its non-GAAP financial measures enhances the user's overall understanding of the company\u2019s historical financial performance. The presentation of the company\u2019s non-GAAP financial measures is not meant to be considered in isolation or as a substitute for the company\u2019s financial results prepared in accordance with GAAP, and the company\u2019s non-GAAP measures may be different from non-GAAP measures used by other companies.", - "type": "NarrativeText" - }, - { - "element_id": "c85c4cd05f2b93e91b40c7b32bfdb052", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 125.0, - 541.4 - ], - [ - 125.0, - 1051.6 - ], - [ - 1582.5, - 1051.6 - ], - [ - 1582.5, - 541.4 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.95266, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 5, - "parent_id": "ca68708d8fe697270b9863b374801fcc" - }, - "text": "Certain statements in this CFO Commentary including, but not limited to, statements as to: our computing platforms providing tremendous acceleration for applications and delivering unique value; markets where our expertise is critical; our expected sales to China and other affected destinations declining significantly in the fourth quarter, and our belief that the decline will be more than offset by strong growth in other regions; and our financial outlook and expected tax rates for the fourth quarter of fiscal 2024 are forward-looking statements that are subject to risks and uncertainties that could cause results to be materially different than expectations. Important factors that could cause actual results to differ materially include: global economic conditions; our reliance on third parties to manufacture, assemble, package and test our products; impact of technological development and competition; development of new products and technologies or the enhancements to our existing product and technologies; market acceptance of our products or our partners\u2019 products; design, manufacturing or software defects; changes in consumer preferences or demands; changes in industry standards and interfaces; and unexpected loss of performance of our products or technologies when integrated into systems; as well as other factors detailed from time to time in the most recent reports NVIDIA files with the Securities and Exchange Commission, or SEC, including, but not limited to, its annual report on Form 10-K and quarterly reports on Form 10-Q. Copies of reports filed with the SEC are posted on the company\u2019s website and are available from NVIDIA without charge. These forward-looking statements are not guarantees of future performance and speak only as of the date hereof, and, except as required by law, NVIDIA disclaims any obligation to update these forward-looking statements to reflect future events or circumstances.", - "type": "NarrativeText" - }, - { - "element_id": "af59e130a13cad5b2a53fa888a807881", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 821.1, - 1078.7 - ], - [ - 821.1, - 1106.6 - ], - [ - 881.1, - 1106.6 - ], - [ - 881.1, - 1078.7 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.75073, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 5 - }, - "text": "# # #", - "type": "Title" - }, - { - "element_id": "b1c081d01de327b70ec3ac0794df8621", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 122.1, - 1133.2 - ], - [ - 122.1, - 1251.6 - ], - [ - 1583.7, - 1251.6 - ], - [ - 1583.7, - 1133.2 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93629, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 5, - "parent_id": "af59e130a13cad5b2a53fa888a807881" - }, - "text": "\u00a9 2023 NVIDIA Corporation. All rights reserved. NVIDIA and the NVIDIA logo are trademarks and/or registered trademarks of NVIDIA Corporation in the U.S. and/or other countries. Other company and product names may be trademarks of the respective companies with which they are associated. Features, pricing, availability, and specifications are subject to change without notice.", - "type": "NarrativeText" - }, - { - "element_id": "c7b1511a5d0aa8423f5809015d82184b", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 370.4, - 129.4 - ], - [ - 370.4, - 244.9 - ], - [ - 1354.4, - 244.9 - ], - [ - 1354.4, - 129.4 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6, - "parent_id": "af59e130a13cad5b2a53fa888a807881" - }, - "text": "NVIDIA CORPORATION RECONCILIATION OF GAAP TO NON-GAAP FINANCIAL MEASURES (In millions, except per share data)", - "type": "UncategorizedText" - }, - { - "element_id": "f9aa17ccb15e3a675fca6f8c1e57bf3d", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 779.1, - 258.8 - ], - [ - 779.1, - 286.6 - ], - [ - 944.7, - 286.6 - ], - [ - 944.7, - 258.8 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6 - }, - "text": "(Unaudited)", - "type": "Title" - }, - { - "element_id": "0033c52bd2baffc7b374715cc58fdb6c", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 149.9, - 348.7 - ], - [ - 149.9, - 1815.5 - ], - [ - 1567.3, - 1815.5 - ], - [ - 1567.3, - 348.7 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.92683, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6, - "parent_id": "f9aa17ccb15e3a675fca6f8c1e57bf3d", - "text_as_html": "
October 29, 2023July 30, 2023October 30, 2022October 29, 2023October 30, 2022
AAP gross profit$13,400$9,462$ 3,177$27,510$11,523
GAAP gross margin74.0 %70.1%53.6%70.9%55.1
Acquisition-related and other costs (A)119119120358335
Stock-based compensation expense (B)38313296108
IP-related costs262_36=
jon-GAAP gross profit$13,583$9,614$ 3,329$28,000$11,966
Non-GAAP gross margin75.0 %71.2%56.1%72.1%57.2
AAP operating expenses$2,983$2,662$ 2,576$8,152$8,555
Stock-based compensation expense (B)(941)(811)(713)(2,459)(1,863)
Acquisition-related and other costs (A)(16)(18)(54)(88)(164)
Acquisition termination cost\u2014\u2014_\u2014\u2014(1,353)
Other (C)_5(16)10(25)
lon-GAAP operating expenses$2,026$1,838$ 1,793$5615$5,150
AAP operating income $ 10,417 $ 6,800 $ 601 $ 19,358 $ 2,968
Total impact of non-GAAP adjustments to operating income1,1409769353,0273,848
ljon-GAAP operating income$11,557 $7,776$ 1,536$22,385$6,816
AAP other income (expense), net$105$181$ 12$354$(75)
(Gains) losses from non-affiliated investments69(62)112336
Interest expense related to amortization of debt discount1133
jon-GAAP other income (expense), net$175$120$ 24$380$(36)
AAP net income$9,243$6,188$ 680$17,475$2,954
Total pre-tax impact of non-GAAP adjustments1,2109159473,0533,887
Income tax impact of non-GAAP adjustments (D)(433)(363)(171)(1,055)(649)
" - }, - "text": "Three Months Ended Nine Months Ended October 29, July 30, October 30, October 29, October 30, 2023 2023 2022 2023 2022 $ 13,400 $ 9,462 $ 3,177 $ 27,510 $ 11,523 GAAP gross margin 74.0 % 70.1 % 53.6 % 70.9 % 55.1 % Acquisition-related and other costs (A) 119 119 120 358 335 Stock-based compensation expense (B) 38 31 32 96 108 IP-related costs 26 2 \u2014 36 \u2014 $ 13,583 $ 9,614 $ 3,329 $ 28,000 $ 11,966 Non-GAAP gross margin 75.0 % 71.2 % 56.1 % 72.1 % 57.2 % $ 2,983 $ 2,662 $ 2,576 $ 8,152 $ 8,555 Stock-based compensation expense (B) (941) (811) (713) (2,459) (1,863) Acquisition-related and other costs (A) (16) (18) (54) (88) (164) Acquisition termination cost \u2014 \u2014 \u2014 \u2014 (1,353) Other (C) \u2014 5 (16) 10 (25) $ 2,026 $ 1,838 $ 1,793 $ 5,615 $ 5,150 $ 10,417 $ 6,800 $ 601 $ 19,358 $ 2,968 1,140 $ 11,557 976 $ 7,776 935 $ 1,536 3,027 $ 22,385 3,848 $ 6,816 $ 105 $ 181 $ 12 $ 354 $ (75) (Gains) losses from non-affiliated investments Interest expense related to amortization of debt discount 69 1 (62) 1 11 1 23 3 36 3 $ 175 $ 120 $ 24 $ 380 $ (36) $ 9,243 $ 6,188 $ 680 $ 17,475 $ 2,954 1,210 (433) $ 10,020 915 (363) $ 6,740 947 (171) $ 1,456 3,053 (1,055) $ 19,473 3,887 (649) $ 6,192", - "type": "Table" - }, - { - "element_id": "4cfb97b0be47ed78dac17cd115423802", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 510.9 - ], - [ - 134.4, - 538.7 - ], - [ - 369.5, - 538.7 - ], - [ - 369.5, - 510.9 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6 - }, - "text": "GAAP gross profit", - "type": "Title" - }, - { - "element_id": "1e819b2510cb7b8c38e1f169eb02fd20", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 719.2 - ], - [ - 134.4, - 747.0 - ], - [ - 441.3, - 747.0 - ], - [ - 441.3, - 719.2 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6 - }, - "text": "Non-GAAP gross profit", - "type": "Title" - }, - { - "element_id": "54fc36f10b5c2bccfd7b9273a5157b02", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 846.3 - ], - [ - 134.4, - 874.1 - ], - [ - 471.6, - 874.1 - ], - [ - 471.6, - 846.3 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6 - }, - "text": "GAAP operating expenses", - "type": "Title" - }, - { - "element_id": "0135a874c74306e23e4bcc104567ab08", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 1054.6 - ], - [ - 134.4, - 1082.4 - ], - [ - 545.4, - 1082.4 - ], - [ - 545.4, - 1054.6 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6 - }, - "text": "Non-GAAP operating expenses", - "type": "Title" - }, - { - "element_id": "98dd97d50bcdddb9f0019553868b2bca", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 1140.1 - ], - [ - 134.4, - 1167.8 - ], - [ - 444.0, - 1167.8 - ], - [ - 444.0, - 1140.1 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6, - "parent_id": "0135a874c74306e23e4bcc104567ab08" - }, - "text": "GAAP operating income", - "type": "NarrativeText" - }, - { - "element_id": "bfb58716b955e6d33be79b424ab1765f", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 1181.4 - ], - [ - 134.4, - 1278.2 - ], - [ - 699.6, - 1278.2 - ], - [ - 699.6, - 1181.4 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6, - "parent_id": "0135a874c74306e23e4bcc104567ab08" - }, - "text": "Total impact of non-GAAP adjustments to operating income Non-GAAP operating income", - "type": "NarrativeText" - }, - { - "element_id": "49c0d0e87bb238c063ffece360749d23", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 1335.9 - ], - [ - 134.4, - 1363.7 - ], - [ - 577.9, - 1363.7 - ], - [ - 577.9, - 1335.9 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6 - }, - "text": "GAAP other income (expense), net", - "type": "Title" - }, - { - "element_id": "240ccd0fb7f0b39954532a1a68dcd5e8", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 1515.1 - ], - [ - 134.4, - 1542.8 - ], - [ - 655.2, - 1542.8 - ], - [ - 655.2, - 1515.1 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6 - }, - "text": "Non-GAAP other income (expense), net", - "type": "Title" - }, - { - "element_id": "2554fe507d5c95f8685182472cdd6269", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 1600.5 - ], - [ - 134.4, - 1628.3 - ], - [ - 361.7, - 1628.3 - ], - [ - 361.7, - 1600.5 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6 - }, - "text": "GAAP net income", - "type": "Title" - }, - { - "element_id": "19b35703f247cd78b12b5f3ceb4aecd0", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 134.4, - 1641.8 - ], - [ - 134.4, - 1807.4 - ], - [ - 630.2, - 1807.4 - ], - [ - 630.2, - 1641.8 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 6, - "parent_id": "2554fe507d5c95f8685182472cdd6269" - }, - "text": "Total pre-tax impact of non-GAAP adjustments Income tax impact of non-GAAP adjustments (D) Non-GAAP net income", - "type": "UncategorizedText" - }, - { - "element_id": "015136f3b935a011bf55bd3b42d71980", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 854.0, - 129.6 - ], - [ - 854.0, - 157.4 - ], - [ - 1129.3, - 157.4 - ], - [ - 1129.3, - 129.6 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7 - }, - "text": "Three Months Ended", - "type": "Title" - }, - { - "element_id": "11000b8d3b30a3734bbc727c9e1b0c0a", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 1282.4, - 130.7 - ], - [ - 1282.4, - 158.5 - ], - [ - 1542.5, - 158.5 - ], - [ - 1542.5, - 130.7 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7 - }, - "text": "Nine Months Ended", - "type": "Title" - }, - { - "element_id": "efc80d9ee0b636bd8a2a4e9b93775ea3", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 139.3, - 151.3 - ], - [ - 139.3, - 769.7 - ], - [ - 1564.0, - 769.7 - ], - [ - 1564.0, - 151.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93295, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7, - "parent_id": "11000b8d3b30a3734bbc727c9e1b0c0a", - "text_as_html": "
Jiluted net income per shareOctober 29, 2023July 30, 2023October 30, 2022October 29, 2023October 30, 2022
GAAP$3.71$2.48$0.27$7.01$1.17
Non-GAAP$402$2.70$058$7.81$2.46
Weighted average shares used in diluted net ncome per share computation2,4942,4992,4992,4942,517
SAAP net cash provided by operating activities$7,333$6,348$392$16591$3,393
Purchases related to property and equipment and intangible assets(278)(289)(530)(815)(1,324
Principal payments on property and equipment and intangible assets Free cash flow$(13)$(11)$(18) (156)$(44)$(54
" - }, - "text": "October 29, July 30, October 30, October 29, October 30, 2023 2023 2022 2023 2022 GAAP $ 3.71 $ 2.48 $ 0.27 $ 7.01 $ Non-GAAP $ 4.02 $ 2.70 $ 0.58 $ 7.81 $ 2,494 2,499 2,499 2,494 $ 7,333 $ 6,348 $ 392 $ 16,591 $ Purchases related to property and equipment and intangible assets Principal payments on property and equipment and intangible assets (278) (13) (289) (11) (530) (18) (815) (44) $ 7,042 $ 6,048 $ (156) $ 15,732 $", - "type": "Table" - }, - { - "element_id": "7b940d330e6358c81641354dbd434f4b", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 132.3, - 876.2 - ], - [ - 132.3, - 936.1 - ], - [ - 1514.9, - 936.1 - ], - [ - 1514.9, - 876.2 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.90828, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7, - "parent_id": "11000b8d3b30a3734bbc727c9e1b0c0a" - }, - "text": "(A) Acquisition-related and other costs are comprised of amortization of intangible assets and transaction costs, and are included in the following line items:", - "type": "NarrativeText" - }, - { - "element_id": "c1fea800bd77ed251229f288d88d3dae", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 158.7, - 943.1 - ], - [ - 158.7, - 1182.1 - ], - [ - 1563.6, - 1182.1 - ], - [ - 1563.6, - 943.1 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.93058, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7, - "parent_id": "11000b8d3b30a3734bbc727c9e1b0c0a", - "text_as_html": "
Cost of revenueOctober 29, 2023July 30, 2023October 30, 2022October 29, 2023October 30, 2022
$119$\u00a2$119$\u00a2$120$358$335
Research and development$12$12$10$37$29
Sales, general and administrative$4$6$44$51$135
" - }, - "text": "Three Months Ended Nine Months Ended October 29, July 30, October 30, October 29, October 30, 2023 2023 2022 2023 2022 Cost of revenue $ 119 $ 119 $ 120 $ 358 $ Research and development $ 12 $ 12 $ 10 $ 37 $ Sales, general and administrative $ 4 $ 6 $ 44 $ 51 $", - "type": "Table" - }, - { - "element_id": "744a9ac2599b8e647e2b5d61995d5c22", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 132.3, - 1236.4 - ], - [ - 132.3, - 1268.4 - ], - [ - 871.6, - 1268.4 - ], - [ - 871.6, - 1236.4 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.79911, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7, - "parent_id": "11000b8d3b30a3734bbc727c9e1b0c0a" - }, - "text": "(B) Stock-based compensation consists of the following:", - "type": "NarrativeText" - }, - { - "element_id": "6cdfc5498142ccc289fc01c7c7ca1500", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 153.5, - 1275.0 - ], - [ - 153.5, - 1513.8 - ], - [ - 1577.0, - 1513.8 - ], - [ - 1577.0, - 1275.0 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.92861, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7, - "parent_id": "11000b8d3b30a3734bbc727c9e1b0c0a", - "text_as_html": "
Cost of revenueOctober 29, 2023July 30, 2023October 30, 2022October 29, 2023October 30, 2022
$38\u00a231$32$96$108
Research and development$701$600$530$1,826$1,365
Sales, general and administrative$240$211$183$633$498
" - }, - "text": "Three Months Ended Nine Months Ended October 29, July 30, October 30, October 29, October 30, 2023 2023 2022 2023 2022 Cost of revenue $ 38 $ 31 $ 32 $ 96 $ 108 Research and development Sales, general and administrative $ $ 701 $ 240 $ 600 $ 211 $ 530 $ 183 $ 1,826 $ 633 $ 1,365 498", - "type": "Table" - }, - { - "element_id": "c90a0fa8e85b5761f64ab374f7b79fb6", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 130.6, - 1565.6 - ], - [ - 130.6, - 1627.1 - ], - [ - 1574.6, - 1627.1 - ], - [ - 1574.6, - 1565.6 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.89978, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7, - "parent_id": "11000b8d3b30a3734bbc727c9e1b0c0a" - }, - "text": "(C) Other consists of costs related to Russia branch office closure, assets held for sale related adjustments, legal settlement costs, and contributions.", - "type": "NarrativeText" - }, - { - "element_id": "f1cc6704887462625bebb4bb41e10cb5", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 131.4, - 1679.3 - ], - [ - 131.4, - 1740.3 - ], - [ - 1438.0, - 1740.3 - ], - [ - 1438.0, - 1679.3 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.91453, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 7, - "parent_id": "11000b8d3b30a3734bbc727c9e1b0c0a" - }, - "text": "(D) Income tax impact of non-GAAP adjustments, including the recognition of excess tax benefits or deficiencies related to stock-based compensation under GAAP accounting standard (ASU 2016-09).", - "type": "NarrativeText" - }, - { - "element_id": "8b5b5fddb31f1e31acb109a562757d69", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 453.7, - 191.1 - ], - [ - 453.7, - 265.4 - ], - [ - 1248.0, - 265.4 - ], - [ - 1248.0, - 191.1 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 8 - }, - "text": "NVIDIA CORPORATION RECONCILIATION OF GAAP TO NON-GAAP OUTLOOK", - "type": "Title" - }, - { - "element_id": "2b2bd347cd867fbd42691ca431db8836", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 163.0, - 314.7 - ], - [ - 163.0, - 758.8 - ], - [ - 1560.9, - 758.8 - ], - [ - 1560.9, - 314.7 - ] - ], - "system": "PixelSpace" - }, - "detection_class_prob": 0.7238, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 8, - "parent_id": "8b5b5fddb31f1e31acb109a562757d69", - "text_as_html": "
\\AP gross margin Impact of stock-based compensation expense, acquisition-related costs, and other costsOutlook ($ in millions) 74.5 Y 1.0 \u00a5 .
n-GAAP gross margin75.5 Y
\\AP operating expenses3,165
Stock-based compensation expense, acquisition-related costs, and other costs(965)
" - }, - "text": "Impact of stock-based compensation expense, acquisition-related costs, and other costs Stock-based compensation expense, acquisition-related costs, and other costs $ $ Q4 FY2024 Outlook ($ in millions) 3,165 (965) 2,200", - "type": "Table" - }, - { - "element_id": "1700f5fbf0ac80bab248c4be060ddb66", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 132.3, - 452.8 - ], - [ - 132.3, - 480.6 - ], - [ - 384.7, - 480.6 - ], - [ - 384.7, - 452.8 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 8 - }, - "text": "GAAP gross margin", - "type": "Title" - }, - { - "element_id": "1d619233a36dacbc9ff3b591ef405fbe", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 132.3, - 563.2 - ], - [ - 132.3, - 591.0 - ], - [ - 456.4, - 591.0 - ], - [ - 456.4, - 563.2 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 8 - }, - "text": "Non-GAAP gross margin", - "type": "Title" - }, - { - "element_id": "54fc36f10b5c2bccfd7b9273a5157b02", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 132.3, - 648.6 - ], - [ - 132.3, - 676.4 - ], - [ - 469.5, - 676.4 - ], - [ - 469.5, - 648.6 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 8 - }, - "text": "GAAP operating expenses", - "type": "Title" - }, - { - "element_id": "0135a874c74306e23e4bcc104567ab08", - "metadata": { - "coordinates": { - "layout_height": 2200, - "layout_width": 1700, - "points": [ - [ - 132.3, - 731.9 - ], - [ - 132.3, - 759.7 - ], - [ - 543.3, - 759.7 - ], - [ - 543.3, - 731.9 - ] - ], - "system": "PixelSpace" - }, - "file_directory": "pdf", - "filename": "Q3FY24-CFO-Commentary.pdf", - "filetype": "application/pdf", - "languages": [ - "eng" - ], - "last_modified": "2024-02-18T21:23:20", - "page_number": 8 - }, - "text": "Non-GAAP operating expenses", - "type": "Title" - } -] \ No newline at end of file diff --git a/workers/pdf/nvidia-yolox.txt b/workers/pdf/nvidia-yolox.txt deleted file mode 100644 index 4d4c8a2b..00000000 --- a/workers/pdf/nvidia-yolox.txt +++ /dev/null @@ -1,18 +0,0 @@ -
fin millions, except earnings perQ3 FY24Q2 FY24Q3 FY23a/aYIY
Revenue$18,120$13,507$5,931Up 34%Up 206%
Gross margin740%70.1%53.6 %Up 3.9 ptsUp 20.4 pts
Operating expenses$2,983$2,662$2,576Up 12%Up 16%
Operating income$10,417$6,800$601Up 53%Up 1,633%
Net income$9,243$6,188$680Up 49%Up 1,259%
Diluted earnings per share$3.71$2.48$0.27Up 50%Up 1,274%
- -
in millions, except earnings perQ3 FY24Q2 FY24Q3 FY23a/aYIY
Revenue$18,120$13,507$5,931Up 34%Up 206%
Gross margin75.0%71.2%56.1%Up 3.8 ptsUp 18.9 pts
Operating expenses$2,026$1,838$1,793Up 10%Up 13%
Operating income$11,557$7,776$1,536Up 49%Up 652%
Net income$10,020$6,740$1,456Up 49%Up 588%
Diluted earnings per share$4.02$2.70$0.58Up 49%Up 593%
- -
($ in millions)Q3 FY24Q2 FY24Q3 FY23a/a7
Compute & Networking$14,645$10,402$3,816Up 41%Up 284%
Graphics3,4753,1052,115Up 12%Up 649
Total$18,120$13,507$5,931Up 34%Up 206%
- -
($ in millions)Q3 FY24Q2 FY24Q3 FY23Q/QYIY
Data Center$14,514$10,323$3,833Up 41%Up 279%
Gaming2,8562,4861,574Up 15%Up 81%
Professional Visualization416379200Up 10%Up 108%
Automotive261253251Up 3%Up 4%
OEM and Other736673Up 11%
Total$18,120$13,507$5,931Up 34%Up 206%
- -
October 29, 2023July 30, 2023October 30, 2022October 29, 2023October 30, 2022
AAP gross profit$13,400$9,462$ 3,177$27,510$11,523
GAAP gross margin74.0 %70.1%53.6%70.9%55.1
Acquisition-related and other costs (A)119119120358335
Stock-based compensation expense (B)38313296108
IP-related costs262_36=
jon-GAAP gross profit$13,583$9,614$ 3,329$28,000$11,966
Non-GAAP gross margin75.0 %71.2%56.1%72.1%57.2
AAP operating expenses$2,983$2,662$ 2,576$8,152$8,555
Stock-based compensation expense (B)(941)(811)(713)(2,459)(1,863)
Acquisition-related and other costs (A)(16)(18)(54)(88)(164)
Acquisition termination cost——_——(1,353)
Other (C)_5(16)10(25)
lon-GAAP operating expenses$2,026$1,838$ 1,793$5615$5,150
AAP operating income $ 10,417 $ 6,800 $ 601 $ 19,358 $ 2,968
Total impact of non-GAAP adjustments to operating income1,1409769353,0273,848
ljon-GAAP operating income$11,557 $7,776$ 1,536$22,385$6,816
AAP other income (expense), net$105$181$ 12$354$(75)
(Gains) losses from non-affiliated investments69(62)112336
Interest expense related to amortization of debt discount1133
jon-GAAP other income (expense), net$175$120$ 24$380$(36)
AAP net income$9,243$6,188$ 680$17,475$2,954
Total pre-tax impact of non-GAAP adjustments1,2109159473,0533,887
Income tax impact of non-GAAP adjustments (D)(433)(363)(171)(1,055)(649)
- -
Jiluted net income per shareOctober 29, 2023July 30, 2023October 30, 2022October 29, 2023October 30, 2022
GAAP$3.71$2.48$0.27$7.01$1.17
Non-GAAP$402$2.70$058$7.81$2.46
Weighted average shares used in diluted net ncome per share computation2,4942,4992,4992,4942,517
SAAP net cash provided by operating activities$7,333$6,348$392$16591$3,393
Purchases related to property and equipment and intangible assets(278)(289)(530)(815)(1,324
Principal payments on property and equipment and intangible assets Free cash flow$(13)$(11)$(18) (156)$(44)$(54
- -
Cost of revenueOctober 29, 2023July 30, 2023October 30, 2022October 29, 2023October 30, 2022
$119$¢$119$¢$120$358$335
Research and development$12$12$10$37$29
Sales, general and administrative$4$6$44$51$135
- -
Cost of revenueOctober 29, 2023July 30, 2023October 30, 2022October 29, 2023October 30, 2022
$38¢31$32$96$108
Research and development$701$600$530$1,826$1,365
Sales, general and administrative$240$211$183$633$498
- -
\AP gross margin Impact of stock-based compensation expense, acquisition-related costs, and other costsOutlook ($ in millions) 74.5 Y 1.0 ¥ .
n-GAAP gross margin75.5 Y
\AP operating expenses3,165
Stock-based compensation expense, acquisition-related costs, and other costs(965)
- diff --git a/workers/pdf/split/documentpage1-20.pdf b/workers/pdf/split/documentpage1-20.pdf deleted file mode 100644 index 89b805e1..00000000 Binary files a/workers/pdf/split/documentpage1-20.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1001-1020.pdf b/workers/pdf/split/documentpage1001-1020.pdf deleted file mode 100644 index 385ffc57..00000000 Binary files a/workers/pdf/split/documentpage1001-1020.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage101-120.pdf b/workers/pdf/split/documentpage101-120.pdf deleted file mode 100644 index 45430635..00000000 Binary files a/workers/pdf/split/documentpage101-120.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1021-1040.pdf b/workers/pdf/split/documentpage1021-1040.pdf deleted file mode 100644 index 59791d20..00000000 Binary files a/workers/pdf/split/documentpage1021-1040.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1041-1060.pdf b/workers/pdf/split/documentpage1041-1060.pdf deleted file mode 100644 index 7a2966bd..00000000 Binary files a/workers/pdf/split/documentpage1041-1060.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1061-1080.pdf b/workers/pdf/split/documentpage1061-1080.pdf deleted file mode 100644 index a2b03adc..00000000 Binary files a/workers/pdf/split/documentpage1061-1080.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1081-1100.pdf b/workers/pdf/split/documentpage1081-1100.pdf deleted file mode 100644 index 3870c565..00000000 Binary files a/workers/pdf/split/documentpage1081-1100.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1101-1120.pdf b/workers/pdf/split/documentpage1101-1120.pdf deleted file mode 100644 index bec73478..00000000 Binary files a/workers/pdf/split/documentpage1101-1120.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1121-1140.pdf b/workers/pdf/split/documentpage1121-1140.pdf deleted file mode 100644 index 650d598a..00000000 Binary files a/workers/pdf/split/documentpage1121-1140.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1141-1160.pdf b/workers/pdf/split/documentpage1141-1160.pdf deleted file mode 100644 index 788a33dc..00000000 Binary files a/workers/pdf/split/documentpage1141-1160.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1161-1180.pdf b/workers/pdf/split/documentpage1161-1180.pdf deleted file mode 100644 index 1f81da70..00000000 Binary files a/workers/pdf/split/documentpage1161-1180.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage1181-1200.pdf b/workers/pdf/split/documentpage1181-1200.pdf deleted file mode 100644 index 59b710cb..00000000 Binary files a/workers/pdf/split/documentpage1181-1200.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage121-140.pdf b/workers/pdf/split/documentpage121-140.pdf deleted file mode 100644 index c95bf791..00000000 Binary files a/workers/pdf/split/documentpage121-140.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage141-160.pdf b/workers/pdf/split/documentpage141-160.pdf deleted file mode 100644 index 42f3efb5..00000000 Binary files a/workers/pdf/split/documentpage141-160.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage161-180.pdf b/workers/pdf/split/documentpage161-180.pdf deleted file mode 100644 index acddf06d..00000000 Binary files a/workers/pdf/split/documentpage161-180.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage181-200.pdf b/workers/pdf/split/documentpage181-200.pdf deleted file mode 100644 index d7455cbc..00000000 Binary files a/workers/pdf/split/documentpage181-200.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage201-220.pdf b/workers/pdf/split/documentpage201-220.pdf deleted file mode 100644 index 96775cc7..00000000 Binary files a/workers/pdf/split/documentpage201-220.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage21-40.pdf b/workers/pdf/split/documentpage21-40.pdf deleted file mode 100644 index 19dbd2fc..00000000 Binary files a/workers/pdf/split/documentpage21-40.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage221-240.pdf b/workers/pdf/split/documentpage221-240.pdf deleted file mode 100644 index f4853b74..00000000 Binary files a/workers/pdf/split/documentpage221-240.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage241-260.pdf b/workers/pdf/split/documentpage241-260.pdf deleted file mode 100644 index 0f14b481..00000000 Binary files a/workers/pdf/split/documentpage241-260.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage261-280.pdf b/workers/pdf/split/documentpage261-280.pdf deleted file mode 100644 index 4d72bdd7..00000000 Binary files a/workers/pdf/split/documentpage261-280.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage281-300.pdf b/workers/pdf/split/documentpage281-300.pdf deleted file mode 100644 index 423faab2..00000000 Binary files a/workers/pdf/split/documentpage281-300.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage301-320.pdf b/workers/pdf/split/documentpage301-320.pdf deleted file mode 100644 index 549408a9..00000000 Binary files a/workers/pdf/split/documentpage301-320.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage321-340.pdf b/workers/pdf/split/documentpage321-340.pdf deleted file mode 100644 index 4f01cbf2..00000000 Binary files a/workers/pdf/split/documentpage321-340.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage341-360.pdf b/workers/pdf/split/documentpage341-360.pdf deleted file mode 100644 index 8a33446f..00000000 Binary files a/workers/pdf/split/documentpage341-360.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage361-380.pdf b/workers/pdf/split/documentpage361-380.pdf deleted file mode 100644 index b1b187db..00000000 Binary files a/workers/pdf/split/documentpage361-380.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage381-400.pdf b/workers/pdf/split/documentpage381-400.pdf deleted file mode 100644 index 54be8bf5..00000000 Binary files a/workers/pdf/split/documentpage381-400.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage401-420.pdf b/workers/pdf/split/documentpage401-420.pdf deleted file mode 100644 index 55f338fa..00000000 Binary files a/workers/pdf/split/documentpage401-420.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage41-60.pdf b/workers/pdf/split/documentpage41-60.pdf deleted file mode 100644 index f8bb9f3a..00000000 Binary files a/workers/pdf/split/documentpage41-60.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage421-440.pdf b/workers/pdf/split/documentpage421-440.pdf deleted file mode 100644 index 5531cbf5..00000000 Binary files a/workers/pdf/split/documentpage421-440.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage441-460.pdf b/workers/pdf/split/documentpage441-460.pdf deleted file mode 100644 index 3e1fa248..00000000 Binary files a/workers/pdf/split/documentpage441-460.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage461-480.pdf b/workers/pdf/split/documentpage461-480.pdf deleted file mode 100644 index 4692e4d4..00000000 Binary files a/workers/pdf/split/documentpage461-480.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage481-500.pdf b/workers/pdf/split/documentpage481-500.pdf deleted file mode 100644 index 66b89989..00000000 Binary files a/workers/pdf/split/documentpage481-500.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage501-520.pdf b/workers/pdf/split/documentpage501-520.pdf deleted file mode 100644 index 57f4ec68..00000000 Binary files a/workers/pdf/split/documentpage501-520.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage521-540.pdf b/workers/pdf/split/documentpage521-540.pdf deleted file mode 100644 index cec887f9..00000000 Binary files a/workers/pdf/split/documentpage521-540.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage541-560.pdf b/workers/pdf/split/documentpage541-560.pdf deleted file mode 100644 index e9c665aa..00000000 Binary files a/workers/pdf/split/documentpage541-560.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage561-580.pdf b/workers/pdf/split/documentpage561-580.pdf deleted file mode 100644 index cc3f2d85..00000000 Binary files a/workers/pdf/split/documentpage561-580.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage581-600.pdf b/workers/pdf/split/documentpage581-600.pdf deleted file mode 100644 index 43420720..00000000 Binary files a/workers/pdf/split/documentpage581-600.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage601-620.pdf b/workers/pdf/split/documentpage601-620.pdf deleted file mode 100644 index f6a350dd..00000000 Binary files a/workers/pdf/split/documentpage601-620.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage61-80.pdf b/workers/pdf/split/documentpage61-80.pdf deleted file mode 100644 index a5dcff45..00000000 Binary files a/workers/pdf/split/documentpage61-80.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage621-640.pdf b/workers/pdf/split/documentpage621-640.pdf deleted file mode 100644 index 0f65f374..00000000 Binary files a/workers/pdf/split/documentpage621-640.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage641-660.pdf b/workers/pdf/split/documentpage641-660.pdf deleted file mode 100644 index 5866774c..00000000 Binary files a/workers/pdf/split/documentpage641-660.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage661-680.pdf b/workers/pdf/split/documentpage661-680.pdf deleted file mode 100644 index 54f1597d..00000000 Binary files a/workers/pdf/split/documentpage661-680.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage681-700.pdf b/workers/pdf/split/documentpage681-700.pdf deleted file mode 100644 index 1532406b..00000000 Binary files a/workers/pdf/split/documentpage681-700.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage701-720.pdf b/workers/pdf/split/documentpage701-720.pdf deleted file mode 100644 index c0bf155f..00000000 Binary files a/workers/pdf/split/documentpage701-720.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage721-740.pdf b/workers/pdf/split/documentpage721-740.pdf deleted file mode 100644 index 51f27e23..00000000 Binary files a/workers/pdf/split/documentpage721-740.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage741-760.pdf b/workers/pdf/split/documentpage741-760.pdf deleted file mode 100644 index a55a03fb..00000000 Binary files a/workers/pdf/split/documentpage741-760.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage761-780.pdf b/workers/pdf/split/documentpage761-780.pdf deleted file mode 100644 index d0430b30..00000000 Binary files a/workers/pdf/split/documentpage761-780.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage781-800.pdf b/workers/pdf/split/documentpage781-800.pdf deleted file mode 100644 index 613bc5e4..00000000 Binary files a/workers/pdf/split/documentpage781-800.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage801-820.pdf b/workers/pdf/split/documentpage801-820.pdf deleted file mode 100644 index b83522d3..00000000 Binary files a/workers/pdf/split/documentpage801-820.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage81-100.pdf b/workers/pdf/split/documentpage81-100.pdf deleted file mode 100644 index 31c430c8..00000000 Binary files a/workers/pdf/split/documentpage81-100.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage821-840.pdf b/workers/pdf/split/documentpage821-840.pdf deleted file mode 100644 index 85f438e3..00000000 Binary files a/workers/pdf/split/documentpage821-840.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage841-860.pdf b/workers/pdf/split/documentpage841-860.pdf deleted file mode 100644 index 593a05f9..00000000 Binary files a/workers/pdf/split/documentpage841-860.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage861-880.pdf b/workers/pdf/split/documentpage861-880.pdf deleted file mode 100644 index 82c30896..00000000 Binary files a/workers/pdf/split/documentpage861-880.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage881-900.pdf b/workers/pdf/split/documentpage881-900.pdf deleted file mode 100644 index fd4c5096..00000000 Binary files a/workers/pdf/split/documentpage881-900.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage901-920.pdf b/workers/pdf/split/documentpage901-920.pdf deleted file mode 100644 index c9e0b5ff..00000000 Binary files a/workers/pdf/split/documentpage901-920.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage921-940.pdf b/workers/pdf/split/documentpage921-940.pdf deleted file mode 100644 index a57a8fa0..00000000 Binary files a/workers/pdf/split/documentpage921-940.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage941-960.pdf b/workers/pdf/split/documentpage941-960.pdf deleted file mode 100644 index 909604d0..00000000 Binary files a/workers/pdf/split/documentpage941-960.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage961-980.pdf b/workers/pdf/split/documentpage961-980.pdf deleted file mode 100644 index 6b771e32..00000000 Binary files a/workers/pdf/split/documentpage961-980.pdf and /dev/null differ diff --git a/workers/pdf/split/documentpage981-1000.pdf b/workers/pdf/split/documentpage981-1000.pdf deleted file mode 100644 index 7be23f67..00000000 Binary files a/workers/pdf/split/documentpage981-1000.pdf and /dev/null differ diff --git a/workers/test-worker/test.py b/workers/test-worker/test.py deleted file mode 100644 index a1428173..00000000 --- a/workers/test-worker/test.py +++ /dev/null @@ -1,27 +0,0 @@ -import sys - -def main(): - print("Starting process...") - # Your code here, e.g., processing data, running a task, etc. - print("Process completed.") - -if __name__ == "__main__": - main() - - - -# print("before import") -# import math - -# print("before function_a") -# def function_a(): -# print("Function A") - -# print("before function_b") -# def function_b(): -# print("Function B {}".format(math.sqrt(100))) - -# print("before __name__ guard") -# if __name__ == '__main__': -# function_a() -# function_b() \ No newline at end of file