diff --git a/.github/workflows/copilot-build-backend.yml b/.github/workflows/copilot-build-backend.yml index bdb4690e1..08ab9d310 100644 --- a/.github/workflows/copilot-build-backend.yml +++ b/.github/workflows/copilot-build-backend.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: include: - - { dotnet: "6.0", configuration: Release, os: windows-latest } + - { dotnet: "8.0", configuration: Release, os: windows-latest } runs-on: ${{ matrix.os }} @@ -57,7 +57,7 @@ jobs: - name: Package Copilot Chat WebAPI run: | - scripts\deploy\package-webapi.ps1 -Configuration Release -DotnetFramework net6.0 -TargetRuntime win-x64 -OutputDirectory ${{ github.workspace }}\scripts\deploy -Version ${{ steps.versiontag.outputs.versiontag }} -InformationalVersion "Built from commit ${{ steps.gitversion.outputs.ShortSha }} on $(Get-Date -Format 'yyyy-MM-dd')" -SkipFrontendFiles ('${{ github.event_name == 'pull_request' }}' -eq 'true') + scripts\deploy\package-webapi.ps1 -Configuration Release -DotnetFramework net8.0 -TargetRuntime win-x64 -OutputDirectory ${{ github.workspace }}\scripts\deploy -Version ${{ steps.versiontag.outputs.versiontag }} -InformationalVersion "Built from commit ${{ steps.gitversion.outputs.ShortSha }} on $(Get-Date -Format 'yyyy-MM-dd')" -SkipFrontendFiles ('${{ github.event_name == 'pull_request' }}' -eq 'true') - name: Upload package to artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/copilot-build-memorypipeline.yml b/.github/workflows/copilot-build-memorypipeline.yml index d45011ad0..791adb4a7 100644 --- a/.github/workflows/copilot-build-memorypipeline.yml +++ b/.github/workflows/copilot-build-memorypipeline.yml @@ -52,11 +52,11 @@ jobs: - name: Set .Net Core version uses: actions/setup-dotnet@v4 with: - dotnet-version: 6.0.x + dotnet-version: 8.0.x - name: Package Copilot Chat Memory Pipeline run: | - scripts\deploy\package-memorypipeline.ps1 -Configuration Release -DotnetFramework net6.0 -TargetRuntime win-x64 -OutputDirectory ${{ github.workspace }}\scripts\deploy -Version ${{ steps.versiontag.outputs.versiontag }} -InformationalVersion "Built from commit ${{ steps.gitversion.outputs.ShortSha }} on $(Get-Date -Format "yyyy-MM-dd")" + scripts\deploy\package-memorypipeline.ps1 -Configuration Release -DotnetFramework net8.0 -TargetRuntime win-x64 -OutputDirectory ${{ github.workspace }}\scripts\deploy -Version ${{ steps.versiontag.outputs.versiontag }} -InformationalVersion "Built from commit ${{ steps.gitversion.outputs.ShortSha }} on $(Get-Date -Format "yyyy-MM-dd")" - name: Upload package to artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/copilot-build-plugins.yml b/.github/workflows/copilot-build-plugins.yml index 57c0c4406..40f0867fd 100644 --- a/.github/workflows/copilot-build-plugins.yml +++ b/.github/workflows/copilot-build-plugins.yml @@ -52,13 +52,13 @@ jobs: - name: Set .Net Core version uses: actions/setup-dotnet@v4 with: - dotnet-version: 6.0.x + dotnet-version: 8.0.x - name: Package Copilot Chat Plugins run: | scripts\deploy\package-plugins.ps1 ` -BuildConfiguration Release ` - -DotNetFramework net6.0 ` + -DotNetFramework net8.0 ` -OutputDirectory ${{ github.workspace }}\scripts\deploy ` -Version ${{ steps.versiontag.outputs.versiontag }} ` -InformationalVersion "Built from commit ${{ steps.gitversion.outputs.ShortSha }} on $(Get-Date -Format "yyyy-MM-dd")" diff --git a/.github/workflows/copilot-deploy-backend.yml b/.github/workflows/copilot-deploy-backend.yml index 83948783e..79f533bd2 100644 --- a/.github/workflows/copilot-deploy-backend.yml +++ b/.github/workflows/copilot-deploy-backend.yml @@ -28,7 +28,7 @@ jobs: fail-fast: false matrix: include: - - { dotnet: "6.0", configuration: Release, os: ubuntu-latest } + - { dotnet: "8.0", configuration: Release, os: ubuntu-latest } # Map the job output to step output outputs: backend-host: ${{steps.app-name.outputs.backend-host}} diff --git a/.github/workflows/copilot-deploy-infra.yml b/.github/workflows/copilot-deploy-infra.yml index c7e13d70a..d8afd5c9f 100644 --- a/.github/workflows/copilot-deploy-infra.yml +++ b/.github/workflows/copilot-deploy-infra.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: include: - - { dotnet: "6.0", configuration: Release, os: ubuntu-latest } + - { dotnet: "8.0", configuration: Release, os: ubuntu-latest } outputs: deployment-id: ${{steps.deployment-id.outputs.deployment_name}} diff --git a/.github/workflows/copilot-deploy-memorypipeline.yml b/.github/workflows/copilot-deploy-memorypipeline.yml index 998751e9c..e32119133 100644 --- a/.github/workflows/copilot-deploy-memorypipeline.yml +++ b/.github/workflows/copilot-deploy-memorypipeline.yml @@ -24,7 +24,7 @@ jobs: fail-fast: false matrix: include: - - { dotnet: "6.0", configuration: Release, os: ubuntu-latest } + - { dotnet: "8.0", configuration: Release, os: ubuntu-latest } runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/copilot-deploy-plugins.yml b/.github/workflows/copilot-deploy-plugins.yml index a5283bf43..ebc866533 100644 --- a/.github/workflows/copilot-deploy-plugins.yml +++ b/.github/workflows/copilot-deploy-plugins.yml @@ -24,7 +24,7 @@ jobs: fail-fast: false matrix: include: - - { dotnet: "6.0", configuration: Release, os: ubuntu-latest } + - { dotnet: "8.0", configuration: Release, os: ubuntu-latest } runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/copilot-test-e2e.yml b/.github/workflows/copilot-test-e2e.yml index 197e5c04e..14751b39d 100644 --- a/.github/workflows/copilot-test-e2e.yml +++ b/.github/workflows/copilot-test-e2e.yml @@ -23,14 +23,14 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 16 + node-version: 18 cache-dependency-path: webapp/yarn.lock cache: "yarn" - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 6.0.x + dotnet-version: 8.0.x - name: Install dependencies run: yarn install diff --git a/.github/workflows/dotnet-format.yml b/.github/workflows/dotnet-format.yml index 39f33aacd..2f56019f2 100644 --- a/.github/workflows/dotnet-format.yml +++ b/.github/workflows/dotnet-format.yml @@ -64,7 +64,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 7.0.x + dotnet-version: 8.0.x - name: Check formatting if: steps.find-csproj.outputs.csproj_files != '' diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 3f8d10c0f..dd4255db4 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -54,7 +54,7 @@ services: - KernelMemory__Services__RabbitMq__Port=5672 - KernelMemory__Services__RabbitMq__Username=chat-copilot - KernelMemory__Services__RabbitMq__Password=chat-copilot - - KernelMemory__ContentStorageType=AzureBlobs + - KernelMemory__DocumentStorageType=AzureBlobs - KernelMemory__ImageOcrType=Tesseract - KernelMemory__TextGeneratorType=AzureOpenAI - KernelMemory__DataIngestion__OrchestrationType=Distributed @@ -123,7 +123,7 @@ services: - KernelMemory__Services__RabbitMq__Port=5672 - KernelMemory__Services__RabbitMq__Username=chat-copilot - KernelMemory__Services__RabbitMq__Password=chat-copilot - - KernelMemory__ContentStorageType=AzureBlobs + - KernelMemory__DocumentStorageType=AzureBlobs - KernelMemory__ImageOcrType=Tesseract - KernelMemory__TextGeneratorType=AzureOpenAI - KernelMemory__DataIngestion__OrchestrationType=Distributed diff --git a/docker/memorypipeline/Dockerfile b/docker/memorypipeline/Dockerfile index 85e575648..904774443 100644 --- a/docker/memorypipeline/Dockerfile +++ b/docker/memorypipeline/Dockerfile @@ -1,7 +1,7 @@ # docker build -f docker/webapi/Dockerfile -t chat-copilot-memorypipeline . # builder -FROM mcr.microsoft.com/dotnet/sdk:7.0 AS builder +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS builder WORKDIR /source # generate dev-certs for https RUN dotnet dev-certs https @@ -17,7 +17,7 @@ RUN cd memorypipeline && \ # final stage/image -FROM mcr.microsoft.com/dotnet/aspnet:7.0 +FROM mcr.microsoft.com/dotnet/aspnet:8.0 WORKDIR /app COPY --from=builder /app . COPY --from=builder /root/.dotnet/corefx/cryptography/x509stores/my/* /root/.dotnet/corefx/cryptography/x509stores/my/ diff --git a/docker/plugins/web-searcher/Dockerfile b/docker/plugins/web-searcher/Dockerfile index 6e590121c..efd286109 100644 --- a/docker/plugins/web-searcher/Dockerfile +++ b/docker/plugins/web-searcher/Dockerfile @@ -1,7 +1,7 @@ # docker build -f docker/plugins/web-searcher/Dockerfile -t web-searcher . # builder -FROM mcr.microsoft.com/dotnet/sdk:6.0 AS installer-env +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS installer-env ARG AZURE_FUNCTION_MASTER_KEY WORKDIR /source COPY plugins/shared shared @@ -13,7 +13,7 @@ RUN cd /source/web-searcher && \ echo "{\"masterKey\":{\"name\":\"master\",\"value\":\"$AZURE_FUNCTION_MASTER_KEY\",\"encrypted\":false},\"functionKeys\":[]}" > /azure-functions-host/Secrets/host.json # final stage/image -FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4.0-dotnet-isolated6.0-appservice +FROM mcr.microsoft.com/azure-functions/dotnet-isolated:4.0-dotnet-isolated8.0-appservice ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ AzureFunctionsJobHost__Logging__Console__IsEnabled=true \ AzureWebJobsSecretStorageType=files diff --git a/docker/webapi/Dockerfile b/docker/webapi/Dockerfile index b0bb2291e..2b1f257cf 100644 --- a/docker/webapi/Dockerfile +++ b/docker/webapi/Dockerfile @@ -1,7 +1,7 @@ # docker build -f docker/webapi/Dockerfile -t chat-copilot-webapi . # builder -FROM mcr.microsoft.com/dotnet/sdk:7.0 AS builder +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS builder WORKDIR /source # generate dev-certs for https RUN dotnet dev-certs https @@ -15,7 +15,7 @@ RUN cd webapi && \ dotnet publish --use-current-runtime --self-contained false --no-restore -o /app # final stage/image -FROM mcr.microsoft.com/dotnet/aspnet:7.0 +FROM mcr.microsoft.com/dotnet/aspnet:8.0 ENV Kestrel__Endpoints__Http__Url=http://0.0.0.0:8080 WORKDIR /app COPY --from=builder /app . diff --git a/integration-tests/ChatCopilotIntegrationTests.csproj b/integration-tests/ChatCopilotIntegrationTests.csproj index 78564602e..ed70bbad2 100644 --- a/integration-tests/ChatCopilotIntegrationTests.csproj +++ b/integration-tests/ChatCopilotIntegrationTests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 enable false true @@ -14,13 +14,13 @@ - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/memorypipeline/CopilotChatMemoryPipeline.csproj b/memorypipeline/CopilotChatMemoryPipeline.csproj index 716e43d4f..55296130d 100644 --- a/memorypipeline/CopilotChatMemoryPipeline.csproj +++ b/memorypipeline/CopilotChatMemoryPipeline.csproj @@ -2,7 +2,7 @@ CopilotChat.MemoryPipeline - net6.0 + net8.0 LatestMajor disable enable @@ -15,8 +15,8 @@ - - + + diff --git a/memorypipeline/appsettings.json b/memorypipeline/appsettings.json index 932afd299..c6917c309 100644 --- a/memorypipeline/appsettings.json +++ b/memorypipeline/appsettings.json @@ -1,7 +1,7 @@ { // // Kernel Memory configuration - https://github.com/microsoft/kernel-memory - // - ContentStorageType is the storage configuration for memory transfer: "AzureBlobs" or "SimpleFileStorage" + // - DocumentStorageType is the storage configuration for memory transfer: "AzureBlobs" or "SimpleFileStorage" // - TextGeneratorType is the AI completion service configuration: "AzureOpenAIText" or "OpenAI" // - ImageOcrType is the image OCR configuration: "None" or "AzureFormRecognizer" or "Tesseract" // - DataIngestion is the configuration section for data ingestion pipelines. @@ -9,7 +9,7 @@ // - Services is the configuration sections for various memory settings. // "KernelMemory": { - "ContentStorageType": "SimpleFileStorage", + "DocumentStorageType": "SimpleFileStorage", "TextGeneratorType": "AzureOpenAIText", "ImageOcrType": "None", // Data ingestion pipelines configuration. diff --git a/scripts/deploy/main.bicep b/scripts/deploy/main.bicep index 756affb15..119f05cab 100644 --- a/scripts/deploy/main.bicep +++ b/scripts/deploy/main.bicep @@ -284,7 +284,7 @@ resource appServiceWebConfig 'Microsoft.Web/sites/config@2022-09-01' = { value: '~2' } { - name: 'KernelMemory:ContentStorageType' + name: 'KernelMemory:DocumentStorageType' value: 'AzureBlobs' } { @@ -442,6 +442,7 @@ resource appServiceMemoryPipeline 'Microsoft.Web/sites@2022-09-01' = { skweb: '1' } properties: { + httpsOnly: true serverFarmId: appServicePlan.id virtualNetworkSubnetId: memoryStore == 'Qdrant' ? virtualNetwork.properties.subnets[0].id : null siteConfig: { @@ -465,7 +466,7 @@ resource appServiceMemoryPipelineConfig 'Microsoft.Web/sites/config@2022-09-01' vnetRouteAllEnabled: true appSettings: [ { - name: 'KernelMemory:ContentStorageType' + name: 'KernelMemory:DocumentStorageType' value: 'AzureBlobs' } { diff --git a/scripts/deploy/main.json b/scripts/deploy/main.json index 27b231f48..6af068690 100644 --- a/scripts/deploy/main.json +++ b/scripts/deploy/main.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.25.53.49325", - "templateHash": "1335279082624698663" + "version": "0.28.1.47646", + "templateHash": "7267170677416144946" } }, "parameters": { @@ -299,7 +299,7 @@ "use32BitWorkerProcess": false, "vnetRouteAllEnabled": true, "webSocketsEnabled": true, - "appSettings": "[concat(createArray(createObject('name', 'Authentication:Type', 'value', 'AzureAd'), createObject('name', 'Authentication:AzureAd:Instance', 'value', parameters('azureAdInstance')), createObject('name', 'Authentication:AzureAd:TenantId', 'value', parameters('azureAdTenantId')), createObject('name', 'Authentication:AzureAd:ClientId', 'value', parameters('webApiClientId')), createObject('name', 'Authentication:AzureAd:Scopes', 'value', 'access_as_user'), createObject('name', 'ChatStore:Type', 'value', if(parameters('deployCosmosDB'), 'cosmos', 'volatile')), createObject('name', 'ChatStore:Cosmos:Database', 'value', 'CopilotChat'), createObject('name', 'ChatStore:Cosmos:ChatSessionsContainer', 'value', 'chatsessions'), createObject('name', 'ChatStore:Cosmos:ChatMessagesContainer', 'value', 'chatmessages'), createObject('name', 'ChatStore:Cosmos:ChatMemorySourcesContainer', 'value', 'chatmemorysources'), createObject('name', 'ChatStore:Cosmos:ChatParticipantsContainer', 'value', 'chatparticipants'), createObject('name', 'ChatStore:Cosmos:ConnectionString', 'value', if(parameters('deployCosmosDB'), listConnectionStrings(resourceId('Microsoft.DocumentDB/databaseAccounts', toLower(format('cosmos-{0}', variables('uniqueName')))), '2023-04-15').connectionStrings[0].connectionString, '')), createObject('name', 'AzureSpeech:Region', 'value', parameters('location')), createObject('name', 'AzureSpeech:Key', 'value', if(parameters('deploySpeechServices'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('cog-speech-{0}', variables('uniqueName'))), '2022-12-01').key1, '')), createObject('name', 'AllowedOrigins', 'value', '[*]'), createObject('name', 'Kestrel:Endpoints:Https:Url', 'value', 'https://localhost:443'), createObject('name', 'Frontend:AadClientId', 'value', parameters('frontendClientId')), createObject('name', 'Logging:LogLevel:Default', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:CopilotChat.WebApi', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.SemanticKernel', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.AspNetCore.Hosting', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.Hosting.Lifetimel', 'value', 'Warning'), createObject('name', 'Logging:ApplicationInsights:LogLevel:Default', 'value', 'Warning'), createObject('name', 'APPLICATIONINSIGHTS_CONNECTION_STRING', 'value', reference(resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName'))), '2020-02-02').ConnectionString), createObject('name', 'ApplicationInsightsAgent_EXTENSION_VERSION', 'value', '~2'), createObject('name', 'KernelMemory:ContentStorageType', 'value', 'AzureBlobs'), createObject('name', 'KernelMemory:TextGeneratorType', 'value', parameters('aiService')), createObject('name', 'KernelMemory:DataIngestion:OrchestrationType', 'value', 'Distributed'), createObject('name', 'KernelMemory:DataIngestion:DistributedOrchestration:QueueType', 'value', 'AzureQueue'), createObject('name', 'KernelMemory:DataIngestion:EmbeddingGeneratorTypes:0', 'value', parameters('aiService')), createObject('name', 'KernelMemory:DataIngestion:MemoryDbTypes:0', 'value', parameters('memoryStore')), createObject('name', 'KernelMemory:Retrieval:MemoryDbType', 'value', parameters('memoryStore')), createObject('name', 'KernelMemory:Retrieval:EmbeddingGeneratorType', 'value', parameters('aiService')), createObject('name', 'KernelMemory:Services:AzureBlobs:Auth', 'value', 'ConnectionString'), createObject('name', 'KernelMemory:Services:AzureBlobs:ConnectionString', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)), createObject('name', 'KernelMemory:Services:AzureBlobs:Container', 'value', 'chatmemory'), createObject('name', 'KernelMemory:Services:AzureQueue:Auth', 'value', 'ConnectionString'), createObject('name', 'KernelMemory:Services:AzureQueue:ConnectionString', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)), createObject('name', 'KernelMemory:Services:AzureAISearch:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureAISearch:Endpoint', 'value', if(equals(parameters('memoryStore'), 'AzureAISearch'), format('https://{0}.search.windows.net', format('acs-{0}', variables('uniqueName'))), '')), createObject('name', 'KernelMemory:Services:AzureAISearch:APIKey', 'value', if(equals(parameters('memoryStore'), 'AzureAISearch'), listAdminKeys(resourceId('Microsoft.Search/searchServices', format('acs-{0}', variables('uniqueName'))), '2022-09-01').primaryKey, '')), createObject('name', 'KernelMemory:Services:Qdrant:Endpoint', 'value', if(equals(parameters('memoryStore'), 'Qdrant'), format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('app-{0}-qdrant', variables('uniqueName'))), '2022-09-01').defaultHostName), '')), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Endpoint', 'value', if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').endpoint, parameters('aiEndpoint'))), createObject('name', 'KernelMemory:Services:AzureOpenAIText:APIKey', 'value', if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').key1, parameters('aiApiKey'))), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Deployment', 'value', parameters('completionModel')), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Endpoint', 'value', if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').endpoint, parameters('aiEndpoint'))), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:APIKey', 'value', if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').key1, parameters('aiApiKey'))), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Deployment', 'value', parameters('embeddingModel')), createObject('name', 'KernelMemory:Services:OpenAI:TextModel', 'value', parameters('completionModel')), createObject('name', 'KernelMemory:Services:OpenAI:EmbeddingModel', 'value', parameters('embeddingModel')), createObject('name', 'KernelMemory:Services:OpenAI:APIKey', 'value', parameters('aiApiKey')), createObject('name', 'Plugins:0:Name', 'value', 'Klarna Shopping'), createObject('name', 'Plugins:0:ManifestDomain', 'value', 'https://www.klarna.com')), if(parameters('deployWebSearcherPlugin'), createArray(createObject('name', 'Plugins:1:Name', 'value', 'WebSearcher'), createObject('name', 'Plugins:1:ManifestDomain', 'value', format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('function-{0}-websearcher-plugin', variables('uniqueName'))), '2022-09-01').defaultHostName)), createObject('name', 'Plugins:1:Key', 'value', listkeys(format('{0}/host/default/', resourceId('Microsoft.Web/sites', format('function-{0}-websearcher-plugin', variables('uniqueName')))), '2022-09-01').functionKeys.default)), createArray()))]" + "appSettings": "[concat(createArray(createObject('name', 'Authentication:Type', 'value', 'AzureAd'), createObject('name', 'Authentication:AzureAd:Instance', 'value', parameters('azureAdInstance')), createObject('name', 'Authentication:AzureAd:TenantId', 'value', parameters('azureAdTenantId')), createObject('name', 'Authentication:AzureAd:ClientId', 'value', parameters('webApiClientId')), createObject('name', 'Authentication:AzureAd:Scopes', 'value', 'access_as_user'), createObject('name', 'ChatStore:Type', 'value', if(parameters('deployCosmosDB'), 'cosmos', 'volatile')), createObject('name', 'ChatStore:Cosmos:Database', 'value', 'CopilotChat'), createObject('name', 'ChatStore:Cosmos:ChatSessionsContainer', 'value', 'chatsessions'), createObject('name', 'ChatStore:Cosmos:ChatMessagesContainer', 'value', 'chatmessages'), createObject('name', 'ChatStore:Cosmos:ChatMemorySourcesContainer', 'value', 'chatmemorysources'), createObject('name', 'ChatStore:Cosmos:ChatParticipantsContainer', 'value', 'chatparticipants'), createObject('name', 'ChatStore:Cosmos:ConnectionString', 'value', if(parameters('deployCosmosDB'), listConnectionStrings(resourceId('Microsoft.DocumentDB/databaseAccounts', toLower(format('cosmos-{0}', variables('uniqueName')))), '2023-04-15').connectionStrings[0].connectionString, '')), createObject('name', 'AzureSpeech:Region', 'value', parameters('location')), createObject('name', 'AzureSpeech:Key', 'value', if(parameters('deploySpeechServices'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('cog-speech-{0}', variables('uniqueName'))), '2022-12-01').key1, '')), createObject('name', 'AllowedOrigins', 'value', '[*]'), createObject('name', 'Kestrel:Endpoints:Https:Url', 'value', 'https://localhost:443'), createObject('name', 'Frontend:AadClientId', 'value', parameters('frontendClientId')), createObject('name', 'Logging:LogLevel:Default', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:CopilotChat.WebApi', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.SemanticKernel', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.AspNetCore.Hosting', 'value', 'Warning'), createObject('name', 'Logging:LogLevel:Microsoft.Hosting.Lifetimel', 'value', 'Warning'), createObject('name', 'Logging:ApplicationInsights:LogLevel:Default', 'value', 'Warning'), createObject('name', 'APPLICATIONINSIGHTS_CONNECTION_STRING', 'value', reference(resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName'))), '2020-02-02').ConnectionString), createObject('name', 'ApplicationInsightsAgent_EXTENSION_VERSION', 'value', '~2'), createObject('name', 'KernelMemory:DocumentStorageType', 'value', 'AzureBlobs'), createObject('name', 'KernelMemory:TextGeneratorType', 'value', parameters('aiService')), createObject('name', 'KernelMemory:DataIngestion:OrchestrationType', 'value', 'Distributed'), createObject('name', 'KernelMemory:DataIngestion:DistributedOrchestration:QueueType', 'value', 'AzureQueue'), createObject('name', 'KernelMemory:DataIngestion:EmbeddingGeneratorTypes:0', 'value', parameters('aiService')), createObject('name', 'KernelMemory:DataIngestion:MemoryDbTypes:0', 'value', parameters('memoryStore')), createObject('name', 'KernelMemory:Retrieval:MemoryDbType', 'value', parameters('memoryStore')), createObject('name', 'KernelMemory:Retrieval:EmbeddingGeneratorType', 'value', parameters('aiService')), createObject('name', 'KernelMemory:Services:AzureBlobs:Auth', 'value', 'ConnectionString'), createObject('name', 'KernelMemory:Services:AzureBlobs:ConnectionString', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)), createObject('name', 'KernelMemory:Services:AzureBlobs:Container', 'value', 'chatmemory'), createObject('name', 'KernelMemory:Services:AzureQueue:Auth', 'value', 'ConnectionString'), createObject('name', 'KernelMemory:Services:AzureQueue:ConnectionString', 'value', format('DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}', format('st{0}', variables('rgIdHash')), listKeys(resourceId('Microsoft.Storage/storageAccounts', format('st{0}', variables('rgIdHash'))), '2022-09-01').keys[1].value)), createObject('name', 'KernelMemory:Services:AzureAISearch:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureAISearch:Endpoint', 'value', if(equals(parameters('memoryStore'), 'AzureAISearch'), format('https://{0}.search.windows.net', format('acs-{0}', variables('uniqueName'))), '')), createObject('name', 'KernelMemory:Services:AzureAISearch:APIKey', 'value', if(equals(parameters('memoryStore'), 'AzureAISearch'), listAdminKeys(resourceId('Microsoft.Search/searchServices', format('acs-{0}', variables('uniqueName'))), '2022-09-01').primaryKey, '')), createObject('name', 'KernelMemory:Services:Qdrant:Endpoint', 'value', if(equals(parameters('memoryStore'), 'Qdrant'), format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('app-{0}-qdrant', variables('uniqueName'))), '2022-09-01').defaultHostName), '')), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Endpoint', 'value', if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').endpoint, parameters('aiEndpoint'))), createObject('name', 'KernelMemory:Services:AzureOpenAIText:APIKey', 'value', if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').key1, parameters('aiApiKey'))), createObject('name', 'KernelMemory:Services:AzureOpenAIText:Deployment', 'value', parameters('completionModel')), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Auth', 'value', 'ApiKey'), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Endpoint', 'value', if(parameters('deployNewAzureOpenAI'), reference(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').endpoint, parameters('aiEndpoint'))), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:APIKey', 'value', if(parameters('deployNewAzureOpenAI'), listKeys(resourceId('Microsoft.CognitiveServices/accounts', format('ai-{0}', variables('uniqueName'))), '2023-05-01').key1, parameters('aiApiKey'))), createObject('name', 'KernelMemory:Services:AzureOpenAIEmbedding:Deployment', 'value', parameters('embeddingModel')), createObject('name', 'KernelMemory:Services:OpenAI:TextModel', 'value', parameters('completionModel')), createObject('name', 'KernelMemory:Services:OpenAI:EmbeddingModel', 'value', parameters('embeddingModel')), createObject('name', 'KernelMemory:Services:OpenAI:APIKey', 'value', parameters('aiApiKey')), createObject('name', 'Plugins:0:Name', 'value', 'Klarna Shopping'), createObject('name', 'Plugins:0:ManifestDomain', 'value', 'https://www.klarna.com')), if(parameters('deployWebSearcherPlugin'), createArray(createObject('name', 'Plugins:1:Name', 'value', 'WebSearcher'), createObject('name', 'Plugins:1:ManifestDomain', 'value', format('https://{0}', reference(resourceId('Microsoft.Web/sites', format('function-{0}-websearcher-plugin', variables('uniqueName'))), '2022-09-01').defaultHostName)), createObject('name', 'Plugins:1:Key', 'value', listkeys(format('{0}/host/default/', resourceId('Microsoft.Web/sites', format('function-{0}-websearcher-plugin', variables('uniqueName')))), '2022-09-01').functionKeys.default)), createArray()))]" }, "dependsOn": [ "[resourceId('Microsoft.Insights/components', format('appins-{0}', variables('uniqueName')))]", @@ -338,6 +338,7 @@ "skweb": "1" }, "properties": { + "httpsOnly": true, "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', format('asp-{0}-webapi', variables('uniqueName')))]", "virtualNetworkSubnetId": "[if(equals(parameters('memoryStore'), 'Qdrant'), reference(resourceId('Microsoft.Network/virtualNetworks', format('vnet-{0}', variables('uniqueName'))), '2021-05-01').subnets[0].id, null())]", "siteConfig": { @@ -362,7 +363,7 @@ "vnetRouteAllEnabled": true, "appSettings": [ { - "name": "KernelMemory:ContentStorageType", + "name": "KernelMemory:DocumentStorageType", "value": "AzureBlobs" }, { diff --git a/shared/ConfigurationBuilderExtensions.cs b/shared/ConfigurationBuilderExtensions.cs index e3c5a0b50..b0006179a 100644 --- a/shared/ConfigurationBuilderExtensions.cs +++ b/shared/ConfigurationBuilderExtensions.cs @@ -4,7 +4,6 @@ using System.IO; using System.Reflection; using Microsoft.Extensions.Configuration; -using Microsoft.KernelMemory.Configuration; namespace CopilotChat.Shared; @@ -35,7 +34,7 @@ public static void AddKMConfigurationSources( var main = Path.Join(settingsDirectory, "appsettings.json"); if (!File.Exists(main)) { - throw new ConfigurationException($"appsettings.json not found. Directory: {settingsDirectory}"); + throw new FileNotFoundException($"appsettings.json not found. Directory: {settingsDirectory}"); } builder.AddJsonFile(main, optional: false); diff --git a/shared/CopilotChatShared.csproj b/shared/CopilotChatShared.csproj index c5d38a85a..25216ca99 100644 --- a/shared/CopilotChatShared.csproj +++ b/shared/CopilotChatShared.csproj @@ -2,17 +2,17 @@ CopilotChat.Shared - net6.0 + net8.0 LatestMajor disable enable - - - - + + + + diff --git a/shared/Ocr/ConfigurationExtensions.cs b/shared/Ocr/ConfigurationExtensions.cs index 3d477e2c0..c53743a42 100644 --- a/shared/Ocr/ConfigurationExtensions.cs +++ b/shared/Ocr/ConfigurationExtensions.cs @@ -3,7 +3,6 @@ using System; using CopilotChat.Shared.Ocr.Tesseract; using Microsoft.Extensions.Configuration; -using Microsoft.KernelMemory.Configuration; using Microsoft.KernelMemory.DataFormats; namespace CopilotChat.Shared.Ocr; @@ -28,7 +27,7 @@ public static class ConfigurationExtensions if (tesseractOptions == null) { - throw new ConfigurationException($"Missing configuration for {ConfigOcrType}: {ocrType}"); + throw new ArgumentNullException($"Missing configuration for {ConfigOcrType}: {ocrType}"); } return new TesseractOcrEngine(tesseractOptions); diff --git a/shared/ServiceConfiguration.cs b/shared/ServiceConfiguration.cs index 7fd116d43..5d189d40d 100644 --- a/shared/ServiceConfiguration.cs +++ b/shared/ServiceConfiguration.cs @@ -6,12 +6,10 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.KernelMemory; using Microsoft.KernelMemory.AI; -using Microsoft.KernelMemory.Configuration; -using Microsoft.KernelMemory.ContentStorage.DevTools; +using Microsoft.KernelMemory.DocumentStorage.DevTools; using Microsoft.KernelMemory.MemoryStorage; using Microsoft.KernelMemory.MemoryStorage.DevTools; using Microsoft.KernelMemory.Pipeline.Queue.DevTools; -using Microsoft.KernelMemory.Postgres; namespace CopilotChat.Shared; @@ -151,17 +149,17 @@ private void ConfigureQueueDependency(IKernelMemoryBuilder builder) private void ConfigureStorageDependency(IKernelMemoryBuilder builder) { - switch (this._memoryConfiguration.ContentStorageType) + switch (this._memoryConfiguration.DocumentStorageType) { case string x1 when x1.Equals("AzureBlob", StringComparison.OrdinalIgnoreCase): case string x2 when x2.Equals("AzureBlobs", StringComparison.OrdinalIgnoreCase): // Check 2 keys for backward compatibility - builder.Services.AddAzureBlobsAsContentStorage(this.GetServiceConfig("AzureBlobs") - ?? this.GetServiceConfig("AzureBlob")); + builder.WithAzureBlobsDocumentStorage(this.GetServiceConfig("AzureBlobs") + ?? this.GetServiceConfig("AzureBlob")); break; case string x when x.Equals("SimpleFileStorage", StringComparison.OrdinalIgnoreCase): - builder.Services.AddSimpleFileStorageAsContentStorage(this.GetServiceConfig("SimpleFileStorage")); + builder.WithSimpleFileStorage(this.GetServiceConfig("SimpleFileStorage")); break; default: diff --git a/tools/importdocument/ImportDocument.csproj b/tools/importdocument/ImportDocument.csproj index 4bd1f2fcf..b8ee82a33 100644 --- a/tools/importdocument/ImportDocument.csproj +++ b/tools/importdocument/ImportDocument.csproj @@ -18,7 +18,7 @@ - + diff --git a/webapi/Auth/PassThroughAuthenticationHandler.cs b/webapi/Auth/PassThroughAuthenticationHandler.cs index 71244966b..8b6f2a1c6 100644 --- a/webapi/Auth/PassThroughAuthenticationHandler.cs +++ b/webapi/Auth/PassThroughAuthenticationHandler.cs @@ -25,8 +25,7 @@ public class PassThroughAuthenticationHandler : AuthenticationHandler options, ILoggerFactory loggerFactory, - UrlEncoder encoder, - ISystemClock clock) : base(options, loggerFactory, encoder, clock) + UrlEncoder encoder) : base(options, loggerFactory, encoder) { } diff --git a/webapi/CopilotChatWebApi.csproj b/webapi/CopilotChatWebApi.csproj index 50a133980..57e804fe0 100644 --- a/webapi/CopilotChatWebApi.csproj +++ b/webapi/CopilotChatWebApi.csproj @@ -1,7 +1,7 @@  CopilotChat.WebApi - net6.0 + net8.0 LatestMajor 10 enable @@ -20,12 +20,11 @@ - - - - - - + + + + + @@ -34,7 +33,7 @@ - + @@ -52,22 +51,22 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/webapi/Extensions/ISemanticMemoryClientExtensions.cs b/webapi/Extensions/ISemanticMemoryClientExtensions.cs index b6782faf7..9d88a8b41 100644 --- a/webapi/Extensions/ISemanticMemoryClientExtensions.cs +++ b/webapi/Extensions/ISemanticMemoryClientExtensions.cs @@ -154,7 +154,7 @@ public static async Task StoreMemoryAsync( using var stream = new MemoryStream(); using var writer = new StreamWriter(stream); await writer.WriteAsync(memory); - await writer.FlushAsync(); + await writer.FlushAsync(cancellationToken); stream.Position = 0; var uploadRequest = new DocumentUploadRequest diff --git a/webapi/README.md b/webapi/README.md index 6f597fc99..c0edfe21d 100644 --- a/webapi/README.md +++ b/webapi/README.md @@ -56,7 +56,7 @@ Running the memory creation pipeline steps in different processes. This means th ### (Optional) Use hosted resources: [Azure Storage Account](https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview), [Azure Cognitive Search](https://learn.microsoft.com/en-us/azure/search/search-what-is-azure-search) -1. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:ContentStorageType` to `AzureBlobs`. +1. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:DocumentStorageType` to `AzureBlobs`. 2. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:DataIngestion:DistributedOrchestration:QueueType` to `AzureQueue`. 3. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:DataIngestion:MemoryDbTypes:0` to `AzureAISearch`. 4. In [./webapi/appsettings.json](./appsettings.json) and [../memorypipeline/appsettings.json](../memorypipeline/appsettings.json), set `SemanticMemory:Retrieval:MemoryDbType` to `AzureAISearch`.