diff --git a/README.md b/README.md index fe5b89ca1..047135bb3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -
+****

English | 中文 @@ -145,22 +145,23 @@ For other providers, change the `provider` & `model` to their respective values. Head over to [Portkey docs](https://portkey.ai/docs/product/ai-gateway-streamline-llm-integrations) for detailed [guides & cookbooks](https://portkey.ai/docs/welcome/integration-guides) on more [provider integrations](https://portkey.ai/docs/welcome/integration-guides). -|| Provider | Support | Stream | -|---|---|---|---| -| | [OpenAI](https://portkey.ai/docs/welcome/integration-guides/openai) | ✅ |✅ | -| | [Azure OpenAI](https://portkey.ai/docs/welcome/integration-guides/azure-openai) | ✅ |✅ | -| | [Anyscale](https://portkey.ai/docs/welcome/integration-guides/anyscale-llama2-mistral-zephyr) | ✅ | ✅ | -| | [Google Gemini & Palm](https://portkey.ai/docs/welcome/integration-guides/gemini) | ✅ |✅ | -| | [Anthropic](https://portkey.ai/docs/welcome/integration-guides/anthropic) | ✅ |✅ | -| | [Cohere](https://portkey.ai/docs/welcome/integration-guides/cohere) | ✅ |✅ | -| | [Together AI](https://portkey.ai/docs/welcome/integration-guides/together-ai) | ✅ |✅ | -| | [Perplexity](https://portkey.ai/docs/welcome/integration-guides/perplexity-ai) | ✅ |✅ | -| | [Mistral](https://portkey.ai/docs/welcome/integration-guides/mistral-ai) | ✅ |✅ | -| | [Nomic](https://portkey.ai/docs/welcome/integration-guides/nomic) | ✅ |✅ | -| | [AI21](https://portkey.ai/docs/welcome/integration-guides) | ✅ |✅ | -| | [Stability AI](https://portkey.ai/docs/welcome/integration-guides/stability-ai) | ✅ |✅ | -| | [DeepInfra](https://portkey.ai/docs/welcome/integration-guides) | ✅ |✅ | -| | [Ollama](https://portkey.ai/docs/welcome/integration-guides/ollama) | ✅ |✅ | +| | Provider | Support | Stream | +| -------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | ------- | ------ | +| | [OpenAI](https://portkey.ai/docs/welcome/integration-guides/openai) | ✅ | ✅ | +| | [Azure OpenAI](https://portkey.ai/docs/welcome/integration-guides/azure-openai) | ✅ | ✅ | +| | [Anyscale](https://portkey.ai/docs/welcome/integration-guides/anyscale-llama2-mistral-zephyr) | ✅ | ✅ | +| | [Google Gemini & Palm](https://portkey.ai/docs/welcome/integration-guides/gemini) | ✅ | ✅ | +| | [Anthropic](https://portkey.ai/docs/welcome/integration-guides/anthropic) | ✅ | ✅ | +| | [Cohere](https://portkey.ai/docs/welcome/integration-guides/cohere) | ✅ | ✅ | +| | [Together AI](https://portkey.ai/docs/welcome/integration-guides/together-ai) | ✅ | ✅ | +| | [Perplexity](https://portkey.ai/docs/welcome/integration-guides/perplexity-ai) | ✅ | ✅ | +| | [Mistral](https://portkey.ai/docs/welcome/integration-guides/mistral-ai) | ✅ | ✅ | +| | [Nomic](https://portkey.ai/docs/welcome/integration-guides/nomic) | ✅ | ✅ | +| | [AI21](https://portkey.ai/docs/welcome/integration-guides) | ✅ | ✅ | +| | [Stability AI](https://portkey.ai/docs/welcome/integration-guides/stability-ai) | ✅ | ✅ | +| | [DeepInfra](https://portkey.ai/docs/welcome/integration-guides) | ✅ | ✅ | +| | [Ollama](https://portkey.ai/docs/welcome/integration-guides/ollama) | ✅ | ✅ | +| | Novita AI | ✅ | ✅ | `/chat/completions`, `/completions` | > [View the complete list of 100+ supported models here](https://portkey.ai/docs/welcome/what-is-portkey#ai-providers-supported)
@@ -302,14 +303,14 @@ Here's a guide to [use the config object in your request](https://portkey.ai/doc ## Supported SDKs -| Language | Supported SDKs | -|---|---| -| Node.js / JS / TS | [Portkey SDK](https://www.npmjs.com/package/portkey-ai)
[OpenAI SDK](https://www.npmjs.com/package/openai)
[LangchainJS](https://www.npmjs.com/package/langchain)
[LlamaIndex.TS](https://www.npmjs.com/package/llamaindex) | -| Python | [Portkey SDK](https://pypi.org/project/portkey-ai/)
[OpenAI SDK](https://portkey.ai/docs/welcome/integration-guides/openai)
[Langchain](https://portkey.ai/docs/welcome/integration-guides/langchain-python)
[LlamaIndex](https://portkey.ai/docs/welcome/integration-guides/llama-index-python) | -| Go | [go-openai](https://github.com/sashabaranov/go-openai) | -| Java | [openai-java](https://github.com/TheoKanning/openai-java) | -| Rust | [async-openai](https://docs.rs/async-openai/latest/async_openai/) | -| Ruby | [ruby-openai](https://github.com/alexrudall/ruby-openai) | +| Language | Supported SDKs | +| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Node.js / JS / TS | [Portkey SDK](https://www.npmjs.com/package/portkey-ai)
[OpenAI SDK](https://www.npmjs.com/package/openai)
[LangchainJS](https://www.npmjs.com/package/langchain)
[LlamaIndex.TS](https://www.npmjs.com/package/llamaindex) | +| Python | [Portkey SDK](https://pypi.org/project/portkey-ai/)
[OpenAI SDK](https://portkey.ai/docs/welcome/integration-guides/openai)
[Langchain](https://portkey.ai/docs/welcome/integration-guides/langchain-python)
[LlamaIndex](https://portkey.ai/docs/welcome/integration-guides/llama-index-python) | +| Go | [go-openai](https://github.com/sashabaranov/go-openai) | +| Java | [openai-java](https://github.com/TheoKanning/openai-java) | +| Rust | [async-openai](https://docs.rs/async-openai/latest/async_openai/) | +| Ruby | [ruby-openai](https://github.com/alexrudall/ruby-openai) |
diff --git a/src/globals.ts b/src/globals.ts index 97d1cd704..33db8b693 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -48,6 +48,7 @@ export const MOONSHOT: string = 'moonshot'; export const OPENROUTER: string = 'openrouter'; export const LINGYI: string = 'lingyi'; export const ZHIPU: string = 'zhipu'; +export const NOVITA_AI: string = 'novita-ai'; export const VALID_PROVIDERS = [ ANTHROPIC, @@ -76,6 +77,7 @@ export const VALID_PROVIDERS = [ OPENROUTER, LINGYI, ZHIPU, + NOVITA_AI, ]; export const CONTENT_TYPES = { diff --git a/src/providers/index.ts b/src/providers/index.ts index 7b6c65e82..0dcb46a81 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -25,6 +25,7 @@ import MoonshotConfig from './moonshot'; import OpenrouterConfig from './openrouter'; import LingYiConfig from './lingyi'; import ZhipuConfig from './zhipu'; +import NovitaAIConfig from './novita-ai'; const Providers: { [key: string]: ProviderConfigs } = { openai: OpenAIConfig, @@ -53,6 +54,7 @@ const Providers: { [key: string]: ProviderConfigs } = { openrouter: OpenrouterConfig, lingyi: LingYiConfig, zhipu: ZhipuConfig, + 'novita-ai': NovitaAIConfig, }; export default Providers; diff --git a/src/providers/novita-ai/api.ts b/src/providers/novita-ai/api.ts new file mode 100644 index 000000000..53f9c8f1f --- /dev/null +++ b/src/providers/novita-ai/api.ts @@ -0,0 +1,20 @@ +import { ProviderAPIConfig } from '../types'; + +const NovitaAIApiConfig: ProviderAPIConfig = { + getBaseURL: () => 'https://api.novita.ai/v3/openai', + headers: ({ providerOptions }) => { + return { Authorization: `Bearer ${providerOptions.apiKey}` }; + }, + getEndpoint: ({ fn }) => { + switch (fn) { + case 'complete': + return '/v1/completions'; + case 'chatComplete': + return '/v1/chat/completions'; + default: + return ''; + } + }, +}; + +export default NovitaAIApiConfig; diff --git a/src/providers/novita-ai/chatComplete.ts b/src/providers/novita-ai/chatComplete.ts new file mode 100644 index 000000000..462dfaaa2 --- /dev/null +++ b/src/providers/novita-ai/chatComplete.ts @@ -0,0 +1,220 @@ +import { NOVITA_AI } from '../../globals'; +import { + ChatCompletionResponse, + ErrorResponse, + ProviderConfig, +} from '../types'; +import { + generateErrorResponse, + generateInvalidProviderResponseError, +} from '../utils'; + +// TODOS: this configuration does not enforce the maximum token limit for the input parameter. If you want to enforce this, you might need to add a custom validation function or a max property to the ParameterConfig interface, and then use it in the input configuration. However, this might be complex because the token count is not a simple length check, but depends on the specific tokenization method used by the model. + +export const NovitaAIChatCompleteConfig: ProviderConfig = { + model: { + param: 'model', + required: true, + default: 'lzlv_70b', + }, + messages: { + param: 'messages', + required: true, + default: '', + }, + max_tokens: { + param: 'max_tokens', + required: true, + default: 128, + min: 1, + }, + stop: { + param: 'stop', + }, + temperature: { + param: 'temperature', + }, + top_p: { + param: 'top_p', + }, + n: { + param: 'n', + }, + top_k: { + param: 'top_k', + }, + presence_penalty: { + param: 'presence_penalty', + min: -2, + max: 2, + }, + frequency_penalty: { + param: 'frequency_penalty', + min: -2, + max: 2, + }, + stream: { + param: 'stream', + default: false, + }, + logprobs: { + param: 'logprobs', + }, + tools: { + param: 'tools', + }, + tool_choice: { + param: 'tool_choice', + }, + response_format: { + param: 'response_format', + }, +}; + +export interface NovitaAIChatCompleteResponse extends ChatCompletionResponse { + usage: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + }; +} + +export interface NovitaAIErrorResponse { + model: string; + job_id: string; + request_id: string; + error: string; + message?: string; + type?: string; +} + +export interface NovitaAIOpenAICompatibleErrorResponse extends ErrorResponse {} + +export interface NovitaAIChatCompletionStreamChunk { + id: string; + request_id: string; + object: string; + choices: { + index: number; + delta: { + content: string; + }; + }[]; +} + +export const NovitaAIErrorResponseTransform: ( + response: NovitaAIErrorResponse | NovitaAIOpenAICompatibleErrorResponse +) => ErrorResponse | false = (response) => { + if ('error' in response && typeof response.error === 'string') { + return generateErrorResponse( + { message: response.error, type: null, param: null, code: null }, + NOVITA_AI + ); + } + + if ('error' in response && typeof response.error === 'object') { + return generateErrorResponse( + { + message: response.error?.message || '', + type: response.error?.type || null, + param: response.error?.param || null, + code: response.error?.code || null, + }, + NOVITA_AI + ); + } + + if ('message' in response && response.message) { + return generateErrorResponse( + { + message: response.message, + type: response.type || null, + param: null, + code: null, + }, + NOVITA_AI + ); + } + + return false; +}; + +export const NovitaAIChatCompleteResponseTransform: ( + response: + | NovitaAIChatCompleteResponse + | NovitaAIErrorResponse + | NovitaAIOpenAICompatibleErrorResponse, + responseStatus: number +) => ChatCompletionResponse | ErrorResponse = (response, responseStatus) => { + if (responseStatus !== 200) { + const errorResponse = NovitaAIErrorResponseTransform( + response as NovitaAIErrorResponse + ); + if (errorResponse) return errorResponse; + } + + if ('choices' in response) { + return { + id: response.id, + object: response.object, + created: response.created, + model: response.model, + provider: NOVITA_AI, + choices: response.choices.map((choice) => { + return { + message: { + role: 'assistant', + content: choice.message.content, + tool_calls: choice.message.tool_calls + ? choice.message.tool_calls.map((toolCall: any) => ({ + id: toolCall.id, + type: toolCall.type, + function: toolCall.function, + })) + : null, + }, + index: 0, + logprobs: null, + finish_reason: choice.finish_reason, + }; + }), + usage: { + prompt_tokens: response.usage?.prompt_tokens, + completion_tokens: response.usage?.completion_tokens, + total_tokens: response.usage?.total_tokens, + }, + }; + } + + return generateInvalidProviderResponseError(response, NOVITA_AI); +}; + +export const NovitaAIChatCompleteStreamChunkTransform: ( + response: string +) => string = (responseChunk) => { + let chunk = responseChunk.trim(); + chunk = chunk.replace(/^data: /, ''); + chunk = chunk.trim(); + if (chunk === '[DONE]') { + return `data: ${chunk}\n\n`; + } + const parsedChunk: NovitaAIChatCompletionStreamChunk = JSON.parse(chunk); + return ( + `data: ${JSON.stringify({ + id: parsedChunk.id, + object: parsedChunk.object, + created: Math.floor(Date.now() / 1000), + model: '', + provider: NOVITA_AI, + choices: [ + { + delta: { + content: parsedChunk.choices[0]?.delta.content, + }, + index: 0, + finish_reason: '', + }, + ], + })}` + '\n\n' + ); +}; diff --git a/src/providers/novita-ai/complete.ts b/src/providers/novita-ai/complete.ts new file mode 100644 index 000000000..a0850c467 --- /dev/null +++ b/src/providers/novita-ai/complete.ts @@ -0,0 +1,131 @@ +import { NOVITA_AI } from '../../globals'; +import { CompletionResponse, ErrorResponse, ProviderConfig } from '../types'; +import { generateInvalidProviderResponseError } from '../utils'; +import { + NovitaAIErrorResponse, + NovitaAIErrorResponseTransform, + NovitaAIOpenAICompatibleErrorResponse, +} from './chatComplete'; + +export const NovitaAICompleteConfig: ProviderConfig = { + model: { + param: 'model', + required: true, + default: 'lzlv_70b', + }, + prompt: { + param: 'prompt', + required: true, + default: '', + }, + max_tokens: { + param: 'max_tokens', + required: true, + default: 128, + min: 1, + }, + stop: { + param: 'stop', + }, + temperature: { + param: 'temperature', + }, + top_p: { + param: 'top_p', + }, + top_k: { + param: 'top_k', + }, + frequency_penalty: { + param: 'repetition_penalty', + }, + stream: { + param: 'stream', + default: false, + }, + logprobs: { + param: 'logprobs', + }, +}; + +interface NovitaAICompleteResponse extends CompletionResponse { + usage: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + }; +} + +interface NovitaAICompletionStreamChunk { + id: string; + request_id: string; + choices: { + text: string; + }[]; +} + +export const NovitaAICompleteResponseTransform: ( + response: + | NovitaAICompleteResponse + | NovitaAIErrorResponse + | NovitaAIOpenAICompatibleErrorResponse, + responseStatus: number +) => CompletionResponse | ErrorResponse = (response, responseStatus) => { + if (responseStatus !== 200) { + const errorResponse = NovitaAIErrorResponseTransform( + response as NovitaAIErrorResponse + ); + if (errorResponse) return errorResponse; + } + + if ('choices' in response) { + return { + id: response.id, + object: response.object, + created: response.created, + model: response.model, + provider: NOVITA_AI, + choices: response.choices.map((choice) => ({ + text: choice.text, + index: choice.index || 0, + logprobs: null, + finish_reason: choice.finish_reason, + })), + usage: { + prompt_tokens: response.usage?.prompt_tokens, + completion_tokens: response.usage?.completion_tokens, + total_tokens: response.usage?.total_tokens, + }, + }; + } + + return generateInvalidProviderResponseError(response, NOVITA_AI); +}; + +export const NovitaAICompleteStreamChunkTransform: ( + response: string +) => string = (responseChunk) => { + let chunk = responseChunk.trim(); + chunk = chunk.replace(/^data: /, ''); + chunk = chunk.trim(); + if (chunk === '[DONE]') { + return `data: ${chunk}\n\n`; + } + const parsedChunk: NovitaAICompletionStreamChunk = JSON.parse(chunk); + return ( + `data: ${JSON.stringify({ + id: parsedChunk.id, + object: 'text_completion', + created: Math.floor(Date.now() / 1000), + model: '', + provider: NOVITA_AI, + choices: [ + { + text: parsedChunk.choices[0]?.text, + index: 0, + finish_reason: '', + }, + ], + })}` + '\n\n' + ); +}; diff --git a/src/providers/novita-ai/index.ts b/src/providers/novita-ai/index.ts new file mode 100644 index 000000000..cb2405c05 --- /dev/null +++ b/src/providers/novita-ai/index.ts @@ -0,0 +1,26 @@ +import { ProviderConfigs } from '../types'; +import NovitaAIApiConfig from './api'; +import { + NovitaAIChatCompleteConfig, + NovitaAIChatCompleteResponseTransform, + NovitaAIChatCompleteStreamChunkTransform, +} from './chatComplete'; +import { + NovitaAICompleteConfig, + NovitaAICompleteResponseTransform, + NovitaAICompleteStreamChunkTransform, +} from './complete'; + +const NovitaAIConfig: ProviderConfigs = { + complete: NovitaAICompleteConfig, + chatComplete: NovitaAIChatCompleteConfig, + api: NovitaAIApiConfig, + responseTransforms: { + 'stream-complete': NovitaAICompleteStreamChunkTransform, + complete: NovitaAICompleteResponseTransform, + chatComplete: NovitaAIChatCompleteResponseTransform, + 'stream-chatComplete': NovitaAIChatCompleteStreamChunkTransform, + }, +}; + +export default NovitaAIConfig;