ENH: Update scores and pricing

gramener · Oct 8, 2024 · 7a49da7 · 7a49da7
1 parent 06b5ff7
commit 7a49da7
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 25 deletions.
diff --git a/README.md b/README.md
@@ -18,6 +18,6 @@ These are shown in green 🟢 and are the best LLMs to use.
 Some LLMs are "pareto suboptimal", i.e. there is no LLM worse in both cost and quality.
 These are shown in red 🔴 and are the LLMs to avoid.
 
-Last updated: **20 Sep 2024**
+Last updated: **8 Oct 2024**
 
 Alternatives: [ArtificialAnalysis.ai](https://artificialanalysis.ai/)
diff --git a/elo.csv b/elo.csv
@@ -1,32 +1,45 @@
 model,elo,cpmi,launch,end,source
-o1-preview,1355,15,2024-09,,https://openai.com/api/pricing/
-chatgpt-4o-latest,1335,5,2024-08,,https://openai.com/api/pricing/
-o1-mini,1324,3,2024-09,,https://openai.com/api/pricing/
+chatgpt-4o-latest,1338,5,2024-08,,https://openai.com/api/pricing/
+o1-preview,1335,15,2024-09,,https://openai.com/api/pricing/
+o1-mini,1314,3,2024-09,,https://openai.com/api/pricing/
+gemini-1.5-pro-002,1304,5,2024-10,,
+grok-2-08-13,1293,,2024-09,,
 gpt-4o-2024-05-13,1285,5,2024-05,,https://openai.com/api/pricing/
 gpt-4o-mini-2024-07-18,1273,0.15,2024-07,,https://openai.com/api/pricing/
-claude-3-5-sonnet-20240620,1271,3,2024-06,,
+claude-3-5-sonnet-20240620,1268,3,2024-06,,
+grok-2-mini-08-13,1267,,2024-08,,
 gemini-advanced-0514,1266,,2024-05,,
-gpt-4o-2024-08-06,1262,2.5,2024-08,,https://openai.com/api/pricing/
-meta-llama-3.1-405b-instruct,1262,2,2024-06,,https://openrouter.ai/models/meta-llama/llama-3.1-405b
-gemini-1.5-pro-api-0514,1260,3.5,2024-05,,https://ai.google.dev/pricing
+meta-llama-3.1-405b-instruct,1266,2,2024-06,,https://openrouter.ai/models/meta-llama/llama-3.1-405b
+gemini-1.5-flash-002,1265,0.075,2024-10,,
+gpt-4o-2024-08-06,1264,2.5,2024-08,,https://openai.com/api/pricing/
+gemini-1.5-pro-001,1259,3.5,2024-05,,https://ai.google.dev/pricing
 gemini-1.5-pro-api-0409-preview,1257,7,2024-04,,https://llmpricecheck.com/google/gemini-1.5-pro
-gpt-4-turbo-2024-04-09,1257,10,2024-04,,https://openai.com/api/pricing/
+qwen-2.5-72b-instruct,1257,0.35,2024-09,,https://openrouter.ai/models/qwen/qwen-2.5-72b-instruct
+gpt-4-turbo-2024-04-09,1256,10,2024-04,,https://openai.com/api/pricing/
+deepseek-v2.5,1252,0.14,2024-08,,https://openrouter.ai/models/deepseek/deepseek-chat
+mistral-large-2407,1251,2,2024-07,,https://openrouter.ai/models/mistralai/mistral-large
 gpt-4-1106-preview,1251,10,2023-11,,https://openai.com/api/pricing/
 mistral-large-2407,1250,3,2024-07,,https://openrouter.ai/models/mistralai/mistral-large
-athene-70b,1250,,,,
+athene-70b,1250,,2024-07,,
 claude-3-opus-20240229,1248,15,2024-02,,
-meta-llama-3.1-70b-instruct,1242,0.52,2024-06,,https://openrouter.ai/models/meta-llama/llama-3.1-70b-instruct
+meta-llama-3.1-70b-instruct,1248,0.52,2024-06,,https://openrouter.ai/models/meta-llama/llama-3.1-70b-instruct
 gpt-4-0125-preview,1245,10,2024-01,,https://openai.com/api/pricing/
-gemini-1.5-flash-api-0514,1227,0.075,2024-08,,https://ai.google.dev/pricing
-gemini-1.5-flash-api-0514,1227,0.35,2024-05,2024-08,https://ai.google.dev/pricing
-deepseek-v2-api-0628,1221,0.14,2024-06,,
-gemma-2-27b-it,1212,0.27,2024-06,,https://openrouter.ai/models/google/gemma-2-27b-it
-yi-large,1213,3,2023-10,,https://openrouter.ai/models/01-ai/yi-large
-jamba-1.5-large,1213,2,2024-08,,https://openrouter.ai/models/ai21/jamba-1-5-large
-bard-jan-24-gemini-pro,1208,7,2024-01,,https://ai.google.dev/pricing
+yi-large-preview,1240,,2024-07,,
+reka-core-20240722,1230,,2024-07,,
+qwen-plus-0828,1228,,2024-08,,
+gemini-1.5-flash-001,1227,0.075,2024-08,,https://ai.google.dev/pricing
+gemini-1.5-flash-001,1227,0.35,2024-05,2024-08,https://ai.google.dev/pricing
+jamba-1.5-large,1221,2,2024-08,,https://openrouter.ai/models/ai21/jamba-1-5-large
+deepseek-v2-api-0628,1219,0.14,2024-06,,
+gemma-2-27b-it,1218,0.27,2024-06,,https://openrouter.ai/models/google/gemma-2-27b-it
+deepseek-coder-v2-0724,1214,0.14,2024-07,,
+yi-large,1212,3,2023-10,,https://openrouter.ai/models/01-ai/yi-large
+command-r-plus-08-2024,1210,,2024-08,,
+gemini-1.5-flash-8b-001,1209,0.04,2024-08,,
 nemotron-4-340b-instruct,1209,4.2,2024-06,,https://openrouter.ai/models/nvidia/nemotron-4-340b-instruct
+bard-jan-24-gemini-pro,1208,7,2024-01,,https://ai.google.dev/pricing
 glm-4-0520,1207,,2024-05,,
-llama-3-70b-instruct,1207,0.59,2024-04,,https://openrouter.ai/models/meta-llama/llama-3-70b-instruct
+llama-3-70b-instruct,1206,0.59,2024-04,,https://openrouter.ai/models/meta-llama/llama-3-70b-instruct
 claude-3-sonnet-20240229,1201,3,2024-02,,
 reka-core-20240501,1201,,2024-05,,
 command-r-plus,1189,3,2024-01,,
@@ -36,18 +49,18 @@ gemma-2-9b-it,1186,0.2,2024-06,,https://openrouter.ai/models/google/gemma-2-9b-i
 glm-4-0116,1183,,2024-01,,
 qwen-max-0428,1183,,2024-04,,
 deepseek-coder-v2-instruct,1178,0.14,2024-07,,https://openrouter.ai/models/deepseek/deepseek-coder
-claude-3-haiku-20240307,1178,0.25,2024-03,,
-jamba-1.5-mini,1171,0.2,2024-08,,https://openrouter.ai/models/ai21/jamba-1-5-mini
-meta-llama-3.1-8b-instruct,1166,0.06,2024-06,,https://openrouter.ai/models/meta-llama/llama-3.1-8b-instruct
+claude-3-haiku-20240307,1179,0.25,2024-03,,
+jamba-1.5-mini,1176,0.2,2024-08,,https://openrouter.ai/models/ai21/jamba-1-5-mini
+meta-llama-3.1-8b-instruct,1172,0.06,2024-06,,https://openrouter.ai/models/meta-llama/llama-3.1-8b-instruct
 qwen1.5-110b-chat,1164,1.62,2024-04,,https://openrouter.ai/models/qwen/qwen-110b-chat
 gpt-4-0613,1161,30,2024-06,,https://platform.openai.com/docs/deprecations/
-yi-1.5-34b-chat,1158,,,,
+yi-1.5-34b-chat,1158,,2024-05,,
 mistral-large-2402,1156,8,2023-11,,
 llama-3-8b-instruct,1153,0.05,2024-04,,
 claude-1,1149,8,2023-03,,
 command-r,1149,0.5,2023-12,,
-reka-flash-21b-online,1148,,,,
-reka-flash-21b,1148,,,,
+reka-flash-21b-online,1148,,2024-02,,
+reka-flash-21b,1148,,2024-02,,
 qwen1.5-72b-chat,1147,0.81,2024-01,,https://openrouter.ai/models/qwen/qwen-72b-chat
 mistral-medium,1146,2.7,2023-11,,
 mixtral-8x22b-instruct-v0.1,1146,0.7,2024-05,,