chore(langchain): remove openai cost span metric (#12233)

This PR removes the `langchain.tokens.total_cost` span metric which estimates the total cost of the llm interaction (and also by extension the wrapped chain span). This was an overly complex implementation that relied on a third party (langchain-community + tiktoken) library which is more-or-less deprecated when we can simply point users to enable LLM Observability to do the cost estimation for OpenAI spans anyway. Note that this shouldn't break anything other than the span metric tag itself, integration metrics are already disabled with 3.0. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
DataDog · Feb 6, 2025 · 9ba79f4 · 9ba79f4
1 parent 25e7e2e
commit 9ba79f4
Show file tree

Hide file tree

Showing 17 changed files with 6 additions and 49 deletions.
diff --git a/ddtrace/contrib/internal/langchain/constants.py b/ddtrace/contrib/internal/langchain/constants.py
@@ -80,7 +80,3 @@
 }
 
 API_KEY = "langchain.request.api_key"
-MODEL = "langchain.request.model"
-COMPLETION_TOKENS = "langchain.tokens.completion_tokens"
-PROMPT_TOKENS = "langchain.tokens.prompt_tokens"
-TOTAL_COST = "langchain.tokens.total_cost"
diff --git a/ddtrace/contrib/internal/langchain/patch.py b/ddtrace/contrib/internal/langchain/patch.py
@@ -41,10 +41,6 @@
 
 from ddtrace import config
 from ddtrace.contrib.internal.langchain.constants import API_KEY
-from ddtrace.contrib.internal.langchain.constants import COMPLETION_TOKENS
-from ddtrace.contrib.internal.langchain.constants import MODEL
-from ddtrace.contrib.internal.langchain.constants import PROMPT_TOKENS
-from ddtrace.contrib.internal.langchain.constants import TOTAL_COST
 from ddtrace.contrib.internal.langchain.constants import agent_output_parser_classes
 from ddtrace.contrib.internal.langchain.constants import text_embedding_models
 from ddtrace.contrib.internal.langchain.constants import vectorstore_classes
@@ -114,9 +110,7 @@ def _extract_api_key(instance: Any) -> str:
     return ""
 
 
-def _tag_openai_token_usage(
-    span: Span, llm_output: Dict[str, Any], propagated_cost: int = 0, propagate: bool = False
-) -> None:
+def _tag_openai_token_usage(span: Span, llm_output: Dict[str, Any]) -> None:
     """
     Extract token usage from llm_output, tag on span.
     Calculate the total cost for each LLM/chat_model, then propagate those values up the trace so that
@@ -126,23 +120,6 @@ def _tag_openai_token_usage(
         current_metric_value = span.get_metric("langchain.tokens.%s_tokens" % token_type) or 0
         metric_value = llm_output["token_usage"].get("%s_tokens" % token_type, 0)
         span.set_metric("langchain.tokens.%s_tokens" % token_type, current_metric_value + metric_value)
-    total_cost = span.get_metric(TOTAL_COST) or 0
-    if not propagate and get_openai_token_cost_for_model:
-        try:
-            completion_cost = get_openai_token_cost_for_model(
-                span.get_tag(MODEL),
-                span.get_metric(COMPLETION_TOKENS),
-                is_completion=True,
-            )
-            prompt_cost = get_openai_token_cost_for_model(span.get_tag(MODEL), span.get_metric(PROMPT_TOKENS))
-            total_cost = completion_cost + prompt_cost
-        except ValueError:
-            # If not in langchain's openai model catalog, the above helpers will raise a ValueError.
-            log.debug("Cannot calculate token/cost as the model is not in LangChain's OpenAI model catalog.")
-    if get_openai_token_cost_for_model:
-        span.set_metric(TOTAL_COST, propagated_cost + total_cost)
-    if span._parent is not None:
-        _tag_openai_token_usage(span._parent, llm_output, propagated_cost=propagated_cost + total_cost, propagate=True)
 
 
 def _is_openai_llm_instance(instance):

diff --git a/releasenotes/notes/deprecate-langchain-openai-cost-042f3a04cda9d23b.yaml b/releasenotes/notes/deprecate-langchain-openai-cost-042f3a04cda9d23b.yaml
@@ -0,0 +1,5 @@
+---
+upgrade:
+  - |
+    langchain: Removes the `langchain.tokens.total_cost` span metric for OpenAI calls. 
+     For continued cost estimation of OpenAI calls, enable `LLM Observability <https://docs.datadoghq.com/llm_observability/>`_.
diff --git a/tests/contrib/langchain/test_langchain.py b/tests/contrib/langchain/test_langchain.py
@@ -23,7 +23,6 @@
     "meta.langchain.request.openai.parameters.logprobs",
     "meta.langchain.request.openai.parameters.seed",  # langchain-openai llm call now includes seed as param
     "meta.langchain.request.openai.parameters.logprobs",  # langchain-openai llm call now includes seed as param
-    "metrics.langchain.tokens.total_cost",  # total_cost depends on if tiktoken is installed
     # these are sometimes named differently
     "meta.langchain.request.openai.parameters.max_tokens",
     "meta.langchain.request.openai.parameters.max_completion_tokens",

diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_batch.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_batch.json
@@ -61,7 +61,6 @@
          "_dd.measured": 1,
          "langchain.tokens.completion_tokens": 16,
          "langchain.tokens.prompt_tokens": 14,
-         "langchain.tokens.total_cost": 5.3e-05,
          "langchain.tokens.total_tokens": 30
        },
        "duration": 6742000,
@@ -95,7 +94,6 @@
          "_dd.measured": 1,
          "langchain.tokens.completion_tokens": 15,
          "langchain.tokens.prompt_tokens": 14,
-         "langchain.tokens.total_cost": 5.1000000000000006e-05,
          "langchain.tokens.total_tokens": 29
        },
        "duration": 3314000,

diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_complicated.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_complicated.json
@@ -25,7 +25,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 19,
       "langchain.tokens.prompt_tokens": 53,
-      "langchain.tokens.total_cost": 0.0001175,
       "langchain.tokens.total_tokens": 72,
       "process_id": 82010
     },
@@ -60,7 +59,6 @@
          "_dd.measured": 1,
          "langchain.tokens.completion_tokens": 19,
          "langchain.tokens.prompt_tokens": 53,
-         "langchain.tokens.total_cost": 0.0001175,
          "langchain.tokens.total_tokens": 72
        },
        "duration": 3680000,

diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_nested.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_nested.json
@@ -25,7 +25,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 53,
       "langchain.tokens.prompt_tokens": 50,
-      "langchain.tokens.total_cost": 0.000181,
       "langchain.tokens.total_tokens": 103,
       "process_id": 82010
     },
@@ -51,7 +50,6 @@
          "_dd.measured": 1,
          "langchain.tokens.completion_tokens": 16,
          "langchain.tokens.prompt_tokens": 18,
-         "langchain.tokens.total_cost": 5.9e-05,
          "langchain.tokens.total_tokens": 34
        },
        "duration": 63808000,
@@ -85,7 +83,6 @@
             "_dd.measured": 1,
             "langchain.tokens.completion_tokens": 16,
             "langchain.tokens.prompt_tokens": 18,
-            "langchain.tokens.total_cost": 5.9e-05,
             "langchain.tokens.total_tokens": 34
           },
           "duration": 61552000,
@@ -119,7 +116,6 @@
          "_dd.measured": 1,
          "langchain.tokens.completion_tokens": 37,
          "langchain.tokens.prompt_tokens": 32,
-         "langchain.tokens.total_cost": 0.000122,
          "langchain.tokens.total_tokens": 69
        },
        "duration": 3288000,

diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_simple.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_simple.json
@@ -24,7 +24,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 101,
       "langchain.tokens.prompt_tokens": 20,
-      "langchain.tokens.total_cost": 0.000232,
       "langchain.tokens.total_tokens": 121,
       "process_id": 82010
     },
@@ -61,7 +60,6 @@
          "_dd.measured": 1,
          "langchain.tokens.completion_tokens": 101,
          "langchain.tokens.prompt_tokens": 20,
-         "langchain.tokens.total_cost": 0.000232,
          "langchain.tokens.total_tokens": 121
        },
        "duration": 6142000,

diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_simple_async.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_chain_simple_async.json
@@ -24,7 +24,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 78,
       "langchain.tokens.prompt_tokens": 20,
-      "langchain.tokens.total_cost": 0.000186,
       "langchain.tokens.total_tokens": 98,
       "process_id": 82010
     },
@@ -61,7 +60,6 @@
          "_dd.measured": 1,
          "langchain.tokens.completion_tokens": 78,
          "langchain.tokens.prompt_tokens": 20,
-         "langchain.tokens.total_cost": 0.000186,
          "langchain.tokens.total_tokens": 98
        },
        "duration": 3399000,

diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_with_tools_openai.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_lcel_with_tools_openai.json
@@ -37,7 +37,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 32,
       "langchain.tokens.prompt_tokens": 85,
-      "langchain.tokens.total_cost": 9.05e-05,
       "langchain.tokens.total_tokens": 117,
       "process_id": 82010
     },

diff --git a/...apshots/tests.contrib.langchain.test_langchain.test_openai_chat_model_async_generate.json b/...apshots/tests.contrib.langchain.test_langchain.test_openai_chat_model_async_generate.json
@@ -43,7 +43,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 150,
       "langchain.tokens.prompt_tokens": 60,
-      "langchain.tokens.total_cost": 0.00038999999999999994,
       "langchain.tokens.total_tokens": 210,
       "process_id": 82010
     },

diff --git a/...s.contrib.langchain.test_langchain.test_openai_chat_model_sync_call_langchain_openai.json b/...s.contrib.langchain.test_langchain.test_openai_chat_model_sync_call_langchain_openai.json
@@ -35,7 +35,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 83,
       "langchain.tokens.prompt_tokens": 20,
-      "langchain.tokens.total_cost": 0.00019600000000000002,
       "langchain.tokens.total_tokens": 103,
       "process_id": 82010
     },

diff --git a/...napshots/tests.contrib.langchain.test_langchain.test_openai_chat_model_sync_generate.json b/...napshots/tests.contrib.langchain.test_langchain.test_openai_chat_model_sync_generate.json
@@ -43,7 +43,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 110,
       "langchain.tokens.prompt_tokens": 60,
-      "langchain.tokens.total_cost": 0.00031,
       "langchain.tokens.total_tokens": 170,
       "process_id": 82010
     },

diff --git a/...pshots/tests.contrib.langchain.test_langchain.test_openai_chat_model_vision_generate.json b/...pshots/tests.contrib.langchain.test_langchain.test_openai_chat_model_vision_generate.json
@@ -35,7 +35,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 56,
       "langchain.tokens.prompt_tokens": 1151,
-      "langchain.tokens.total_cost": 0,
       "langchain.tokens.total_tokens": 1207,
       "process_id": 34354
     },

diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_llm_async.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_llm_async.json
@@ -36,7 +36,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 12,
       "langchain.tokens.prompt_tokens": 10,
-      "langchain.tokens.total_cost": 3.9e-05,
       "langchain.tokens.total_tokens": 22,
       "process_id": 82010
     },

diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_llm_sync.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_llm_sync.json
@@ -36,7 +36,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 256,
       "langchain.tokens.prompt_tokens": 17,
-      "langchain.tokens.total_cost": 0.0005375,
       "langchain.tokens.total_tokens": 273,
       "process_id": 82010
     },

diff --git a/...apshots/tests.contrib.langchain.test_langchain.test_openai_llm_sync_multiple_prompts.json b/...apshots/tests.contrib.langchain.test_langchain.test_openai_llm_sync_multiple_prompts.json
@@ -40,7 +40,6 @@
       "_sampling_priority_v1": 1,
       "langchain.tokens.completion_tokens": 271,
       "langchain.tokens.prompt_tokens": 23,
-      "langchain.tokens.total_cost": 0.0005765000000000001,
       "langchain.tokens.total_tokens": 294,
       "process_id": 82010
     },