Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tokens to TRTLLM Backend #73

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Tokens to TRTLLM Backend #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions genai-perf/genai_perf/inputs/input_constants.py
View file
Open in desktop
Copy link
Contributor

@matthewkotila matthewkotila Sep 23, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@IzzyPutterman is this PR still something we want merged?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes I think we do. We can query the trtllm backend with this, when hosted by triton.

Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class OutputFormat(Enum):
TENSORRTLLM = auto()
VLLM = auto()
TENSORRTLLM_ENGINE = auto()
TENSORRTLLM_BACKEND = auto()

def to_lowercase(self):
return self.name.lower()
Expand Down
1 change: 1 addition & 0 deletions genai-perf/genai_perf/inputs/output_format_converter_factory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ def create(output_format: OutputFormat):
OutputFormat.RANKINGS: RankingsConverter,
OutputFormat.VLLM: VLLMConverter,
OutputFormat.TENSORRTLLM: TensorRTLLMConverter,
OutputFormat.TENSORRTLLM_BACKEND: TensorRTLLMEngineConverter,
OutputFormat.TENSORRTLLM_ENGINE: TensorRTLLMEngineConverter,
}
if output_format not in converters:
Expand Down
18 changes: 12 additions & 6 deletions genai-perf/genai_perf/profile_data_parser/llm_profile_data_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,13 @@ def _preprocess_response(

def _get_input_token_count(self, req_inputs: dict) -> int:
"""Deserialize the request input and return tokenized inputs."""
if self._service_kind == "triton":
input_text = req_inputs["text_input"]
elif self._service_kind == "triton_c_api":
if (
self._service_kind == "triton_c_api"
or self._response_format == ResponseFormat.TENSORRTLLM_BACKEND
):
return len(req_inputs["input_ids"]) # no tokenizer required
elif self._service_kind == "triton":
input_text = req_inputs["text_input"]
elif self._service_kind == "openai":
input_text = self._get_openai_input_text(req_inputs)
else:
Expand All @@ -259,11 +262,14 @@ def _get_output_token_counts(
self, res_outputs: List[Dict]
) -> Tuple[List[int], int]:
"""Return response-level token counts and total token count."""
if self._service_kind == "triton":
output_texts = self._get_triton_output_tokens(res_outputs)
elif self._service_kind == "triton_c_api":
if (
self._service_kind == "triton_c_api"
or self._response_format == ResponseFormat.TENSORRTLLM_BACKEND
):
# No tokenizer is need to get the token counts.
return self._get_tensorrtllm_engine_token_counts(res_outputs)
elif self._service_kind == "triton":
output_texts = self._get_triton_output_tokens(res_outputs)
elif self._service_kind == "openai":
output_texts = self._get_openai_output_tokens(res_outputs)
else:
Expand Down
3 changes: 3 additions & 0 deletions genai-perf/genai_perf/profile_data_parser/profile_data_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class ResponseFormat(Enum):
RANKINGS = auto()
IMAGE_RETRIEVAL = auto()
TRITON = auto()
TENSORRTLLM_BACKEND = auto()


class ProfileDataParser:
Expand Down Expand Up @@ -109,6 +110,8 @@ def _get_profile_metadata(self, data: dict) -> None:

elif self._service_kind == "triton":
self._response_format = ResponseFormat.TRITON
if "input_ids" in data["experiments"][0]["requests"][0]["request_inputs"]:
self._response_format = ResponseFormat.TENSORRTLLM_BACKEND
elif self._service_kind == "triton_c_api":
pass # ignore
else:
Expand Down
Loading