Bump the default max_num_seq to 2048 (#226)

octoml · Mar 7, 2024 · ce08442 · ce08442
1 parent 72adea5
commit ce08442
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/serve/mlc_serve/engine/base.py b/serve/mlc_serve/engine/base.py
@@ -31,7 +31,7 @@ class MLCServeEngineConfig:
     # TODO(@sunggg): figure out better defaults
     use_staging_engine: bool = True
     max_num_batched_tokens: int = 4096
-    max_num_seq: int = 256
+    max_num_seq: int = 2048
     max_num_seq_per_request: Optional[int] = None # default to `max_num_seq / 4`
     min_decode_steps: int = 32
     max_decode_steps: int = 48

diff --git a/serve/mlc_serve/utils.py b/serve/mlc_serve/utils.py
@@ -30,7 +30,7 @@ def get_default_mlc_serve_argparser(description="", allow_override=False):
     parser.add_argument("--use-sync-engine", action="store_true")
     parser.add_argument("--num-sequences-to-sample", type=int, default=1)
     parser.add_argument("--max-num-batched-tokens", type=int, default=4096)
-    parser.add_argument("--max-num-seq", type=int, default=256)
+    parser.add_argument("--max-num-seq", type=int, default=2048)
     parser.add_argument("--min-decode-steps", type=int, default=32)
     parser.add_argument("--max-decode-steps", type=int, default=56)
     parser.add_argument("--gpu-memory-utilization", type=float, default=0.9)