SGLang相关参数

SGLANG_USE_MODELSCOPE=true python -m sglang.bench_latency --model-path /data/hub/LLM-Research/Meta-Llama-3___1-70B-Instruct-AWQ-INT4 --batch 128 --input-len 256 --output-len 32 --tp 8 --mem-fraction-static 0.9 --dtype float16 --context-length 512 --enable-p2p-check --quantization Marlin
usage: bench_latency.py [-h] [--model-path MODEL_PATH] [--tokenizer-path TOKENIZER_PATH]
                        [--host HOST] [--port PORT] [--additional-ports [ADDITIONAL_PORTS ...]]
                        [--tokenizer-mode {auto,slow}] [--skip-tokenizer-init]
                        [--load-format {auto,pt,safetensors,npcache,dummy}]
                        [--dtype {auto,half,float16,bfloat16,float,float32}]
                        [--kv-cache-dtype {auto,fp8_e5m2}] [--trust-remote-code] [--is-embedding]
                        [--context-length CONTEXT_LENGTH]
                        [--quantization {awq,fp8,gptq,marlin,gptq_marlin,awq_marlin,squeezellm,bitsandbytes}]
                        [--served-model-name SERVED_MODEL_NAME] [--chat-template CHAT_TEMPLATE]
                        [--mem-fraction-static MEM_FRACTION_STATIC]
                        [--max-running-requests MAX_RUNNING_REQUESTS] [--max-num-reqs MAX_NUM_REQS]
                        [--max-total-tokens MAX_TOTAL_TOKENS]
                        [--chunked-prefill-size CHUNKED_PREFILL_SIZE]
                        [--max-prefill-tokens MAX_PREFILL_TOKENS]
                        [--schedule-policy {lpm,random,fcfs,dfs-weight}]
                        [--schedule-conservativeness SCHEDULE_CONSERVATIVENESS]
                        [--tensor-parallel-size TENSOR_PARALLEL_SIZE]
                        [--stream-interval STREAM_INTERVAL] [--random-seed RANDOM_SEED]
                        [--log-level LOG_LEVEL] [--log-level-http LOG_LEVEL_HTTP] [--log-requests]
                        [--show-time-cost] [--api-key API_KEY] [--file-storage-pth FILE_STORAGE_PTH]
                        [--data-parallel-size DATA_PARALLEL_SIZE]
                        [--load-balance-method {round_robin,shortest_queue}]
                        [--nccl-init-addr NCCL_INIT_ADDR] [--nnodes NNODES] [--node-rank NODE_RANK]
                        [--disable-flashinfer] [--disable-flashinfer-sampling]
                        [--disable-radix-cache] [--disable-regex-jump-forward]
                        [--disable-cuda-graph] [--disable-cuda-graph-padding] [--disable-disk-cache]
                        [--disable-custom-all-reduce] [--enable-mixed-chunk]
                        [--enable-torch-compile] [--enable-p2p-check] [--enable-mla]
                        [--triton-attention-reduce-in-fp32] [--efficient-weight-load]

                        [--run-name RUN_NAME] [--batch-size BATCH_SIZE [BATCH_SIZE ...]]
                        [--input-len INPUT_LEN [INPUT_LEN ...]]
                        [--output-len OUTPUT_LEN [OUTPUT_LEN ...]]
                        [--result-filename RESULT_FILENAME] [--correctness-test] [--cut-len CUT_LEN]
                        [--graph-sql GRAPH_SQL] [--graph-filename GRAPH_FILENAME]
bench_latency.py: error: argument --quantization: invalid choice: 'Marlin' (choose from 'awq', 'fp8', 'gptq', 'marlin', 'gptq_marlin', 'awq_marlin', 'squeezellm', 'bitsandbytes')