MiMo-V2-Flash
在 Clore.ai 上部署 MiMo-V2-Flash（309B MoE）并使用推测解码——超高速推理，达到 150+ tok/s
最后更新于
这有帮助吗？
这有帮助吗？
pip install "sglang[all]>=0.3.0"
# 或 最新 版本
pip install git+https://github.com/sgl-project/sglang.gitpython -m sglang.launch_server \
--model-path mimo-ai/MiMo-V2-Flash \
--tp-size 8 \
--enable-mtp \
--mtp-max-draft-tokens 8 \
--mtp-acceptance-rate 0.8 \
--mem-fraction-static 0.85 \
--dtype float16 \
--context-length 32768 \
--served-model-name mimo-v2-flashfrom openai import OpenAI
client = OpenAI(
base_url="http://localhost:30000/v1",
api_key="EMPTY"
)
response = client.chat.completions.create(
model="mimo-v2-flash",
messages=[
{"role": "system", "content": "You are an expert AI researcher."},
{"role": "user", "content": "Explain the EAGLE speculative decoding algorithm and why it enables faster inference"}
],
max_tokens=1024,
temperature=0.7,
stream=True # 推荐以获得最佳延迟
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end='', flush=True)pip install vllm>=0.6.0
vllm serve mimo-ai/MiMo-V2-Flash \
--tensor-parallel-size 8 \
--speculative-model mimo-ai/MiMo-V2-Flash-Draft \
--speculative-max-model-len 32768 \
--speculative-draft-tensor-parallel-size 2 \
--use-v2-block-manager \
--dtype float16 \
--served-model-name mimo-v2-flash \
--trust-remote-codeFROM nvidia/cuda:12.1-devel-ubuntu22.04
# 安装依赖
RUN apt-get update && \
apt-get install -y python3.10 python3-pip git && \
rm -rf /var/lib/apt/lists/*
# 安装具有 MTP 支持的 SGLang
RUN pip install "sglang[all]>=0.3.0" transformers
# 设置环境变量
ENV PYTHONUNBUFFERED=1
ENV CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
# 预下载模型（可选，可节省启动时间）
# RUN python3 -c "from transformers import AutoModel; AutoModel.from_pretrained('mimo-ai/MiMo-V2-Flash', trust_remote_code=True)"
EXPOSE 30000
CMD ["python", "-m", "sglang.launch_server", \
"--model-path", "mimo-ai/MiMo-V2-Flash", \
"--host", "0.0.0.0", \
"--port", "30000", \
"--tp-size", "8", \
"--enable-mtp", \
"--mtp-max-draft-tokens", "8", \
"--dtype", "float16"]docker build -t mimo-v2-flash .
docker run --gpus all -p 30000:30000 \
--shm-size=64g \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
mimo-v2-flash# 用于代码生成（更高接受率）
python -m sglang.launch_server \
--model-path mimo-ai/MiMo-V2-Flash \
--tp-size 8 \
--enable-mtp \
--mtp-max-draft-tokens 12 \
--mtp-acceptance-rate 0.9 \
--temperature 0.1
# 用于创意写作（较低接受率）
python -m sglang.launch_server \
--model-path mimo-ai/MiMo-V2-Flash \
--tp-size 8 \
--enable-mtp \
--mtp-max-draft-tokens 6 \
--mtp-acceptance-rate 0.7 \
--temperature 0.8# 降低内存使用（较慢但适配 4×A100）
python -m sglang.launch_server \
--model-path mimo-ai/MiMo-V2-Flash \
--tp-size 4 \
--mem-fraction-static 0.75 \
--context-length 16384 \
--dtype float16 \
--disable-cuda-graph # 节省显存import time
from openai import OpenAI
client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
def benchmark_generation():
start_time = time.time()
response = client.chat.completions.create(
model="mimo-v2-flash",
messages=[
{"role": "user", "content": "Write a detailed explanation of quantum computing in exactly 500 words"}
],
max_tokens=600,
temperature=0.1,
stream=False
)
end_time = time.time()
content = response.choices[0].message.content
tokens = len(content.split()) # 粗略的标记估计
duration = end_time - start_time
tokens_per_second = tokens / duration
print(f"Generated {tokens} tokens in {duration:.2f}s")
print(f"Speed: {tokens_per_second:.1f} tokens/second")
return tokens_per_second
# 运行基准
speed = benchmark_generation()
print(f"\nMiMo-V2-Flash achieved {speed:.1f} tok/s")