diff --git a/pipeline_setup.py b/pipeline_setup.py index e27eb31..46f4faf 100644 --- a/pipeline_setup.py +++ b/pipeline_setup.py @@ -9,11 +9,12 @@ pipe = pipeline( model, backend_config=TurbomindEngineConfig( model_format="awq", - tp=4, + tp=2, + device_ids=[0, 1], session_len=12864, max_batch_size=1, cache_max_entry_count=0.05, cache_block_seq_len=32768, quant_policy=4 ) -) +) \ No newline at end of file