add qwen inference

This commit is contained in:
Shivering Ocelot committed 2025-12-31 00:51:18 -05:00
1 parent 93703a78c1
commit 56deeed52f
2 files changed
+4 -4

No files matched your search

+2 -2
View File
@@ -52,9 +52,9 @@ class ExpertAgent(Agent):
model = self.config['model'] model = self.config['model']
if model == "qwen-big": if model == "qwen-big":
response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.quen3_235b) response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_235b)
elif model == "qwen-small": elif model == "qwen-small":
response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.quen3_8b) response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_8b)
elif model == "llama-big": elif model == "llama-big":
response = await inference.replicate_llama_completion(self.app, messages, model=inference.Models.llama_70b) response = await inference.replicate_llama_completion(self.app, messages, model=inference.Models.llama_70b)
elif model == "llama-small": elif model == "llama-small":
+2 -2
View File
@@ -15,7 +15,7 @@ class Models:
llama_8b = "meta/meta-llama-3-8b-instruct" llama_8b = "meta/meta-llama-3-8b-instruct"
llama_70b = "meta/meta-llama-3-70b-instruct" llama_70b = "meta/meta-llama-3-70b-instruct"
qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11" qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
quen3_235b = "qwen/qwen3-235b-a22b-instruct-2507" qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
async def replicate_llama_completion(app, messages, model=None, opts={}): async def replicate_llama_completion(app, messages, model=None, opts={}):
if model is None: if model is None:
@@ -39,7 +39,7 @@ async def replicate_qwen_completion(app, messages, model=None, opts={}):
"temperature": opts.get('temperature', 0.75), "temperature": opts.get('temperature', 0.75),
"max_new_tokens": opts.get('max_tokens', 2048), "max_new_tokens": opts.get('max_tokens', 2048),
} }
elif model == Models.quen3_235b: elif model == Models.qwen3_235b:
input = { input = {
"prompt": format_instruct(messages), "prompt": format_instruct(messages),
"temperature": opts.get('temperature', 0.75), "temperature": opts.get('temperature', 0.75),