add qwen inference
This commit is contained in:
1 parent
93703a78c1
commit
56deeed52f
2 files changed
+4
-4
No files matched your search
@@ -52,9 +52,9 @@ class ExpertAgent(Agent):
|
|||||||
model = self.config['model']
|
model = self.config['model']
|
||||||
|
|
||||||
if model == "qwen-big":
|
if model == "qwen-big":
|
||||||
response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.quen3_235b)
|
response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_235b)
|
||||||
elif model == "qwen-small":
|
elif model == "qwen-small":
|
||||||
response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.quen3_8b)
|
response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_8b)
|
||||||
elif model == "llama-big":
|
elif model == "llama-big":
|
||||||
response = await inference.replicate_llama_completion(self.app, messages, model=inference.Models.llama_70b)
|
response = await inference.replicate_llama_completion(self.app, messages, model=inference.Models.llama_70b)
|
||||||
elif model == "llama-small":
|
elif model == "llama-small":
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ class Models:
|
|||||||
llama_8b = "meta/meta-llama-3-8b-instruct"
|
llama_8b = "meta/meta-llama-3-8b-instruct"
|
||||||
llama_70b = "meta/meta-llama-3-70b-instruct"
|
llama_70b = "meta/meta-llama-3-70b-instruct"
|
||||||
qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
|
qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
|
||||||
quen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
|
qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
|
||||||
|
|
||||||
async def replicate_llama_completion(app, messages, model=None, opts={}):
|
async def replicate_llama_completion(app, messages, model=None, opts={}):
|
||||||
if model is None:
|
if model is None:
|
||||||
@@ -39,7 +39,7 @@ async def replicate_qwen_completion(app, messages, model=None, opts={}):
|
|||||||
"temperature": opts.get('temperature', 0.75),
|
"temperature": opts.get('temperature', 0.75),
|
||||||
"max_new_tokens": opts.get('max_tokens', 2048),
|
"max_new_tokens": opts.get('max_tokens', 2048),
|
||||||
}
|
}
|
||||||
elif model == Models.quen3_235b:
|
elif model == Models.qwen3_235b:
|
||||||
input = {
|
input = {
|
||||||
"prompt": format_instruct(messages),
|
"prompt": format_instruct(messages),
|
||||||
"temperature": opts.get('temperature', 0.75),
|
"temperature": opts.get('temperature', 0.75),
|
||||||
|
|||||||
Reference in new issue
Block a user