diff --git a/backend/src/agents/expert.py b/backend/src/agents/expert.py index 72d7e30..f886b91 100644 --- a/backend/src/agents/expert.py +++ b/backend/src/agents/expert.py @@ -31,17 +31,17 @@ async def chat_completion_loop(app, run, messages, model_id=None, tools=None): opts=run.input, ) + pre_len = len(messages) async for msg in generator: messages.append(msg) + run.emit_event('inference', msg) if msg.role == 'tool': tool_calls.append(msg.content) if msg.role == 'assistant': turn_response = True - run.emit_event('inference', msg) - - # Broken generator with no output - else: break + # Avoid infinite loops from server failures + if len(messages) == pre_len: break for tc in tool_calls: tool = next((t for t in tools if t.name == tc.tool_name), None) @@ -53,9 +53,8 @@ async def chat_completion_loop(app, run, messages, model_id=None, tools=None): else: tc.tool = tool - # If not calling must output something useful - else: - if turn_response: break + # If not calling, respond to end the turn + if not (tool_calls and execute_tools) and turn_response: break run.close() diff --git a/backend/src/features/web.py b/backend/src/features/web.py index 0e85ed8..fb8f0a9 100644 --- a/backend/src/features/web.py +++ b/backend/src/features/web.py @@ -6,6 +6,7 @@ class's automatic ``ToolSpec`` generation (docstring -> description, """ import re +import urllib.parse import urllib.request from html import unescape @@ -28,24 +29,68 @@ def _strip_html(html: str) -> str: class WebSearch(Tool): """Search the web with DuckDuckGo. Returns titles, URLs and snippets.""" + ENDPOINT = "https://html.duckduckgo.com/html/" + USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0 Safari/537.36" + ) + + # Result anchors look like: + # Title + _RESULT_RE = re.compile( + r']*class="[^"]*\bresult__a\b[^"]*"[^>]*href="(?P[^"]+)"[^>]*>' + r"(?P.*?)</a>", + re.DOTALL, + ) + _SNIPPET_RE = re.compile( + r'<a[^>]*class="[^"]*\bresult__snippet\b[^"]*"[^>]*>(?P<snippet>.*?)</a>', + re.DOTALL, + ) + def run(self, query: str, max_results: int = 5): - # `ddgs` is the current package name; `duckduckgo_search` is the legacy one. - try: - from ddgs import DDGS - except ImportError: # pragma: no cover - fallback for older installs - from duckduckgo_search import DDGS + body = urllib.parse.urlencode({"q": query}).encode("utf-8") + req = urllib.request.Request( + self.ENDPOINT, + data=body, + method="POST", + headers={ + "User-Agent": self.USER_AGENT, + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "text/html", + }, + ) + with urllib.request.urlopen(req, timeout=15) as resp: + page = resp.read().decode("utf-8", errors="replace") - with DDGS() as ddgs: - results = list(ddgs.text(query, max_results=max_results)) + snippets = [m.group("snippet") for m in self._SNIPPET_RE.finditer(page)] - return [ - { - "title": r.get("title", ""), - "url": r.get("href", ""), - "snippet": r.get("body", ""), - } - for r in results - ] + results = [] + for i, match in enumerate(self._RESULT_RE.finditer(page)): + if len(results) >= max_results: + break + results.append({ + "title": self._clean(match.group("title")), + "url": self._unwrap(match.group("href")), + "snippet": self._clean(snippets[i]) if i < len(snippets) else "", + }) + return results + + @staticmethod + def _clean(value: str) -> str: + return unescape(re.sub(r"<[^>]+>", "", value)).strip() + + @staticmethod + def _unwrap(href: str) -> str: + # DDG wraps result links as //duckduckgo.com/l/?uddg=<url-encoded-target> + if "uddg=" in href: + target = urllib.parse.parse_qs(urllib.parse.urlparse(href).query).get( + "uddg", [""] + )[0] + if target: + return target + if href.startswith("//"): + return "https:" + href + return href class WebFetch(Tool): diff --git a/backend/src/tool.py b/backend/src/tool.py index a501322..493ae32 100644 --- a/backend/src/tool.py +++ b/backend/src/tool.py @@ -149,7 +149,7 @@ class Tool: @classmethod def tool_name(cls): - return ''.join(['-'+c.lower() if c.isupper() else c for c in cls.__name__]).lstrip('_') + return ''.join(['-'+c.lower() if c.isupper() else c for c in cls.__name__]).lstrip('-') def _generate_spec(self):