now the agent is able to transmit and bring browser use logs to telegram in a human-readable form
This commit is contained in:
parent
69106ec711
commit
b90fb85ab3
10 changed files with 1498 additions and 73 deletions
|
|
@ -71,6 +71,38 @@ class FakeAgent:
|
|||
}
|
||||
|
||||
|
||||
class FakeBrowserLiveAgent(FakeAgent):
|
||||
def run_conversation(self, message, conversation_history=None, task_id=None):
|
||||
self.tool_progress_callback(
|
||||
"internet_browser",
|
||||
"📍 Я на странице: example.com",
|
||||
{"_browser_live": True},
|
||||
)
|
||||
time.sleep(0.35)
|
||||
return {
|
||||
"final_response": "done",
|
||||
"messages": [],
|
||||
"api_calls": 1,
|
||||
}
|
||||
|
||||
|
||||
class FakeLongBrowserLiveAgent(FakeAgent):
|
||||
def run_conversation(self, message, conversation_history=None, task_id=None):
|
||||
for index in range(8):
|
||||
self.tool_progress_callback(
|
||||
"internet_browser",
|
||||
f"📍 Событие браузера номер {index}: " + ("x" * 50),
|
||||
{"_browser_live": True},
|
||||
)
|
||||
time.sleep(0.05)
|
||||
time.sleep(0.35)
|
||||
return {
|
||||
"final_response": "done",
|
||||
"messages": [],
|
||||
"api_calls": 1,
|
||||
}
|
||||
|
||||
|
||||
def _make_runner(adapter):
|
||||
gateway_run = importlib.import_module("gateway.run")
|
||||
GatewayRunner = gateway_run.GatewayRunner
|
||||
|
|
@ -133,3 +165,81 @@ async def test_run_agent_progress_stays_in_originating_topic(monkeypatch, tmp_pa
|
|||
]
|
||||
assert adapter.edits
|
||||
assert all(call["metadata"] == {"thread_id": "17585"} for call in adapter.typing)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_live_progress_uses_raw_message(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("HERMES_TOOL_PROGRESS_MODE", "all")
|
||||
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
fake_dotenv.load_dotenv = lambda *args, **kwargs: None
|
||||
monkeypatch.setitem(sys.modules, "dotenv", fake_dotenv)
|
||||
|
||||
fake_run_agent = types.ModuleType("run_agent")
|
||||
fake_run_agent.AIAgent = FakeBrowserLiveAgent
|
||||
monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent)
|
||||
|
||||
adapter = ProgressCaptureAdapter()
|
||||
runner = _make_runner(adapter)
|
||||
gateway_run = importlib.import_module("gateway.run")
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path)
|
||||
monkeypatch.setattr(gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"})
|
||||
source = SessionSource(
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_id="-1001",
|
||||
chat_type="group",
|
||||
thread_id="17585",
|
||||
)
|
||||
|
||||
result = await runner._run_agent(
|
||||
message="hello",
|
||||
context_prompt="",
|
||||
history=[],
|
||||
source=source,
|
||||
session_id="sess-1",
|
||||
session_key="agent:main:telegram:group:-1001:17585",
|
||||
)
|
||||
|
||||
assert result["final_response"] == "done"
|
||||
assert adapter.sent[0]["content"] == "📍 Я на странице: example.com"
|
||||
assert "internet_browser" not in adapter.sent[0]["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_live_progress_is_capped_for_telegram(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("HERMES_TOOL_PROGRESS_MODE", "all")
|
||||
monkeypatch.setenv("HERMES_TOOL_PROGRESS_MAX_CHARS", "180")
|
||||
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
fake_dotenv.load_dotenv = lambda *args, **kwargs: None
|
||||
monkeypatch.setitem(sys.modules, "dotenv", fake_dotenv)
|
||||
|
||||
fake_run_agent = types.ModuleType("run_agent")
|
||||
fake_run_agent.AIAgent = FakeLongBrowserLiveAgent
|
||||
monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent)
|
||||
|
||||
adapter = ProgressCaptureAdapter()
|
||||
runner = _make_runner(adapter)
|
||||
gateway_run = importlib.import_module("gateway.run")
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path)
|
||||
monkeypatch.setattr(gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"})
|
||||
source = SessionSource(
|
||||
platform=Platform.TELEGRAM,
|
||||
chat_id="-1001",
|
||||
chat_type="group",
|
||||
thread_id="17585",
|
||||
)
|
||||
|
||||
result = await runner._run_agent(
|
||||
message="hello",
|
||||
context_prompt="",
|
||||
history=[],
|
||||
source=source,
|
||||
session_id="sess-1",
|
||||
session_key="agent:main:telegram:group:-1001:17585",
|
||||
)
|
||||
|
||||
assert result["final_response"] == "done"
|
||||
assert adapter.edits
|
||||
assert all(len(call["content"]) <= 180 for call in adapter.edits)
|
||||
assert adapter.edits[-1]["content"].startswith("…\n")
|
||||
|
|
|
|||
154
hermes_code/tests/tools/test_browser_use_live_events.py
Normal file
154
hermes_code/tests/tools/test_browser_use_live_events.py
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
import asyncio
|
||||
|
||||
from tools import browser_use_tool
|
||||
|
||||
|
||||
def test_build_events_url_replaces_run_endpoint(monkeypatch):
|
||||
monkeypatch.delenv("BROWSER_USE_EVENTS_URL", raising=False)
|
||||
|
||||
assert (
|
||||
browser_use_tool._build_events_url("http://browser:8787/run")
|
||||
== "http://browser:8787/events"
|
||||
)
|
||||
assert (
|
||||
browser_use_tool._build_events_url("http://browser:8787/api/run?x=1")
|
||||
== "http://browser:8787/api/events"
|
||||
)
|
||||
|
||||
|
||||
def test_build_events_url_uses_override(monkeypatch):
|
||||
monkeypatch.setenv("BROWSER_USE_EVENTS_URL", "http://agent/events")
|
||||
|
||||
assert browser_use_tool._build_events_url("http://browser:8787/run") == "http://agent/events"
|
||||
|
||||
|
||||
def test_format_event_for_progress_adds_human_help_link():
|
||||
text = browser_use_tool._format_event_for_progress(
|
||||
{
|
||||
"phase": "human_help",
|
||||
"level": "help",
|
||||
"message": "Похоже, нужна капча.",
|
||||
},
|
||||
vnc_url="https://vnc.example",
|
||||
)
|
||||
|
||||
assert text == "🧑💻 Нужна помощь: Похоже, нужна капча. Экран: https://vnc.example"
|
||||
|
||||
|
||||
def test_emit_unseen_events_keeps_high_priority_after_limit():
|
||||
calls = []
|
||||
|
||||
def progress_callback(name, preview, args):
|
||||
calls.append((name, preview, args))
|
||||
|
||||
last_seq = {"value": 0}
|
||||
emitted = browser_use_tool._emit_unseen_events(
|
||||
[
|
||||
{"seq": 1, "phase": "page", "message": "Я на странице: example.com"},
|
||||
{"seq": 2, "phase": "page", "message": "Я на странице: example.org"},
|
||||
{
|
||||
"seq": 3,
|
||||
"phase": "human_help",
|
||||
"level": "help",
|
||||
"message": "Нужна капча.",
|
||||
},
|
||||
],
|
||||
last_seq,
|
||||
progress_callback,
|
||||
"https://vnc.example",
|
||||
max_events=1,
|
||||
)
|
||||
|
||||
assert emitted == 2
|
||||
assert last_seq["value"] == 3
|
||||
assert [call[1] for call in calls] == [
|
||||
"📍 Я на странице: example.com",
|
||||
"🧑💻 Нужна помощь: Нужна капча. Экран: https://vnc.example",
|
||||
]
|
||||
assert all(call[0] == "internet_browser" for call in calls)
|
||||
assert all(call[2]["_browser_live"] is True for call in calls)
|
||||
|
||||
|
||||
def test_poll_live_events_emits_until_done(monkeypatch):
|
||||
responses = [
|
||||
{
|
||||
"success": True,
|
||||
"events": [{"seq": 1, "phase": "start", "message": "Запускаю."}],
|
||||
"done": False,
|
||||
},
|
||||
{
|
||||
"success": True,
|
||||
"events": [{"seq": 2, "phase": "done", "level": "done", "message": "Готово."}],
|
||||
"done": True,
|
||||
},
|
||||
]
|
||||
|
||||
def fake_fetch(events_url, run_id, after):
|
||||
return responses.pop(0)
|
||||
|
||||
calls = []
|
||||
|
||||
def progress_callback(name, preview, args):
|
||||
calls.append((name, preview, args))
|
||||
|
||||
monkeypatch.setattr(browser_use_tool, "_fetch_browser_events", fake_fetch)
|
||||
monkeypatch.setenv("BROWSER_LIVE_LOG_POLL_INTERVAL", "0.1")
|
||||
|
||||
async def run_poll():
|
||||
stop_event = asyncio.Event()
|
||||
last_seq = {"value": 0}
|
||||
await browser_use_tool._poll_live_events(
|
||||
"http://browser:8787/events",
|
||||
"run-1",
|
||||
progress_callback,
|
||||
stop_event,
|
||||
"",
|
||||
last_seq,
|
||||
)
|
||||
return last_seq
|
||||
|
||||
last_seq = asyncio.run(run_poll())
|
||||
|
||||
assert last_seq["value"] == 2
|
||||
assert [call[1] for call in calls] == ["🌐 Запускаю.", "✅ Готово."]
|
||||
|
||||
|
||||
def test_poll_live_events_applies_event_limit_per_poll(monkeypatch):
|
||||
responses = [
|
||||
{
|
||||
"success": True,
|
||||
"events": [{"seq": 1, "phase": "page", "message": "Первая страница."}],
|
||||
"done": False,
|
||||
},
|
||||
{
|
||||
"success": True,
|
||||
"events": [{"seq": 2, "phase": "action", "message": "Нажимаю play."}],
|
||||
"done": True,
|
||||
},
|
||||
]
|
||||
|
||||
def fake_fetch(events_url, run_id, after):
|
||||
return responses.pop(0)
|
||||
|
||||
calls = []
|
||||
|
||||
def progress_callback(name, preview, args):
|
||||
calls.append(preview)
|
||||
|
||||
monkeypatch.setattr(browser_use_tool, "_fetch_browser_events", fake_fetch)
|
||||
monkeypatch.setenv("BROWSER_LIVE_LOG_POLL_INTERVAL", "0.1")
|
||||
monkeypatch.setenv("BROWSER_LIVE_LOG_MAX_EVENTS", "1")
|
||||
|
||||
async def run_poll():
|
||||
await browser_use_tool._poll_live_events(
|
||||
"http://browser:8787/events",
|
||||
"run-1",
|
||||
progress_callback,
|
||||
asyncio.Event(),
|
||||
"",
|
||||
{"value": 0},
|
||||
)
|
||||
|
||||
asyncio.run(run_poll())
|
||||
|
||||
assert calls == ["📍 Первая страница.", "🖱️ Нажимаю play."]
|
||||
216
hermes_code/tests/tools/test_browser_use_runner_events.py
Normal file
216
hermes_code/tests/tools/test_browser_use_runner_events.py
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
import importlib.util
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _load_runner(monkeypatch):
|
||||
fake_browser_use = types.ModuleType("browser_use")
|
||||
fake_browser_use.Agent = object
|
||||
fake_browser_use.Browser = object
|
||||
fake_browser_use.ChatOpenAI = object
|
||||
monkeypatch.setitem(sys.modules, "browser_use", fake_browser_use)
|
||||
|
||||
path = Path(__file__).resolve().parents[3] / "browser_env" / "browser_use_runner.py"
|
||||
spec = importlib.util.spec_from_file_location("test_browser_use_runner", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_event_store_returns_incremental_events(monkeypatch):
|
||||
runner = _load_runner(monkeypatch)
|
||||
runner._RUNS.clear()
|
||||
|
||||
runner._append_event("run-1", "start", "Запускаю.")
|
||||
runner._append_event("run-1", "page", "Я на странице: example.com")
|
||||
runner._finish_run("run-1")
|
||||
|
||||
payload = runner._get_events("run-1", after=1)
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["done"] is True
|
||||
assert [event["message"] for event in payload["events"]] == ["Я на странице: example.com"]
|
||||
|
||||
|
||||
def test_browser_use_log_translation_detects_actions_and_captcha(monkeypatch):
|
||||
runner = _load_runner(monkeypatch)
|
||||
|
||||
assert runner._browser_log_to_event("Step 3: deciding next action", logging.INFO) == (
|
||||
"step",
|
||||
"Шаг 3: анализирую страницу и выбираю следующее действие.",
|
||||
"info",
|
||||
)
|
||||
assert runner._browser_log_to_event("Action: click button Sign in", logging.INFO) == (
|
||||
"action",
|
||||
"Кликаю по элементу на странице.",
|
||||
"info",
|
||||
)
|
||||
assert runner._browser_log_to_event("Cloudflare captcha detected", logging.INFO) == (
|
||||
"human_help",
|
||||
"Похоже, на странице проверка или капча. Откройте экран браузера и помогите пройти её.",
|
||||
"help",
|
||||
)
|
||||
|
||||
|
||||
def test_model_action_formatter_reports_specific_actions(monkeypatch):
|
||||
runner = _load_runner(monkeypatch)
|
||||
|
||||
assert runner._format_model_action({"go_to_url": {"url": "https://music.yandex.ru/search?text=Дора"}}) == (
|
||||
"navigation",
|
||||
"Перехожу на music.yandex.ru/search.",
|
||||
)
|
||||
assert runner._format_model_action({"input_text": {"text": "Дора", "index": 4}}) == (
|
||||
"input",
|
||||
"Ввожу в поле: Дора.",
|
||||
)
|
||||
assert runner._format_model_action({"click_element_by_index": {"index": 12}}) is None
|
||||
assert runner._format_model_action({"click": {"label": "Playback"}}) == (
|
||||
"action",
|
||||
"Нажимаю: Playback.",
|
||||
)
|
||||
assert runner._format_model_action({"send_keys": {"keys": "Enter"}}) == (
|
||||
"action",
|
||||
"Нажимаю клавиши: Enter.",
|
||||
)
|
||||
assert runner._format_model_action({"scroll_down": {"amount": 614}}) == (
|
||||
"action",
|
||||
"Прокручиваю страницу вниз.",
|
||||
)
|
||||
assert runner._format_model_action({"action": [{"click_element_by_index": {"index": 5}}]}) is None
|
||||
assert runner._format_model_action({"action": "evaluate", "code": "document.querySelector('#q')"}) == (
|
||||
"action",
|
||||
"Проверяю страницу скриптом.",
|
||||
)
|
||||
|
||||
|
||||
def test_action_result_formatter_humanizes_browser_use_content(monkeypatch):
|
||||
runner = _load_runner(monkeypatch)
|
||||
|
||||
assert runner._format_action_result_content("Navigated to https://youtube.com") == (
|
||||
"navigation",
|
||||
"Открыл страницу: youtube.com/.",
|
||||
)
|
||||
assert runner._format_action_result_content("🔗 Navigated to https://music.vk.com") == (
|
||||
"navigation",
|
||||
"Открыл страницу: music.vk.com/.",
|
||||
)
|
||||
assert runner._format_action_result_content(
|
||||
"🔗 Opened new tab with url https://yandex.ru/search/?text=ВК+Музыка+Дора"
|
||||
) == (
|
||||
"navigation",
|
||||
"Открыл новую вкладку: yandex.ru/search/.",
|
||||
)
|
||||
assert runner._format_action_result_content("https://music.yandex.ru/search?text=Дора") == (
|
||||
"navigation",
|
||||
"Открыл страницу: music.yandex.ru/search.",
|
||||
)
|
||||
assert runner._format_action_result_content('Clicked button "Accept all" aria-label=Accept the use') == (
|
||||
"action",
|
||||
"Нажал: Accept all.",
|
||||
)
|
||||
assert runner._format_action_result_content('Clicked div "Меню" id=header__burger_menu') == (
|
||||
"action",
|
||||
"Нажал: Меню.",
|
||||
)
|
||||
assert runner._format_action_result_content("Clicked a aria-label=Home") == (
|
||||
"action",
|
||||
"Нажал: Home.",
|
||||
)
|
||||
assert runner._format_action_result_content("Clicked button aria-label=Playback") == (
|
||||
"action",
|
||||
"Нажал: Playback.",
|
||||
)
|
||||
assert runner._format_action_result_content("Clicked button") == (
|
||||
"action",
|
||||
"Нажал кнопку.",
|
||||
)
|
||||
assert runner._format_action_result_content(
|
||||
"Typed 'Дора' 💡 This is an autocomplete field. Wait for suggestions to appear"
|
||||
) == (
|
||||
"input",
|
||||
"Ввёл в поиск: Дора.",
|
||||
)
|
||||
assert runner._format_action_result_content("Waited for 3 seconds") == (
|
||||
"action",
|
||||
"Жду загрузку: 3 сек.",
|
||||
)
|
||||
assert runner._format_action_result_content("🔍 Scrolled up 1.5 pages") == (
|
||||
"action",
|
||||
"Прокрутил страницу вверх на 1.5 pages.",
|
||||
)
|
||||
assert runner._format_action_result_content("🔍 Scrolled down 613px") == (
|
||||
"action",
|
||||
"Прокрутил страницу вниз на 613px.",
|
||||
)
|
||||
assert runner._format_action_result_content(
|
||||
"No elements found matching \"input[type='text'], [role='searchbox']\"."
|
||||
) == (
|
||||
"read",
|
||||
"Не нашёл подходящее поле или кнопку на странице.",
|
||||
)
|
||||
assert runner._format_action_result_content("Task Completed Successfully! I found band 'Дора'") == (
|
||||
"done",
|
||||
"Browser-use сообщил, что задача выполнена.",
|
||||
)
|
||||
|
||||
|
||||
def test_step_end_hook_emits_history_actions(monkeypatch):
|
||||
runner = _load_runner(monkeypatch)
|
||||
runner._RUNS.clear()
|
||||
|
||||
class FakeHistory:
|
||||
def model_actions(self):
|
||||
return [
|
||||
{"go_to_url": {"url": "https://music.yandex.ru"}},
|
||||
{"input_text": {"text": "Дора", "index": 3}},
|
||||
{"click_element_by_index": {"index": 7}},
|
||||
]
|
||||
|
||||
def action_results(self):
|
||||
return []
|
||||
|
||||
agent = types.SimpleNamespace(history=FakeHistory())
|
||||
hook = runner._make_step_end_hook("run-2", {"actions": 0, "results": 0})
|
||||
|
||||
asyncio.run(hook(agent))
|
||||
payload = runner._get_events("run-2", after=0)
|
||||
|
||||
assert [event["message"] for event in payload["events"]] == [
|
||||
"Перехожу на music.yandex.ru/.",
|
||||
"Ввожу в поле: Дора.",
|
||||
]
|
||||
|
||||
asyncio.run(hook(agent))
|
||||
assert runner._get_events("run-2", after=2)["events"] == []
|
||||
|
||||
|
||||
def test_step_end_hook_emits_action_results(monkeypatch):
|
||||
runner = _load_runner(monkeypatch)
|
||||
runner._RUNS.clear()
|
||||
|
||||
class FakeHistory:
|
||||
def model_actions(self):
|
||||
return ()
|
||||
|
||||
def action_results(self):
|
||||
return (
|
||||
{"extracted_content": "Clicked button \"Play (k)\" aria-label=Play (k)"},
|
||||
{"error": "Element not found"},
|
||||
{"extracted_content": "Cloudflare captcha"},
|
||||
)
|
||||
|
||||
agent = types.SimpleNamespace(history=FakeHistory())
|
||||
hook = runner._make_step_end_hook("run-3", {"actions": 0, "results": 0})
|
||||
|
||||
asyncio.run(hook(agent))
|
||||
payload = runner._get_events("run-3", after=0)
|
||||
|
||||
assert [event["phase"] for event in payload["events"]] == ["action", "error", "human_help"]
|
||||
assert [event["message"] for event in payload["events"]] == [
|
||||
"Нажал: Play (k).",
|
||||
"Ошибка действия: Element not found",
|
||||
"Похоже, страница просит проверку человека. Откройте экран браузера и помогите пройти её.",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue