197 lines
No EOL
8 KiB
Python
197 lines
No EOL
8 KiB
Python
import requests
|
||
import time
|
||
import json
|
||
from datasets import load_dataset
|
||
from datetime import datetime
|
||
|
||
# Конфигурация API
|
||
API_URL = "http://localhost:8088/api/browser/tasks"
|
||
HEADERS = {"Content-Type": "application/json"}
|
||
|
||
# Загружаем датасет
|
||
dataset = load_dataset("iMeanAI/Mind2Web-Live", split="train")
|
||
|
||
# Для теста берем первые N задач (замените на полный датасет при необходимости)
|
||
TEST_SIZE = 10 # или len(dataset) для полного бенчмарка
|
||
dataset = dataset.select(range(TEST_SIZE))
|
||
|
||
print(f"Загружено задач: {len(dataset)}")
|
||
print(f"Поля: {dataset[0].keys()}\n")
|
||
cnt = 3
|
||
results = []
|
||
|
||
for idx, item in enumerate(dataset):
|
||
if cnt > 0:
|
||
cnt -=1
|
||
continue
|
||
# Поля из датасета
|
||
task_desc = item['task'] # Описание задачи
|
||
ref_length = item['reference_task_length'] # Эталонная длина в шагах
|
||
evaluation = item['evaluation'] # Критерии оценки
|
||
|
||
# ID задачи (используем index + timestamp для уникальности)
|
||
task_id_orig = f"mind2web_{idx}_{int(time.time())}"
|
||
|
||
print(f"\n[{idx + 1}/{len(dataset)}] Task: {task_desc[:70]}...")
|
||
print(f" Эталонная длина: {ref_length} шагов")
|
||
|
||
start_time = time.time()
|
||
|
||
# 1. Создаем задачу через API
|
||
try:
|
||
resp = requests.post(
|
||
API_URL,
|
||
json={
|
||
"task": task_desc,
|
||
"timeout": 300, # Увеличим таймаут для сложных задач
|
||
"metadata": {
|
||
"source": "mind2web",
|
||
"reference_length": ref_length
|
||
}
|
||
},
|
||
headers=HEADERS,
|
||
timeout=10
|
||
)
|
||
|
||
if resp.status_code != 202:
|
||
print(f" ❌ Ошибка создания задачи: {resp.status_code}")
|
||
print(f" Ответ: {resp.text}")
|
||
continue
|
||
|
||
api_task_id = resp.json()["task_id"]
|
||
created_at = time.time()
|
||
queue_time = created_at - start_time
|
||
|
||
print(f" 📝 Task ID: {api_task_id} | Очередь: {queue_time:.2f}с")
|
||
|
||
# 2. Ожидание завершения с прогрессом
|
||
status = "queued"
|
||
poll_count = 0
|
||
while status in ["queued", "running"]:
|
||
time.sleep(2) # Интервал опроса
|
||
poll_count += 1
|
||
|
||
try:
|
||
status_resp = requests.get(f"{API_URL}/{api_task_id}", timeout=5)
|
||
if status_resp.status_code == 200:
|
||
status_data = status_resp.json()
|
||
status = status_data.get("status", "unknown")
|
||
|
||
# Показываем прогресс каждые 5 опросов
|
||
if poll_count % 5 == 0:
|
||
elapsed = time.time() - start_time
|
||
print(f" ⏳ Статус: {status} | Прошло: {elapsed:.1f}с")
|
||
except Exception as e:
|
||
print(f" ⚠️ Ошибка опроса: {e}")
|
||
pass
|
||
|
||
end_time = time.time()
|
||
execution_time = end_time - start_time
|
||
|
||
# 3. Получение результата
|
||
result_resp = requests.get(f"{API_URL}/{api_task_id}/result", timeout=10)
|
||
|
||
result_data = None
|
||
if result_resp.status_code == 200:
|
||
try:
|
||
result_data = result_resp.json()
|
||
except:
|
||
result_data = result_resp.text
|
||
|
||
# 4. Запись метрик
|
||
result = {
|
||
"index": idx,
|
||
"original_task_id": task_id_orig,
|
||
"api_task_id": api_task_id,
|
||
"task_description": task_desc,
|
||
"reference_length": ref_length,
|
||
"status": status,
|
||
"queue_time_sec": round(queue_time, 2),
|
||
"execution_time_sec": round(execution_time, 2),
|
||
"total_time_sec": round(end_time - start_time, 2),
|
||
"result": result_data,
|
||
"timestamp": datetime.now().isoformat()
|
||
}
|
||
results.append(result)
|
||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
filename = f"mind2web_benchmark.json"
|
||
with open(filename, "w", encoding="utf-8") as f:
|
||
json.dump(results, f, indent=2, ensure_ascii=False)
|
||
# Эмодзи статуса
|
||
status_emoji = "✅" if status == "succeeded" else "❌"
|
||
print(f" {status_emoji} Статус: {status} | Время: {execution_time:.1f}с")
|
||
|
||
except requests.exceptions.Timeout:
|
||
print(f" ❌ Таймаут при создании задачи")
|
||
except Exception as e:
|
||
print(f" ❌ Ошибка: {type(e).__name__}: {e}")
|
||
continue
|
||
|
||
# Сохранение детальных результатов
|
||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
filename = f"mind2web_benchmark_{timestamp}.json"
|
||
|
||
with open(filename, "w", encoding="utf-8") as f:
|
||
json.dump(results, f, indent=2, ensure_ascii=False)
|
||
|
||
print("\n" + "=" * 60)
|
||
print("📊 ИТОГОВЫЕ МЕТРИКИ СКОРОСТИ")
|
||
print("=" * 60)
|
||
|
||
# Статистика по статусам
|
||
completed = [r for r in results if r["status"] == "completed"]
|
||
failed = [r for r in results if r["status"] == "failed"]
|
||
unknown = [r for r in results if r["status"] not in ["completed", "failed"]]
|
||
|
||
print(f"\n📈 СТАТУСЫ:")
|
||
print(f" Всего задач: {len(results)}")
|
||
print(f" ✅ Успешно: {len(completed)} ({len(completed) / max(len(results), 1) * 100:.1f}%)")
|
||
print(f" ❌ Провалено: {len(failed)} ({len(failed) / max(len(results), 1) * 100:.1f}%)")
|
||
if unknown:
|
||
print(f" ❓ Неизвестный статус: {len(unknown)}")
|
||
|
||
if completed:
|
||
total_times = [r["total_time_sec"] for r in completed]
|
||
queue_times = [r["queue_time_sec"] for r in completed]
|
||
exec_times = [r["execution_time_sec"] for r in completed]
|
||
|
||
print(f"\n⏱️ ВРЕМЯ ВЫПОЛНЕНИЯ:")
|
||
print(f" Среднее: {sum(total_times) / len(total_times):.2f} сек")
|
||
print(f" Медиана (p50): {sorted(total_times)[len(total_times) // 2]:.2f} сек")
|
||
if len(total_times) >= 20:
|
||
print(f" p95: {sorted(total_times)[int(len(total_times) * 0.95)]:.2f} сек")
|
||
print(f" Мин: {min(total_times):.2f} сек")
|
||
print(f" Макс: {max(total_times):.2f} сек")
|
||
|
||
print(f"\n📊 ПРОИЗВОДИТЕЛЬНОСТЬ:")
|
||
print(f" Среднее время в очереди: {sum(queue_times) / len(queue_times):.2f} сек")
|
||
tasks_per_hour = 3600 / (sum(total_times) / len(total_times))
|
||
print(f" Скорость выполнения: {tasks_per_hour:.1f} задач/час")
|
||
|
||
# Эффективность относительно эталонной длины
|
||
if all("reference_length" in r for r in completed):
|
||
avg_ref_length = sum(r["reference_length"] for r in completed) / len(completed)
|
||
time_per_step = (sum(total_times) / len(total_times)) / avg_ref_length
|
||
print(f" Среднее время на шаг: {time_per_step:.2f} сек")
|
||
|
||
print(f"\n💾 Результаты сохранены в: {filename}")
|
||
|
||
# Создание краткого отчета для сравнения
|
||
summary = {
|
||
"benchmark": "Online-Mind2Web",
|
||
"timestamp": timestamp,
|
||
"api_endpoint": API_URL,
|
||
"total_tasks": len(results),
|
||
"completed": len(completed),
|
||
"failed": len(failed),
|
||
"success_rate": len(completed) / max(len(results), 1) * 100,
|
||
"avg_time_sec": sum(total_times) / len(total_times) if completed else None,
|
||
"median_time_sec": sorted(total_times)[len(total_times) // 2] if completed else None,
|
||
"tasks_per_hour": 3600 / (sum(total_times) / len(total_times)) if completed else None
|
||
}
|
||
|
||
summary_file = f"mind2web_summary_{timestamp}.json"
|
||
with open(summary_file, "w", encoding="utf-8") as f:
|
||
json.dump(summary, f, indent=2, ensure_ascii=False)
|
||
|
||
print(f"📋 Краткий отчет сохранен в: {summary_file}") |