add sandbox observability tests
This commit is contained in:
Azamat
parent
8d3a080d45
commit
dff28efecf
3 changed files with 621 additions and 1 deletions
|
|
@ -1,11 +1,15 @@
|
|||
import threading
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import TracebackType
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
|
||||
from adapter.observability.noop import NoopMetrics, NoopTracer
|
||||
from domain.sandbox import SandboxSession, SandboxStatus
|
||||
from repository.sandbox_lock import ProcessLocalSandboxLifecycleLocker
|
||||
from repository.sandbox_session import InMemorySandboxSessionRepository
|
||||
from usecase.interface import AttrValue, Attrs
|
||||
from usecase.sandbox import CleanupExpiredSandboxes, CreateSandbox, CreateSandboxCommand
|
||||
|
||||
CHAT_ID = UUID('11111111-1111-1111-1111-111111111111')
|
||||
|
|
@ -53,6 +57,130 @@ class FakeLogger:
|
|||
self.messages.append(('error', message, attrs))
|
||||
|
||||
|
||||
class RecordingMetrics:
|
||||
def __init__(self) -> None:
|
||||
self.increment_calls: list[tuple[str, int, Attrs | None]] = []
|
||||
self.record_calls: list[tuple[str, float, Attrs | None]] = []
|
||||
self.set_calls: list[tuple[str, int | float, Attrs | None]] = []
|
||||
|
||||
def increment(
|
||||
self,
|
||||
name: str,
|
||||
value: int = 1,
|
||||
attrs: Attrs | None = None,
|
||||
) -> None:
|
||||
self.increment_calls.append((name, value, attrs))
|
||||
|
||||
def record(
|
||||
self,
|
||||
name: str,
|
||||
value: float,
|
||||
attrs: Attrs | None = None,
|
||||
) -> None:
|
||||
self.record_calls.append((name, value, attrs))
|
||||
|
||||
def set(
|
||||
self,
|
||||
name: str,
|
||||
value: int | float,
|
||||
attrs: Attrs | None = None,
|
||||
) -> None:
|
||||
self.set_calls.append((name, value, attrs))
|
||||
|
||||
|
||||
class RecordingSpan:
|
||||
def __init__(self) -> None:
|
||||
self.attrs: dict[str, AttrValue] = {}
|
||||
self.errors: list[Exception] = []
|
||||
|
||||
def set_attribute(self, name: str, value: AttrValue) -> None:
|
||||
self.attrs[name] = value
|
||||
|
||||
def record_error(self, error: Exception) -> None:
|
||||
self.errors.append(error)
|
||||
|
||||
|
||||
class RecordingSpanContext:
|
||||
def __init__(self, span: RecordingSpan) -> None:
|
||||
self._span = span
|
||||
|
||||
def __enter__(self) -> RecordingSpan:
|
||||
return self._span
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> bool | None:
|
||||
return None
|
||||
|
||||
|
||||
class RecordingTracer:
|
||||
def __init__(self) -> None:
|
||||
self.spans: list[tuple[str, Attrs | None, RecordingSpan]] = []
|
||||
|
||||
def start_span(
|
||||
self,
|
||||
name: str,
|
||||
attrs: Attrs | None = None,
|
||||
) -> RecordingSpanContext:
|
||||
span = RecordingSpan()
|
||||
self.spans.append((name, attrs, span))
|
||||
return RecordingSpanContext(span)
|
||||
|
||||
|
||||
def _attrs_include(
|
||||
actual: Attrs | dict[str, AttrValue] | None,
|
||||
expected: dict[str, AttrValue],
|
||||
) -> bool:
|
||||
if actual is None:
|
||||
return False
|
||||
|
||||
return all(actual.get(name) == value for name, value in expected.items())
|
||||
|
||||
|
||||
def _find_span(
|
||||
tracer: RecordingTracer,
|
||||
name: str,
|
||||
attrs: dict[str, AttrValue] | None = None,
|
||||
span_attrs: dict[str, AttrValue] | None = None,
|
||||
) -> RecordingSpan:
|
||||
for recorded_name, recorded_attrs, span in tracer.spans:
|
||||
if recorded_name != name:
|
||||
continue
|
||||
if attrs is not None and not _attrs_include(recorded_attrs, attrs):
|
||||
continue
|
||||
if span_attrs is not None and not _attrs_include(span.attrs, span_attrs):
|
||||
continue
|
||||
return span
|
||||
|
||||
raise AssertionError(f'missing span {name}')
|
||||
|
||||
|
||||
def _assert_increment_metric_present(
|
||||
metrics: RecordingMetrics,
|
||||
name: str,
|
||||
*,
|
||||
value: int = 1,
|
||||
attrs: dict[str, AttrValue] | None = None,
|
||||
) -> None:
|
||||
for recorded_name, recorded_value, recorded_attrs in metrics.increment_calls:
|
||||
if recorded_name != name or recorded_value != value:
|
||||
continue
|
||||
if attrs is not None and not _attrs_include(recorded_attrs, attrs):
|
||||
continue
|
||||
return
|
||||
|
||||
raise AssertionError(f'missing increment metric {name}')
|
||||
|
||||
|
||||
def _active_count_values(metrics: RecordingMetrics) -> list[int | float]:
|
||||
return [
|
||||
value for name, value, _ in metrics.set_calls if name == 'sandbox.active.count'
|
||||
]
|
||||
|
||||
|
||||
class FakeLockContext:
|
||||
def __enter__(self) -> None:
|
||||
return None
|
||||
|
|
@ -198,6 +326,30 @@ class FailingStopRuntime(FakeRuntime):
|
|||
raise RuntimeError('stop_failed')
|
||||
|
||||
|
||||
class FailingCreateRuntime(FakeRuntime):
|
||||
def __init__(self, error: Exception) -> None:
|
||||
super().__init__()
|
||||
self._error = error
|
||||
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
session_id: UUID,
|
||||
chat_id: UUID,
|
||||
created_at: datetime,
|
||||
expires_at: datetime,
|
||||
) -> SandboxSession:
|
||||
self.create_calls.append(
|
||||
{
|
||||
'session_id': session_id,
|
||||
'chat_id': chat_id,
|
||||
'created_at': created_at,
|
||||
'expires_at': expires_at,
|
||||
}
|
||||
)
|
||||
raise self._error
|
||||
|
||||
|
||||
def test_create_sandbox_reuses_active_session_when_not_expired() -> None:
|
||||
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
||||
session = SandboxSession(
|
||||
|
|
@ -244,6 +396,104 @@ def test_create_sandbox_reuses_active_session_when_not_expired() -> None:
|
|||
]
|
||||
|
||||
|
||||
def test_create_sandbox_reuse_records_observability() -> None:
|
||||
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
||||
session = SandboxSession(
|
||||
session_id=SESSION_REUSED_ID,
|
||||
chat_id=CHAT_ID,
|
||||
container_id='container-1',
|
||||
status=SandboxStatus.RUNNING,
|
||||
created_at=now - timedelta(minutes=1),
|
||||
expires_at=now + timedelta(minutes=4),
|
||||
)
|
||||
repository = InMemorySandboxSessionRepository()
|
||||
repository.save(session)
|
||||
metrics = RecordingMetrics()
|
||||
tracer = RecordingTracer()
|
||||
usecase = CreateSandbox(
|
||||
repository=repository,
|
||||
locker=FakeLocker(),
|
||||
runtime=FakeRuntime(),
|
||||
clock=FakeClock(now),
|
||||
logger=FakeLogger(),
|
||||
metrics=metrics,
|
||||
tracer=tracer,
|
||||
ttl=timedelta(minutes=5),
|
||||
)
|
||||
|
||||
result = usecase.execute(CreateSandboxCommand(chat_id=CHAT_ID))
|
||||
|
||||
assert result == session
|
||||
_assert_increment_metric_present(
|
||||
metrics,
|
||||
'sandbox.create.total',
|
||||
attrs={'result': 'reused'},
|
||||
)
|
||||
span = _find_span(
|
||||
tracer,
|
||||
'usecase.create_sandbox',
|
||||
{'chat.id': str(CHAT_ID)},
|
||||
{
|
||||
'session.id': str(SESSION_REUSED_ID),
|
||||
'container.id': 'container-1',
|
||||
'sandbox.result': 'reused',
|
||||
},
|
||||
)
|
||||
assert not span.errors
|
||||
|
||||
|
||||
def test_create_sandbox_replace_records_observability_and_final_active_count(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
||||
expired_session = SandboxSession(
|
||||
session_id=SESSION_OLD_ID,
|
||||
chat_id=CHAT_ID,
|
||||
container_id='container-old',
|
||||
status=SandboxStatus.RUNNING,
|
||||
created_at=now - timedelta(minutes=10),
|
||||
expires_at=now,
|
||||
)
|
||||
repository = InMemorySandboxSessionRepository()
|
||||
repository.save(expired_session)
|
||||
metrics = RecordingMetrics()
|
||||
tracer = RecordingTracer()
|
||||
usecase = CreateSandbox(
|
||||
repository=repository,
|
||||
locker=FakeLocker(),
|
||||
runtime=FakeRuntime(),
|
||||
clock=FakeClock(now),
|
||||
logger=FakeLogger(),
|
||||
metrics=metrics,
|
||||
tracer=tracer,
|
||||
ttl=timedelta(minutes=5),
|
||||
)
|
||||
monkeypatch.setattr('usecase.sandbox._new_session_id', lambda: SESSION_NEW_ID)
|
||||
|
||||
result = usecase.execute(CreateSandboxCommand(chat_id=CHAT_ID))
|
||||
|
||||
assert result.session_id == SESSION_NEW_ID
|
||||
assert repository.count_active() == 1
|
||||
_assert_increment_metric_present(
|
||||
metrics,
|
||||
'sandbox.create.total',
|
||||
attrs={'result': 'replaced'},
|
||||
)
|
||||
assert _active_count_values(metrics)
|
||||
assert _active_count_values(metrics)[-1] == 1
|
||||
span = _find_span(
|
||||
tracer,
|
||||
'usecase.create_sandbox',
|
||||
{'chat.id': str(CHAT_ID)},
|
||||
{
|
||||
'session.id': str(SESSION_NEW_ID),
|
||||
'container.id': f'container-{SESSION_NEW_ID}',
|
||||
'sandbox.result': 'replaced',
|
||||
},
|
||||
)
|
||||
assert not span.errors
|
||||
|
||||
|
||||
def test_create_sandbox_replaces_expired_session_and_creates_new_one(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
|
|
@ -363,6 +613,42 @@ def test_create_sandbox_creates_new_session_when_none_exists() -> None:
|
|||
]
|
||||
|
||||
|
||||
def test_create_sandbox_error_records_observability(monkeypatch) -> None:
|
||||
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
||||
metrics = RecordingMetrics()
|
||||
tracer = RecordingTracer()
|
||||
usecase = CreateSandbox(
|
||||
repository=InMemorySandboxSessionRepository(),
|
||||
locker=FakeLocker(),
|
||||
runtime=FailingCreateRuntime(RuntimeError('create_failed')),
|
||||
clock=FakeClock(now),
|
||||
logger=FakeLogger(),
|
||||
metrics=metrics,
|
||||
tracer=tracer,
|
||||
ttl=timedelta(minutes=5),
|
||||
)
|
||||
monkeypatch.setattr('usecase.sandbox._new_session_id', lambda: SESSION_NEW_ID)
|
||||
|
||||
with pytest.raises(RuntimeError, match='create_failed') as excinfo:
|
||||
usecase.execute(CreateSandboxCommand(chat_id=CHAT_ID))
|
||||
|
||||
_assert_increment_metric_present(
|
||||
metrics,
|
||||
'sandbox.create.total',
|
||||
attrs={'result': 'error'},
|
||||
)
|
||||
span = _find_span(
|
||||
tracer,
|
||||
'usecase.create_sandbox',
|
||||
{'chat.id': str(CHAT_ID)},
|
||||
{
|
||||
'session.id': str(SESSION_NEW_ID),
|
||||
'sandbox.result': 'error',
|
||||
},
|
||||
)
|
||||
assert excinfo.value in span.errors
|
||||
|
||||
|
||||
def test_create_sandbox_serializes_duplicate_concurrent_create_for_chat_id(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
|
|
@ -516,6 +802,64 @@ def test_cleanup_expired_sandboxes_stops_and_deletes_only_expired_sessions() ->
|
|||
]
|
||||
|
||||
|
||||
def test_cleanup_expired_sandboxes_records_observability_on_cleaned_session() -> None:
|
||||
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
||||
expired_session = SandboxSession(
|
||||
session_id=SESSION_EXPIRED_ID,
|
||||
chat_id=EXPIRED_CHAT_ID,
|
||||
container_id='container-expired',
|
||||
status=SandboxStatus.RUNNING,
|
||||
created_at=now - timedelta(minutes=10),
|
||||
expires_at=now - timedelta(seconds=1),
|
||||
)
|
||||
repository = InMemorySandboxSessionRepository()
|
||||
repository.save(expired_session)
|
||||
metrics = RecordingMetrics()
|
||||
tracer = RecordingTracer()
|
||||
usecase = CleanupExpiredSandboxes(
|
||||
repository=repository,
|
||||
locker=FakeLocker(),
|
||||
runtime=FakeRuntime(),
|
||||
clock=FakeClock(now),
|
||||
logger=FakeLogger(),
|
||||
metrics=metrics,
|
||||
tracer=tracer,
|
||||
)
|
||||
|
||||
result = usecase.execute()
|
||||
|
||||
assert result == [expired_session]
|
||||
_assert_increment_metric_present(
|
||||
metrics,
|
||||
'sandbox.cleanup.total',
|
||||
attrs={'result': 'cleaned'},
|
||||
)
|
||||
assert _active_count_values(metrics)
|
||||
assert _active_count_values(metrics)[-1] == 0
|
||||
root_span = _find_span(
|
||||
tracer,
|
||||
'usecase.cleanup_expired_sandboxes',
|
||||
span_attrs={
|
||||
'sandbox.expired_count': 1,
|
||||
'sandbox.cleaned_count': 1,
|
||||
'sandbox.error_count': 0,
|
||||
'sandbox.result': 'completed',
|
||||
},
|
||||
)
|
||||
assert not root_span.errors
|
||||
cleanup_span = _find_span(
|
||||
tracer,
|
||||
'usecase.cleanup_expired_sandbox',
|
||||
{
|
||||
'chat.id': str(EXPIRED_CHAT_ID),
|
||||
'session.id': str(SESSION_EXPIRED_ID),
|
||||
'container.id': 'container-expired',
|
||||
},
|
||||
{'sandbox.result': 'cleaned'},
|
||||
)
|
||||
assert not cleanup_span.errors
|
||||
|
||||
|
||||
def test_cleanup_expired_sandboxes_skips_replaced_session_from_stale_snapshot() -> None:
|
||||
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
||||
expired_snapshot = SandboxSession(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue