add sandbox rollback regression tests

test/test_create_http.py

@@ -3,6 +3,7 @@ import json
 from datetime import UTC, datetime, timedelta
 from uuid import UUID
 
+import pytest
 from docker import DockerClient
 from fastapi import FastAPI
 from starlette.types import Message, Scope
@@ -226,6 +227,16 @@ class FixedSandboxState:
         return list(self._sessions)
 
 
+class FailingSandboxState:
+    def __init__(self, error: Exception) -> None:
+        self._error = error
+        self.calls = 0
+
+    def list_active_sessions(self) -> list[SandboxSession]:
+        self.calls += 1
+        raise self._error
+
+
 class CountingRegistry:
     def __init__(self, count_active_result: int) -> None:
         self._count_active_result = count_active_result
@@ -238,6 +249,25 @@ class CountingRegistry:
         return self._count_active_result
 
 
+class FailingRegistry:
+    def __init__(self, error: Exception, *, fail_on: str = 'replace_all') -> None:
+        self._error = error
+        self._fail_on = fail_on
+        self.replaced_sessions: list[SandboxSession] = []
+        self.count_calls = 0
+
+    def replace_all(self, sessions: list[SandboxSession]) -> None:
+        self.replaced_sessions = list(sessions)
+        if self._fail_on == 'replace_all':
+            raise self._error
+
+    def count_active(self) -> int:
+        self.count_calls += 1
+        if self._fail_on == 'count_active':
+            raise self._error
+        return 0
+
+
 def build_config() -> AppConfig:
     return AppConfig(
         app=AppSectionConfig(name='master', env='test'),
@@ -719,6 +749,124 @@ def test_reconciliation_uses_registry_backed_active_count_metric() -> None:
     assert tracer.spans[0][2].attrs['sandbox.active_count'] == 7
 
 
+def test_reconciliation_records_error_when_state_source_fails() -> None:
+    logger = FakeLogger()
+    metrics = RecordingMetrics()
+    tracer = RecordingTracer()
+    state_error = RuntimeError('state_failed')
+    state_source = FailingSandboxState(state_error)
+    reconciler = SandboxSessionReconciler(
+        state_source=state_source,
+        registry=CountingRegistry(count_active_result=7),
+        logger=logger,
+        metrics=metrics,
+        tracer=tracer,
+    )
+
+    with pytest.raises(RuntimeError, match='state_failed') as excinfo:
+        reconciler.execute()
+
+    assert state_source.calls == 1
+    assert metrics.set_calls == []
+    spans = [
+        span
+        for name, _, span in tracer.spans
+        if name == 'adapter.sandbox.reconcile_sessions'
+    ]
+    assert spans
+    span = spans[0]
+    assert span.attrs['sandbox.result'] == 'error'
+    assert 'sandbox.discovered_count' not in span.attrs
+    assert 'sandbox.active_count' not in span.attrs
+    assert excinfo.value in span.errors
+
+
+def test_reconciliation_records_error_without_active_count_metric_on_registry_failure() -> (
+    None
+):
+    logger = FakeLogger()
+    metrics = RecordingMetrics()
+    tracer = RecordingTracer()
+    created_at = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
+    session = SandboxSession(
+        session_id=SESSION_ID,
+        chat_id=CHAT_ID,
+        container_id='container-123',
+        status=SandboxStatus.RUNNING,
+        created_at=created_at,
+        expires_at=created_at + timedelta(minutes=5),
+    )
+    registry_error = RuntimeError('replace_failed')
+    registry = FailingRegistry(registry_error)
+    reconciler = SandboxSessionReconciler(
+        state_source=FixedSandboxState([session]),
+        registry=registry,
+        logger=logger,
+        metrics=metrics,
+        tracer=tracer,
+    )
+
+    with pytest.raises(RuntimeError, match='replace_failed') as excinfo:
+        reconciler.execute()
+
+    assert registry.replaced_sessions == [session]
+    assert registry.count_calls == 0
+    assert metrics.set_calls == []
+    spans = [
+        span
+        for name, _, span in tracer.spans
+        if name == 'adapter.sandbox.reconcile_sessions'
+    ]
+    assert spans
+    span = spans[0]
+    assert span.attrs['sandbox.discovered_count'] == 1
+    assert span.attrs['sandbox.result'] == 'error'
+    assert 'sandbox.active_count' not in span.attrs
+    assert excinfo.value in span.errors
+
+
+def test_reconciliation_records_error_when_registry_count_active_fails() -> None:
+    logger = FakeLogger()
+    metrics = RecordingMetrics()
+    tracer = RecordingTracer()
+    created_at = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
+    session = SandboxSession(
+        session_id=SESSION_ID,
+        chat_id=CHAT_ID,
+        container_id='container-123',
+        status=SandboxStatus.RUNNING,
+        created_at=created_at,
+        expires_at=created_at + timedelta(minutes=5),
+    )
+    registry_error = RuntimeError('count_failed')
+    registry = FailingRegistry(registry_error, fail_on='count_active')
+    reconciler = SandboxSessionReconciler(
+        state_source=FixedSandboxState([session]),
+        registry=registry,
+        logger=logger,
+        metrics=metrics,
+        tracer=tracer,
+    )
+
+    with pytest.raises(RuntimeError, match='count_failed') as excinfo:
+        reconciler.execute()
+
+    assert registry.replaced_sessions == [session]
+    assert registry.count_calls == 1
+    assert metrics.set_calls == []
+    spans = [
+        span
+        for name, _, span in tracer.spans
+        if name == 'adapter.sandbox.reconcile_sessions'
+    ]
+    assert spans
+    span = spans[0]
+    assert span.attrs['sandbox.discovered_count'] == 1
+    assert 'sandbox.active_count' not in span.attrs
+    assert span.attrs['sandbox.result'] == 'error'
+    assert excinfo.value in span.errors
+
+
 def test_build_container_wires_observability_into_runtime_and_reconciler(
     monkeypatch,
 ) -> None: