607 lines
19 KiB
Python
607 lines
19 KiB
Python
import threading
|
|
from datetime import UTC, datetime, timedelta
|
|
from uuid import UUID
|
|
|
|
from domain.sandbox import SandboxSession, SandboxStatus
|
|
from repository.sandbox_lock import ProcessLocalSandboxLifecycleLocker
|
|
from repository.sandbox_session import InMemorySandboxSessionRepository
|
|
from usecase.sandbox import CleanupExpiredSandboxes, CreateSandbox, CreateSandboxCommand
|
|
|
|
CHAT_ID = UUID('11111111-1111-1111-1111-111111111111')
|
|
NON_CANONICAL_CHAT_ID = '11111111111111111111111111111111'
|
|
EXPIRED_CHAT_ID = UUID('22222222-2222-2222-2222-222222222222')
|
|
BOUNDARY_CHAT_ID = UUID('33333333-3333-3333-3333-333333333333')
|
|
ACTIVE_CHAT_ID = UUID('44444444-4444-4444-4444-444444444444')
|
|
FAIL_CHAT_ID = UUID('55555555-5555-5555-5555-555555555555')
|
|
CLEAN_CHAT_ID = UUID('66666666-6666-6666-6666-666666666666')
|
|
SESSION_REUSED_ID = UUID('00000000-0000-0000-0000-000000000001')
|
|
SESSION_OLD_ID = UUID('00000000-0000-0000-0000-000000000002')
|
|
SESSION_NEW_ID = UUID('00000000-0000-0000-0000-000000000003')
|
|
SESSION_EXPIRED_ID = UUID('00000000-0000-0000-0000-000000000004')
|
|
SESSION_BOUNDARY_ID = UUID('00000000-0000-0000-0000-000000000005')
|
|
SESSION_ACTIVE_ID = UUID('00000000-0000-0000-0000-000000000006')
|
|
SESSION_FAIL_ID = UUID('00000000-0000-0000-0000-000000000007')
|
|
SESSION_CLEAN_ID = UUID('00000000-0000-0000-0000-000000000008')
|
|
SESSION_REPLACEMENT_ID = UUID('00000000-0000-0000-0000-000000000009')
|
|
|
|
|
|
class FakeClock:
|
|
def __init__(self, now: datetime) -> None:
|
|
self._now = now
|
|
|
|
def now(self) -> datetime:
|
|
return self._now
|
|
|
|
|
|
class FakeLogger:
|
|
def __init__(self) -> None:
|
|
self.messages: list[
|
|
tuple[str, str, dict[str, str | int | float | bool] | None]
|
|
] = []
|
|
|
|
def debug(self, message: str, attrs=None) -> None:
|
|
self.messages.append(('debug', message, attrs))
|
|
|
|
def info(self, message: str, attrs=None) -> None:
|
|
self.messages.append(('info', message, attrs))
|
|
|
|
def warning(self, message: str, attrs=None) -> None:
|
|
self.messages.append(('warning', message, attrs))
|
|
|
|
def error(self, message: str, attrs=None) -> None:
|
|
self.messages.append(('error', message, attrs))
|
|
|
|
|
|
class FakeLockContext:
|
|
def __enter__(self) -> None:
|
|
return None
|
|
|
|
def __exit__(self, exc_type, exc, traceback) -> None:
|
|
return None
|
|
|
|
|
|
class FakeLocker:
|
|
def __init__(self) -> None:
|
|
self.chat_ids: list[UUID] = []
|
|
|
|
def lock(self, chat_id: UUID) -> FakeLockContext:
|
|
self.chat_ids.append(chat_id)
|
|
return FakeLockContext()
|
|
|
|
|
|
class TrackingLockContext:
|
|
def __init__(
|
|
self,
|
|
locker: 'TrackingLocker',
|
|
chat_id: UUID,
|
|
inner_context,
|
|
) -> None:
|
|
self._locker = locker
|
|
self._chat_id = chat_id
|
|
self._inner_context = inner_context
|
|
|
|
def __enter__(self) -> None:
|
|
with self._locker._state_lock:
|
|
self._locker.chat_ids.append(self._chat_id)
|
|
self._locker._attempts += 1
|
|
if self._locker._attempts == 2:
|
|
self._locker.second_attempted.set()
|
|
|
|
self._inner_context.__enter__()
|
|
|
|
def __exit__(self, exc_type, exc, traceback) -> bool | None:
|
|
return self._inner_context.__exit__(exc_type, exc, traceback)
|
|
|
|
|
|
class TrackingLocker:
|
|
def __init__(self) -> None:
|
|
self._locker = ProcessLocalSandboxLifecycleLocker()
|
|
self._state_lock = threading.Lock()
|
|
self._attempts = 0
|
|
self.second_attempted = threading.Event()
|
|
self.chat_ids: list[UUID] = []
|
|
|
|
def lock(self, chat_id: UUID) -> TrackingLockContext:
|
|
return TrackingLockContext(self, chat_id, self._locker.lock(chat_id))
|
|
|
|
|
|
class BlockingCreateRuntime:
|
|
def __init__(self) -> None:
|
|
self.create_calls: list[dict[str, object]] = []
|
|
self.stop_calls: list[str] = []
|
|
self.create_started = threading.Event()
|
|
self.allow_create = threading.Event()
|
|
|
|
def create(
|
|
self,
|
|
*,
|
|
session_id: UUID,
|
|
chat_id: UUID,
|
|
created_at: datetime,
|
|
expires_at: datetime,
|
|
) -> SandboxSession:
|
|
self.create_calls.append(
|
|
{
|
|
'session_id': session_id,
|
|
'chat_id': chat_id,
|
|
'created_at': created_at,
|
|
'expires_at': expires_at,
|
|
}
|
|
)
|
|
self.create_started.set()
|
|
assert self.allow_create.wait(timeout=1)
|
|
return SandboxSession(
|
|
session_id=session_id,
|
|
chat_id=chat_id,
|
|
container_id=f'container-{session_id}',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=created_at,
|
|
expires_at=expires_at,
|
|
)
|
|
|
|
def stop(self, container_id: str) -> None:
|
|
self.stop_calls.append(container_id)
|
|
|
|
|
|
class StaleSnapshotRepository(InMemorySandboxSessionRepository):
|
|
def __init__(self, snapshot: SandboxSession) -> None:
|
|
super().__init__()
|
|
self._snapshot = snapshot
|
|
|
|
def list_expired(self, now: datetime) -> list[SandboxSession]:
|
|
return [self._snapshot]
|
|
|
|
|
|
class FakeRuntime:
|
|
def __init__(self) -> None:
|
|
self.create_calls: list[dict[str, object]] = []
|
|
self.stop_calls: list[str] = []
|
|
|
|
def create(
|
|
self,
|
|
*,
|
|
session_id: UUID,
|
|
chat_id: UUID,
|
|
created_at: datetime,
|
|
expires_at: datetime,
|
|
) -> SandboxSession:
|
|
self.create_calls.append(
|
|
{
|
|
'session_id': session_id,
|
|
'chat_id': chat_id,
|
|
'created_at': created_at,
|
|
'expires_at': expires_at,
|
|
}
|
|
)
|
|
return SandboxSession(
|
|
session_id=session_id,
|
|
chat_id=chat_id,
|
|
container_id=f'container-{session_id}',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=created_at,
|
|
expires_at=expires_at,
|
|
)
|
|
|
|
def stop(self, container_id: str) -> None:
|
|
self.stop_calls.append(container_id)
|
|
|
|
|
|
class FailingStopRuntime(FakeRuntime):
|
|
def __init__(self, failing_container_id: str) -> None:
|
|
super().__init__()
|
|
self._failing_container_id = failing_container_id
|
|
|
|
def stop(self, container_id: str) -> None:
|
|
self.stop_calls.append(container_id)
|
|
if container_id == self._failing_container_id:
|
|
raise RuntimeError('stop_failed')
|
|
|
|
|
|
def test_create_sandbox_reuses_active_session_when_not_expired() -> None:
|
|
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
|
session = SandboxSession(
|
|
session_id=SESSION_REUSED_ID,
|
|
chat_id=CHAT_ID,
|
|
container_id='container-1',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(minutes=1),
|
|
expires_at=now + timedelta(minutes=4),
|
|
)
|
|
repository = InMemorySandboxSessionRepository()
|
|
repository.save(session)
|
|
runtime = FakeRuntime()
|
|
logger = FakeLogger()
|
|
locker = FakeLocker()
|
|
usecase = CreateSandbox(
|
|
repository=repository,
|
|
locker=locker,
|
|
runtime=runtime,
|
|
clock=FakeClock(now),
|
|
logger=logger,
|
|
ttl=timedelta(minutes=5),
|
|
)
|
|
|
|
result = usecase.execute(CreateSandboxCommand(chat_id=CHAT_ID))
|
|
|
|
assert result == session
|
|
assert runtime.create_calls == []
|
|
assert runtime.stop_calls == []
|
|
assert repository.get_active_by_chat_id(CHAT_ID) == session
|
|
assert locker.chat_ids == [CHAT_ID]
|
|
assert logger.messages == [
|
|
(
|
|
'info',
|
|
'sandbox_reused',
|
|
{
|
|
'chat_id': str(CHAT_ID),
|
|
'session_id': str(SESSION_REUSED_ID),
|
|
'container_id': 'container-1',
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_create_sandbox_replaces_expired_session_and_creates_new_one(
|
|
monkeypatch,
|
|
) -> None:
|
|
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
|
expired_session = SandboxSession(
|
|
session_id=SESSION_OLD_ID,
|
|
chat_id=CHAT_ID,
|
|
container_id='container-old',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(minutes=10),
|
|
expires_at=now,
|
|
)
|
|
repository = InMemorySandboxSessionRepository()
|
|
repository.save(expired_session)
|
|
runtime = FakeRuntime()
|
|
logger = FakeLogger()
|
|
locker = FakeLocker()
|
|
usecase = CreateSandbox(
|
|
repository=repository,
|
|
locker=locker,
|
|
runtime=runtime,
|
|
clock=FakeClock(now),
|
|
logger=logger,
|
|
ttl=timedelta(minutes=5),
|
|
)
|
|
monkeypatch.setattr('usecase.sandbox._new_session_id', lambda: SESSION_NEW_ID)
|
|
|
|
result = usecase.execute(CreateSandboxCommand(chat_id=CHAT_ID))
|
|
|
|
assert runtime.stop_calls == ['container-old']
|
|
assert runtime.create_calls == [
|
|
{
|
|
'session_id': SESSION_NEW_ID,
|
|
'chat_id': CHAT_ID,
|
|
'created_at': now,
|
|
'expires_at': now + timedelta(minutes=5),
|
|
}
|
|
]
|
|
assert result == SandboxSession(
|
|
session_id=SESSION_NEW_ID,
|
|
chat_id=CHAT_ID,
|
|
container_id=f'container-{SESSION_NEW_ID}',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now,
|
|
expires_at=now + timedelta(minutes=5),
|
|
)
|
|
assert repository.get_active_by_chat_id(CHAT_ID) == result
|
|
assert locker.chat_ids == [CHAT_ID]
|
|
assert logger.messages == [
|
|
(
|
|
'info',
|
|
'sandbox_replaced',
|
|
{
|
|
'chat_id': str(CHAT_ID),
|
|
'session_id': str(SESSION_OLD_ID),
|
|
'container_id': 'container-old',
|
|
},
|
|
),
|
|
(
|
|
'info',
|
|
'sandbox_created',
|
|
{
|
|
'chat_id': str(CHAT_ID),
|
|
'session_id': str(SESSION_NEW_ID),
|
|
'container_id': f'container-{SESSION_NEW_ID}',
|
|
},
|
|
),
|
|
]
|
|
|
|
|
|
def test_create_sandbox_creates_new_session_when_none_exists() -> None:
|
|
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
|
repository = InMemorySandboxSessionRepository()
|
|
runtime = FakeRuntime()
|
|
logger = FakeLogger()
|
|
locker = FakeLocker()
|
|
usecase = CreateSandbox(
|
|
repository=repository,
|
|
locker=locker,
|
|
runtime=runtime,
|
|
clock=FakeClock(now),
|
|
logger=logger,
|
|
ttl=timedelta(minutes=5),
|
|
)
|
|
|
|
result = usecase.execute(CreateSandboxCommand(chat_id=UUID(NON_CANONICAL_CHAT_ID)))
|
|
|
|
assert result.chat_id == CHAT_ID
|
|
assert result.container_id == f'container-{result.session_id}'
|
|
assert result.status is SandboxStatus.RUNNING
|
|
assert result.created_at == now
|
|
assert result.expires_at == now + timedelta(minutes=5)
|
|
assert len(runtime.create_calls) == 1
|
|
assert runtime.create_calls[0] == {
|
|
'session_id': result.session_id,
|
|
'chat_id': CHAT_ID,
|
|
'created_at': now,
|
|
'expires_at': now + timedelta(minutes=5),
|
|
}
|
|
assert runtime.stop_calls == []
|
|
assert repository.get_active_by_chat_id(CHAT_ID) == result
|
|
assert locker.chat_ids == [CHAT_ID]
|
|
assert logger.messages == [
|
|
(
|
|
'info',
|
|
'sandbox_created',
|
|
{
|
|
'chat_id': str(CHAT_ID),
|
|
'session_id': str(result.session_id),
|
|
'container_id': result.container_id,
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_create_sandbox_serializes_duplicate_concurrent_create_for_chat_id(
|
|
monkeypatch,
|
|
) -> None:
|
|
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
|
repository = InMemorySandboxSessionRepository()
|
|
runtime = BlockingCreateRuntime()
|
|
logger = FakeLogger()
|
|
locker = TrackingLocker()
|
|
usecase = CreateSandbox(
|
|
repository=repository,
|
|
locker=locker,
|
|
runtime=runtime,
|
|
clock=FakeClock(now),
|
|
logger=logger,
|
|
ttl=timedelta(minutes=5),
|
|
)
|
|
monkeypatch.setattr('usecase.sandbox._new_session_id', lambda: SESSION_NEW_ID)
|
|
|
|
results: list[SandboxSession | None] = [None, None]
|
|
errors: list[Exception] = []
|
|
|
|
def run_create(index: int) -> None:
|
|
try:
|
|
results[index] = usecase.execute(CreateSandboxCommand(chat_id=CHAT_ID))
|
|
except Exception as exc:
|
|
errors.append(exc)
|
|
|
|
first_thread = threading.Thread(target=run_create, args=(0,))
|
|
second_thread = threading.Thread(target=run_create, args=(1,))
|
|
|
|
first_thread.start()
|
|
assert runtime.create_started.wait(timeout=1)
|
|
|
|
second_thread.start()
|
|
assert locker.second_attempted.wait(timeout=1)
|
|
assert len(runtime.create_calls) == 1
|
|
|
|
runtime.allow_create.set()
|
|
|
|
first_thread.join(timeout=1)
|
|
second_thread.join(timeout=1)
|
|
|
|
assert errors == []
|
|
assert results[0] == results[1]
|
|
assert results[0] == SandboxSession(
|
|
session_id=SESSION_NEW_ID,
|
|
chat_id=CHAT_ID,
|
|
container_id=f'container-{SESSION_NEW_ID}',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now,
|
|
expires_at=now + timedelta(minutes=5),
|
|
)
|
|
assert len(runtime.create_calls) == 1
|
|
assert runtime.stop_calls == []
|
|
assert repository.get_active_by_chat_id(CHAT_ID) == results[0]
|
|
assert locker.chat_ids == [CHAT_ID, CHAT_ID]
|
|
assert logger.messages == [
|
|
(
|
|
'info',
|
|
'sandbox_created',
|
|
{
|
|
'chat_id': str(CHAT_ID),
|
|
'session_id': str(SESSION_NEW_ID),
|
|
'container_id': f'container-{SESSION_NEW_ID}',
|
|
},
|
|
),
|
|
(
|
|
'info',
|
|
'sandbox_reused',
|
|
{
|
|
'chat_id': str(CHAT_ID),
|
|
'session_id': str(SESSION_NEW_ID),
|
|
'container_id': f'container-{SESSION_NEW_ID}',
|
|
},
|
|
),
|
|
]
|
|
|
|
|
|
def test_cleanup_expired_sandboxes_stops_and_deletes_only_expired_sessions() -> None:
|
|
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
|
expired_session = SandboxSession(
|
|
session_id=SESSION_EXPIRED_ID,
|
|
chat_id=EXPIRED_CHAT_ID,
|
|
container_id='container-expired',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(minutes=10),
|
|
expires_at=now - timedelta(seconds=1),
|
|
)
|
|
boundary_session = SandboxSession(
|
|
session_id=SESSION_BOUNDARY_ID,
|
|
chat_id=BOUNDARY_CHAT_ID,
|
|
container_id='container-boundary',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(minutes=5),
|
|
expires_at=now,
|
|
)
|
|
active_session = SandboxSession(
|
|
session_id=SESSION_ACTIVE_ID,
|
|
chat_id=ACTIVE_CHAT_ID,
|
|
container_id='container-active',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(minutes=1),
|
|
expires_at=now + timedelta(minutes=5),
|
|
)
|
|
repository = InMemorySandboxSessionRepository()
|
|
repository.save(expired_session)
|
|
repository.save(boundary_session)
|
|
repository.save(active_session)
|
|
runtime = FakeRuntime()
|
|
logger = FakeLogger()
|
|
locker = FakeLocker()
|
|
usecase = CleanupExpiredSandboxes(
|
|
repository=repository,
|
|
locker=locker,
|
|
runtime=runtime,
|
|
clock=FakeClock(now),
|
|
logger=logger,
|
|
)
|
|
|
|
result = usecase.execute()
|
|
|
|
assert result == [expired_session, boundary_session]
|
|
assert runtime.stop_calls == ['container-expired', 'container-boundary']
|
|
assert repository.get_active_by_chat_id(EXPIRED_CHAT_ID) is None
|
|
assert repository.get_active_by_chat_id(BOUNDARY_CHAT_ID) is None
|
|
assert repository.get_active_by_chat_id(ACTIVE_CHAT_ID) == active_session
|
|
assert locker.chat_ids == [EXPIRED_CHAT_ID, BOUNDARY_CHAT_ID]
|
|
assert logger.messages == [
|
|
(
|
|
'info',
|
|
'sandbox_cleaned',
|
|
{
|
|
'chat_id': str(EXPIRED_CHAT_ID),
|
|
'session_id': str(SESSION_EXPIRED_ID),
|
|
'container_id': 'container-expired',
|
|
},
|
|
),
|
|
(
|
|
'info',
|
|
'sandbox_cleaned',
|
|
{
|
|
'chat_id': str(BOUNDARY_CHAT_ID),
|
|
'session_id': str(SESSION_BOUNDARY_ID),
|
|
'container_id': 'container-boundary',
|
|
},
|
|
),
|
|
]
|
|
|
|
|
|
def test_cleanup_expired_sandboxes_skips_replaced_session_from_stale_snapshot() -> None:
|
|
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
|
expired_snapshot = SandboxSession(
|
|
session_id=SESSION_EXPIRED_ID,
|
|
chat_id=CHAT_ID,
|
|
container_id='container-expired',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(minutes=10),
|
|
expires_at=now - timedelta(seconds=1),
|
|
)
|
|
replacement_session = SandboxSession(
|
|
session_id=SESSION_REPLACEMENT_ID,
|
|
chat_id=CHAT_ID,
|
|
container_id='container-new',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(seconds=30),
|
|
expires_at=now + timedelta(minutes=5),
|
|
)
|
|
repository = StaleSnapshotRepository(expired_snapshot)
|
|
repository.save(replacement_session)
|
|
runtime = FakeRuntime()
|
|
logger = FakeLogger()
|
|
locker = FakeLocker()
|
|
usecase = CleanupExpiredSandboxes(
|
|
repository=repository,
|
|
locker=locker,
|
|
runtime=runtime,
|
|
clock=FakeClock(now),
|
|
logger=logger,
|
|
)
|
|
|
|
result = usecase.execute()
|
|
|
|
assert result == []
|
|
assert runtime.stop_calls == []
|
|
assert repository.get_active_by_chat_id(CHAT_ID) == replacement_session
|
|
assert locker.chat_ids == [CHAT_ID]
|
|
assert logger.messages == []
|
|
|
|
|
|
def test_cleanup_expired_sandboxes_continues_after_stop_failure() -> None:
|
|
now = datetime(2026, 4, 2, 12, 0, tzinfo=UTC)
|
|
failing_session = SandboxSession(
|
|
session_id=SESSION_FAIL_ID,
|
|
chat_id=FAIL_CHAT_ID,
|
|
container_id='container-fail',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(minutes=10),
|
|
expires_at=now - timedelta(minutes=1),
|
|
)
|
|
cleaned_session = SandboxSession(
|
|
session_id=SESSION_CLEAN_ID,
|
|
chat_id=CLEAN_CHAT_ID,
|
|
container_id='container-clean',
|
|
status=SandboxStatus.RUNNING,
|
|
created_at=now - timedelta(minutes=9),
|
|
expires_at=now - timedelta(seconds=1),
|
|
)
|
|
repository = InMemorySandboxSessionRepository()
|
|
repository.save(failing_session)
|
|
repository.save(cleaned_session)
|
|
runtime = FailingStopRuntime('container-fail')
|
|
logger = FakeLogger()
|
|
locker = FakeLocker()
|
|
usecase = CleanupExpiredSandboxes(
|
|
repository=repository,
|
|
locker=locker,
|
|
runtime=runtime,
|
|
clock=FakeClock(now),
|
|
logger=logger,
|
|
)
|
|
|
|
result = usecase.execute()
|
|
|
|
assert result == [cleaned_session]
|
|
assert runtime.stop_calls == ['container-fail', 'container-clean']
|
|
assert repository.get_active_by_chat_id(FAIL_CHAT_ID) == failing_session
|
|
assert repository.get_active_by_chat_id(CLEAN_CHAT_ID) is None
|
|
assert locker.chat_ids == [FAIL_CHAT_ID, CLEAN_CHAT_ID]
|
|
assert logger.messages == [
|
|
(
|
|
'error',
|
|
'sandbox_clean_failed',
|
|
{
|
|
'chat_id': str(FAIL_CHAT_ID),
|
|
'session_id': str(SESSION_FAIL_ID),
|
|
'container_id': 'container-fail',
|
|
'error': 'RuntimeError',
|
|
},
|
|
),
|
|
(
|
|
'info',
|
|
'sandbox_cleaned',
|
|
{
|
|
'chat_id': str(CLEAN_CHAT_ID),
|
|
'session_id': str(SESSION_CLEAN_ID),
|
|
'container_id': 'container-clean',
|
|
},
|
|
),
|
|
]
|