[fix] cleanup task to other thread
This commit is contained in:
parent
f5d13feaf9
commit
776b513858
3 changed files with 61 additions and 29 deletions
|
|
@ -75,13 +75,25 @@ def _build_shutdown_handler(
|
||||||
container: AppContainer,
|
container: AppContainer,
|
||||||
) -> Callable[[], Awaitable[None]]:
|
) -> Callable[[], Awaitable[None]]:
|
||||||
async def shutdown() -> None:
|
async def shutdown() -> None:
|
||||||
|
errors: list[Exception] = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await _stop_cleanup_loop(app)
|
await _stop_cleanup_loop(app)
|
||||||
finally:
|
except Exception as exc:
|
||||||
try:
|
errors.append(exc)
|
||||||
_uninstrument_app(app)
|
|
||||||
finally:
|
try:
|
||||||
container.shutdown()
|
_uninstrument_app(app)
|
||||||
|
except Exception as exc:
|
||||||
|
errors.append(exc)
|
||||||
|
|
||||||
|
try:
|
||||||
|
container.shutdown()
|
||||||
|
except Exception as exc:
|
||||||
|
errors.append(exc)
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
raise ExceptionGroup('app shutdown failed', errors)
|
||||||
|
|
||||||
return shutdown
|
return shutdown
|
||||||
|
|
||||||
|
|
@ -94,7 +106,9 @@ async def _run_cleanup_loop(
|
||||||
|
|
||||||
while not stop_event.is_set():
|
while not stop_event.is_set():
|
||||||
try:
|
try:
|
||||||
container.usecases.cleanup_expired_sandboxes.execute()
|
await asyncio.to_thread(
|
||||||
|
container.usecases.cleanup_expired_sandboxes.execute
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
container.observability.logger.error(
|
container.observability.logger.error(
|
||||||
'sandbox_cleanup_failed',
|
'sandbox_cleanup_failed',
|
||||||
|
|
|
||||||
2
tasks.md
2
tasks.md
|
|
@ -131,7 +131,7 @@
|
||||||
### M10. Устойчивый cleanup и вынос blocking cleanup из event loop
|
### M10. Устойчивый cleanup и вынос blocking cleanup из event loop
|
||||||
|
|
||||||
- Субагент: `feature-developer`
|
- Субагент: `feature-developer`
|
||||||
- Статус: pending
|
- Статус: completed
|
||||||
- Зависимости: `M09`
|
- Зависимости: `M09`
|
||||||
- Commit required: no
|
- Commit required: no
|
||||||
- Scope: сделать cleanup устойчивым к частичным ошибкам и не блокировать FastAPI event loop синхронным Docker stop
|
- Scope: сделать cleanup устойчивым к частичным ошибкам и не блокировать FastAPI event loop синхронным Docker stop
|
||||||
|
|
|
||||||
|
|
@ -102,34 +102,52 @@ class CleanupExpiredSandboxes:
|
||||||
cleaned_sessions: list[SandboxSession] = []
|
cleaned_sessions: list[SandboxSession] = []
|
||||||
|
|
||||||
for session in expired_sessions:
|
for session in expired_sessions:
|
||||||
with self._locker.lock(session.chat_id):
|
try:
|
||||||
current_session = self._repository.get_active_by_chat_id(
|
cleaned_session = self._cleanup_session(session)
|
||||||
session.chat_id
|
except Exception as exc:
|
||||||
)
|
self._logger.error(
|
||||||
now = self._clock.now()
|
'sandbox_clean_failed',
|
||||||
if current_session is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if current_session.session_id != session.session_id:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if current_session.expires_at > now:
|
|
||||||
continue
|
|
||||||
|
|
||||||
self._runtime.stop(current_session.container_id)
|
|
||||||
self._repository.delete(current_session.session_id)
|
|
||||||
cleaned_sessions.append(current_session)
|
|
||||||
self._logger.info(
|
|
||||||
'sandbox_cleaned',
|
|
||||||
attrs={
|
attrs={
|
||||||
'chat_id': current_session.chat_id,
|
'chat_id': session.chat_id,
|
||||||
'session_id': current_session.session_id,
|
'session_id': session.session_id,
|
||||||
'container_id': current_session.container_id,
|
'container_id': session.container_id,
|
||||||
|
'error': type(exc).__name__,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if cleaned_session is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
cleaned_sessions.append(cleaned_session)
|
||||||
|
self._logger.info(
|
||||||
|
'sandbox_cleaned',
|
||||||
|
attrs={
|
||||||
|
'chat_id': cleaned_session.chat_id,
|
||||||
|
'session_id': cleaned_session.session_id,
|
||||||
|
'container_id': cleaned_session.container_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
return cleaned_sessions
|
return cleaned_sessions
|
||||||
|
|
||||||
|
def _cleanup_session(self, session: SandboxSession) -> SandboxSession | None:
|
||||||
|
with self._locker.lock(session.chat_id):
|
||||||
|
current_session = self._repository.get_active_by_chat_id(session.chat_id)
|
||||||
|
now = self._clock.now()
|
||||||
|
if current_session is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if current_session.session_id != session.session_id:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if current_session.expires_at > now:
|
||||||
|
return None
|
||||||
|
|
||||||
|
self._runtime.stop(current_session.container_id)
|
||||||
|
self._repository.delete(current_session.session_id)
|
||||||
|
return current_session
|
||||||
|
|
||||||
|
|
||||||
def _new_session_id() -> str:
|
def _new_session_id() -> str:
|
||||||
return uuid4().hex
|
return uuid4().hex
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue