Refactor batch processing with rich progress tracking and update logging in AIAgent

- Replaced tqdm with rich for enhanced visual progress tracking in batch processing.
- Adjusted logging levels in AIAgent to suppress asyncio debug messages.
- Modified datagen script to reduce number of workers for improved performance.
This commit is contained in:
teknium 2026-01-14 14:02:59 +00:00
parent 6e3dbb8d8b
commit b32cc4b09d
4 changed files with 37 additions and 12 deletions

View file

@ -127,7 +127,8 @@ class AIAgent:
logging.getLogger('openai._base_client').setLevel(logging.WARNING)
logging.getLogger('httpx').setLevel(logging.WARNING)
logging.getLogger('httpcore').setLevel(logging.WARNING)
print("🔍 Verbose logging enabled (OpenAI/httpx internal logs suppressed)")
logging.getLogger('asyncio').setLevel(logging.WARNING) # Suppress asyncio debug
print("🔍 Verbose logging enabled (OpenAI/httpx/asyncio internal logs suppressed)")
else:
# Set logging to INFO level for important messages only
logging.basicConfig(