refactor: remove obsolete Nous API test scripts
- Deleted test scripts for Nous API limits, patterns, and temperature checks to streamline the testing suite. - These scripts were no longer necessary and their removal helps maintain a cleaner codebase.
This commit is contained in:
teknium1
parent
a885d2f240
commit
cbff1b818c
3 changed files with 0 additions and 416 deletions
|
|
@ -1,176 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script to diagnose Nous API 400 errors with gemini-2.5-flash model.
|
||||
This tests various content lengths and parameters to identify what causes failures.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from openai import AsyncOpenAI
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv()
|
||||
|
||||
# Initialize the Nous API client
|
||||
nous_client = AsyncOpenAI(
|
||||
api_key=os.getenv("NOUS_API_KEY"),
|
||||
base_url="https://inference-api.nousresearch.com/v1"
|
||||
)
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
|
||||
async def test_api_call(test_name: str, content_length: int, **kwargs):
|
||||
"""Test an API call with specific parameters."""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Test: {test_name}")
|
||||
print(f"Content length: {content_length:,} characters")
|
||||
print(f"Additional params: {kwargs}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Generate test content
|
||||
content = "A" * content_length
|
||||
|
||||
system_prompt = """You are an expert content analyst. Your job is to process web content and create a comprehensive yet concise summary that preserves all important information while dramatically reducing bulk.
|
||||
|
||||
Create a well-structured markdown summary that includes:
|
||||
1. Key excerpts (quotes, code snippets, important facts) in their original format
|
||||
2. Comprehensive summary of all other important information
|
||||
3. Proper markdown formatting with headers, bullets, and emphasis
|
||||
|
||||
Your goal is to preserve ALL important information while reducing length. Never lose key facts, figures, insights, or actionable information. Make it scannable and well-organized."""
|
||||
|
||||
user_prompt = f"""Please process this web content and create a comprehensive markdown summary:
|
||||
|
||||
CONTENT TO PROCESS:
|
||||
{content}
|
||||
|
||||
Create a markdown summary that captures all key information in a well-organized, scannable format. Include important quotes and code snippets in their original formatting. Focus on actionable information, specific details, and unique insights."""
|
||||
|
||||
try:
|
||||
response = await nous_client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
**kwargs
|
||||
)
|
||||
|
||||
result = response.choices[0].message.content
|
||||
print(f"✅ SUCCESS")
|
||||
print(f" Response length: {len(result)} characters")
|
||||
print(f" Model used: {response.model}")
|
||||
print(f" Usage: {response.usage}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ FAILED: {str(e)}")
|
||||
return False
|
||||
|
||||
async def main():
|
||||
"""Run all tests."""
|
||||
print("Testing Nous API with gemini-2.5-flash model")
|
||||
print(f"API Key present: {'Yes' if os.getenv('NOUS_API_KEY') else 'No'}")
|
||||
|
||||
results = {}
|
||||
|
||||
# Test 1: Small content (should always work)
|
||||
results['small'] = await test_api_call(
|
||||
"Small content (5,000 chars)",
|
||||
5000,
|
||||
temperature=0.1,
|
||||
max_tokens=4000
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test 2: Medium content (around what was failing)
|
||||
results['medium'] = await test_api_call(
|
||||
"Medium content (20,000 chars)",
|
||||
20000,
|
||||
temperature=0.1,
|
||||
max_tokens=4000
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test 3: Large content (79,625 chars like the error)
|
||||
results['large'] = await test_api_call(
|
||||
"Large content (79,625 chars)",
|
||||
79625,
|
||||
temperature=0.1,
|
||||
max_tokens=4000
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test 4: Very large content (100k chars)
|
||||
results['very_large'] = await test_api_call(
|
||||
"Very large content (100,000 chars)",
|
||||
100000,
|
||||
temperature=0.1,
|
||||
max_tokens=4000
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test 5: Same as working case but different max_tokens
|
||||
results['diff_max_tokens'] = await test_api_call(
|
||||
"Medium content with higher max_tokens",
|
||||
20000,
|
||||
temperature=0.1,
|
||||
max_tokens=8000
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test 6: No max_tokens specified
|
||||
results['no_max_tokens'] = await test_api_call(
|
||||
"Medium content without max_tokens",
|
||||
20000,
|
||||
temperature=0.1
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test 7: With actual web content (mixed characters)
|
||||
mixed_content = """
|
||||
This is a test of web content with various characters:
|
||||
- Unicode: 你好世界 🌍
|
||||
- Special chars: <>&"'
|
||||
- Numbers: 123456789
|
||||
- Markdown: **bold** _italic_ `code`
|
||||
- URLs: https://example.com
|
||||
""" * 1000 # Repeat to make it ~79k chars
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Test: Mixed content (real-world scenario)")
|
||||
print(f"Content length: {len(mixed_content):,} characters")
|
||||
print(f"{'='*60}")
|
||||
|
||||
try:
|
||||
response = await nous_client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": "Summarize this content."},
|
||||
{"role": "user", "content": mixed_content}
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=4000
|
||||
)
|
||||
print(f"✅ SUCCESS")
|
||||
results['mixed_content'] = True
|
||||
except Exception as e:
|
||||
print(f"❌ FAILED: {str(e)}")
|
||||
results['mixed_content'] = False
|
||||
|
||||
# Summary
|
||||
print(f"\n{'='*60}")
|
||||
print("SUMMARY OF RESULTS:")
|
||||
print(f"{'='*60}")
|
||||
for test, passed in results.items():
|
||||
status = "✅ PASS" if passed else "❌ FAIL"
|
||||
print(f"{test:20s}: {status}")
|
||||
|
||||
passed = sum(results.values())
|
||||
total = len(results)
|
||||
print(f"\nTotal: {passed}/{total} tests passed")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
|
|
@ -1,131 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test to understand the pattern of failures - it's not about content length!
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from openai import AsyncOpenAI
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
nous_client = AsyncOpenAI(
|
||||
api_key=os.getenv("NOUS_API_KEY"),
|
||||
base_url="https://inference-api.nousresearch.com/v1"
|
||||
)
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
|
||||
async def quick_test(description: str, content: str, **kwargs):
|
||||
"""Quick API test."""
|
||||
print(f"\n{description} ({len(content):,} chars)...", end=" ")
|
||||
|
||||
try:
|
||||
response = await nous_client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": "Summarize this."},
|
||||
{"role": "user", "content": content}
|
||||
],
|
||||
**kwargs
|
||||
)
|
||||
print(f"✅ SUCCESS")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ FAILED: {str(e)[:80]}")
|
||||
return False
|
||||
|
||||
async def main():
|
||||
print("Testing different content types and parameters...")
|
||||
|
||||
# Theory 1: Repeated characters trigger validation
|
||||
print("\n" + "="*60)
|
||||
print("THEORY 1: Repeated characters")
|
||||
print("="*60)
|
||||
await quick_test("Repeated 'A's (5k)", "A" * 5000, temperature=0.1, max_tokens=4000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("Repeated 'A's (79k)", "A" * 79625, temperature=0.1, max_tokens=4000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("Varied text (5k)", "Test content. " * 400, temperature=0.1, max_tokens=4000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("Varied text (79k)", "Test content with variety. " * 3000, temperature=0.1, max_tokens=4000)
|
||||
|
||||
# Theory 2: max_tokens parameter
|
||||
print("\n" + "="*60)
|
||||
print("THEORY 2: max_tokens parameter")
|
||||
print("="*60)
|
||||
content = "Test " * 4000 # 20k chars
|
||||
await quick_test("max_tokens=4000", content, temperature=0.1, max_tokens=4000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("max_tokens=8000", content, temperature=0.1, max_tokens=8000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("max_tokens=2000", content, temperature=0.1, max_tokens=2000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("No max_tokens", content, temperature=0.1)
|
||||
|
||||
# Theory 3: Temperature parameter
|
||||
print("\n" + "="*60)
|
||||
print("THEORY 3: Temperature parameter")
|
||||
print("="*60)
|
||||
content = "Test " * 4000
|
||||
await quick_test("temperature=0.1", content, temperature=0.1, max_tokens=4000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("temperature=0.0", content, temperature=0.0, max_tokens=4000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("temperature=0.5", content, temperature=0.5, max_tokens=4000)
|
||||
await asyncio.sleep(0.5)
|
||||
await quick_test("No temperature", content, max_tokens=4000)
|
||||
|
||||
# Theory 4: System prompt impact
|
||||
print("\n" + "="*60)
|
||||
print("THEORY 4: System prompt length")
|
||||
print("="*60)
|
||||
|
||||
short_system = "Summarize this."
|
||||
long_system = """You are an expert content analyst. Your job is to process web content and create a comprehensive yet concise summary that preserves all important information while dramatically reducing bulk.
|
||||
|
||||
Create a well-structured markdown summary that includes:
|
||||
1. Key excerpts (quotes, code snippets, important facts) in their original format
|
||||
2. Comprehensive summary of all other important information
|
||||
3. Proper markdown formatting with headers, bullets, and emphasis
|
||||
|
||||
Your goal is to preserve ALL important information while reducing length."""
|
||||
|
||||
content = "A" * 5000
|
||||
|
||||
print(f"\nShort system prompt...", end=" ")
|
||||
try:
|
||||
response = await nous_client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": short_system},
|
||||
{"role": "user", "content": content}
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=4000
|
||||
)
|
||||
print(f"✅ SUCCESS")
|
||||
except Exception as e:
|
||||
print(f"❌ FAILED")
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
print(f"Long system prompt...", end=" ")
|
||||
try:
|
||||
response = await nous_client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": long_system},
|
||||
{"role": "user", "content": content}
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=4000
|
||||
)
|
||||
print(f"✅ SUCCESS")
|
||||
except Exception as e:
|
||||
print(f"❌ FAILED")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test to confirm: temperature < 0.3 causes failures on Nous API
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from openai import AsyncOpenAI
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
nous_client = AsyncOpenAI(
|
||||
api_key=os.getenv("NOUS_API_KEY"),
|
||||
base_url="https://inference-api.nousresearch.com/v1"
|
||||
)
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
|
||||
async def test_temp(temp_value):
|
||||
"""Test a specific temperature value."""
|
||||
content = "Test content. " * 1000 # 14k chars
|
||||
|
||||
print(f"Testing temperature={temp_value}...", end=" ")
|
||||
|
||||
try:
|
||||
response = await nous_client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": "Summarize this content."},
|
||||
{"role": "user", "content": content}
|
||||
],
|
||||
temperature=temp_value,
|
||||
max_tokens=4000
|
||||
)
|
||||
print(f"✅ SUCCESS")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ FAILED")
|
||||
return False
|
||||
|
||||
async def main():
|
||||
print("Testing temperature threshold for Nous API...")
|
||||
print("="*60)
|
||||
|
||||
temps = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 1.0]
|
||||
|
||||
for temp in temps:
|
||||
await test_temp(temp)
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
print("="*60)
|
||||
print("\nNow testing with ACTUAL web_tools.py content and parameters:")
|
||||
print("="*60)
|
||||
|
||||
# Simulate the actual web_tools.py call
|
||||
system_prompt = """You are an expert content analyst. Your job is to process web content and create a comprehensive yet concise summary that preserves all important information while dramatically reducing bulk.
|
||||
|
||||
Create a well-structured markdown summary that includes:
|
||||
1. Key excerpts (quotes, code snippets, important facts) in their original format
|
||||
2. Comprehensive summary of all other important information
|
||||
3. Proper markdown formatting with headers, bullets, and emphasis
|
||||
|
||||
Your goal is to preserve ALL important information while reducing length. Never lose key facts, figures, insights, or actionable information. Make it scannable and well-organized."""
|
||||
|
||||
content = "Sample web page content. " * 3000 # ~75k chars like the real failures
|
||||
|
||||
user_prompt = f"""Please process this web content and create a comprehensive markdown summary:
|
||||
|
||||
CONTENT TO PROCESS:
|
||||
{content}
|
||||
|
||||
Create a markdown summary that captures all key information in a well-organized, scannable format. Include important quotes and code snippets in their original formatting. Focus on actionable information, specific details, and unique insights."""
|
||||
|
||||
print(f"\nActual web_tools call (temp=0.1, {len(content):,} chars)...", end=" ")
|
||||
try:
|
||||
response = await nous_client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=4000
|
||||
)
|
||||
print(f"✅ SUCCESS")
|
||||
except Exception:
|
||||
print(f"❌ FAILED")
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
print(f"Same call but with temp=0.3...", end=" ")
|
||||
try:
|
||||
response = await nous_client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
temperature=0.3,
|
||||
max_tokens=4000
|
||||
)
|
||||
print(f"✅ SUCCESS")
|
||||
except Exception:
|
||||
print(f"❌ FAILED")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue