Cleanup time!
This commit is contained in:
parent
9a19fe1f50
commit
70dd3a16dc
38 changed files with 150 additions and 351 deletions
|
|
@ -108,7 +108,7 @@ def _count_tokens_for_entry(entry: Dict) -> Tuple[Dict, int]:
|
|||
if value:
|
||||
try:
|
||||
total += len(_TOKENIZER.encode(value))
|
||||
except:
|
||||
except Exception:
|
||||
# Fallback to character estimate
|
||||
total += len(value) // 4
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue