FSSCoding
53174a34eb
Project Reorganization: - Created docs/ directory and moved all documentation - Created scripts/ directory for shell scripts - Created scripts/experimental/ for research scripts - Updated .gitignore for new structure - Updated README.md with MVP status and new structure New Features: - Category verification system (verify_model_categories) - --verify-categories flag for mailbox compatibility check - --no-llm-fallback flag for pure ML classification - Trained model saved in src/models/calibrated/ Threshold Optimization: - Reduced default threshold from 0.75 to 0.55 - Updated all category thresholds to 0.55 - Reduces LLM fallback rate by 40% (35% -> 21%) Documentation: - SYSTEM_FLOW.html - Complete system architecture - VERIFY_CATEGORIES_FEATURE.html - Feature documentation - LABEL_TRAINING_PHASE_DETAIL.html - Calibration breakdown - FAST_ML_ONLY_WORKFLOW.html - Pure ML guide - PROJECT_STATUS_AND_NEXT_STEPS.html - Roadmap - ROOT_CAUSE_ANALYSIS.md - Bug fixes MVP Status: - 10k emails in 4 minutes, 72.7% accuracy, 0 LLM calls - LLM-driven category discovery working - Embedding-based transfer learning confirmed - All model paths verified and working
51 lines
1.3 KiB
Bash
Executable File
51 lines
1.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Clean 10k test with all fixes applied
|
|
# Run this when ready: ./run_clean_10k.sh
|
|
|
|
set -e
|
|
|
|
echo "=========================================="
|
|
echo "CLEAN 10K TEST - Fixed Category System"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "Fixes applied:"
|
|
echo " ✓ Removed hardcoded category pollution"
|
|
echo " ✓ LLM-only category discovery"
|
|
echo " ✓ Intelligent scaling (3% cal, 1% val)"
|
|
echo ""
|
|
echo "Expected results:"
|
|
echo " - ~11 clean categories (not 29)"
|
|
echo " - No duplicates (Work vs work)"
|
|
echo " - Realistic confidence scores"
|
|
echo ""
|
|
echo "Starting at: $(date)"
|
|
echo ""
|
|
|
|
# Activate venv
|
|
if [ -z "$VIRTUAL_ENV" ]; then
|
|
source venv/bin/activate
|
|
fi
|
|
|
|
# Clean start
|
|
rm -rf results_10k/
|
|
rm -f src/models/calibrated/classifier.pkl
|
|
rm -f src/models/category_cache.json
|
|
|
|
# Run with progress visible
|
|
python -m src.cli run \
|
|
--source enron \
|
|
--limit 10000 \
|
|
--output results_10k/ \
|
|
--verbose
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "COMPLETE at: $(date)"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "Check results:"
|
|
echo " - Categories: cat src/models/category_cache.json | python3 -m json.tool"
|
|
echo " - Model: ls -lh src/models/calibrated/"
|
|
echo " - Results: ls -lh results_10k/"
|
|
echo ""
|