BobAi
4166d0a362
🎯 Complete transformation from 5.9GB bloated system to 70MB optimized solution ✨ Key Features: - Hybrid embedding system (Ollama + ML fallback + hash backup) - Intelligent chunking with language-aware parsing - Semantic + BM25 hybrid search with rich context - Zero-config portable design with graceful degradation - Beautiful TUI for beginners + powerful CLI for experts - Comprehensive documentation with 8+ Mermaid diagrams - Professional animated demo (183KB optimized GIF) 🏗️ Architecture Highlights: - LanceDB vector storage with streaming indexing - Smart file tracking (size/mtime) to avoid expensive rehashing - Progressive chunking: Markdown headers → Python functions → fixed-size - Quality filtering: 200+ chars, 20+ words, 30% alphanumeric content - Concurrent batch processing with error recovery 📦 Package Contents: - Core engine: claude_rag/ (11 modules, 2,847 lines) - Entry points: rag-mini (unified), rag-tui (beginner interface) - Documentation: README + 6 guides with visual diagrams - Assets: 3D icon, optimized demo GIF, recording tools - Tests: 8 comprehensive integration and validation tests - Examples: Usage patterns, config templates, dependency analysis 🎥 Demo System: - Scripted demonstration showing 12 files → 58 chunks indexing - Semantic search with multi-line result previews - Complete workflow from TUI startup to CLI mastery - Professional recording pipeline with asciinema + GIF conversion 🛡️ Security & Quality: - Complete .gitignore with personal data protection - Dependency optimization (removed python-dotenv) - Code quality validation and educational test suite - Agent-reviewed architecture and documentation Ready for production use - copy folder, run ./rag-mini, start searching\!
87 lines
2.6 KiB
Python
87 lines
2.6 KiB
Python
"""
|
|
Performance monitoring for RAG system.
|
|
Track loading times, query times, and resource usage.
|
|
"""
|
|
|
|
import time
|
|
import psutil
|
|
import os
|
|
from contextlib import contextmanager
|
|
from typing import Dict, Any, Optional
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class PerformanceMonitor:
|
|
"""Track performance metrics for RAG operations."""
|
|
|
|
def __init__(self):
|
|
self.metrics = {}
|
|
self.process = psutil.Process(os.getpid())
|
|
|
|
@contextmanager
|
|
def measure(self, operation: str):
|
|
"""Context manager to measure operation time and memory."""
|
|
# Get initial state
|
|
start_time = time.time()
|
|
start_memory = self.process.memory_info().rss / 1024 / 1024 # MB
|
|
|
|
try:
|
|
yield self
|
|
finally:
|
|
# Calculate metrics
|
|
end_time = time.time()
|
|
end_memory = self.process.memory_info().rss / 1024 / 1024 # MB
|
|
|
|
duration = end_time - start_time
|
|
memory_delta = end_memory - start_memory
|
|
|
|
# Store metrics
|
|
self.metrics[operation] = {
|
|
'duration_seconds': duration,
|
|
'memory_delta_mb': memory_delta,
|
|
'final_memory_mb': end_memory,
|
|
}
|
|
|
|
logger.info(
|
|
f"[PERF] {operation}: {duration:.2f}s, "
|
|
f"Memory: {end_memory:.1f}MB (+{memory_delta:+.1f}MB)"
|
|
)
|
|
|
|
def get_summary(self) -> Dict[str, Any]:
|
|
"""Get performance summary."""
|
|
total_time = sum(m['duration_seconds'] for m in self.metrics.values())
|
|
|
|
return {
|
|
'total_time_seconds': total_time,
|
|
'operations': self.metrics,
|
|
'current_memory_mb': self.process.memory_info().rss / 1024 / 1024,
|
|
}
|
|
|
|
def print_summary(self):
|
|
"""Print a formatted summary."""
|
|
print("\n" + "="*50)
|
|
print("PERFORMANCE SUMMARY")
|
|
print("="*50)
|
|
|
|
for op, metrics in self.metrics.items():
|
|
print(f"\n{op}:")
|
|
print(f" Time: {metrics['duration_seconds']:.2f}s")
|
|
print(f" Memory: +{metrics['memory_delta_mb']:+.1f}MB")
|
|
|
|
summary = self.get_summary()
|
|
print(f"\nTotal Time: {summary['total_time_seconds']:.2f}s")
|
|
print(f"Current Memory: {summary['current_memory_mb']:.1f}MB")
|
|
print("="*50)
|
|
|
|
|
|
# Global instance for easy access
|
|
_monitor = None
|
|
|
|
def get_monitor() -> PerformanceMonitor:
|
|
"""Get or create global monitor instance."""
|
|
global _monitor
|
|
if _monitor is None:
|
|
_monitor = PerformanceMonitor()
|
|
return _monitor |