Fss-Rag-Mini/tests/test_command_help_systems.py

#!/usr/bin/env python3
"""
Comprehensive test suite for all rag-mini commands and help systems.

Tests that ALL commands support:
- --help 
- -h
- help (where applicable)

And verifies help menu clarity and completeness.

⚠️  IMPORTANT: Run with venv activated:
    source .venv/bin/activate
    PYTHONPATH=. python tests/test_command_help_systems.py
"""

import unittest
import tempfile
import os
from pathlib import Path
from click.testing import CliRunner
from mini_rag.cli import cli


class TestAllCommandHelp(unittest.TestCase):
    """Test help systems for all rag-mini commands."""
    
    def setUp(self):
        self.runner = CliRunner()
        # Get all available commands from CLI
        result = self.runner.invoke(cli, ['--help'])
        self.help_output = result.output
        
        # Extract command names from help output
        lines = self.help_output.split('\n')
        commands_section = False
        self.commands = []
        
        for line in lines:
            if line.strip() == 'Commands:':
                commands_section = True
                continue
            if commands_section and line.strip():
                if line.startswith('  ') and not line.startswith('   '):
                    # Command line format: "  command-name  Description..."
                    parts = line.strip().split()
                    if parts:
                        self.commands.append(parts[0])
                elif not line.startswith('  '):
                    # End of commands section
                    break

    def test_main_command_help_formats(self):
        """Test main rag-mini command supports all help formats."""
        # Test --help
        result = self.runner.invoke(cli, ['--help'])
        self.assertEqual(result.exit_code, 0)
        self.assertIn('Mini RAG', result.output)
        
        # Note: -h and help subcommand not applicable to main command

    def test_all_subcommands_support_help_flag(self):
        """Test all subcommands support --help flag."""
        for cmd in self.commands:
            with self.subTest(command=cmd):
                result = self.runner.invoke(cli, [cmd, '--help'])
                self.assertEqual(result.exit_code, 0, f"Command {cmd} failed --help")
                self.assertIn('Usage:', result.output, f"Command {cmd} help missing usage")
                
    def test_all_subcommands_support_h_flag(self):
        """Test all subcommands support -h flag."""
        for cmd in self.commands:
            with self.subTest(command=cmd):
                result = self.runner.invoke(cli, [cmd, '-h'])
                self.assertEqual(result.exit_code, 0, f"Command {cmd} failed -h")
                self.assertIn('Usage:', result.output, f"Command {cmd} -h missing usage")

    def test_help_menu_completeness(self):
        """Test that help menus are complete and clear."""
        required_commands = ['init', 'search', 'status', 'info']
        
        for cmd in required_commands:
            with self.subTest(command=cmd):
                self.assertIn(cmd, self.commands, f"Required command {cmd} not found in CLI")
                
                result = self.runner.invoke(cli, [cmd, '--help'])
                self.assertEqual(result.exit_code, 0)
                
                help_text = result.output.lower()
                # Each command should have usage and options sections
                self.assertIn('usage:', help_text, f"{cmd} help missing usage")
                self.assertIn('options:', help_text, f"{cmd} help missing options")

    def test_help_consistency(self):
        """Test help output consistency across commands."""
        for cmd in self.commands:
            with self.subTest(command=cmd):
                # Test --help and -h produce same output
                result1 = self.runner.invoke(cli, [cmd, '--help'])
                result2 = self.runner.invoke(cli, [cmd, '-h'])
                
                self.assertEqual(result1.exit_code, result2.exit_code, 
                               f"{cmd}: --help and -h exit codes differ")
                self.assertEqual(result1.output, result2.output,
                               f"{cmd}: --help and -h output differs")


class TestCommandFunctionality(unittest.TestCase):
    """Test actual command functionality."""
    
    def setUp(self):
        self.runner = CliRunner()
        self.temp_dir = tempfile.mkdtemp()
        self.temp_path = Path(self.temp_dir)
        
        # Create test files
        (self.temp_path / "test.py").write_text('''
def hello_world():
    """Say hello to the world"""
    print("Hello, World!")
    return "success"

class TestClass:
    """A test class for demonstration"""
    def __init__(self):
        self.name = "test"
        
    def method_example(self):
        """Example method"""
        return self.name
''')
        
        (self.temp_path / "config.json").write_text('''
{
    "name": "test_project",
    "version": "1.0.0",
    "settings": {
        "debug": true,
        "api_key": "test123"
    }
}
''')

    def tearDown(self):
        import shutil
        shutil.rmtree(self.temp_dir, ignore_errors=True)

    def test_init_command_functionality(self):
        """Test init command creates proper index."""
        result = self.runner.invoke(cli, ['init', '-p', str(self.temp_path)])
        # Should complete without crashing
        self.assertIn(result.exit_code, [0, 1])  # May fail due to model config but shouldn't crash
        
        # Check that .mini-rag directory was created
        rag_dir = self.temp_path / '.mini-rag'
        if result.exit_code == 0:
            self.assertTrue(rag_dir.exists(), "Init should create .mini-rag directory")

    def test_search_command_functionality(self):
        """Test search command basic functionality."""
        # First try to init
        init_result = self.runner.invoke(cli, ['init', '-p', str(self.temp_path)])
        
        # Then search (may fail if no index, but shouldn't crash)
        result = self.runner.invoke(cli, ['search', '-p', str(self.temp_path), 'hello world'])
        
        # Should complete without crashing  
        self.assertNotIn('Traceback', result.output, "Search should not crash with traceback")

    def test_status_command_functionality(self):
        """Test status command functionality."""
        result = self.runner.invoke(cli, ['status', '-p', str(self.temp_path)])
        # Should complete without crashing
        self.assertNotIn('Traceback', result.output, "Status should not crash")

    def test_info_command_functionality(self):
        """Test info command functionality."""
        result = self.runner.invoke(cli, ['info'])
        self.assertEqual(result.exit_code, 0, "Info command should succeed")
        self.assertNotIn('Traceback', result.output, "Info should not crash")

    def test_stats_command_functionality(self):
        """Test stats command functionality."""
        result = self.runner.invoke(cli, ['stats', '-p', str(self.temp_path)])
        # Should complete without crashing even if no index
        self.assertNotIn('Traceback', result.output, "Stats should not crash")


class TestHelpMenuClarity(unittest.TestCase):
    """Test help menu clarity and user experience."""
    
    def setUp(self):
        self.runner = CliRunner()

    def test_main_help_is_clear(self):
        """Test main help menu is clear and informative."""
        result = self.runner.invoke(cli, ['--help'])
        self.assertEqual(result.exit_code, 0)
        
        help_text = result.output
        # Should contain clear description
        self.assertIn('Mini RAG', help_text)
        # Check for semantic search concept (appears in multiple forms)
        help_lower = help_text.lower()
        semantic_found = ('semantic search' in help_lower or 
                         'semantic code search' in help_lower or
                         'semantic similarity' in help_lower)
        self.assertTrue(semantic_found, f"No semantic search concept found in help: {help_lower}")
        
        # Should list main commands clearly
        self.assertIn('Commands:', help_text)
        self.assertIn('init', help_text)
        self.assertIn('search', help_text)

    def test_init_help_is_clear(self):
        """Test init command help is clear."""
        result = self.runner.invoke(cli, ['init', '--help'])
        self.assertEqual(result.exit_code, 0)
        
        help_text = result.output
        self.assertIn('Initialize RAG index', help_text)
        self.assertIn('-p, --path', help_text)  # Should explain path option

    def test_search_help_is_clear(self):
        """Test search command help is clear."""
        result = self.runner.invoke(cli, ['search', '--help'])
        self.assertEqual(result.exit_code, 0)
        
        help_text = result.output
        self.assertIn('Search', help_text)
        self.assertIn('query', help_text.lower())  # Should mention query
        # Should have key options
        self.assertIn('--top-k', help_text)
        self.assertIn('--show-perf', help_text)

    def test_error_messages_are_helpful(self):
        """Test error messages provide helpful guidance."""
        # Test command without required arguments
        result = self.runner.invoke(cli, ['search'])
        # Should show usage help, not just crash
        if result.exit_code != 0:
            self.assertIn('Usage:', result.output)


def run_comprehensive_help_test():
    """Run all help system tests with detailed reporting."""
    from rich.console import Console
    from rich.table import Table
    from rich.panel import Panel
    
    console = Console()
    
    console.print(Panel.fit(
        "[bold cyan]FSS-Mini-RAG Command Help System Test Suite[/bold cyan]\n"
        "[dim]Testing all commands support -h, --help, and help functionality[/dim]",
        border_style="cyan"
    ))
    
    # Test suites to run
    test_suites = [
        ("Help System Support", TestAllCommandHelp),
        ("Command Functionality", TestCommandFunctionality), 
        ("Help Menu Clarity", TestHelpMenuClarity)
    ]
    
    total_tests = 0
    passed_tests = 0
    failed_tests = []
    
    for suite_name, test_class in test_suites:
        console.print(f"\n[bold yellow]Testing {suite_name}[/bold yellow]")
        
        suite = unittest.TestLoader().loadTestsFromTestCase(test_class)
        
        for test in suite:
            total_tests += 1
            try:
                result = unittest.TestResult()
                test.run(result)
                
                if result.wasSuccessful():
                    passed_tests += 1
                    console.print(f"  [green]✓[/green] {test._testMethodName}")
                else:
                    failed_tests.append(f"{test_class.__name__}.{test._testMethodName}")
                    console.print(f"  [red]✗[/red] {test._testMethodName}")
                    for error in result.errors + result.failures:
                        console.print(f"    [red]{error[1].split('AssertionError:')[-1].strip()}[/red]")
                        
            except Exception as e:
                failed_tests.append(f"{test_class.__name__}.{test._testMethodName}")
                console.print(f"  [red]✗[/red] {test._testMethodName}: {e}")
    
    # Results summary
    console.print("\n" + "="*60)
    console.print(f"[bold]Test Results Summary[/bold]")
    
    table = Table()
    table.add_column("Metric", style="cyan")
    table.add_column("Value", style="green" if len(failed_tests) == 0 else "red")
    
    table.add_row("Total Tests", str(total_tests))
    table.add_row("Passed", str(passed_tests))
    table.add_row("Failed", str(len(failed_tests)))
    table.add_row("Success Rate", f"{(passed_tests/total_tests)*100:.1f}%" if total_tests > 0 else "0%")
    
    console.print(table)
    
    if failed_tests:
        console.print(f"\n[red]Failed Tests:[/red]")
        for test in failed_tests[:10]:  # Show first 10
            console.print(f"  • {test}")
        if len(failed_tests) > 10:
            console.print(f"  ... and {len(failed_tests) - 10} more")
    else:
        console.print(f"\n[green]🎉 All help system tests passed![/green]")
    
    # Show available commands
    from click.testing import CliRunner
    from mini_rag.cli import cli
    
    runner = CliRunner()
    result = runner.invoke(cli, ['--help'])
    if result.exit_code == 0:
        console.print(f"\n[cyan]Available Commands Tested:[/cyan]")
        lines = result.output.split('\n')
        commands_section = False
        
        for line in lines:
            if line.strip() == 'Commands:':
                commands_section = True
                continue
            if commands_section and line.strip():
                if line.startswith('  ') and not line.startswith('   '):
                    parts = line.strip().split(None, 1)
                    if len(parts) >= 2:
                        console.print(f"  • [bold]{parts[0]}[/bold] - {parts[1]}")
                elif not line.startswith('  '):
                    break
    
    console.print(f"\n[dim]Command help system verification complete.[/dim]")
    return passed_tests == total_tests


if __name__ == "__main__":
    import sys
    
    if len(sys.argv) > 1 and sys.argv[1] == "--comprehensive":
        # Run with rich output
        success = run_comprehensive_help_test()
        sys.exit(0 if success else 1)
    else:
        # Run standard unittest
        unittest.main()