run_benchmarks.py

"""
Run all sparsity benchmarks and generate performance comparison report.
"""

import subprocess
import os
import re

BENCHMARK_DIR = "benchmarks"
SIMULATOR_EXE = "simulator.exe" if os.name == 'nt' else "./simulator"

def run_simulator(asm_file, single_cycle=False):
    try:
        cmd = [SIMULATOR_EXE, asm_file]
        if single_cycle:
            cmd.append("--single-cycle")
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
        return result.stdout
    except Exception as e:
        return f"Error: {e}"

def parse_output(output):
    """Extract key metrics from simulator output."""
    metrics = {}
    
    # Total Cycles
    match = re.search(r'Total Cycles:\s+(\d+)', output)
    if match:
        metrics['cycles'] = int(match.group(1))
    
    # Instructions Retired
    match = re.search(r'Instructions Retired:\s+(\d+)', output)
    if match:
        metrics['instructions'] = int(match.group(1))
    
    # CPI
    match = re.search(r'CPI \(Cycles Per Instr\):\s+([\d.]+)', output)
    if match:
        metrics['cpi'] = float(match.group(1))
    
    # Zero-Skipping Savings
    match = re.search(r'Zero-Skipping Savings:\s+(\d+)', output)
    if match:
        metrics['zero_skips'] = int(match.group(1))
    
    # Stall Cycles
    match = re.search(r'Stall Cycles.*:\s+(\d+)', output)
    if match:
        metrics['stalls'] = int(match.group(1))
    
    # Get final register values (t5 = LNZ sum, t6 = SW sum)
    match = re.search(r'x30\s*:\s*(-?\d+)', output)
    if match:
        metrics['lnz_sum'] = int(match.group(1))
    
    match = re.search(r'x31\s*:\s*(-?\d+)', output)
    if match:
        metrics['sw_sum'] = int(match.group(1))
    
    return metrics

def main():
    print("=" * 80)
    print("  SparseFlow Performance Benchmark Suite")
    print("  Comparing LNZ (Hardware Skip) vs Standard LW (Software Loop)")
    print("=" * 80)
    
    sparsity_levels = [60, 70, 80, 90, 99]
    
    results = []
    
    for sparsity in sparsity_levels:
        filename = os.path.join(BENCHMARK_DIR, f"sparse_{sparsity}pct.asm")
        
        if not os.path.exists(filename):
            print(f"[SKIP] {filename} not found")
            continue
        
        print(f"\n[Running] {filename}...")
        
        # Run Pipelined Mode
        output_pipe = run_simulator(filename, single_cycle=False)
        metrics_pipe = parse_output(output_pipe)
        
        # Run Single-Cycle Mode
        output_single = run_simulator(filename, single_cycle=True)
        metrics_single = parse_output(output_single)
        
        results.append({
            'sparsity': sparsity,
            'pipelined': metrics_pipe,
            'single': metrics_single
        })
        
        print(f"  Pipelined: {metrics_pipe.get('cycles', 'N/A')} cycles, {metrics_pipe.get('zero_skips', 0)} skips")
        print(f"  Single:    {metrics_single.get('cycles', 'N/A')} cycles")
    
    # Generate Report
    print("\n" + "=" * 80)
    print("  PERFORMANCE COMPARISON REPORT")
    print("=" * 80)
    
    print(f"\n{'Sparsity':<10} {'Pipelined':<12} {'Single-Cycle':<14} {'Speedup':<10} {'Zero-Skips':<12} {'CPI (Pipe)'}")
    print("-" * 80)
    
    for r in results:
        sp = r['sparsity']
        pipe_cycles = r['pipelined'].get('cycles', 0)
        single_cycles = r['single'].get('cycles', 0)
        speedup = single_cycles / pipe_cycles if pipe_cycles > 0 else 0
        skips = r['pipelined'].get('zero_skips', 0)
        cpi = r['pipelined'].get('cpi', 0)
        
        print(f"{sp}%{'':<8} {pipe_cycles:<12} {single_cycles:<14} {speedup:.2f}x{'':<6} {skips:<12} {cpi:.2f}")
    
    # Save to file
    report_file = os.path.join(BENCHMARK_DIR, "benchmark_results.txt")
    with open(report_file, 'w') as f:
        f.write("=" * 80 + "\n")
        f.write("  SparseFlow Performance Benchmark Results\n")
        f.write("=" * 80 + "\n\n")
        
        f.write(f"{'Sparsity':<10} {'Pipelined':<12} {'Single-Cycle':<14} {'Speedup':<10} {'Zero-Skips':<12} {'CPI'}\n")
        f.write("-" * 80 + "\n")
        
        for r in results:
            sp = r['sparsity']
            pipe_cycles = r['pipelined'].get('cycles', 0)
            single_cycles = r['single'].get('cycles', 0)
            speedup = single_cycles / pipe_cycles if pipe_cycles > 0 else 0
            skips = r['pipelined'].get('zero_skips', 0)
            cpi = r['pipelined'].get('cpi', 0)
            
            f.write(f"{sp}%{'':<8} {pipe_cycles:<12} {single_cycles:<14} {speedup:.2f}x{'':<6} {skips:<12} {cpi:.2f}\n")
        
        f.write("\n" + "=" * 80 + "\n")
        f.write("Analysis:\n")
        f.write("- Higher sparsity = More zeros = More skips = Better LNZ performance\n")
        f.write("- At 60% sparsity, LNZ still provides benefit by skipping 600 zeros\n")
        f.write("- At 99% sparsity, LNZ skips 990 zeros, dramatically reducing cycles\n")
        f.write("=" * 80 + "\n")
    
    print(f"\nResults saved to: {report_file}")

if __name__ == "__main__":
    main()