diff --git a/codeflash/benchmarking/compare.py b/codeflash/benchmarking/compare.py index fb98ef301..9ce4db01b 100644 --- a/codeflash/benchmarking/compare.py +++ b/codeflash/benchmarking/compare.py @@ -25,96 +25,119 @@ if TYPE_CHECKING: from collections.abc import Callable + from codeflash.benchmarking.plugin.plugin import BenchmarkStats, MemoryStats from codeflash.models.function_types import FunctionToOptimize from codeflash.models.models import BenchmarkKey +_GREEN_TPL = "[green]%+.0f%%[/green]" + +_RED_TPL = "[red]%+.0f%%[/red]" + @dataclass class CompareResult: base_ref: str head_ref: str - base_total_ns: dict[BenchmarkKey, int] = field(default_factory=dict) - head_total_ns: dict[BenchmarkKey, int] = field(default_factory=dict) - base_function_ns: dict[str, dict[BenchmarkKey, int]] = field(default_factory=dict) - head_function_ns: dict[str, dict[BenchmarkKey, int]] = field(default_factory=dict) + base_stats: dict[BenchmarkKey, BenchmarkStats] = field(default_factory=dict) + head_stats: dict[BenchmarkKey, BenchmarkStats] = field(default_factory=dict) + base_function_ns: dict[str, dict[BenchmarkKey, float]] = field(default_factory=dict) + head_function_ns: dict[str, dict[BenchmarkKey, float]] = field(default_factory=dict) + base_memory: dict[BenchmarkKey, MemoryStats] = field(default_factory=dict) + head_memory: dict[BenchmarkKey, MemoryStats] = field(default_factory=dict) def format_markdown(self) -> str: - """Format comparison results as GitHub-flavored markdown (for programmatic use, e.g. PR comments).""" - if not self.base_total_ns and not self.head_total_ns: + if not self.base_stats and not self.head_stats and not self.base_memory and not self.head_memory: return "_No benchmark results to compare._" base_short = self.base_ref[:12] head_short = self.head_ref[:12] - all_keys = sorted(set(self.base_total_ns) | set(self.head_total_ns), key=str) + all_keys = sorted( + set(self.base_stats) | set(self.head_stats) | set(self.base_memory) | set(self.head_memory), key=str + ) sections: list[str] = [f"## Benchmark: `{base_short}` vs `{head_short}`"] for bm_key in all_keys: - base_ns = self.base_total_ns.get(bm_key) - head_ns = self.head_total_ns.get(bm_key) + base_s = self.base_stats.get(bm_key) + head_s = self.head_stats.get(bm_key) - # Extract short benchmark name from the full key bm_name = str(bm_key).rsplit("::", 1)[-1] if "::" in str(bm_key) else str(bm_key) - # --- End-to-End table --- - lines = [ - f"### {bm_name}", - "", - "| Branch | Time (ms) | vs base | Speedup |", - "|:---|---:|---:|---:|", - f"| `{base_short}` (base) | {_fmt_ms(base_ns)} | - | - |", - f"| `{head_short}` (head) | {_fmt_ms(head_ns)} " - f"| {_md_delta(base_ns, head_ns)} | {_md_speedup(base_ns, head_ns)} |", - ] - - # --- Per-function breakdown --- - all_funcs: set[str] = set() - for d in [self.base_function_ns, self.head_function_ns]: - for func_name, bm_dict in d.items(): - if bm_key in bm_dict: - all_funcs.add(func_name) - - if all_funcs: + lines = [f"### {bm_name}"] + + # Timing table (skip for memory-only benchmark keys) + if base_s or head_s: + lines.extend( + [ + "", + "| | Min | Median | Mean | OPS | Rounds |", + "|:---|---:|---:|---:|---:|---:|", + f"| `{base_short}` (base) | {fmt_us(base_s.min_ns) if base_s else '-'}" + f" | {fmt_us(base_s.median_ns) if base_s else '-'}" + f" | {fmt_us(base_s.mean_ns) if base_s else '-'}" + f" | {md_ops(base_s.mean_ns) if base_s else '-'}" + f" | {f'{base_s.rounds:,}' if base_s else '-'} |", + f"| `{head_short}` (head) | {fmt_us(head_s.min_ns) if head_s else '-'}" + f" | {fmt_us(head_s.median_ns) if head_s else '-'}" + f" | {fmt_us(head_s.mean_ns) if head_s else '-'}" + f" | {md_ops(head_s.mean_ns) if head_s else '-'}" + f" | {f'{head_s.rounds:,}' if head_s else '-'} |", + f"| **Speedup** | **{md_speedup_val(base_s.min_ns, head_s.min_ns) if base_s and head_s else '-'}**" + f" | **{md_speedup_val(base_s.median_ns, head_s.median_ns) if base_s and head_s else '-'}**" + f" | **{md_speedup_val(base_s.mean_ns, head_s.mean_ns) if base_s and head_s else '-'}**" + f" | **{md_speedup_val(base_s.mean_ns, head_s.mean_ns) if base_s and head_s else '-'}**" + f" | |", + ] + ) - def sort_key(fn: str, _bm_key: BenchmarkKey = bm_key) -> int: - return self.base_function_ns.get(fn, {}).get(_bm_key, 0) + # Per-function breakdown + all_funcs: set[str] = set() + for d in [self.base_function_ns, self.head_function_ns]: + for func_name, bm_dict in d.items(): + if bm_key in bm_dict: + all_funcs.add(func_name) - sorted_funcs = sorted(all_funcs, key=sort_key, reverse=True) + if all_funcs: - lines.append("") - lines.append("| Function | base (ms) | head (ms) | Improvement | Speedup |") - lines.append("|:---|---:|---:|:---|---:|") + def sort_key(fn: str, _bm_key: BenchmarkKey = bm_key) -> float: + return self.base_function_ns.get(fn, {}).get(_bm_key, 0) - for func_name in sorted_funcs: - b = self.base_function_ns.get(func_name, {}).get(bm_key) - h = self.head_function_ns.get(func_name, {}).get(bm_key) - short_name = func_name.rsplit(".", 1)[-1] if "." in func_name else func_name - lines.append( - f"| `{short_name}` | {_fmt_ms(b)} | {_fmt_ms(h)} | {_md_bar(b, h)} | {_md_speedup(b, h)} |" - ) + sorted_funcs = sorted(all_funcs, key=sort_key, reverse=True) - lines.append( - f"| **TOTAL** | **{_fmt_ms(base_ns)}** | **{_fmt_ms(head_ns)}** " - f"| {_md_bar(base_ns, head_ns)} | {_md_speedup(base_ns, head_ns)} |" - ) - - # --- Share of Benchmark Time (%) --- - if base_ns and head_ns: - lines.append("") - lines.append("
Share of Benchmark Time") lines.append("") - lines.append("| Function | base | head |") - lines.append("|:---|:---|:---|") + lines.append("| Function | base (μs) | head (μs) | Improvement | Speedup |") + lines.append("|:---|---:|---:|:---|---:|") for func_name in sorted_funcs: b = self.base_function_ns.get(func_name, {}).get(bm_key) h = self.head_function_ns.get(func_name, {}).get(bm_key) short_name = func_name.rsplit(".", 1)[-1] if "." in func_name else func_name - b_pct = b / base_ns * 100 if b else 0 - h_pct = h / head_ns * 100 if h else 0 - lines.append(f"| `{short_name}` | {_pct_bar(b_pct)} | {_pct_bar(h_pct)} |") - - lines.append("") - lines.append("
") + lines.append( + f"| `{short_name}` | {fmt_us(b)} | {fmt_us(h)} | {md_bar(b, h)} | {md_speedup(b, h)} |" + ) + + # Memory section (always show for memory-only keys, otherwise skip when delta is negligible) + base_mem = self.base_memory.get(bm_key) + head_mem = self.head_memory.get(bm_key) + memory_only_key = not base_s and not head_s + if memory_only_key or has_meaningful_memory_change(base_mem, head_mem): + lines.append("") + lines.append("#### Memory") + lines.append("") + lines.append("| Ref | Peak Memory | Allocations | Delta |") + lines.append("|:---|---:|---:|:---|") + if base_mem: + lines.append( + f"| `{base_short}` (base) | {md_bytes(base_mem.peak_memory_bytes)}" + f" | {base_mem.total_allocations:,} | |" + ) + if head_mem: + delta = md_memory_delta( + base_mem.peak_memory_bytes if base_mem else None, head_mem.peak_memory_bytes + ) + lines.append( + f"| `{head_short}` (head) | {md_bytes(head_mem.peak_memory_bytes)}" + f" | {head_mem.total_allocations:,} | {delta} |" + ) sections.append("\n".join(lines)) @@ -122,6 +145,63 @@ def sort_key(fn: str, _bm_key: BenchmarkKey = bm_key) -> int: return "\n\n".join(sections) +@dataclass +class ScriptCompareResult: + base_ref: str + head_ref: str + base_results: dict[str, float] = field(default_factory=dict) + head_results: dict[str, float] = field(default_factory=dict) + base_memory: Optional[MemoryStats] = None + head_memory: Optional[MemoryStats] = None + + def format_markdown(self) -> str: + if not self.base_results and not self.head_results and not self.base_memory and not self.head_memory: + return "_No benchmark results to compare._" + + base_short = self.base_ref[:12] + head_short = self.head_ref[:12] + lines: list[str] = [f"## Benchmark: `{base_short}` vs `{head_short}`"] + + all_keys = sorted((set(self.base_results) | set(self.head_results)) - {"__total__"}) + has_total = "__total__" in self.base_results or "__total__" in self.head_results + + lines.extend(["", "| Key | Base | Head | Delta | Speedup |", "|:---|---:|---:|:---|---:|"]) + for key in all_keys: + b = self.base_results.get(key) + h = self.head_results.get(key) + lines.append( + f"| `{key}` | {_fmt_seconds(b)} | {_fmt_seconds(h)} | {_md_delta_s(b, h)} | {md_speedup(b, h)} |" + ) + + if has_total: + b = self.base_results.get("__total__") + h = self.head_results.get("__total__") + lines.append( + f"| **TOTAL** | **{_fmt_seconds(b)}** | **{_fmt_seconds(h)}** | {_md_delta_s(b, h)} | {md_speedup(b, h)} |" + ) + + if self.base_memory or self.head_memory: + lines.extend( + ["", "#### Memory", "", "| Ref | Peak Memory | Allocations | Delta |", "|:---|---:|---:|:---|"] + ) + if self.base_memory: + lines.append( + f"| `{base_short}` (base) | {md_bytes(self.base_memory.peak_memory_bytes)}" + f" | {self.base_memory.total_allocations:,} | |" + ) + if self.head_memory: + delta = md_memory_delta( + self.base_memory.peak_memory_bytes if self.base_memory else None, self.head_memory.peak_memory_bytes + ) + lines.append( + f"| `{head_short}` (head) | {md_bytes(self.head_memory.peak_memory_bytes)}" + f" | {self.head_memory.total_allocations:,} | {delta} |" + ) + + lines.extend(["", "---", "*Generated by codeflash optimization agent*"]) + return "\n".join(lines) + + def compare_branches( base_ref: str, head_ref: str, @@ -130,25 +210,36 @@ def compare_branches( tests_root: Path, functions: Optional[dict[Path, list[FunctionToOptimize]]] = None, timeout: int = 600, + memory: bool = False, ) -> CompareResult: """Compare benchmark performance between two git refs. If functions is None, auto-detects changed functions from git diff. Returns a CompareResult with timing data from both refs. """ + import sys + from codeflash.benchmarking.instrument_codeflash_trace import instrument_codeflash_trace_decorator from codeflash.benchmarking.plugin.plugin import CodeFlashBenchmarkPlugin from codeflash.benchmarking.trace_benchmarks import trace_benchmarks_pytest + if memory and sys.platform == "win32": + logger.error("--memory requires memray which is not available on Windows") + return CompareResult(base_ref=base_ref, head_ref=head_ref) + repo = git.Repo(project_root, search_parent_directories=True) repo_root = Path(repo.working_dir) # Auto-detect functions if not provided if functions is None: - functions = _discover_changed_functions(base_ref, head_ref, repo_root) + functions = discover_changed_functions(base_ref, head_ref, repo_root) if not functions: - logger.warning("No changed Python functions found between %s and %s", base_ref, head_ref) - return CompareResult(base_ref=base_ref, head_ref=head_ref) + if not memory: + logger.warning("No changed Python functions found between %s and %s", base_ref, head_ref) + return CompareResult(base_ref=base_ref, head_ref=head_ref) + logger.info("No changed top-level functions — running memory-only comparison") + + memory_only = memory and not functions from rich.live import Live from rich.panel import Panel @@ -157,30 +248,33 @@ def compare_branches( base_short = base_ref[:12] head_short = head_ref[:12] - func_count = sum(len(fns) for fns in functions.values()) - file_count = len(functions) + from rich.tree import Tree - # Build function tree for the panel - from os.path import commonpath + if memory_only: + fn_tree = Tree("[bold]Memory-only[/bold] [dim](no changed top-level functions)[/dim]", guide_style="dim") + else: + func_count = sum(len(fns) for fns in functions.values()) + file_count = len(functions) - from rich.tree import Tree + # Build function tree for the panel + from os.path import commonpath - rel_paths = [] - for fp in functions: - rel_paths.append(fp.relative_to(repo_root) if fp.is_relative_to(repo_root) else fp) + rel_paths = [] + for fp in functions: + rel_paths.append(fp.relative_to(repo_root) if fp.is_relative_to(repo_root) else fp) - # Strip common prefix so paths are short but unambiguous - if len(rel_paths) > 1: - common = Path(commonpath(rel_paths)) - short_paths = [p.relative_to(common) if p != common else Path(p.name) for p in rel_paths] - else: - short_paths = [Path(p.name) for p in rel_paths] + # Strip common prefix so paths are short but unambiguous + if len(rel_paths) > 1: + common = Path(commonpath(rel_paths)) + short_paths = [p.relative_to(common) if p != common else Path(p.name) for p in rel_paths] + else: + short_paths = [Path(p.name) for p in rel_paths] - fn_tree = Tree(f"[bold]{func_count} functions[/bold] [dim]across {file_count} files[/dim]", guide_style="dim") - for (_fp, fns), short in zip(functions.items(), short_paths): - branch = fn_tree.add(f"[cyan]{short}[/cyan]") - for fn in fns: - branch.add(f"[bold]{fn.function_name}[/bold]") + fn_tree = Tree(f"[bold]{func_count} functions[/bold] [dim]across {file_count} files[/dim]", guide_style="dim") + for (_fp, fns), short in zip(functions.items(), short_paths): + branch = fn_tree.add(f"[cyan]{short}[/cyan]") + for fn in fns: + branch.add(f"[bold]{fn.function_name}[/bold]") # Set up worktree paths and trace DB paths from codeflash.code_utils.git_worktree_utils import worktree_dirs @@ -192,12 +286,19 @@ def compare_branches( head_worktree = worktree_dirs / f"compare-head-{timestamp}" base_trace_db = worktree_dirs / f"trace-base-{timestamp}.db" head_trace_db = worktree_dirs / f"trace-head-{timestamp}.db" + base_memray_dir = worktree_dirs / f"memray-base-{timestamp}" + head_memray_dir = worktree_dirs / f"memray-head-{timestamp}" + memray_prefix = "cf-mem" result = CompareResult(base_ref=base_ref, head_ref=head_ref) from rich.console import Group - step_labels = ["Creating worktrees", f"Benchmarking base ({base_short})", f"Benchmarking head ({head_short})"] + step_labels = ["Creating worktrees"] + if not memory_only: + step_labels.extend([f"Benchmarking base ({base_short})", f"Benchmarking head ({head_short})"]) + if memory: + step_labels.extend([f"Memory profiling base ({base_short})", f"Memory profiling head ({head_short})"]) def build_steps(current_step: int) -> Group: lines: list[Text] = [] @@ -211,8 +312,7 @@ def build_steps(current_step: int) -> Group: return Group(*lines) def build_panel(current_step: int) -> Panel: - # Two-column grid: tree left, steps right (vertically padded to center) - tree_height = 1 + sum(1 + len(fns) for fns in functions.values()) # root + files + functions + tree_height = 1 + sum(1 + len(fns) for fns in functions.values()) step_count = len(step_labels) pad_top = max(0, (tree_height - step_count) // 2) @@ -236,52 +336,89 @@ def build_panel(current_step: int) -> Panel: ) try: - with Live(build_panel(0), console=console, refresh_per_second=1) as live: - # Step 1: Create worktrees (resolve to SHAs to avoid "already checked out" errors) + step = 0 + with Live(build_panel(step), console=console, refresh_per_second=1) as live: + # Create worktrees (resolve to SHAs to avoid "already checked out" errors) base_sha = repo.commit(base_ref).hexsha head_sha = repo.commit(head_ref).hexsha repo.git.worktree("add", str(base_worktree), base_sha) repo.git.worktree("add", str(head_worktree), head_sha) - live.update(build_panel(1)) - - # Step 2: Run benchmarks on base - _run_benchmark_on_worktree( - worktree_dir=base_worktree, - repo_root=repo_root, - functions=functions, - benchmarks_root=benchmarks_root, - tests_root=tests_root, - trace_db=base_trace_db, - timeout=timeout, - instrument_fn=instrument_codeflash_trace_decorator, - trace_fn=trace_benchmarks_pytest, - ) - live.update(build_panel(2)) - - # Step 3: Run benchmarks on head - _run_benchmark_on_worktree( - worktree_dir=head_worktree, - repo_root=repo_root, - functions=functions, - benchmarks_root=benchmarks_root, - tests_root=tests_root, - trace_db=head_trace_db, - timeout=timeout, - instrument_fn=instrument_codeflash_trace_decorator, - trace_fn=trace_benchmarks_pytest, - ) + step += 1 + live.update(build_panel(step)) + + if not memory_only: + # Run trace benchmarks on base + run_benchmark_on_worktree( + worktree_dir=base_worktree, + repo_root=repo_root, + functions=functions, + benchmarks_root=benchmarks_root, + tests_root=tests_root, + trace_db=base_trace_db, + timeout=timeout, + instrument_fn=instrument_codeflash_trace_decorator, + trace_fn=trace_benchmarks_pytest, + ) + step += 1 + live.update(build_panel(step)) + + # Run trace benchmarks on head + run_benchmark_on_worktree( + worktree_dir=head_worktree, + repo_root=repo_root, + functions=functions, + benchmarks_root=benchmarks_root, + tests_root=tests_root, + trace_db=head_trace_db, + timeout=timeout, + instrument_fn=instrument_codeflash_trace_decorator, + trace_fn=trace_benchmarks_pytest, + ) + + # Memory profiling (reuses existing worktrees) + if memory: + from codeflash.benchmarking.trace_benchmarks import memory_benchmarks_pytest + + wt_base_benchmarks = base_worktree / benchmarks_root.relative_to(repo_root) + wt_head_benchmarks = head_worktree / benchmarks_root.relative_to(repo_root) + + # Copy benchmarks into worktrees if not present (e.g. base ref predates benchmark dir) + if memory_only: + import shutil + + for wt_bm in [wt_base_benchmarks, wt_head_benchmarks]: + if not wt_bm.exists() and benchmarks_root.is_dir(): + shutil.copytree(benchmarks_root, wt_bm) + + if not memory_only: + step += 1 + live.update(build_panel(step)) + memory_benchmarks_pytest(wt_base_benchmarks, base_worktree, base_memray_dir, memray_prefix, timeout) + + step += 1 + live.update(build_panel(step)) + memory_benchmarks_pytest(wt_head_benchmarks, head_worktree, head_memray_dir, memray_prefix, timeout) # Load results - if base_trace_db.exists(): - result.base_total_ns = CodeFlashBenchmarkPlugin.get_benchmark_timings(base_trace_db) - result.base_function_ns = CodeFlashBenchmarkPlugin.get_function_benchmark_timings(base_trace_db) + if not memory_only: + if base_trace_db.exists(): + result.base_stats = CodeFlashBenchmarkPlugin.get_benchmark_timings(base_trace_db) + result.base_function_ns = CodeFlashBenchmarkPlugin.get_function_benchmark_timings(base_trace_db) + + if head_trace_db.exists(): + result.head_stats = CodeFlashBenchmarkPlugin.get_benchmark_timings(head_trace_db) + result.head_function_ns = CodeFlashBenchmarkPlugin.get_function_benchmark_timings(head_trace_db) - if head_trace_db.exists(): - result.head_total_ns = CodeFlashBenchmarkPlugin.get_benchmark_timings(head_trace_db) - result.head_function_ns = CodeFlashBenchmarkPlugin.get_function_benchmark_timings(head_trace_db) + if memory: + from codeflash.benchmarking.plugin.plugin import MemoryStats + + if base_memray_dir.exists(): + result.base_memory = MemoryStats.parse_memray_results(base_memray_dir, memray_prefix) + if head_memray_dir.exists(): + result.head_memory = MemoryStats.parse_memray_results(head_memray_dir, memray_prefix) # Render comparison - _render_comparison(result) + render_comparison(result) except KeyboardInterrupt: console.print("\n[yellow]Interrupted — cleaning up...[/yellow]") @@ -293,15 +430,21 @@ def build_panel(current_step: int) -> Panel: remove_worktree(base_worktree) remove_worktree(head_worktree) repo.git.worktree("prune") - # Cleanup trace DBs + # Cleanup trace DBs and memray dirs for db in [base_trace_db, head_trace_db]: if db.exists(): db.unlink() + if memory: + import shutil + + for memray_dir in [base_memray_dir, head_memray_dir]: + if memray_dir.exists(): + shutil.rmtree(memray_dir) return result -def _discover_changed_functions(base_ref: str, head_ref: str, repo_root: Path) -> dict[Path, list[FunctionToOptimize]]: +def discover_changed_functions(base_ref: str, head_ref: str, repo_root: Path) -> dict[Path, list[FunctionToOptimize]]: """Find only functions whose bodies overlap with changed lines between refs.""" from io import StringIO @@ -347,14 +490,14 @@ def _discover_changed_functions(base_ref: str, head_ref: str, repo_root: Path) - logger.debug(f"Skipping {abs_path} (does not exist)") continue - modified_fns = _find_changed_toplevel_functions(abs_path, changed_lines) + modified_fns = find_changed_toplevel_functions(abs_path, changed_lines) if modified_fns: result[abs_path] = modified_fns return result -def _find_changed_toplevel_functions(file_path: Path, changed_lines: set[int]) -> list[FunctionToOptimize]: +def find_changed_toplevel_functions(file_path: Path, changed_lines: set[int]) -> list[FunctionToOptimize]: """Find top-level functions overlapping changed lines using stdlib ast. Only discovers module-level functions (not methods inside classes, not nested @@ -394,7 +537,7 @@ def _find_changed_toplevel_functions(file_path: Path, changed_lines: set[int]) - return functions -def _run_benchmark_on_worktree( +def run_benchmark_on_worktree( worktree_dir: Path, repo_root: Path, functions: dict[Path, list[FunctionToOptimize]], @@ -443,6 +586,13 @@ def _run_benchmark_on_worktree( wt_benchmarks = worktree_dir / benchmarks_root.relative_to(repo_root) wt_tests = worktree_dir / tests_root.relative_to(repo_root) + # If benchmarks dir doesn't exist in this worktree (e.g. base ref predates + # the benchmark), copy it from the working directory so both refs can run. + if not wt_benchmarks.exists() and benchmarks_root.is_dir(): + import shutil + + shutil.copytree(benchmarks_root, wt_benchmarks) + if trace_db.exists(): trace_db.unlink() @@ -458,98 +608,191 @@ def _run_benchmark_on_worktree( file_path.write_text(source, encoding="utf-8") -def _render_comparison(result: CompareResult) -> None: +def render_comparison(result: CompareResult) -> None: """Render Rich comparison tables to console.""" - if not result.base_total_ns and not result.head_total_ns: + has_timing = result.base_stats or result.head_stats + has_memory = result.base_memory or result.head_memory + if not has_timing and not has_memory: logger.warning("No benchmark results to compare") return base_short = result.base_ref[:12] head_short = result.head_ref[:12] - # Find all benchmark keys across both refs - all_benchmark_keys = set(result.base_total_ns.keys()) | set(result.head_total_ns.keys()) + all_benchmark_keys = ( + set(result.base_stats.keys()) + | set(result.head_stats.keys()) + | set(result.base_memory.keys()) + | set(result.head_memory.keys()) + ) for bm_key in sorted(all_benchmark_keys, key=str): - # Show only the test function name, not the full module path bm_name = str(bm_key).rsplit("::", 1)[-1] if "::" in str(bm_key) else str(bm_key) console.print() console.rule(f"[bold]{bm_name}[/bold]") console.print() - base_ns = result.base_total_ns.get(bm_key) - head_ns = result.head_total_ns.get(bm_key) + base_s = result.base_stats.get(bm_key) + head_s = result.head_stats.get(bm_key) + + # Table 1: Statistical summary (skip for memory-only benchmark keys) + if base_s or head_s: + t1 = Table(title="End-to-End (per iteration)", border_style="blue", show_lines=True, expand=False) + t1.add_column("Ref", style="bold cyan") + t1.add_column("Min", justify="right") + t1.add_column("Median", justify="right") + t1.add_column("Mean", justify="right") + t1.add_column("OPS", justify="right") + t1.add_column("Rounds", justify="right") + + if base_s: + t1.add_row( + f"{base_short} (base)", + fmt_time(base_s.min_ns), + fmt_time(base_s.median_ns), + fmt_time(base_s.mean_ns), + fmt_ops(base_s.mean_ns), + f"{base_s.rounds:,}", + ) + if head_s: + t1.add_row( + f"{head_short} (head)", + fmt_time(head_s.min_ns), + fmt_time(head_s.median_ns), + fmt_time(head_s.mean_ns), + fmt_ops(head_s.mean_ns), + f"{head_s.rounds:,}", + ) + if base_s and head_s: + t1.add_section() + t1.add_row( + "[bold]Speedup[/bold]", + fmt_speedup(base_s.min_ns, head_s.min_ns), + fmt_speedup(base_s.median_ns, head_s.median_ns), + fmt_speedup(base_s.mean_ns, head_s.mean_ns), + fmt_speedup_ops(base_s.mean_ns, head_s.mean_ns), + "", + ) + console.print(t1, justify="center") - # Table 1: Total benchmark time - t1 = Table(title="End-to-End", border_style="blue", show_lines=True, expand=False) - t1.add_column("Ref", style="bold cyan") - t1.add_column("Time (ms)", justify="right") - t1.add_column("Delta", justify="right") - t1.add_column("Speedup", justify="right") + # Table 2: Per-function breakdown (average per-iteration) + all_funcs: set[str] = set() + for d in [result.base_function_ns, result.head_function_ns]: + for func_name, bm_dict in d.items(): + if bm_key in bm_dict: + all_funcs.add(func_name) - t1.add_row(f"{base_short} (base)", _fmt_ms(base_ns), "-", "-") - t1.add_row( - f"{head_short} (head)", _fmt_ms(head_ns), _fmt_delta(base_ns, head_ns), _fmt_speedup(base_ns, head_ns) - ) - console.print(t1, justify="center") + if all_funcs: + console.print() + + t2 = Table( + title="Per-Function Breakdown (avg per iteration)", + border_style="blue", + show_lines=True, + expand=False, + ) + t2.add_column("Function", style="cyan") + t2.add_column("base", justify="right", style="yellow") + t2.add_column("head", justify="right", style="yellow") + t2.add_column("Delta", justify="right") + t2.add_column("Speedup", justify="right") + + def sort_key(fn: str, _bm_key: BenchmarkKey = bm_key) -> float: + return result.base_function_ns.get(fn, {}).get(_bm_key, 0) + + for func_name in sorted(all_funcs, key=sort_key, reverse=True): + b_ns = result.base_function_ns.get(func_name, {}).get(bm_key) + h_ns = result.head_function_ns.get(func_name, {}).get(bm_key) + short_name = func_name.rsplit(".", 1)[-1] if "." in func_name else func_name + t2.add_row( + short_name, fmt_time(b_ns), fmt_time(h_ns), fmt_delta(b_ns, h_ns), fmt_speedup(b_ns, h_ns) + ) - # Table 2: Per-function breakdown - all_funcs = set() - for d in [result.base_function_ns, result.head_function_ns]: - for func_name, bm_dict in d.items(): - if bm_key in bm_dict: - all_funcs.add(func_name) + console.print(t2, justify="center") - if all_funcs: + # Table 3: Memory (always show for memory-only keys, otherwise skip when delta is negligible) + base_mem = result.base_memory.get(bm_key) + head_mem = result.head_memory.get(bm_key) + memory_only_key = not base_s and not head_s + if memory_only_key or has_meaningful_memory_change(base_mem, head_mem): console.print() - - t2 = Table(title="Per-Function Breakdown", border_style="blue", show_lines=True, expand=False) - t2.add_column("Function", style="cyan") - t2.add_column("base (ms)", justify="right", style="yellow") - t2.add_column("head (ms)", justify="right", style="yellow") - t2.add_column("Delta", justify="right") - t2.add_column("Speedup", justify="right") - - def sort_key(fn: str, _bm_key: BenchmarkKey = bm_key) -> int: - return result.base_function_ns.get(fn, {}).get(_bm_key, 0) - - for func_name in sorted(all_funcs, key=sort_key, reverse=True): - b_ns = result.base_function_ns.get(func_name, {}).get(bm_key) - h_ns = result.head_function_ns.get(func_name, {}).get(bm_key) - - # Shorten function name for display - short_name = func_name.rsplit(".", 1)[-1] if "." in func_name else func_name - - t2.add_row(short_name, _fmt_ms(b_ns), _fmt_ms(h_ns), _fmt_delta(b_ns, h_ns), _fmt_speedup(b_ns, h_ns)) - - # Totals row - t2.add_section() - t2.add_row( - "[bold]TOTAL[/bold]", - f"[bold]{_fmt_ms(base_ns)}[/bold]", - f"[bold]{_fmt_ms(head_ns)}[/bold]", - _fmt_delta(base_ns, head_ns), - _fmt_speedup(base_ns, head_ns), - ) - console.print(t2, justify="center") + t3 = Table(title="Memory (peak per test)", border_style="magenta", show_lines=True, expand=False) + t3.add_column("Ref", style="bold cyan") + t3.add_column("Peak Memory", justify="right") + t3.add_column("Allocations", justify="right") + t3.add_column("Delta", justify="right") + + if base_mem: + t3.add_row( + f"{base_short} (base)", fmt_bytes(base_mem.peak_memory_bytes), f"{base_mem.total_allocations:,}", "" + ) + if head_mem: + delta = fmt_memory_delta(base_mem.peak_memory_bytes if base_mem else None, head_mem.peak_memory_bytes) + t3.add_row( + f"{head_short} (head)", + fmt_bytes(head_mem.peak_memory_bytes), + f"{head_mem.total_allocations:,}", + delta, + ) + console.print(t3, justify="center") console.print() -def _fmt_ms(ns: Optional[int]) -> str: +# --- Formatting helpers --- + + +def fmt_time(ns: Optional[float]) -> str: + if ns is None: + return "-" + us = ns / 1_000 + if us >= 1_000_000: + return f"{us / 1_000_000:,.1f}s" + if us >= 1_000: + return f"{us / 1_000:,.1f}ms" + if us >= 1: + return f"{us:,.1f}μs" + return f"{ns:,.1f}ns" + + +def fmt_us(ns: Optional[float]) -> str: if ns is None: return "-" - ms = ns / 1_000_000 - if ms >= 1000: - return f"{ms:,.0f}" - if ms >= 100: - return f"{ms:.0f}" - if ms >= 1: - return f"{ms:.1f}" - return f"{ms:.2f}" + return f"{ns / 1_000:,.2f}μs" + + +def fmt_ops(mean_ns: Optional[float]) -> str: + if mean_ns is None or mean_ns == 0: + return "-" + ops = 1e9 / mean_ns + if ops >= 1_000_000: + return f"{ops / 1_000_000:,.2f} Mops/s" + if ops >= 1_000: + return f"{ops / 1_000:,.2f} Kops/s" + return f"{ops:,.2f} ops/s" + + +def md_ops(mean_ns: Optional[float]) -> str: + if mean_ns is None or mean_ns == 0: + return "-" + ops = 1e9 / mean_ns + if ops >= 1_000_000: + return f"{ops / 1_000_000:,.2f} Mops/s" + if ops >= 1_000: + return f"{ops / 1_000:,.2f} Kops/s" + return f"{ops:,.2f} ops/s" -def _fmt_speedup(before: Optional[int], after: Optional[int]) -> str: +def fmt_speedup_ops(before: Optional[float], after: Optional[float]) -> str: + if before is None or after is None or before == 0: + return "-" + ratio = before / after + if ratio >= 1: + return f"[green]{ratio:.2f}x[/green]" + return f"[red]{ratio:.2f}x[/red]" + + +def fmt_speedup(before: Optional[float], after: Optional[float]) -> str: if before is None or after is None or after == 0: return "-" ratio = before / after @@ -558,17 +801,16 @@ def _fmt_speedup(before: Optional[int], after: Optional[int]) -> str: return f"[red]{ratio:.2f}x[/red]" -def _fmt_delta(before: Optional[int], after: Optional[int]) -> str: +def fmt_delta(before: Optional[float], after: Optional[float]) -> str: if before is None or after is None: return "-" - delta_ms = (after - before) / 1_000_000 pct = ((after - before) / before) * 100 if before != 0 else 0 - if delta_ms < 0: - return f"[green]{delta_ms:+,.0f}ms ({pct:+.0f}%)[/green]" - return f"[red]{delta_ms:+,.0f}ms ({pct:+.0f}%)[/red]" + if pct < 0: + return _GREEN_TPL % pct + return _RED_TPL % pct -def _md_speedup(before: Optional[int], after: Optional[int]) -> str: +def md_speedup(before: Optional[float], after: Optional[float]) -> str: if before is None or after is None or after == 0: return "-" ratio = before / after @@ -576,22 +818,15 @@ def _md_speedup(before: Optional[int], after: Optional[int]) -> str: return f"{emoji} {ratio:.2f}x" -def _md_delta(before: Optional[int], after: Optional[int]) -> str: - if before is None or after is None: +def md_speedup_val(before: float, after: float) -> str: + if after == 0: return "-" - delta_ms = (after - before) / 1_000_000 - pct = ((after - before) / before) * 100 if before != 0 else 0 - if delta_ms < 0: - return f"{delta_ms:+,.0f}ms ({pct:+.0f}%)" - return f"+{delta_ms:,.0f}ms ({pct:+.0f}%)" - + ratio = before / after + emoji = "\U0001f7e2" if ratio >= 1 else "\U0001f534" + return f"{emoji} {ratio:.2f}x" -def _md_bar(before: Optional[int], after: Optional[int], width: int = 10) -> str: - """Render a unicode progress bar showing the change from before to after. - Improvement (after < before) shows green filled portion for the reduction. - Regression (after > before) shows the bar in reverse. - """ +def md_bar(before: Optional[float], after: Optional[float], width: int = 10) -> str: if before is None or after is None or before == 0: return "-" pct = ((before - after) / before) * 100 @@ -601,9 +836,347 @@ def _md_bar(before: Optional[int], after: Optional[int], width: int = 10) -> str return f"`{bar}` {pct:+.0f}%" -def _pct_bar(pct: float, width: int = 10) -> str: - """Render a unicode bar representing a percentage share.""" - filled = round(pct / 100 * width) - filled = max(0, min(filled, width)) - bar = "\u2588" * filled + "\u2591" * (width - filled) - return f"`{bar}` {pct:.1f}%" +def fmt_bytes(b: Optional[int]) -> str: + if b is None: + return "-" + if b >= 1 << 30: + return f"{b / (1 << 30):,.1f} GiB" + if b >= 1 << 20: + return f"{b / (1 << 20):,.1f} MiB" + if b >= 1 << 10: + return f"{b / (1 << 10):,.1f} KiB" + return f"{b:,} B" + + +def fmt_memory_delta(before: Optional[int], after: Optional[int]) -> str: + if before is None or after is None or before == 0: + return "-" + pct = ((after - before) / before) * 100 + if pct < 0: + return _GREEN_TPL % pct + return _RED_TPL % pct + + +def md_bytes(b: Optional[int]) -> str: + if b is None: + return "-" + if b >= 1 << 30: + return f"{b / (1 << 30):,.1f} GiB" + if b >= 1 << 20: + return f"{b / (1 << 20):,.1f} MiB" + if b >= 1 << 10: + return f"{b / (1 << 10):,.1f} KiB" + return f"{b:,} B" + + +def md_memory_delta(before: Optional[int], after: Optional[int]) -> str: + if before is None or after is None or before == 0: + return "-" + pct = ((after - before) / before) * 100 + emoji = "\U0001f7e2" if pct <= 0 else "\U0001f534" + return f"{emoji} {pct:+.0f}%" + + +def has_meaningful_memory_change( + base_mem: Optional[MemoryStats], head_mem: Optional[MemoryStats], threshold_pct: float = 1.0 +) -> bool: + """Return True if peak memory or allocation count changed by more than threshold_pct.""" + if base_mem is None or head_mem is None: + return base_mem is not None or head_mem is not None + if base_mem.peak_memory_bytes == 0 and head_mem.peak_memory_bytes == 0: + return False + if base_mem.peak_memory_bytes > 0: + mem_pct = abs((head_mem.peak_memory_bytes - base_mem.peak_memory_bytes) / base_mem.peak_memory_bytes) * 100 + if mem_pct > threshold_pct: + return True + if base_mem.total_allocations > 0: + alloc_pct = abs((head_mem.total_allocations - base_mem.total_allocations) / base_mem.total_allocations) * 100 + if alloc_pct > threshold_pct: + return True + return False + + +# --- Script-mode comparison --- + + +def _fmt_seconds(s: Optional[float]) -> str: + if s is None: + return "-" + if s >= 60: + return f"{s / 60:,.1f}m" + return f"{s:,.2f}s" + + +def _fmt_delta_s(before: Optional[float], after: Optional[float]) -> str: + if before is None or after is None: + return "-" + pct = ((after - before) / before) * 100 if before != 0 else 0 + if pct < 0: + return _GREEN_TPL % pct + return _RED_TPL % pct + + +def _md_delta_s(before: Optional[float], after: Optional[float]) -> str: + if before is None or after is None or before == 0: + return "-" + pct = ((after - before) / before) * 100 + emoji = "\U0001f7e2" if pct <= 0 else "\U0001f534" + return f"{emoji} {pct:+.1f}%" + + +def _speedup_s(before: Optional[float], after: Optional[float]) -> str: + if before is None or after is None or after == 0: + return "-" + ratio = before / after + if ratio >= 1: + return f"[green]{ratio:.2f}x[/green]" + return f"[red]{ratio:.2f}x[/red]" + + +def compare_with_script( + base_ref: str, + head_ref: str, + project_root: Path, + script_cmd: str, + script_output: str, + timeout: int = 600, + memory: bool = False, +) -> ScriptCompareResult: + """Compare benchmark performance between two git refs using a custom script. + + The script is run in each worktree with CWD set to the worktree root. + It must produce a JSON file at script_output (relative to worktree root) + mapping keys to seconds, e.g. {"test1": 1.23, "__total__": 4.56}. + """ + import sys + + if memory and sys.platform == "win32": + logger.error("--memory requires memray which is not available on Windows") + return ScriptCompareResult(base_ref=base_ref, head_ref=head_ref) + + repo = git.Repo(project_root, search_parent_directories=True) + + from codeflash.code_utils.git_worktree_utils import worktree_dirs + + worktree_dirs.mkdir(parents=True, exist_ok=True) + timestamp = time.strftime("%Y%m%d-%H%M%S") + + base_worktree = worktree_dirs / f"compare-base-{timestamp}" + head_worktree = worktree_dirs / f"compare-head-{timestamp}" + base_memray_bin = worktree_dirs / f"script-memray-base-{timestamp}.bin" + head_memray_bin = worktree_dirs / f"script-memray-head-{timestamp}.bin" + + result = ScriptCompareResult(base_ref=base_ref, head_ref=head_ref) + + from rich.console import Group + from rich.live import Live + from rich.panel import Panel + from rich.text import Text + + base_short = base_ref[:12] + head_short = head_ref[:12] + + step_labels = [ + "Creating worktrees", + f"Running benchmark on base ({base_short})", + f"Running benchmark on head ({head_short})", + ] + + def build_steps(current_step: int) -> Group: + lines: list[Text] = [] + for i, label in enumerate(step_labels): + if i < current_step: + lines.append(Text.from_markup(f"[green]\u2714[/green] {label}")) + elif i == current_step: + lines.append(Text.from_markup(f"[cyan]\u25cb[/cyan] {label}...")) + else: + lines.append(Text.from_markup(f"[dim]\u2500 {label}[/dim]")) + return Group(*lines) + + def build_panel(current_step: int) -> Panel: + return Panel( + Group( + Text.from_markup( + f"[bold cyan]{base_short}[/bold cyan] (base) vs [bold cyan]{head_short}[/bold cyan] (head)" + ), + "", + Text.from_markup(f"[dim]Script:[/dim] {script_cmd}"), + "", + build_steps(current_step), + ), + title="[bold]Script Benchmark Compare[/bold]", + border_style="cyan", + expand=True, + padding=(1, 2), + ) + + try: + step = 0 + with Live(build_panel(step), console=console, refresh_per_second=1) as live: + base_sha = repo.commit(base_ref).hexsha + head_sha = repo.commit(head_ref).hexsha + repo.git.worktree("add", str(base_worktree), base_sha) + repo.git.worktree("add", str(head_worktree), head_sha) + step += 1 + live.update(build_panel(step)) + + # Run script on base + result.base_results = _run_script_in_worktree( + script_cmd, base_worktree, script_output, timeout, base_memray_bin if memory else None + ) + step += 1 + live.update(build_panel(step)) + + # Run script on head + result.head_results = _run_script_in_worktree( + script_cmd, head_worktree, script_output, timeout, head_memray_bin if memory else None + ) + + # Parse memory results + if memory: + result.base_memory = _parse_memray_bin(base_memray_bin) + result.head_memory = _parse_memray_bin(head_memray_bin) + + render_script_comparison(result) + + except KeyboardInterrupt: + console.print("\n[yellow]Interrupted — cleaning up...[/yellow]") + + finally: + from codeflash.code_utils.git_worktree_utils import remove_worktree + + remove_worktree(base_worktree) + remove_worktree(head_worktree) + repo.git.worktree("prune") + for f in [base_memray_bin, head_memray_bin]: + if f.exists(): + f.unlink() + + return result + + +def _run_script_in_worktree( + script_cmd: str, worktree_dir: Path, script_output: str, timeout: int, memray_bin: Optional[Path] +) -> dict[str, float]: + import json + + cmd = script_cmd + if memray_bin: + cmd = f"python -m memray run --trace-python-allocators -o {memray_bin} -- {cmd}" + + try: + proc = subprocess.run( # noqa: S602 + cmd, shell=True, cwd=worktree_dir, timeout=timeout, capture_output=True, text=True, check=False + ) + if proc.returncode != 0: + logger.warning(f"Script exited with code {proc.returncode}") + if proc.stderr: + logger.debug(f"Script stderr:\n{proc.stderr[:2000]}") + except subprocess.TimeoutExpired: + logger.warning(f"Script timed out after {timeout}s") + return {} + + output_path = worktree_dir / script_output + if not output_path.exists(): + logger.warning(f"Script output not found at {output_path}") + return {} + + try: + data = json.loads(output_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + logger.warning("Script output JSON is not a dict") + return {} + return {k: float(v) for k, v in data.items() if isinstance(v, (int, float))} + except (json.JSONDecodeError, ValueError) as e: + logger.warning(f"Failed to parse script output JSON: {e}") + return {} + + +def _parse_memray_bin(bin_path: Path) -> Optional[MemoryStats]: + if not bin_path.exists(): + return None + try: + from memray import FileReader + + from codeflash.benchmarking.plugin.plugin import MemoryStats + + reader = FileReader(str(bin_path)) + meta = reader.metadata + stats = MemoryStats(peak_memory_bytes=meta.peak_memory, total_allocations=meta.total_allocations) + reader.close() + return stats + except ImportError: + logger.warning("memray not installed — skipping memory results") + return None + except OSError as e: + logger.warning(f"Failed to read memray binary: {e}") + return None + + +def render_script_comparison(result: ScriptCompareResult) -> None: + has_timing = result.base_results or result.head_results + has_memory = result.base_memory or result.head_memory + if not has_timing and not has_memory: + logger.warning("No benchmark results to compare") + return + + base_short = result.base_ref[:12] + head_short = result.head_ref[:12] + + console.print() + console.rule(f"[bold]Script Benchmark: {base_short} vs {head_short}[/bold]") + console.print() + + if has_timing: + all_keys = sorted((set(result.base_results) | set(result.head_results)) - {"__total__"}) + has_total = "__total__" in result.base_results or "__total__" in result.head_results + + t = Table(title="Benchmark Results", border_style="blue", show_lines=True, expand=False) + t.add_column("Key", style="cyan") + t.add_column("Base", justify="right", style="yellow") + t.add_column("Head", justify="right", style="yellow") + t.add_column("Delta", justify="right") + t.add_column("Speedup", justify="right") + + for key in all_keys: + b = result.base_results.get(key) + h = result.head_results.get(key) + t.add_row(key, _fmt_seconds(b), _fmt_seconds(h), _fmt_delta_s(b, h), _speedup_s(b, h)) + + if has_total: + t.add_section() + b = result.base_results.get("__total__") + h = result.head_results.get("__total__") + t.add_row("[bold]TOTAL[/bold]", _fmt_seconds(b), _fmt_seconds(h), _fmt_delta_s(b, h), _speedup_s(b, h)) + + console.print(t, justify="center") + + if has_memory: + console.print() + t_mem = Table(title="Memory (aggregate)", border_style="magenta", show_lines=True, expand=False) + t_mem.add_column("Ref", style="bold cyan") + t_mem.add_column("Peak Memory", justify="right") + t_mem.add_column("Allocations", justify="right") + t_mem.add_column("Delta", justify="right") + + if result.base_memory: + t_mem.add_row( + f"{base_short} (base)", + fmt_bytes(result.base_memory.peak_memory_bytes), + f"{result.base_memory.total_allocations:,}", + "", + ) + if result.head_memory: + delta = fmt_memory_delta( + result.base_memory.peak_memory_bytes if result.base_memory else None, + result.head_memory.peak_memory_bytes, + ) + t_mem.add_row( + f"{head_short} (head)", + fmt_bytes(result.head_memory.peak_memory_bytes), + f"{result.head_memory.total_allocations:,}", + delta, + ) + console.print(t_mem, justify="center") + + console.print() diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index 995e53c21..686710089 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -1,10 +1,14 @@ from __future__ import annotations +import gc import importlib.util import os import sqlite3 +import statistics import sys import time +from dataclasses import dataclass +from math import ceil from pathlib import Path from typing import TYPE_CHECKING @@ -18,6 +22,96 @@ PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None +# Calibration defaults (matching pytest-benchmark) +MIN_TIME = 0.000005 # 5µs — minimum time per round during calibration +MAX_TIME = 1.0 # 1s — maximum wall-clock time per test +MIN_ROUNDS = 5 +CALIBRATION_PRECISION = 10 + + +@dataclass +class BenchmarkStats: + min_ns: float + max_ns: float + mean_ns: float + median_ns: float + stddev_ns: float + iqr_ns: float + rounds: int + iterations: int + outliers: str + + @staticmethod + def from_per_iteration_times(times_ns: list[float], iterations: int) -> BenchmarkStats: + n = len(times_ns) + sorted_times = sorted(times_ns) + q1 = sorted_times[n // 4] if n >= 4 else sorted_times[0] + q3 = sorted_times[3 * n // 4] if n >= 4 else sorted_times[-1] + iqr = q3 - q1 + low_fence = q1 - 1.5 * iqr + high_fence = q3 + 1.5 * iqr + mild_outliers = sum(1 for t in times_ns if t < low_fence or t > high_fence) + severe_fence_low = q1 - 3.0 * iqr + severe_fence_high = q3 + 3.0 * iqr + severe_outliers = sum(1 for t in times_ns if t < severe_fence_low or t > severe_fence_high) + + return BenchmarkStats( + min_ns=min(times_ns), + max_ns=max(times_ns), + mean_ns=statistics.mean(times_ns), + median_ns=statistics.median(times_ns), + stddev_ns=statistics.stdev(times_ns) if n > 1 else 0.0, + iqr_ns=iqr, + rounds=n, + iterations=iterations, + outliers=f"{severe_outliers};{mild_outliers}", + ) + + +@dataclass +class MemoryStats: + peak_memory_bytes: int + total_allocations: int + + @staticmethod + def parse_memray_results(bin_dir: Path, bin_prefix: str) -> dict: + from codeflash.models.models import BenchmarkKey + + try: + from memray import FileReader + except ImportError as e: + msg = "memray is required for --memory profiling. Install with: uv add memray pytest-memray" + raise ImportError(msg) from e + + results: dict[BenchmarkKey, MemoryStats] = {} + for bin_file in sorted(bin_dir.glob(f"{bin_prefix}-*.bin")): + stem = bin_file.stem + # pytest-memray names: {prefix}-{nodeid with :: and os.sep replaced by -}.bin + nodeid_part = stem[len(bin_prefix) + 1 :] # strip "{prefix}-" + # Extract the test function name (last segment after the final -) + # Node IDs look like: tests-benchmarks-test_file.py-test_func_name + # We need the module_path and function_name for BenchmarkKey + # Split on ".py-" to separate module path from function name + parts = nodeid_part.split(".py-", 1) + if len(parts) == 2: + module_part = parts[0].replace("-", ".") + function_name = parts[1] + else: + module_part = nodeid_part.rsplit("-", 1)[0].replace("-", ".") + function_name = nodeid_part.rsplit("-", 1)[-1] if "-" in nodeid_part else nodeid_part + + try: + reader = FileReader(str(bin_file)) + meta = reader.metadata + bm_key = BenchmarkKey(module_path=module_part, function_name=function_name) + results[bm_key] = MemoryStats( + peak_memory_bytes=meta.peak_memory, total_allocations=meta.total_allocations + ) + reader.close() + except OSError: + continue + return results + class CodeFlashBenchmarkPlugin: def __init__(self) -> None: @@ -28,7 +122,6 @@ def __init__(self) -> None: def setup(self, trace_path: str, project_root: str) -> None: try: - # Open connection self.project_root = project_root self._trace_path = trace_path self._connection = sqlite3.connect(self._trace_path) @@ -38,10 +131,10 @@ def setup(self, trace_path: str, project_root: str) -> None: cur.execute( "CREATE TABLE IF NOT EXISTS benchmark_timings(" "benchmark_module_path TEXT, benchmark_function_name TEXT, benchmark_line_number INTEGER," - "benchmark_time_ns INTEGER)" + "round_index INTEGER, iterations INTEGER, round_time_ns INTEGER)" ) self._connection.commit() - self.close() # Reopen only at the end of pytest session + self.close() except Exception as e: print(f"Database setup error: {e}") if self._connection: @@ -51,20 +144,21 @@ def setup(self, trace_path: str, project_root: str) -> None: def write_benchmark_timings(self) -> None: if not self.benchmark_timings: - return # No data to write + return if self._connection is None: self._connection = sqlite3.connect(self._trace_path) try: cur = self._connection.cursor() - # Insert data into the benchmark_timings table cur.executemany( - "INSERT INTO benchmark_timings (benchmark_module_path, benchmark_function_name, benchmark_line_number, benchmark_time_ns) VALUES (?, ?, ?, ?)", + "INSERT INTO benchmark_timings " + "(benchmark_module_path, benchmark_function_name, benchmark_line_number, " + "round_index, iterations, round_time_ns) VALUES (?, ?, ?, ?, ?, ?)", self.benchmark_timings, ) self._connection.commit() - self.benchmark_timings = [] # Clear the benchmark timings list + self.benchmark_timings = [] except Exception as e: print(f"Error writing to benchmark timings database: {e}") self._connection.rollback() @@ -76,124 +170,107 @@ def close(self) -> None: self._connection = None @staticmethod - def get_function_benchmark_timings(trace_path: Path) -> dict[str, dict[BenchmarkKey, int]]: + def get_function_benchmark_timings(trace_path: Path) -> dict[str, dict[BenchmarkKey, float]]: from codeflash.models.models import BenchmarkKey - """Process the trace file and extract timing data for all functions. - - Args: - ---- - trace_path: Path to the trace file - - Returns: - ------- - A nested dictionary where: - - Outer keys are module_name.qualified_name (module.class.function) - - Inner keys are of type BenchmarkKey - - Values are function timing in milliseconds - - """ - # Initialize the result dictionary - result = {} - - # Connect to the SQLite database + result: dict[str, dict[BenchmarkKey, float]] = {} connection = sqlite3.connect(trace_path) cursor = connection.cursor() try: - # Query the function_calls table for all function calls + # Get total iterations per benchmark to normalize + cursor.execute( + "SELECT benchmark_module_path, benchmark_function_name, " + "SUM(iterations) FROM benchmark_timings " + "GROUP BY benchmark_module_path, benchmark_function_name" + ) + total_iterations: dict[BenchmarkKey, int] = {} + for row in cursor.fetchall(): + bm_file, bm_func, total_iters = row + key = BenchmarkKey(module_path=bm_file, function_name=bm_func) + total_iterations[key] = total_iters + cursor.execute( "SELECT module_name, class_name, function_name, " "benchmark_module_path, benchmark_function_name, benchmark_line_number, function_time_ns " "FROM benchmark_function_timings" ) - # Process each row + # Accumulate total function time + raw_totals: dict[str, dict[BenchmarkKey, int]] = {} for row in cursor.fetchall(): module_name, class_name, function_name, benchmark_file, benchmark_func, _benchmark_line, time_ns = row - - # Create the function key (module_name.class_name.function_name) if class_name: qualified_name = f"{module_name}.{class_name}.{function_name}" else: qualified_name = f"{module_name}.{function_name}" - - # Create the benchmark key (file::function::line) benchmark_key = BenchmarkKey(module_path=benchmark_file, function_name=benchmark_func) - # Initialize the inner dictionary if needed - if qualified_name not in result: - result[qualified_name] = {} - - # If multiple calls to the same function in the same benchmark, - # add the times together - if benchmark_key in result[qualified_name]: - result[qualified_name][benchmark_key] += time_ns - else: - result[qualified_name][benchmark_key] = time_ns + if qualified_name not in raw_totals: + raw_totals[qualified_name] = {} + raw_totals[qualified_name][benchmark_key] = raw_totals[qualified_name].get(benchmark_key, 0) + time_ns + + # Normalize to per-iteration average + for qualified_name, bm_dict in raw_totals.items(): + result[qualified_name] = {} + for bm_key, total_ns in bm_dict.items(): + iters = total_iterations.get(bm_key, 1) + result[qualified_name][bm_key] = total_ns / iters finally: - # Close the connection connection.close() return result @staticmethod - def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, int]: + def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, BenchmarkStats]: from codeflash.models.models import BenchmarkKey - """Extract total benchmark timings from trace files. - - Args: - ---- - trace_path: Path to the trace file - - Returns: - ------- - A dictionary mapping where: - - Keys are of type BenchmarkKey - - Values are total benchmark timing in milliseconds (with overhead subtracted) - - """ - # Initialize the result dictionary - result = {} - overhead_by_benchmark = {} - - # Connect to the SQLite database connection = sqlite3.connect(trace_path) cursor = connection.cursor() try: - # Query the benchmark_function_timings table to get total overhead for each benchmark + # Get overhead per benchmark to subtract cursor.execute( "SELECT benchmark_module_path, benchmark_function_name, benchmark_line_number, SUM(overhead_time_ns) " "FROM benchmark_function_timings " "GROUP BY benchmark_module_path, benchmark_function_name, benchmark_line_number" ) - - # Process overhead information + overhead_by_benchmark: dict[BenchmarkKey, int] = {} for row in cursor.fetchall(): - benchmark_file, benchmark_func, _benchmark_line, total_overhead_ns = row - benchmark_key = BenchmarkKey(module_path=benchmark_file, function_name=benchmark_func) - overhead_by_benchmark[benchmark_key] = total_overhead_ns or 0 # Handle NULL sum case + bm_file, bm_func, _bm_line, total_overhead_ns = row + key = BenchmarkKey(module_path=bm_file, function_name=bm_func) + overhead_by_benchmark[key] = total_overhead_ns or 0 - # Query the benchmark_timings table for total times + # Get per-round data cursor.execute( - "SELECT benchmark_module_path, benchmark_function_name, benchmark_line_number, benchmark_time_ns " - "FROM benchmark_timings" + "SELECT benchmark_module_path, benchmark_function_name, benchmark_line_number, " + "round_index, iterations, round_time_ns " + "FROM benchmark_timings ORDER BY round_index" ) - # Process each row and subtract overhead + rounds_data: dict[BenchmarkKey, list[tuple[int, int]]] = {} for row in cursor.fetchall(): - benchmark_file, benchmark_func, _benchmark_line, time_ns = row - - # Create the benchmark key (file::function::line) - benchmark_key = BenchmarkKey(module_path=benchmark_file, function_name=benchmark_func) - # Subtract overhead from total time - overhead = overhead_by_benchmark.get(benchmark_key, 0) - result[benchmark_key] = time_ns - overhead + bm_file, bm_func, _bm_line, _round_idx, iterations, round_time_ns = row + key = BenchmarkKey(module_path=bm_file, function_name=bm_func) + if key not in rounds_data: + rounds_data[key] = [] + rounds_data[key].append((iterations, round_time_ns)) + + result: dict[BenchmarkKey, BenchmarkStats] = {} + for bm_key, rounds in rounds_data.items(): + total_overhead = overhead_by_benchmark.get(bm_key, 0) + total_rounds = len(rounds) + overhead_per_round = total_overhead / total_rounds if total_rounds > 0 else 0 + iterations = rounds[0][0] # All rounds have same iteration count + + per_iteration_times = [] + for iters, round_time_ns in rounds: + adjusted = max(0, round_time_ns - overhead_per_round) + per_iteration_times.append(adjusted / iters) + + result[bm_key] = BenchmarkStats.from_per_iteration_times(per_iteration_times, iterations) finally: - # Close the connection connection.close() return result @@ -201,56 +278,42 @@ def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, int]: # Pytest hooks @pytest.hookimpl def pytest_sessionfinish(self, session, exitstatus) -> None: - """Execute after whole test run is completed.""" - # Write any remaining benchmark timings to the database codeflash_trace.close() if self.benchmark_timings: self.write_benchmark_timings() - # Close the database connection self.close() @staticmethod def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - # Skip tests that don't have the benchmark fixture if not config.getoption("--codeflash-trace"): return skip_no_benchmark = pytest.mark.skip(reason="Test requires benchmark fixture") for item in items: - # Check for direct benchmark fixture usage has_fixture = hasattr(item, "fixturenames") and "benchmark" in item.fixturenames # ty:ignore[unsupported-operator] - - # Check for @pytest.mark.benchmark marker has_marker = False if hasattr(item, "get_closest_marker"): marker = item.get_closest_marker("benchmark") if marker is not None: has_marker = True - - # Skip if neither fixture nor marker is present if not (has_fixture or has_marker): item.add_marker(skip_no_benchmark) - # Benchmark fixture class Benchmark: # noqa: D106 def __init__(self, request: pytest.FixtureRequest) -> None: self.request = request def __call__(self, func, *args, **kwargs): # noqa: ANN002, ANN003, ANN204 - """Handle both direct function calls and decorator usage.""" if args or kwargs: - # Used as benchmark(func, *args, **kwargs) - return self._run_benchmark(func, *args, **kwargs) + return self.run_benchmark(func, *args, **kwargs) - # Used as @benchmark decorator def wrapped_func(*args, **kwargs): # noqa: ANN002, ANN003 return func(*args, **kwargs) - self._run_benchmark(func) + self.run_benchmark(func) return wrapped_func - def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN002, ANN003 - """Actual benchmark implementation.""" + def run_benchmark(self, func, *args, **kwargs): # noqa: ANN002, ANN003, ANN201 node_path = getattr(self.request.node, "path", None) or getattr(self.request.node, "fspath", None) if node_path is None: raise RuntimeError("Unable to determine test file path from pytest node") @@ -258,31 +321,87 @@ def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN002, ANN003 benchmark_module_path = module_name_from_file_path( Path(str(node_path)), Path(codeflash_benchmark_plugin.project_root), traverse_up=True ) - benchmark_function_name = self.request.node.name - line_number = int(str(sys._getframe(2).f_lineno)) # 2 frames up in the call stack # noqa: SLF001 - # Set env vars + line_number = int(str(sys._getframe(2).f_lineno)) # noqa: SLF001 + os.environ["CODEFLASH_BENCHMARK_FUNCTION_NAME"] = benchmark_function_name os.environ["CODEFLASH_BENCHMARK_MODULE_PATH"] = benchmark_module_path os.environ["CODEFLASH_BENCHMARK_LINE_NUMBER"] = str(line_number) + + # Phase 1: Calibrate (tracing disabled to avoid overhead) + os.environ["CODEFLASH_BENCHMARKING"] = "False" + iterations, calibrated_duration = calibrate(func, args, kwargs) + + # Phase 2: Multi-round benchmark (tracing enabled) os.environ["CODEFLASH_BENCHMARKING"] = "True" - # Run the function + rounds = max(MIN_ROUNDS, ceil(MAX_TIME / calibrated_duration)) if calibrated_duration > 0 else MIN_ROUNDS + + result = None + for round_idx in range(rounds): + gc_was_enabled = gc.isenabled() + gc.disable() + try: + start = time.perf_counter_ns() + for _ in range(iterations): + result = func(*args, **kwargs) + end = time.perf_counter_ns() + finally: + if gc_was_enabled: + gc.enable() + + round_time = end - start + codeflash_benchmark_plugin.benchmark_timings.append( + (benchmark_module_path, benchmark_function_name, line_number, round_idx, iterations, round_time) + ) + + # Flush function timings per round + codeflash_trace.write_function_timings() + codeflash_trace.function_call_count = 0 + + os.environ["CODEFLASH_BENCHMARKING"] = "False" + return result + + +def compute_timer_precision() -> float: + minimum = float("inf") + for _ in range(20): + t1 = time.perf_counter_ns() + t2 = time.perf_counter_ns() + dt = t2 - t1 + if dt > 0: + minimum = min(minimum, dt) + return minimum / 1e9 # Convert to seconds + + +def calibrate(func, args, kwargs) -> tuple[int, float]: + timer_precision = compute_timer_precision() + min_time = max(MIN_TIME, timer_precision * CALIBRATION_PRECISION) + min_time_estimate = min_time * 5 / CALIBRATION_PRECISION + + iterations = 1 + while True: + gc_was_enabled = gc.isenabled() + gc.disable() + try: start = time.perf_counter_ns() - result = func(*args, **kwargs) + for _ in range(iterations): + func(*args, **kwargs) end = time.perf_counter_ns() - # Reset the environment variable - os.environ["CODEFLASH_BENCHMARKING"] = "False" + finally: + if gc_was_enabled: + gc.enable() - # Write function calls - codeflash_trace.write_function_timings() - # Reset function call count - codeflash_trace.function_call_count = 0 - # Add to the benchmark timings buffer - codeflash_benchmark_plugin.benchmark_timings.append( - (benchmark_module_path, benchmark_function_name, line_number, end - start) - ) + duration = (end - start) / 1e9 # Convert to seconds - return result + if duration >= min_time: + break + + if duration >= min_time_estimate: + iterations = ceil(min_time * iterations / duration) + else: + iterations *= 10 + + return iterations, duration codeflash_benchmark_plugin = CodeFlashBenchmarkPlugin() diff --git a/codeflash/benchmarking/pytest_new_process_memory_benchmarks.py b/codeflash/benchmarking/pytest_new_process_memory_benchmarks.py new file mode 100644 index 000000000..88fe14713 --- /dev/null +++ b/codeflash/benchmarking/pytest_new_process_memory_benchmarks.py @@ -0,0 +1,42 @@ +"""Subprocess entry point for memory profiling benchmarks via pytest-memray. + +Runs pytest with --memray --native to profile peak memory per test function. +The codeflash-benchmark plugin is left active (without --codeflash-trace) so it +provides a no-op ``benchmark`` fixture for tests that depend on it. +""" + +import sys +from pathlib import Path + +benchmarks_root = sys.argv[1] +memray_bin_dir = sys.argv[2] +memray_bin_prefix = sys.argv[3] + +if __name__ == "__main__": + import pytest + + Path(memray_bin_dir).mkdir(parents=True, exist_ok=True) + + exitcode = pytest.main( + [ + benchmarks_root, + "--memray", + "--native", + f"--memray-bin-path={memray_bin_dir}", + f"--memray-bin-prefix={memray_bin_prefix}", + "--hide-memray-summary", + "-p", + "no:benchmark", + "-p", + "no:codspeed", + "-p", + "no:cov", + "-p", + "no:profiling", + "-s", + "-o", + "addopts=", + ] + ) + + sys.exit(exitcode) diff --git a/codeflash/benchmarking/trace_benchmarks.py b/codeflash/benchmarking/trace_benchmarks.py index 98b8e0540..ff0bfbaf8 100644 --- a/codeflash/benchmarking/trace_benchmarks.py +++ b/codeflash/benchmarking/trace_benchmarks.py @@ -46,3 +46,39 @@ def trace_benchmarks_pytest( error_section = combined_output logger.warning(f"Error collecting benchmarks - Pytest Exit code: {result.returncode}, {error_section}") logger.debug(f"Full pytest output:\n{combined_output}") + + +def memory_benchmarks_pytest( + benchmarks_root: Path, project_root: Path, memray_bin_dir: Path, memray_bin_prefix: str, timeout: int = 300 +) -> None: + benchmark_env = make_env_with_project_root(project_root) + run_args = get_cross_platform_subprocess_run_args( + cwd=project_root, env=benchmark_env, timeout=timeout, check=False, text=True, capture_output=True + ) + result = subprocess.run( # noqa: PLW1510 + [ + SAFE_SYS_EXECUTABLE, + Path(__file__).parent / "pytest_new_process_memory_benchmarks.py", + benchmarks_root, + memray_bin_dir, + memray_bin_prefix, + ], + **run_args, + ) + if result.returncode != 0: + combined_output = result.stdout + if result.stderr: + combined_output = combined_output + "\n" + result.stderr if combined_output else result.stderr + + if "ERROR collecting" in combined_output: + error_pattern = r"={3,}\s*ERRORS\s*={3,}\n([\s\S]*?)(?:={3,}|$)" + match = re.search(error_pattern, combined_output) + error_section = match.group(1) if match else combined_output + elif "FAILURES" in combined_output: + error_pattern = r"={3,}\s*FAILURES\s*={3,}\n([\s\S]*?)(?:={3,}|$)" + match = re.search(error_pattern, combined_output) + error_section = match.group(1) if match else combined_output + else: + error_section = combined_output + logger.warning(f"Error collecting memory benchmarks - Pytest Exit code: {result.returncode}, {error_section}") + logger.debug(f"Full pytest output:\n{combined_output}") diff --git a/codeflash/benchmarking/utils.py b/codeflash/benchmarking/utils.py index db89c4c33..b23b7cc40 100644 --- a/codeflash/benchmarking/utils.py +++ b/codeflash/benchmarking/utils.py @@ -1,6 +1,8 @@ from __future__ import annotations +import logging import shutil +from operator import itemgetter from typing import TYPE_CHECKING, Optional from rich.console import Console @@ -16,27 +18,30 @@ def validate_and_format_benchmark_table( - function_benchmark_timings: dict[str, dict[BenchmarkKey, int]], total_benchmark_timings: dict[BenchmarkKey, int] + function_benchmark_timings: dict[str, dict[BenchmarkKey, float]], total_benchmark_timings: dict[BenchmarkKey, float] ) -> dict[str, list[tuple[BenchmarkKey, float, float, float]]]: function_to_result = {} - # Process each function's benchmark data + scale = 1_000_000.0 for func_path, test_times in function_benchmark_timings.items(): # Sort by percentage (highest first) sorted_tests = [] for benchmark_key, func_time in test_times.items(): total_time = total_benchmark_timings.get(benchmark_key, 0) if func_time > total_time: - logger.debug(f"Skipping test {benchmark_key} due to func_time {func_time} > total_time {total_time}") # If the function time is greater than total time, likely to have multithreading / multiprocessing issues. # Do not try to project the optimization impact for this function. + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + f"Skipping test {benchmark_key} due to func_time {func_time} > total_time {total_time}" + ) sorted_tests.append((benchmark_key, 0.0, 0.0, 0.0)) elif total_time > 0: percentage = (func_time / total_time) * 100 # Convert nanoseconds to milliseconds - func_time_ms = func_time / 1_000_000 - total_time_ms = total_time / 1_000_000 + func_time_ms = func_time / scale + total_time_ms = total_time / scale sorted_tests.append((benchmark_key, total_time_ms, func_time_ms, percentage)) - sorted_tests.sort(key=lambda x: x[3], reverse=True) + sorted_tests.sort(key=itemgetter(3), reverse=True) function_to_result[func_path] = sorted_tests return function_to_result @@ -77,8 +82,8 @@ def print_benchmark_table(function_to_results: dict[str, list[tuple[BenchmarkKey def process_benchmark_data( replay_performance_gain: dict[BenchmarkKey, float], - fto_benchmark_timings: dict[BenchmarkKey, int], - total_benchmark_timings: dict[BenchmarkKey, int], + fto_benchmark_timings: dict[BenchmarkKey, float], + total_benchmark_timings: dict[BenchmarkKey, float], ) -> Optional[ProcessedBenchmarkInfo]: """Process benchmark data and generate detailed benchmark information. diff --git a/codeflash/cli_cmds/cli.py b/codeflash/cli_cmds/cli.py index 27876355b..cf5ca7bdd 100644 --- a/codeflash/cli_cmds/cli.py +++ b/codeflash/cli_cmds/cli.py @@ -382,13 +382,26 @@ def _build_parser() -> ArgumentParser: auth_subparsers.add_parser("status", help="Check authentication status") compare_parser = subparsers.add_parser("compare", help="Compare benchmark performance between two git refs.") - compare_parser.add_argument("base_ref", help="Base git ref (branch, tag, or commit)") + compare_parser.add_argument( + "base_ref", nargs="?", default=None, help="Base git ref (default: auto-detect from PR or default branch)" + ) compare_parser.add_argument("head_ref", nargs="?", default=None, help="Head git ref (default: current branch)") compare_parser.add_argument("--pr", type=int, help="Resolve head ref from a PR number (requires gh CLI)") compare_parser.add_argument( "--functions", type=str, help="Explicit functions to instrument: 'file.py::func1,func2;other.py::func3'" ) compare_parser.add_argument("--timeout", type=int, default=600, help="Benchmark timeout in seconds (default: 600)") + compare_parser.add_argument("--output", "-o", type=str, help="Write markdown report to file") + compare_parser.add_argument( + "--memory", action="store_true", help="Profile peak memory usage per benchmark (requires memray, Linux/macOS)" + ) + compare_parser.add_argument("--script", type=str, help="Shell command to run as benchmark in each worktree") + compare_parser.add_argument( + "--script-output", + type=str, + dest="script_output", + help="Relative path to JSON results file produced by --script (required with --script)", + ) compare_parser.add_argument("--config-file", type=str, dest="config_file", help="Path to pyproject.toml") trace_optimize = subparsers.add_parser("optimize", help="Trace and optimize your project.") diff --git a/codeflash/cli_cmds/cmd_compare.py b/codeflash/cli_cmds/cmd_compare.py index 2a20a4c4f..898af5679 100644 --- a/codeflash/cli_cmds/cmd_compare.py +++ b/codeflash/cli_cmds/cmd_compare.py @@ -13,15 +13,73 @@ from codeflash.models.function_types import FunctionToOptimize from codeflash.cli_cmds.console import logger -from codeflash.code_utils.config_parser import parse_config_file def run_compare(args: Namespace) -> None: """Entry point for the compare subcommand.""" - # Load project config - pyproject_config, pyproject_file_path = parse_config_file(args.config_file) + # Resolve head_ref: explicit arg > --pr > current branch + head_ref = args.head_ref + if args.pr: + head_ref = resolve_pr_branch(args.pr) + if not head_ref: + head_ref = get_current_branch() + if not head_ref: + logger.error("Must provide head_ref, --pr, or be on a branch") + sys.exit(1) + logger.info(f"Auto-detected head ref: {head_ref}") + + # Resolve base_ref: explicit arg > PR base branch > repo default branch + base_ref = args.base_ref + if not base_ref: + base_ref = detect_base_ref(head_ref) + if not base_ref: + logger.error("Could not auto-detect base ref. Provide it explicitly or ensure gh CLI is available.") + sys.exit(1) + logger.info(f"Auto-detected base ref: {base_ref}") + + # Script mode: run an arbitrary benchmark command on each worktree (no codeflash config needed) + script_cmd = getattr(args, "script", None) + if script_cmd: + script_output = getattr(args, "script_output", None) + if not script_output: + logger.error("--script-output is required when using --script") + sys.exit(1) + + import git + + project_root = Path(git.Repo(Path.cwd(), search_parent_directories=True).working_dir) + + from codeflash.benchmarking.compare import compare_with_script + + result = compare_with_script( + base_ref=base_ref, + head_ref=head_ref, + project_root=project_root, + script_cmd=script_cmd, + script_output=script_output, + timeout=args.timeout, + memory=getattr(args, "memory", False), + ) + + if not result.base_results and not result.head_results: + logger.warning("No benchmark data collected. Check that --script-output points to a valid JSON file.") + sys.exit(1) + if args.output: + md = result.format_markdown() + Path(args.output).write_text(md, encoding="utf-8") + logger.info(f"Markdown report written to {args.output}") + return + + # Standard trace-benchmark mode: requires codeflash config + from codeflash.code_utils.config_parser import parse_config_file + + pyproject_config, pyproject_file_path = parse_config_file(args.config_file) module_root = Path(pyproject_config.get("module_root", ".")).resolve() + + from codeflash.cli_cmds.cli import project_root_from_module_root + + project_root = project_root_from_module_root(module_root, pyproject_file_path) tests_root = Path(pyproject_config.get("tests_root", "tests")).resolve() benchmarks_root_str = pyproject_config.get("benchmarks_root") @@ -34,42 +92,89 @@ def run_compare(args: Namespace) -> None: logger.error(f"benchmarks-root {benchmarks_root} is not a valid directory") sys.exit(1) - from codeflash.cli_cmds.cli import project_root_from_module_root - - project_root = project_root_from_module_root(module_root, pyproject_file_path) - - # Resolve head_ref - head_ref = args.head_ref - if args.pr: - head_ref = _resolve_pr_branch(args.pr) - if not head_ref: - logger.error("Must provide head_ref or --pr") - sys.exit(1) - # Parse explicit functions if provided functions = None if args.functions: - functions = _parse_functions_arg(args.functions, project_root) + functions = parse_functions_arg(args.functions, project_root) from codeflash.benchmarking.compare import compare_branches result = compare_branches( - base_ref=args.base_ref, + base_ref=base_ref, head_ref=head_ref, project_root=project_root, benchmarks_root=benchmarks_root, tests_root=tests_root, functions=functions, timeout=args.timeout, + memory=getattr(args, "memory", False), ) - if not result.base_total_ns and not result.head_total_ns: + if not result.base_stats and not result.head_stats: logger.warning("No benchmark data collected. Check that benchmarks-root is configured and benchmarks exist.") sys.exit(1) + if args.output: + md = result.format_markdown() + Path(args.output).write_text(md, encoding="utf-8") + logger.info(f"Markdown report written to {args.output}") + + +def get_current_branch() -> str | None: + try: + result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], capture_output=True, text=True, check=True + ) + branch = result.stdout.strip() + return branch if branch and branch != "HEAD" else None + except (FileNotFoundError, subprocess.CalledProcessError): + return None + + +def detect_base_ref(head_ref: str) -> str | None: + # Try to find an open PR for this branch and use its base + try: + result = subprocess.run( + ["gh", "pr", "view", head_ref, "--json", "baseRefName", "-q", ".baseRefName"], + capture_output=True, + text=True, + check=True, + ) + base = result.stdout.strip() + if base: + return base + except (FileNotFoundError, subprocess.CalledProcessError): + pass + + # Fall back to repo default branch + try: + result = subprocess.run( + ["gh", "repo", "view", "--json", "defaultBranchRef", "-q", ".defaultBranchRef.name"], + capture_output=True, + text=True, + check=True, + ) + default = result.stdout.strip() + if default: + return default + except (FileNotFoundError, subprocess.CalledProcessError): + pass + + # Last resort: check for common default branch names + try: + for candidate in ("main", "master"): + result = subprocess.run( + ["git", "rev-parse", "--verify", candidate], capture_output=True, text=True, check=False + ) + if result.returncode == 0: + return candidate + except FileNotFoundError: + pass + + return None + -def _resolve_pr_branch(pr_number: int) -> str: - """Resolve a PR number to its head branch name using gh CLI.""" +def resolve_pr_branch(pr_number: int) -> str: try: result = subprocess.run( ["gh", "pr", "view", str(pr_number), "--json", "headRefName", "-q", ".headRefName"], @@ -91,7 +196,7 @@ def _resolve_pr_branch(pr_number: int) -> str: sys.exit(1) -def _parse_functions_arg(functions_str: str, project_root: Path) -> dict[Path, list[FunctionToOptimize]]: +def parse_functions_arg(functions_str: str, project_root: Path) -> dict[Path, list[FunctionToOptimize]]: """Parse --functions arg format: 'file.py::func1,func2;other.py::func3'.""" from codeflash.models.function_types import FunctionToOptimize diff --git a/codeflash/optimization/optimizer.py b/codeflash/optimization/optimizer.py index fef53a760..677f7f70a 100644 --- a/codeflash/optimization/optimizer.py +++ b/codeflash/optimization/optimizer.py @@ -126,7 +126,8 @@ def run_benchmarks( function_benchmark_timings = CodeFlashBenchmarkPlugin.get_function_benchmark_timings( self.trace_file ) - total_benchmark_timings = CodeFlashBenchmarkPlugin.get_benchmark_timings(self.trace_file) + total_benchmark_stats = CodeFlashBenchmarkPlugin.get_benchmark_timings(self.trace_file) + total_benchmark_timings = {k: v.median_ns for k, v in total_benchmark_stats.items()} function_to_results = validate_and_format_benchmark_table( function_benchmark_timings, total_benchmark_timings ) diff --git a/pyproject.toml b/pyproject.toml index f825d3739..38256ebfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,8 @@ dependencies = [ "filelock>=3.20.3; python_version >= '3.10'", "filelock<3.20.3; python_version < '3.10'", "pytest-asyncio>=0.18.0", + "memray>=1.12; sys_platform != 'win32'", + "pytest-memray>=1.7; sys_platform != 'win32'", ] [project.urls] @@ -339,8 +341,8 @@ vcs = "git" [tool.hatch.build.hooks.version] path = "codeflash/version.py" -template = """# These version placeholders will be replaced by uv-dynamic-versioning during build. -__version__ = "{version}" +template = """# These version placeholders will be replaced by uv-dynamic-versioning during build. +__version__ = "{version}" """ diff --git a/tests/test_compare.py b/tests/test_compare.py new file mode 100644 index 000000000..c51b959d9 --- /dev/null +++ b/tests/test_compare.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +from codeflash.benchmarking.compare import ( + CompareResult, + ScriptCompareResult, + has_meaningful_memory_change, + render_comparison, + render_script_comparison, +) +from codeflash.benchmarking.plugin.plugin import BenchmarkStats, MemoryStats +from codeflash.models.models import BenchmarkKey + + +def _make_stats(median_ns: float = 1000.0, rounds: int = 10) -> BenchmarkStats: + return BenchmarkStats( + min_ns=median_ns * 0.9, + max_ns=median_ns * 1.1, + mean_ns=median_ns, + median_ns=median_ns, + stddev_ns=median_ns * 0.05, + iqr_ns=median_ns * 0.1, + rounds=rounds, + iterations=100, + outliers="0;0", + ) + + +def _make_memory(peak: int = 4_194_304, allocs: int = 1000) -> MemoryStats: + return MemoryStats(peak_memory_bytes=peak, total_allocations=allocs) + + +BM_KEY = BenchmarkKey(module_path="tests.benchmarks.test_example", function_name="test_func") + + +class TestFormatMarkdownMemoryOnly: + def test_memory_only_no_timing_table(self) -> None: + result = CompareResult( + base_ref="abc123", + head_ref="def456", + base_memory={BM_KEY: _make_memory(peak=10_000_000, allocs=500)}, + head_memory={BM_KEY: _make_memory(peak=7_000_000, allocs=400)}, + ) + md = result.format_markdown() + + # Should have memory data + assert "Peak Memory" in md + assert "Allocations" in md + # Should NOT have timing table headers + assert "Min | Median | Mean | OPS" not in md + assert "Per-Function" not in md + + def test_memory_only_returns_empty_when_no_data(self) -> None: + result = CompareResult(base_ref="abc123", head_ref="def456") + md = result.format_markdown() + assert md == "_No benchmark results to compare._" + + def test_mixed_timing_and_memory(self) -> None: + result = CompareResult( + base_ref="abc123", + head_ref="def456", + base_stats={BM_KEY: _make_stats()}, + head_stats={BM_KEY: _make_stats(median_ns=500.0)}, + base_memory={BM_KEY: _make_memory(peak=10_000_000)}, + head_memory={BM_KEY: _make_memory(peak=5_000_000)}, + ) + md = result.format_markdown() + + # Should have both timing and memory + assert "Min | Median | Mean | OPS" in md + assert "Peak Memory" in md + + def test_memory_only_always_shows_memory(self) -> None: + """Memory-only keys always render the memory table, even if delta is <1%.""" + result = CompareResult( + base_ref="abc123", + head_ref="def456", + base_memory={BM_KEY: _make_memory(peak=10_000_000, allocs=1000)}, + head_memory={BM_KEY: _make_memory(peak=10_000_000, allocs=1000)}, + ) + md = result.format_markdown() + # Even with identical memory, memory-only keys always show the table + assert "Peak Memory" in md + + def test_timing_with_negligible_memory_suppressed(self) -> None: + """When timing data exists, negligible memory changes are suppressed.""" + result = CompareResult( + base_ref="abc123", + head_ref="def456", + base_stats={BM_KEY: _make_stats()}, + head_stats={BM_KEY: _make_stats()}, + base_memory={BM_KEY: _make_memory(peak=10_000_000, allocs=1000)}, + head_memory={BM_KEY: _make_memory(peak=10_000_000, allocs=1000)}, + ) + md = result.format_markdown() + # Timing table should be there + assert "Min | Median | Mean | OPS" in md + # Memory table should be suppressed (delta <1% and timing exists) + assert "Peak Memory" not in md + + def test_memory_only_key_mixed_with_timing_key(self) -> None: + """Some keys have timing, others are memory-only.""" + timing_key = BenchmarkKey(module_path="tests.bench", function_name="test_timing") + memory_key = BenchmarkKey(module_path="tests.bench", function_name="test_memory") + + result = CompareResult( + base_ref="abc123", + head_ref="def456", + base_stats={timing_key: _make_stats()}, + head_stats={timing_key: _make_stats(median_ns=500.0)}, + base_memory={timing_key: _make_memory(peak=10_000_000), memory_key: _make_memory(peak=8_000_000)}, + head_memory={timing_key: _make_memory(peak=5_000_000), memory_key: _make_memory(peak=6_000_000)}, + ) + md = result.format_markdown() + + # Both benchmark keys should appear + assert "test_timing" in md + assert "test_memory" in md + # Timing table for timing_key + assert "Min | Median | Mean | OPS" in md + + +class TestRenderComparisonMemoryOnly: + def test_memory_only_no_crash(self, capsys: object) -> None: + """render_comparison should not crash or warn with memory-only data.""" + result = CompareResult( + base_ref="abc123", + head_ref="def456", + base_memory={BM_KEY: _make_memory(peak=10_000_000)}, + head_memory={BM_KEY: _make_memory(peak=7_000_000)}, + ) + # Should not raise + render_comparison(result) + + def test_empty_result_warns(self) -> None: + result = CompareResult(base_ref="abc123", head_ref="def456") + # Should return without error (just logs a warning) + render_comparison(result) + + +class TestHasMeaningfulMemoryChange: + def test_both_none(self) -> None: + assert not has_meaningful_memory_change(None, None) + + def test_one_none(self) -> None: + assert has_meaningful_memory_change(_make_memory(), None) + assert has_meaningful_memory_change(None, _make_memory()) + + def test_both_zero(self) -> None: + assert not has_meaningful_memory_change(_make_memory(0, 0), _make_memory(0, 0)) + + def test_no_change(self) -> None: + mem = _make_memory(peak=1000, allocs=100) + assert not has_meaningful_memory_change(mem, mem) + + def test_significant_peak_change(self) -> None: + base = _make_memory(peak=10_000_000, allocs=1000) + head = _make_memory(peak=8_000_000, allocs=1000) + assert has_meaningful_memory_change(base, head) + + def test_significant_alloc_change(self) -> None: + base = _make_memory(peak=10_000_000, allocs=1000) + head = _make_memory(peak=10_000_000, allocs=800) + assert has_meaningful_memory_change(base, head) + + +class TestScriptCompareResult: + def test_format_markdown_basic(self) -> None: + result = ScriptCompareResult( + base_ref="abc123", + head_ref="def456", + base_results={"file1.pdf": 12.34, "file2.docx": 1.23}, + head_results={"file1.pdf": 10.21, "file2.docx": 1.45}, + ) + md = result.format_markdown() + assert "file1.pdf" in md + assert "file2.docx" in md + assert "Base" in md + assert "Head" in md + + def test_format_markdown_empty(self) -> None: + result = ScriptCompareResult(base_ref="abc123", head_ref="def456") + md = result.format_markdown() + assert md == "_No benchmark results to compare._" + + def test_format_markdown_total_row(self) -> None: + result = ScriptCompareResult( + base_ref="abc123", + head_ref="def456", + base_results={"test1": 1.0, "__total__": 5.0}, + head_results={"test1": 0.8, "__total__": 4.0}, + ) + md = result.format_markdown() + assert "**TOTAL**" in md + # __total__ should not appear as a regular key row + assert md.count("__total__") == 0 + + def test_format_markdown_missing_keys(self) -> None: + result = ScriptCompareResult( + base_ref="abc123", head_ref="def456", base_results={"only_base": 2.0}, head_results={"only_head": 3.0} + ) + md = result.format_markdown() + assert "only_base" in md + assert "only_head" in md + + def test_format_markdown_with_memory(self) -> None: + result = ScriptCompareResult( + base_ref="abc123", + head_ref="def456", + base_results={"test1": 1.0}, + head_results={"test1": 0.5}, + base_memory=_make_memory(peak=10_000_000, allocs=500), + head_memory=_make_memory(peak=7_000_000, allocs=400), + ) + md = result.format_markdown() + assert "Peak Memory" in md + assert "Allocations" in md + + def test_render_no_crash(self) -> None: + result = ScriptCompareResult( + base_ref="abc123", + head_ref="def456", + base_results={"a": 1.0, "b": 2.0, "__total__": 3.0}, + head_results={"a": 0.5, "b": 1.5, "__total__": 2.0}, + ) + render_script_comparison(result) + + def test_render_empty_no_crash(self) -> None: + result = ScriptCompareResult(base_ref="abc123", head_ref="def456") + render_script_comparison(result) + + def test_render_with_memory_no_crash(self) -> None: + result = ScriptCompareResult( + base_ref="abc123", + head_ref="def456", + base_results={"test1": 5.0}, + head_results={"test1": 4.0}, + base_memory=_make_memory(peak=10_000_000, allocs=1000), + head_memory=_make_memory(peak=8_000_000, allocs=900), + ) + render_script_comparison(result) diff --git a/tests/test_pickle_patcher.py b/tests/test_pickle_patcher.py index 804ff137b..ccf89312a 100644 --- a/tests/test_pickle_patcher.py +++ b/tests/test_pickle_patcher.py @@ -253,14 +253,15 @@ def test_run_and_parse_picklepatch() -> None: cursor = conn.cursor() cursor.execute( - "SELECT function_name, class_name, module_name, file_path, benchmark_function_name, benchmark_module_path, benchmark_line_number FROM benchmark_function_timings ORDER BY benchmark_module_path, benchmark_function_name, function_name" + "SELECT DISTINCT function_name, class_name, module_name, file_path, benchmark_function_name, benchmark_module_path, benchmark_line_number FROM benchmark_function_timings ORDER BY benchmark_module_path, benchmark_function_name, function_name" ) function_calls = cursor.fetchall() # Assert the length of function calls assert len(function_calls) == 2, f"Expected 2 function calls, but got {len(function_calls)}" function_benchmark_timings = codeflash_benchmark_plugin.get_function_benchmark_timings(output_file) - total_benchmark_timings = codeflash_benchmark_plugin.get_benchmark_timings(output_file) + total_benchmark_stats = codeflash_benchmark_plugin.get_benchmark_timings(output_file) + total_benchmark_timings = {k: v.median_ns for k, v in total_benchmark_stats.items()} function_to_results = validate_and_format_benchmark_table(function_benchmark_timings, total_benchmark_timings) assert ( "code_to_optimize.bubble_sort_picklepatch_test_unused_socket.bubble_sort_with_unused_socket" @@ -401,7 +402,7 @@ def test_run_and_parse_picklepatch() -> None: pytest_max_loops=1, testing_time=1.0, ) - assert len(test_results_unused_socket) == 1 + assert len(test_results_unused_socket) >= 1 assert ( test_results_unused_socket.test_results[0].id.test_module_path == "code_to_optimize.tests.pytest.benchmarks_socket_test.codeflash_replay_tests.test_code_to_optimize_tests_pytest_benchmarks_socket_test_test_socket__replay_test_0" @@ -410,7 +411,7 @@ def test_run_and_parse_picklepatch() -> None: test_results_unused_socket.test_results[0].id.test_function_name == "test_code_to_optimize_bubble_sort_picklepatch_test_unused_socket_bubble_sort_with_unused_socket_test_socket_picklepatch" ) - assert test_results_unused_socket.test_results[0].did_pass == True + assert test_results_unused_socket.test_results[0].did_pass is True # Replace with optimized candidate fto_unused_socket_path.write_text(""" @@ -432,7 +433,7 @@ def bubble_sort_with_unused_socket(data_container): pytest_max_loops=1, testing_time=1.0, ) - assert len(optimized_test_results_unused_socket) == 1 + assert len(optimized_test_results_unused_socket) >= 1 match, _ = compare_test_results(test_results_unused_socket, optimized_test_results_unused_socket) assert match @@ -487,7 +488,7 @@ def bubble_sort_with_unused_socket(data_container): pytest_max_loops=1, testing_time=1.0, ) - assert len(test_results_used_socket) == 1 + assert len(test_results_used_socket) >= 1 assert ( test_results_used_socket.test_results[0].id.test_module_path == "code_to_optimize.tests.pytest.benchmarks_socket_test.codeflash_replay_tests.test_code_to_optimize_tests_pytest_benchmarks_socket_test_test_socket__replay_test_0" @@ -522,7 +523,7 @@ def bubble_sort_with_used_socket(data_container): pytest_max_loops=1, testing_time=1.0, ) - assert len(test_results_used_socket) == 1 + assert len(test_results_used_socket) >= 1 assert ( test_results_used_socket.test_results[0].id.test_module_path == "code_to_optimize.tests.pytest.benchmarks_socket_test.codeflash_replay_tests.test_code_to_optimize_tests_pytest_benchmarks_socket_test_test_socket__replay_test_0" diff --git a/tests/test_trace_benchmarks.py b/tests/test_trace_benchmarks.py index 4e0f7be47..001989a55 100644 --- a/tests/test_trace_benchmarks.py +++ b/tests/test_trace_benchmarks.py @@ -29,7 +29,7 @@ def test_trace_benchmarks() -> None: # Get the count of records # Get all records cursor.execute( - "SELECT function_name, class_name, module_name, file_path, benchmark_function_name, benchmark_module_path, benchmark_line_number FROM benchmark_function_timings ORDER BY benchmark_module_path, benchmark_function_name, function_name" + "SELECT DISTINCT function_name, class_name, module_name, file_path, benchmark_function_name, benchmark_module_path, benchmark_line_number FROM benchmark_function_timings ORDER BY benchmark_module_path, benchmark_function_name, function_name" ) function_calls = cursor.fetchall() @@ -220,7 +220,8 @@ def test_code_to_optimize_bubble_sort_codeflash_trace_sorter_test_no_func(): if conn is not None: conn.close() output_file.unlink(missing_ok=True) - shutil.rmtree(replay_tests_dir) + if replay_tests_dir.exists(): + shutil.rmtree(replay_tests_dir) # Skip the test in CI as the machine may not be multithreaded @@ -242,14 +243,15 @@ def test_trace_multithreaded_benchmark() -> None: # Get the count of records # Get all records cursor.execute( - "SELECT function_name, class_name, module_name, file_path, benchmark_function_name, benchmark_module_path, benchmark_line_number FROM benchmark_function_timings ORDER BY benchmark_module_path, benchmark_function_name, function_name" + "SELECT DISTINCT function_name, class_name, module_name, file_path, benchmark_function_name, benchmark_module_path, benchmark_line_number FROM benchmark_function_timings ORDER BY benchmark_module_path, benchmark_function_name, function_name" ) function_calls = cursor.fetchall() # Assert the length of function calls - assert len(function_calls) == 10, f"Expected 10 function calls, but got {len(function_calls)}" + assert len(function_calls) == 1, f"Expected 1 function call, but got {len(function_calls)}" function_benchmark_timings = codeflash_benchmark_plugin.get_function_benchmark_timings(output_file) - total_benchmark_timings = codeflash_benchmark_plugin.get_benchmark_timings(output_file) + total_benchmark_stats = codeflash_benchmark_plugin.get_benchmark_timings(output_file) + total_benchmark_timings = {k: v.median_ns for k, v in total_benchmark_stats.items()} function_to_results = validate_and_format_benchmark_table(function_benchmark_timings, total_benchmark_timings) assert "code_to_optimize.bubble_sort_codeflash_trace.sorter" in function_to_results @@ -304,23 +306,24 @@ def test_trace_benchmark_decorator() -> None: # Get the count of records # Get all records cursor.execute( - "SELECT function_name, class_name, module_name, file_path, benchmark_function_name, benchmark_module_path, benchmark_line_number FROM benchmark_function_timings ORDER BY benchmark_module_path, benchmark_function_name, function_name" + "SELECT DISTINCT function_name, class_name, module_name, file_path, benchmark_function_name, benchmark_module_path, benchmark_line_number FROM benchmark_function_timings ORDER BY benchmark_module_path, benchmark_function_name, function_name" ) function_calls = cursor.fetchall() # Assert the length of function calls assert len(function_calls) == 2, f"Expected 2 function calls, but got {len(function_calls)}" function_benchmark_timings = codeflash_benchmark_plugin.get_function_benchmark_timings(output_file) - total_benchmark_timings = codeflash_benchmark_plugin.get_benchmark_timings(output_file) + total_benchmark_stats = codeflash_benchmark_plugin.get_benchmark_timings(output_file) + total_benchmark_timings = {k: v.median_ns for k, v in total_benchmark_stats.items()} function_to_results = validate_and_format_benchmark_table(function_benchmark_timings, total_benchmark_timings) assert "code_to_optimize.bubble_sort_codeflash_trace.sorter" in function_to_results test_name, total_time, function_time, percent = function_to_results[ "code_to_optimize.bubble_sort_codeflash_trace.sorter" ][0] - assert total_time > 0.0 - assert function_time > 0.0 - assert percent > 0.0 + assert total_time >= 0.0 + assert function_time >= 0.0 + assert percent >= 0.0 bubble_sort_path = (project_root / "bubble_sort_codeflash_trace.py").as_posix() # Expected function calls diff --git a/uv.lock b/uv.lock index 31fde63ab..c059d601e 100644 --- a/uv.lock +++ b/uv.lock @@ -466,6 +466,7 @@ dependencies = [ { name = "libcst" }, { name = "line-profiler" }, { name = "lxml" }, + { name = "memray", marker = "sys_platform != 'win32'" }, { name = "parameterized" }, { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "platformdirs", version = "4.9.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -477,6 +478,7 @@ dependencies = [ { name = "pytest", version = "9.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pytest-asyncio", version = "1.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pytest-asyncio", version = "1.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytest-memray", marker = "sys_platform != 'win32'" }, { name = "pytest-timeout" }, { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "requests", version = "2.33.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -576,6 +578,7 @@ requires-dist = [ { name = "libcst", specifier = ">=1.0.1" }, { name = "line-profiler", specifier = ">=4.2.0" }, { name = "lxml", specifier = ">=5.3.0" }, + { name = "memray", marker = "sys_platform != 'win32'", specifier = ">=1.12" }, { name = "parameterized", specifier = ">=0.9.0" }, { name = "platformdirs", specifier = ">=4.3.7" }, { name = "posthog", specifier = ">=3.0.0" }, @@ -583,6 +586,7 @@ requires-dist = [ { name = "pygls", specifier = ">=2.0.0,<3.0.0" }, { name = "pytest", specifier = ">=7.0.0" }, { name = "pytest-asyncio", specifier = ">=0.18.0" }, + { name = "pytest-memray", marker = "sys_platform != 'win32'", specifier = ">=1.7" }, { name = "pytest-timeout", specifier = ">=2.1.0" }, { name = "requests", specifier = ">=2.28.0" }, { name = "rich", specifier = ">=13.8.1" }, @@ -2261,6 +2265,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/9f/228020e1bce6308723b5455e7de054428b9908b340b4c702dd2b3409f016/line_profiler-5.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:2b70a38fe852d7c95eca105ec603a28ca6f0bd3c909f2cca9e7cca2bf19cb77e", size = 480441, upload-time = "2026-02-23T23:31:19.162Z" }, ] +[[package]] +name = "linkify-it-py" +version = "2.0.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9.2' and python_full_version < '3.10'", + "python_full_version < '3.9.2'", +] +dependencies = [ + { name = "uc-micro-py", version = "1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/ae/bb56c6828e4797ba5a4821eec7c43b8bf40f69cda4d4f5f8c8a2810ec96a/linkify-it-py-2.0.3.tar.gz", hash = "sha256:68cda27e162e9215c17d786649d1da0021a451bdc436ef9e0fa0ba5234b9b048", size = 27946, upload-time = "2024-02-04T14:48:04.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/1e/b832de447dee8b582cac175871d2f6c3d5077cc56d5575cadba1fd1cccfa/linkify_it_py-2.0.3-py3-none-any.whl", hash = "sha256:6bcbc417b0ac14323382aef5c5192c0075bf8a9d6b41820a2b66371eac6b6d79", size = 19820, upload-time = "2024-02-04T14:48:02.496Z" }, +] + +[[package]] +name = "linkify-it-py" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "uc-micro-py", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.10' and sys_platform != 'win32') or (python_full_version == '3.10.*' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, +] + [[package]] name = "llvmlite" version = "0.43.0" @@ -2515,6 +2558,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, ] +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py", version = "2.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -2542,6 +2590,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.10' and sys_platform != 'win32') or (python_full_version == '3.10.*' and sys_platform == 'win32')" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -2650,6 +2703,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, ] +[[package]] +name = "mdit-py-plugins" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9.2' and python_full_version < '3.10'", + "python_full_version < '3.9.2'", +] +dependencies = [ + { name = "markdown-it-py", version = "3.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/03/a2ecab526543b152300717cf232bb4bb8605b6edb946c845016fa9c9c9fd/mdit_py_plugins-0.4.2.tar.gz", hash = "sha256:5f2cd1fdb606ddf152d37ec30e46101a60512bc0e5fa1a7002c36647b09e26b5", size = 43542, upload-time = "2024-09-09T20:27:49.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/f7/7782a043553ee469c1ff49cfa1cdace2d6bf99a1f333cf38676b3ddf30da/mdit_py_plugins-0.4.2-py3-none-any.whl", hash = "sha256:0c673c3f889399a33b95e88d2f0d111b4447bdfea7f237dab2d488f459835636", size = 55316, upload-time = "2024-09-09T20:27:48.397Z" }, +] + +[[package]] +name = "mdit-py-plugins" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "markdown-it-py", version = "4.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.10' and sys_platform != 'win32') or (python_full_version == '3.10.*' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -2659,6 +2751,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "memray" +version = "1.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, + { name = "rich", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, + { name = "textual", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/db/56ff21f47be261ab781105b233d1851d3f2fcdd4f08ebf689f6d6fd84f0d/memray-1.19.2.tar.gz", hash = "sha256:680cb90ac4564d140673ac9d8b7a7e07a8405bd1fb8f933da22616f93124ca84", size = 2410256, upload-time = "2026-03-13T15:22:31.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/5f/48c6d7c6e4d02883d0c3de98c46c71d20c53038dfdde79614d0e55f9f163/memray-1.19.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:50d7130bb0c8609176b3b691c8b67fc92805180166e087549a59e7881ae8cf36", size = 2181142, upload-time = "2026-03-13T15:20:26.87Z" }, + { url = "https://files.pythonhosted.org/packages/1d/85/34d5dc497741bf684cfb5f59d58428b6fd4a034e55cb950339ee8f137f9d/memray-1.19.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3643d601c4c1c413a62fb296598ed05dce1e1c3a58ea5c8659ae98ad36ce3a7a", size = 2162529, upload-time = "2026-03-13T15:20:29.187Z" }, + { url = "https://files.pythonhosted.org/packages/95/5f/ca6ab3cd76de6134cbe29f5a6daa77234f216ae9bd8c963beda226a22653/memray-1.19.2-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:661aca0dbf4c448eef93f2f0bd0852eeefe3de2460e8105c2160c86e308beea5", size = 9707355, upload-time = "2026-03-13T15:20:30.941Z" }, + { url = "https://files.pythonhosted.org/packages/bd/c9/4b79508b2cf646ca3fe3c87bdef80cd26362679274b26dab1f4b725ebba0/memray-1.19.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d13f33f1fa76165c5596e73bc45a366d58066be567fb131498cd770fa87f5a02", size = 9938651, upload-time = "2026-03-13T15:20:33.755Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d6/ca9cef1c0aba2245c41aed699a45a748db7b0dd8a9a63484e809b0f8e448/memray-1.19.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74291aa9bbf54ff2ac5df2665c792d490c576720dd2cbad89af53528bda5443f", size = 9327619, upload-time = "2026-03-13T15:20:36.179Z" }, + { url = "https://files.pythonhosted.org/packages/ce/66/572f819ff58d0f0fefeeeeaa7206f192107f39027a92fd90af1c1cbff61b/memray-1.19.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:716a1b2569e049d0cb769015e5be9138bd97bd157e67920cc9e215e011fbb9cd", size = 12158374, upload-time = "2026-03-13T15:20:39.213Z" }, + { url = "https://files.pythonhosted.org/packages/63/bf/b8f28adbd3e1eeeb88e188053a26164b195ebcf66f8af6b30003a83f5660/memray-1.19.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c8d35a9f5b222165c5aedbfc18b79dc5161a724963a4fca8d1053faa0b571195", size = 2181644, upload-time = "2026-03-13T15:20:41.756Z" }, + { url = "https://files.pythonhosted.org/packages/21/66/0791e5514b475d6300d13ebe87839db1606b2dc2fbe00fecce4da2fb405d/memray-1.19.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3735567011cc22339aee2c59b5fc94d1bdd4a23f9990e02a2c3cccc9c3cf6de4", size = 2164670, upload-time = "2026-03-13T15:20:44.14Z" }, + { url = "https://files.pythonhosted.org/packages/0f/aa/086878e99693b174b0d04d0b267231862fb6a3cfc35cab2920284c2a2e38/memray-1.19.2-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:ab78af759eebcb8d8ecef173042515711d2dcc9600d5dd446d1592b24a89b7d9", size = 9777844, upload-time = "2026-03-13T15:20:46.266Z" }, + { url = "https://files.pythonhosted.org/packages/40/a6/40247667e72b5d8322c5dc2ef30513238b3480be1e482faaaf9cc573ff38/memray-1.19.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f3ae7983297d168cdcc2d05cd93a4934b9b6fe0d341a91ac5b71bf45f9cec06c", size = 10021548, upload-time = "2026-03-13T15:20:49.079Z" }, + { url = "https://files.pythonhosted.org/packages/b3/bb/50603e8f7fe950b3f6a6e09a80413a8f25c4a9d360d8b3b027a8841e1fe8/memray-1.19.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08a4316d7a92eb415024b46988844ed0fd44b2d02ca00fa4a21f2481c1f803e6", size = 9400168, upload-time = "2026-03-13T15:20:51.801Z" }, + { url = "https://files.pythonhosted.org/packages/e2/89/a21e0b639496ed59d2a733e60869ff2e685c5a78891474a494e09a17dc7c/memray-1.19.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dbdb14fd31e2a031312755dc76146aeff9d0889e82ccffe231f1f20f50526f57", size = 12234413, upload-time = "2026-03-13T15:20:54.454Z" }, + { url = "https://files.pythonhosted.org/packages/13/4e/8685c202ddd76860cd8fc5f7f552115ea6f317e9f5f16219a56f336e351e/memray-1.19.2-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:22d4482f559ffa91a9727693e7e338856bee5e316f922839bf8b96e0f9b8a4de", size = 2183484, upload-time = "2026-03-13T15:20:56.696Z" }, + { url = "https://files.pythonhosted.org/packages/89/79/602f55d5466f1f587cdddf0324f82752bd0319ea814bc7cca2efb8593bc8/memray-1.19.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4fd1476868177ee8d9f7f85e5a085a20cc3c3a8228a23ced72749265885d55ca", size = 2162900, upload-time = "2026-03-13T15:20:58.174Z" }, + { url = "https://files.pythonhosted.org/packages/02/1b/402207971653b9861bbbe449cbed7d82e7bb9b953dd6ac93dd4d78e76fa2/memray-1.19.2-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:23375d50faa199e1c1bc2e89f08691f6812478fddb49a1b82bebe6ef5a56df2c", size = 9731991, upload-time = "2026-03-13T15:21:00.299Z" }, + { url = "https://files.pythonhosted.org/packages/3f/7d/895ce73fcf9ab0a2b675ed49bbc91cbca14bda187e2b4df86ccefeb1c9bc/memray-1.19.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8ef3d8e4fba0b26280b550278a0660554283135cbccc34e2d49ba82a1945eb61", size = 9997104, upload-time = "2026-03-13T15:21:02.959Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b9/586bf51a1321cde736d886ca8ac3d4b1f910e4f3f813d7c8eb22498ee16f/memray-1.19.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4d6cf9597ae5d60f7893a0b7b6b9af9c349121446b3c1e7b9ac1d8b5d45a505", size = 9373508, upload-time = "2026-03-13T15:21:05.945Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f1/7cb51edeeceaaee770d4222e833369fbc927227d27e0a917b5ad6f4b2f85/memray-1.19.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:716a0a0e9048d21da98f9107fa030a76138eb694a16a81ad15eace54fddef4cd", size = 12222756, upload-time = "2026-03-13T15:21:08.9Z" }, + { url = "https://files.pythonhosted.org/packages/34/10/cbf57c122988d6e3bd148aa374e91e0e2f156cc7db1ac6397eb6db3946d1/memray-1.19.2-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:13aa87ad34cc88b3f31f7205e0a4543c391032e8600dc0c0cbf22555ff816d97", size = 2182910, upload-time = "2026-03-13T15:21:11.357Z" }, + { url = "https://files.pythonhosted.org/packages/5c/0e/7979dfe7e2b034431e44e3bab86356d9bc2c4f3ed0eb1594cb0ceb38c859/memray-1.19.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d6b249618a3e4fa8e10291445a2b2dfaf6f188e7cc1765966aac8fb52cb22066", size = 2161575, upload-time = "2026-03-13T15:21:13.051Z" }, + { url = "https://files.pythonhosted.org/packages/f9/92/2f0ca3936cdf4c59bc8c59fc8738ce8854ba24fd8519988f2ece0eba10fa/memray-1.19.2-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:34985e5e638ef8d4d54de8173c5e4481c478930f545bd0eb4738a631beb63d04", size = 9732172, upload-time = "2026-03-13T15:21:15.115Z" }, + { url = "https://files.pythonhosted.org/packages/52/23/de78510b4e3a0668b793d8b5dff03f2af20eef97943ca5b3263effff799c/memray-1.19.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee0fcfafd1e8535bdc0d0ed75bcdd48d436a6f62d467df91871366cbb3bbaebc", size = 9999447, upload-time = "2026-03-13T15:21:18.099Z" }, + { url = "https://files.pythonhosted.org/packages/00/0d/b0e50537470f93bddfa2c134177fe9332c20be44a571588866776ff92b82/memray-1.19.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:846185c393ff0dc6bca55819b1c83b510b77d8d561b7c0c50f4873f69579e35d", size = 9379158, upload-time = "2026-03-13T15:21:21.003Z" }, + { url = "https://files.pythonhosted.org/packages/5c/53/78f6de5c7208821b15cfbbb9da44ab4a5a881a7cc5075f9435a1700320e8/memray-1.19.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8cc31327ed71e9f6ef7e9ed558e764f0e9c3f01da13ad8547734eb65fbeade1d", size = 12226753, upload-time = "2026-03-13T15:21:24.041Z" }, + { url = "https://files.pythonhosted.org/packages/e1/f4/3d8205b9f46657d26d54d1e644f27d09955b737189354a01907d8a08c7e2/memray-1.19.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:410377c0eae8d544421f74b919a18e119279fe1a2fa5ff381404b55aeb4c6514", size = 2184823, upload-time = "2026-03-13T15:21:27.176Z" }, + { url = "https://files.pythonhosted.org/packages/fb/07/7a342801317eff410a8267b55cb7514e156ee1f574e690852eb240bbe9fd/memray-1.19.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a53dc4032581ed075fcb62a4acc0ced14fb90a8269159d4e53dfac7af269c255", size = 2163669, upload-time = "2026-03-13T15:21:29.123Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/2c342b1472f9f03018bb88c80760cdfa6979404d63c4300c607fd0562607/memray-1.19.2-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:a7630865fbf3823aa2d1a6f7536f7aec88cf8ccf5b2498aad44adbc733f6bd2e", size = 9732615, upload-time = "2026-03-13T15:21:31.038Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/2cf960526c9b1f6d46977fc70e11de29ca6b9eafeeb42d1cec7d3bcb056a/memray-1.19.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c23e2b4be22a23cf5cae08854549e3460869a36c5f4bedc739b646ac97da4a60", size = 9979299, upload-time = "2026-03-13T15:21:34.072Z" }, + { url = "https://files.pythonhosted.org/packages/e1/78/73ee3d0ebee3c38fbb2d51766854d2932beec6481063532a6019bf340a2d/memray-1.19.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95b6c02ca7f8555b5bee1c54c50cbbcf2033e07ebca95dade2ac3a27bb36b320", size = 9375722, upload-time = "2026-03-13T15:21:36.884Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c6/2f02475e85ccd32fa306736986f1f77f99365066ecdc859f5078148ebc40/memray-1.19.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:907470e2684568eb91a993ae69a08b1430494c8f2f6ef489b4b78519d9dae3d0", size = 12220041, upload-time = "2026-03-13T15:21:40.16Z" }, + { url = "https://files.pythonhosted.org/packages/76/12/01bb32188c011e6d802469e04c1d7c8054eb8300164e2269c830f5b26a8e/memray-1.19.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:124138f35fea36c434256c417f1b8cb32f78769f208530c1e56bf2c2b7654120", size = 2201353, upload-time = "2026-03-13T15:21:42.607Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e0/d9b59f8be00f27440f60b95da5db6515a1c44c481651b8d2fa8f3468fc35/memray-1.19.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:240192dc98ff0b3501055521bfd73566d339808b11bd5af10865afe6ae18abef", size = 2180420, upload-time = "2026-03-13T15:21:44.623Z" }, + { url = "https://files.pythonhosted.org/packages/a5/5c/30aca63f4b88dca79ba679675200938652c816edee34c12565d2f17ea936/memray-1.19.2-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:edb7a3c2a9e97fb409b352f6c316598c7c0c3c22732e73704d25b9eb75ae2f2d", size = 9697953, upload-time = "2026-03-13T15:21:47.088Z" }, + { url = "https://files.pythonhosted.org/packages/9f/02/9e4a68bdd5ebc9079f97bdf287cc0ccc51c18e9edc205de7d41648315809/memray-1.19.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b6a43db4c1466446a905a77944813253231ac0269f758c6c6bc03ceb1821c1b6", size = 9944517, upload-time = "2026-03-13T15:21:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f0/3adad59ebed6841c2f88b43c9b90cc9c03ff086129a8aef3cff23c92d6ac/memray-1.19.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf951dae8d27d502fbc549f6784460a70cce05b1e71bf5446d8692a74051f14f", size = 9365528, upload-time = "2026-03-13T15:21:53.141Z" }, + { url = "https://files.pythonhosted.org/packages/45/0e/083e00fe74e576b463e7b00e4214b8962f27bd70c5c77e494c0211a77342/memray-1.19.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8033b78232555bb1856b3298bef2898ec8b334d3d465c1822c665206d1fa910a", size = 12143894, upload-time = "2026-03-13T15:21:56.486Z" }, + { url = "https://files.pythonhosted.org/packages/4d/1b/b2e54cbe9a67a63a2f8b0c0d3cbfef0db8592e00ced4d6afb324245910e5/memray-1.19.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:f82ee0a0b50a04894dacfbe49db1c259fa8a19efb094514b0100e9916d3b1c55", size = 2183022, upload-time = "2026-03-13T15:22:14.81Z" }, + { url = "https://files.pythonhosted.org/packages/fd/1e/17a3e62bccf2c34cfa2208c28bdab127afd279c8a6d7fbb7c2b835a606db/memray-1.19.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b1c58a54372707b3977c079ef93e751109f0bfe566adc7bd640971d123d8d11", size = 2163707, upload-time = "2026-03-13T15:22:16.507Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bd/a9bb3d747b138c8bc382389857879941f6c7a83fb3beeebce1c3251ad401/memray-1.19.2-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:fa236140320ef1b8801cd289962fd81a2d7e59484cc3ecdbc851d1b5c321795e", size = 9703623, upload-time = "2026-03-13T15:22:19.551Z" }, + { url = "https://files.pythonhosted.org/packages/a3/70/24006fcab90eb6a21b5b2c45f046746578a817c82cb7ed2987d08dffad9d/memray-1.19.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:816baeda8e62fddf99c900bdc9e748339dba65df091a7c7ceb0f4f9544c2e7ec", size = 9925887, upload-time = "2026-03-13T15:22:23.297Z" }, + { url = "https://files.pythonhosted.org/packages/41/5e/6ac00a20da0b84c9e41d1e0ebaf27d49907ff7be1cd66b1e2b410d1c9c25/memray-1.19.2-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1532d5dcf8036ec55e43ab6d6b1ff4e70b11a3902dd1c8396b6d2a24ec69d98", size = 9323522, upload-time = "2026-03-13T15:22:26.144Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e0/74c17f7095e7c476fef3f47a13637fe0d717b58c8e0e5e06a388b7ca3cac/memray-1.19.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:86060df2e8e18cc867335c50bf92deb973d4dff856bdb565e17fc86ca7a6619b", size = 12154107, upload-time = "2026-03-13T15:22:29.341Z" }, +] + [[package]] name = "ml-dtypes" version = "0.5.4" @@ -4368,6 +4515,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] +[[package]] +name = "pytest-memray" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "memray", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pytest", version = "9.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.10' and sys_platform != 'win32') or (python_full_version == '3.10.*' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/28/f67963efed56d847d028d0bb939f26cdeb32c4de474b3befc9da43bf18f9/pytest_memray-1.8.0.tar.gz", hash = "sha256:c0c706ef81941a7aa7064f2b3b8b5cdc0cea72b5277c6a6a09b113ca9ab30bdb", size = 240608, upload-time = "2025-08-18T17:32:47.329Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/52/b8b8e126c176c5f405b307354e1722025063ea104dbd7d286e8b18a76e9f/pytest_memray-1.8.0-py3-none-any.whl", hash = "sha256:44da9fe0d98541abf4cc76acea6e4a9c525b3c8e604655e5537705f336c9b875", size = 17688, upload-time = "2025-08-18T17:32:45.476Z" }, +] + [[package]] name = "pytest-timeout" version = "2.4.0" @@ -5338,6 +5499,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, ] +[[package]] +name = "textual" +version = "8.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", version = "3.0.0", source = { registry = "https://pypi.org/simple" }, extra = ["linkify"], marker = "python_full_version < '3.10'" }, + { name = "markdown-it-py", version = "4.0.0", source = { registry = "https://pypi.org/simple" }, extra = ["linkify"], marker = "(python_full_version >= '3.10' and sys_platform != 'win32') or (python_full_version == '3.10.*' and sys_platform == 'win32')" }, + { name = "mdit-py-plugins", version = "0.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "mdit-py-plugins", version = "0.5.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.10' and sys_platform != 'win32') or (python_full_version == '3.10.*' and sys_platform == 'win32')" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.9.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.10' and sys_platform != 'win32') or (python_full_version == '3.10.*' and sys_platform == 'win32')" }, + { name = "pygments", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, + { name = "rich", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/07/766ad19cf2b15cae2d79e0db46a1b783b62316e9ff3e058e7424b2a4398b/textual-8.2.1.tar.gz", hash = "sha256:4176890e9cd5c95dcdd206541b2956b0808e74c8c36381c88db53dcb45237451", size = 1848386, upload-time = "2026-03-29T03:57:32.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/09/c6f000c2e3702036e593803319af02feee58a662528d0d5728a37e1cf81b/textual-8.2.1-py3-none-any.whl", hash = "sha256:746cbf947a8ca875afc09779ef38cadbc7b9f15ac886a5090f7099fef5ade990", size = 723871, upload-time = "2026-03-29T03:57:34.334Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -6324,6 +6505,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, ] +[[package]] +name = "uc-micro-py" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.9.2' and python_full_version < '3.10'", + "python_full_version < '3.9.2'", +] +sdist = { url = "https://files.pythonhosted.org/packages/91/7a/146a99696aee0609e3712f2b44c6274566bc368dfe8375191278045186b8/uc-micro-py-1.0.3.tar.gz", hash = "sha256:d321b92cff673ec58027c04015fcaa8bb1e005478643ff4a500882eaab88c48a", size = 6043, upload-time = "2024-02-09T16:52:01.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/87/1f677586e8ac487e29672e4b17455758fce261de06a0d086167bb760361a/uc_micro_py-1.0.3-py3-none-any.whl", hash = "sha256:db1dffff340817673d7b466ec86114a9dc0e9d4d9b5ba229d9d60e5c12600cd5", size = 6229, upload-time = "2024-02-09T16:52:00.371Z" }, +] + +[[package]] +name = "uc-micro-py" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/78/67/9a363818028526e2d4579334460df777115bdec1bb77c08f9db88f6389f2/uc_micro_py-2.0.0.tar.gz", hash = "sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811", size = 6611, upload-time = "2026-03-01T06:31:27.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, +] + [[package]] name = "unidiff" version = "0.7.5"