From a469f45a63a5a1d6b8b599e0a7964d4c7949b647 Mon Sep 17 00:00:00 2001 From: guanqin-123 Date: Tue, 7 Jul 2026 09:34:07 +1000 Subject: [PATCH] align cli with yaml --- .codecov.yml | 2 +- act/back_end/cli.py | 132 ++++++++++++++++++++++++++++-- act/pipeline/cli.py | 193 +++++++++++++++++++++++++++++++++++++------- 3 files changed, 290 insertions(+), 37 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index 95839a134..1abbe3cbd 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -14,7 +14,7 @@ coverage: status: project: default: - target: 72% + target: 74% threshold: 1% informational: false patch: diff --git a/act/back_end/cli.py b/act/back_end/cli.py index 462d1afd6..9da8597cb 100644 --- a/act/back_end/cli.py +++ b/act/back_end/cli.py @@ -13,6 +13,7 @@ """ import argparse +from dataclasses import fields import datetime import glob import json @@ -21,7 +22,7 @@ import sys import time from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional, Union, cast +from typing import Any, Dict, List, NamedTuple, Optional, Union, cast, get_args, get_origin, get_type_hints from act.back_end.config import VALID_BERT_METHODS, _VALID_SOLVERS from act.back_end.layer_schema import LayerKind @@ -33,6 +34,111 @@ _SOLVERS: tuple[str, ...] = tuple(sorted(_VALID_SOLVERS)) +def _strip_optional(tp: Any) -> Any: + origin = get_origin(tp) + if origin is Union: + args = [arg for arg in get_args(tp) if arg is not type(None)] + if len(args) == 1: + return args[0] + return tp + + +def _parse_config_value(tp: Any): + base = _strip_optional(tp) + origin = get_origin(base) + if base in (int, float, str): + return base + if origin in (list, List): + def _parse_list(raw: str) -> list[Any]: + if raw.strip().startswith("["): + value = json.loads(raw) + if not isinstance(value, list): + raise argparse.ArgumentTypeError("expected a JSON list") + return value + return [item for item in raw.split(",") if item] + return _parse_list + return str + + +def _add_dataclass_config_args(parser: argparse.ArgumentParser) -> None: + """Expose all BackendConfig dataclass fields without hand-maintained drift.""" + from act.back_end.config import BackendConfig, BaBConfig, GenerationConfig, HybridZConfig + + existing_options = { + option + for action in parser._actions + for option in action.option_strings + } + + def add_group(cls: type[Any], title: str, flag_prefix: str, dest_prefix: str, skip: set[str]) -> None: + group = parser.add_argument_group(title) + type_hints = get_type_hints(cls) + for fld in fields(cls): + if fld.name in skip: + continue + flag = f"--{flag_prefix}{fld.name.replace('_', '-')}" + dest = f"{dest_prefix}{fld.name}" + field_type = type_hints.get(fld.name, fld.type) + base = _strip_optional(field_type) + help_text = f"Override config field {cls.__name__}.{fld.name} (default: from config.yaml)" + if flag in existing_options: + no_flag = f"--no-{flag[2:]}" + if base is bool and no_flag not in existing_options: + group.add_argument( + no_flag, + action="store_false", + default=None, + dest=dest, + help=help_text, + ) + existing_options.add(no_flag) + continue + if base is bool: + group.add_argument( + flag, + action=argparse.BooleanOptionalAction, + default=None, + dest=dest, + help=help_text, + ) + existing_options.add(flag) + existing_options.add(f"--no-{flag[2:]}") + else: + group.add_argument( + flag, + type=_parse_config_value(field_type), + default=None, + dest=dest, + help=help_text, + ) + existing_options.add(flag) + + add_group( + BackendConfig, + "Backend Config Overrides (generated)", + "", + "", + {"bab", "generation", "hybridz"}, + ) + add_group(BaBConfig, "BaB Config Overrides (generated)", "bab-", "bab_", set()) + add_group(GenerationConfig, "Generation Config Overrides (generated)", "gen-", "gen_", set()) + add_group(HybridZConfig, "HybridZ Config Overrides (generated)", "hz-", "hybridz_", set()) + + +def _backend_override_keys_from_dataclasses() -> set[str]: + from act.back_end.config import BackendConfig, BaBConfig, GenerationConfig, HybridZConfig + + keys = { + fld.name + for fld in fields(BackendConfig) + if fld.name not in {"bab", "generation", "hybridz"} + } + keys.update(f"bab_{fld.name}" for fld in fields(BaBConfig)) + keys.update(f"gen_{fld.name}" for fld in fields(GenerationConfig)) + keys.update(f"hybridz_{fld.name}" for fld in fields(HybridZConfig)) + return keys + + class _SkipUnsupported(NamedTuple): """Tagged-union result for nets the active TF cannot verify. @@ -1147,6 +1253,10 @@ def main(): # Add standard device/dtype arguments (shared across all ACT CLIs) add_device_args(parser) + # Add missing config-backed flags from dataclass fields. Explicit legacy + # flags above win on spelling/dest behavior; generated flags fill gaps. + _add_dataclass_config_args(parser) + # Detect user-provided flags BEFORE parsing so env vars / config.yaml # can serve as fallbacks without overriding explicit CLI flags. argv = sys.argv[1:] @@ -1224,7 +1334,7 @@ def main(): # (override_key, args_attr, env_var, env_cast, cli_check) # cli_check="user_set" for flags with non-None defaults (--device/--dtype/--registry-mode) -_BACKEND_OVERRIDE_SPEC: list[tuple[str, str, Optional[str], Any, str]] = [ +_BACKEND_ALIAS_OVERRIDE_SPEC: list[tuple[str, str, Optional[str], Any, str]] = [ ("solver", "solver", "ACT_SOLVER", None, "not_none"), ("device", "device", "ACT_DEVICE", None, "user_set"), ("dtype", "dtype", "ACT_DTYPE", None, "user_set"), @@ -1272,19 +1382,31 @@ def main(): ] +def _build_backend_override_spec() -> list[tuple[str, str, Optional[str], Any, str]]: + spec = list(_BACKEND_ALIAS_OVERRIDE_SPEC) + aliased_keys = {key for key, *_ in spec} + for key in sorted(_backend_override_keys_from_dataclasses() - aliased_keys): + spec.append((key, key, None, None, "not_none")) + return spec + + +_BACKEND_OVERRIDE_SPEC = _build_backend_override_spec() + + def _collect_backend_overrides(args: Any, _user_set: Any) -> dict[str, Any]: """Build overrides dict from CLI flags + env vars (precedence: CLI > env > yaml).""" overrides: dict[str, Any] = {} - for key, attr, env, cast, check in _BACKEND_OVERRIDE_SPEC: + for key, attr, env, env_cast, check in _BACKEND_OVERRIDE_SPEC: cli_val = getattr(args, attr, None) if check == "user_set": - cli_provided = _user_set(f"--{attr.replace('_', '-')}") + attr_flag = attr.replace('_', '-') + cli_provided = _user_set(f"--{attr_flag}") or _user_set(f"--no-{attr_flag}") else: cli_provided = cli_val is not None if cli_provided: overrides[key] = cli_val elif env is not None and os.environ.get(env): - overrides[key] = cast(os.environ[env]) if cast else os.environ[env] + overrides[key] = env_cast(os.environ[env]) if env_cast else os.environ[env] if args.verbose: overrides["verbose"] = True diff --git a/act/pipeline/cli.py b/act/pipeline/cli.py index c601365eb..69d114b38 100644 --- a/act/pipeline/cli.py +++ b/act/pipeline/cli.py @@ -35,6 +35,140 @@ from act.pipeline.verification.per_neuron_bounds import PerNeuronCheckConfig +_FUZZ_MUTATION_WEIGHT_KEYS = frozenset( + {"gradient", "pgd", "activation", "boundary", "random"} +) + + +_FUZZ_OVERRIDE_SPEC: list[tuple[str, str, str, Any]] = [ + ("max_iterations", "--iterations", "iterations", int), + ("timeout_seconds", "--timeout", "timeout", float), + ("coverage_strategy", "--coverage-strategy", "coverage_strategy", str), + ("activation_threshold", "--activation-threshold", "activation_threshold", float), + ("mutation_weights", "--mutation-weights", "mutation_weights", dict), + ("perturb_mode", "--perturb-mode", "perturb_mode", str), + ("perturb_scale", "--perturb-scale", "perturb_scale", float), + ("seed_selection_strategy", "--seed-selection", "seed_selection_strategy", str), + ("save_counterexamples", "--no-save", "save_counterexamples", bool), + ("output_dir", "--output", "output", Path), + ("report_interval", "--report-interval", "report_interval", int), + ("verbose", "--fuzz-verbose", "fuzz_verbose", int), + ("trace_level", "--trace-level", "trace_level", int), + ("trace_sample_rate", "--trace-sample", "trace_sample", int), + ("trace_storage", "--trace-storage", "trace_storage", str), + ("trace_output", "--trace-output", "trace_output", Path), + ("stop_on_first_violation", "--stop-on-first-violation", "stop_on_first_violation", bool), +] + + +def _parse_mutation_weights(raw: str) -> dict[str, float]: + """Parse a complete mutation-weight override from key=value CSV text.""" + weights: dict[str, float] = {} + try: + for item in raw.split(","): + key, value = item.split("=", 1) + key = key.strip() + weights[key] = float(value.strip()) + except ValueError as exc: + raise argparse.ArgumentTypeError( + "expected comma-separated key=value pairs, e.g. " + "gradient=0,pgd=0.4,activation=0.3,boundary=0.2,random=0.1" + ) from exc + + missing = _FUZZ_MUTATION_WEIGHT_KEYS - weights.keys() + extra = weights.keys() - _FUZZ_MUTATION_WEIGHT_KEYS + if missing or extra: + details = [] + if missing: + details.append(f"missing: {', '.join(sorted(missing))}") + if extra: + details.append(f"unknown: {', '.join(sorted(extra))}") + raise argparse.ArgumentTypeError( + "mutation weights must provide exactly " + f"{', '.join(sorted(_FUZZ_MUTATION_WEIGHT_KEYS))} ({'; '.join(details)})" + ) + + total = sum(weights.values()) + if abs(total - 1.0) > 1e-9: + raise argparse.ArgumentTypeError( + f"mutation weights must sum to 1.0, got {total:g}" + ) + return weights + + +def _add_fuzz_config_args(parser: argparse.ArgumentParser) -> None: + group = parser.add_argument_group("Fuzzing Config Overrides") + group.add_argument( + "--coverage-strategy", + type=str, + default=None, + help="Coverage strategy (default: from config.yaml)", + ) + group.add_argument( + "--activation-threshold", + type=float, + default=None, + help="Neuron activation threshold for coverage (default: from config.yaml)", + ) + group.add_argument( + "--mutation-weights", + type=str, + default=None, + metavar="K=V,...", + help="Full mutation weights override, e.g. gradient=0,pgd=0.4,activation=0.3,boundary=0.2,random=0.1", + ) + group.add_argument( + "--perturb-mode", + type=str, + default=None, + help="Perturbation sizing mode (default: from config.yaml)", + ) + group.add_argument( + "--perturb-scale", + type=float, + default=None, + help="Fraction of feasible range per mutation (default: from config.yaml)", + ) + group.add_argument( + "--seed-selection", + type=str, + default=None, + help="Seed selection strategy (default: from config.yaml)", + ) + group.add_argument( + "--fuzz-verbose", + type=int, + default=None, + help="Fuzzing verbosity: 0=silent, 1=progress violations, 2=each violation (default: from config.yaml)", + ) + group.add_argument( + "--stop-on-first-violation", + action="store_true", + default=None, + help="Stop after the first counterexample (default: from config.yaml/FuzzingConfig default)", + ) + + +def _collect_fuzzing_overrides(args: Any) -> dict[str, Any]: + overrides: dict[str, Any] = {} + for key, _flag, attr, cast_fn in _FUZZ_OVERRIDE_SPEC: + value = getattr(args, attr, None) + if value is None: + continue + if key == "save_counterexamples": + value = False + elif key == "mutation_weights": + value = _parse_mutation_weights(value) + elif cast_fn is Path and not isinstance(value, Path): + value = Path(value) + elif cast_fn is int and not isinstance(value, int): + value = int(str(value)) + elif cast_fn is float and not isinstance(value, float): + value = float(str(value)) + overrides[key] = value + return overrides + + def print_header(): """Print simple header.""" @@ -305,19 +439,9 @@ def cmd_fuzz(args): print(f"⚠️ Strict mode enabled: Errors will be raised on constraint violations") print() - # Load configuration from YAML with CLI overrides - overrides: dict[str, Any] = dict( - max_iterations=args.iterations, - timeout_seconds=args.timeout, - save_counterexamples=not args.no_save, - output_dir=Path(args.output), - report_interval=args.report_interval, - # Tracing configuration - trace_level=args.trace_level, - trace_sample_rate=args.trace_sample, - trace_storage=args.trace_storage, - trace_output=Path(args.trace_output) if args.trace_output else None, - ) + # Load configuration from YAML with CLI overrides. Unset CLI options remain + # None sentinels and therefore do not clobber config.yaml values. + overrides = _collect_fuzzing_overrides(args) config = FuzzingConfig.from_yaml(**overrides) # Create spec creator and load data-model pairs @@ -495,19 +619,20 @@ def cmd_fuzz(args): print(f"Seeds explored: {report.seeds_explored}") print(f"{'=' * 80}\n") - if report.counterexamples and not args.no_save: + if report.counterexamples and config.save_counterexamples: import os import torch as _torch from act.front_end.vnnlib_loader.vnnlib_parser import write_vnncomp_result - os.makedirs(args.output, exist_ok=True) + os.makedirs(config.output_dir, exist_ok=True) ce0 = report.counterexamples[0] - x = ce0.input if hasattr(ce0, "input") else ce0 + x = cast(Any, getattr(ce0, "input", ce0)) with _torch.no_grad(): - y = wrapped_model(x) + y_raw = wrapped_model(x) + y = cast(Any, y_raw["output"] if isinstance(y_raw, dict) else y_raw) fname = "_".join(map(str, model_id)) if isinstance(model_id, tuple) else str(model_id) write_vnncomp_result( - os.path.join(args.output, f"{fname}_result.txt"), + os.path.join(config.output_dir, f"{fname}_result.txt"), "sat", x=x, y=y, in_decl=("X", "float32", tuple(x.shape)), out_decl=("Y", "float32", tuple(y.shape)), @@ -1325,29 +1450,33 @@ def main(): fuzz_group.add_argument( "--iterations", type=int, - default=10000, - help="Max fuzzing iterations (default: 10000)", + default=None, + help="Max fuzzing iterations (default: from config.yaml)", ) fuzz_group.add_argument( "--timeout", type=float, - default=3600.0, - help="Timeout in seconds (default: 3600)", + default=None, + help="Timeout in seconds (default: from config.yaml)", ) fuzz_group.add_argument( "--output", type=str, - default="fuzzing_results", - help="Output directory (default: fuzzing_results)", + default=None, + help="Output directory (default: from config.yaml)", ) fuzz_group.add_argument( - "--no-save", action="store_true", help="Don't save counterexamples to disk" + "--no-save", + action="store_false", + default=None, + dest="save_counterexamples", + help="Don't save counterexamples to disk", ) fuzz_group.add_argument( "--report-interval", type=int, - default=100, - help="Report progress every N iterations (default: 100)", + default=None, + help="Report progress every N iterations (default: from config.yaml)", ) fuzz_group.add_argument( "--strict-mode", @@ -1361,14 +1490,14 @@ def main(): "--trace-level", type=int, choices=[0, 1, 2, 3], - default=0, + default=None, help="Tracing detail level: 0=disabled (default), 1=basic (iteration metrics + inputs), " "2=full (+ layer activations), 3=debug (+ gradients and loss)", ) trace_group.add_argument( "--trace-sample", type=int, - default=1, + default=None, metavar="N", help="Capture every Nth iteration (default: 1 = all iterations). " "Use higher values to reduce overhead (e.g., 10 = every 10th iteration)", @@ -1377,8 +1506,8 @@ def main(): "--trace-storage", type=str, choices=["hdf5", "json"], - default="json", - help="Storage backend: json=text/readable (default), hdf5=binary/compressed", + default=None, + help="Storage backend: json=text/readable, hdf5=binary/compressed (default: from config.yaml)", ) trace_group.add_argument( "--trace-output", @@ -1466,6 +1595,8 @@ def main(): # Add standard device/dtype arguments (shared across all ACT CLIs) add_device_args(parser) + _add_fuzz_config_args(parser) + args = parser.parse_args() # Initialize device manager from CLI arguments