diff --git a/CMakeLists.txt b/CMakeLists.txt index 29643264fc..0e67235020 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -438,6 +438,7 @@ set(SeQuant_eval_src SeQuant/core/eval/eval_expr.hpp SeQuant/core/eval/eval_node.hpp SeQuant/core/eval/eval_node_compare.hpp + SeQuant/core/eval/node_batch_annotation.hpp SeQuant/core/eval/result.cpp SeQuant/core/eval/result.hpp SeQuant/core/eval/fwd.hpp diff --git a/SeQuant/.DS_Store b/SeQuant/.DS_Store new file mode 100644 index 0000000000..ef1d771575 Binary files /dev/null and b/SeQuant/.DS_Store differ diff --git a/SeQuant/core/batch_policy.hpp b/SeQuant/core/batch_policy.hpp index 48c3c5cd45..b34dfb0658 100644 --- a/SeQuant/core/batch_policy.hpp +++ b/SeQuant/core/batch_policy.hpp @@ -3,16 +3,64 @@ #include #include +#include namespace sequant { class Index; class Tensor; +/// The three RUNTIME EXECUTION MODELS for batched evaluation (Task 6 of the +/// whole-scope batched DAG execution design, `doc/dev/specs/2026-08-10- +/// whole-scope-batched-dag-execution-design.md`, plus its SP3 follow-on +/// `doc/dev/specs/2026-08-05-dryrun-wetrun-schedule-equivalence-design.md`, +/// the ordered-scope batched-eval design): +/// - \c forest_descent (default): one tree at a time, +/// `sequant::evaluate(Nodes const&, ...)`, unchanged. +/// - \c whole_scope: one fused scope-tree walk over the whole forest, +/// `sequant::eval::evaluate_whole_scope`, so a value shared across trees +/// is built once per home block and reused, rather than rebuilt per tree. +/// - \c ordered: driven by the SP2 `eval::OrderedSchedule` IR rather than +/// the narrow `ScopeSchedule` scope tree, +/// `sequant::eval::evaluate_ordered_schedule`. +enum class BatchScheduler { forest_descent, whole_scope, ordered }; + /// One batchability policy shared by the single-term optimizer and the runtime /// batched evaluator (make_evaluator, Task A3). All predicates default empty. struct BatchPolicy { - std::function is_batchable_index = {}; + /// Spaces batchable in the CONTRACTED role: a mode of such a space is + /// batchable where it is summed. Companion to \ref + /// is_batchable_external_index (the EXTERNAL role). Splitting batchability by + /// role lets a caller admit a space only where batching it is meaningful -- + /// e.g. a space batchable only as an external spectator contributes none of + /// its contracted occurrences to the optimizer's 2^m search. Building block; + /// the derived "batchable in any role" query is \ref is_batchable_index(). + /// Defaults to decline every index; a caller opts spaces in explicitly. + std::function is_batchable_contracted_index = + [](Index const&) { return false; }; + /// Spaces batchable in the EXTERNAL role: a mode of such a space is batchable + /// where it is open on the term root (a spectator carried to the result), not + /// where it is contracted. Building block; declared adjacent to its + /// contracted companion. Defaults to decline every index; a caller that wants + /// external batching sets this predicate explicitly (there is no fallback to + /// the contracted role). + std::function is_batchable_external_index = + [](Index const&) { return false; }; + + /// Derived "batchable in ANY role": the union of the two building-block + /// predicates. This is NEVER a settable field -- it is computed from + /// \ref is_batchable_contracted_index and \ref is_batchable_external_index. + /// The runtime batched evaluator's accept predicate is this union (a mode is + /// accepted at runtime if it is batchable in either role); the factorizer's + /// role filters instead consume the individual building blocks. The building + /// blocks default-decline, so both are always callable here. + std::function is_batchable_index() const { + auto contracted = is_batchable_contracted_index; + auto external = is_batchable_external_index; + return [contracted, external](Index const& ix) { + return contracted(ix) || external(ix); + }; + } /// Per-index per-batch slice size (in elements) for a batchable index -- an /// UPPER BOUND, not a goal. Both the single-term optimizer and the runtime /// batched evaluator treat it as a ceiling: the realized whole-tile batch is @@ -21,6 +69,41 @@ struct BatchPolicy { std::function batch_target_size = {}; std::function is_volatile_leaf = {}; + /// If true, an external/spectator index -- open on the whole network's result + /// yet contracted at no node -- is eligible for batching; its per-slice size + /// comes from \c batch_target_size(ix) like any batchable index. Default + /// false = no spectator batching (byte-identical to non-spectator behavior). + /// Necessary but not sufficient: spectator axes are emitted only under a + /// TIME-FIRST objective (DenseTimeSpaceBatched) and only when the selected + /// root's modeled peak exceeds \c peak_threshold. Spectator batching is + /// therefore currently unavailable under the space-first objectives. + bool batch_spectator_indices = false; + + /// Enable the order-aware multilevel recompute cost model (resident-scan peak + /// + ordered-key flops recompute). SELECTION knob ONLY: it makes the DP + /// charge recompute realistically and thus pick a different (better-batching) + /// factorization. It does NOT control external-mode EMISSION -- that is the + /// independent \ref node_level_placement. Consulted only by the batched + /// objectives (threaded via CostParams). Default TRUE: the recompute-aware + /// model is the more realistic cost for selection. This is SAFE precisely + /// because it is now selection-only -- the node-level emission it used to + /// force is separately gated by \ref node_level_placement (default off), so + /// the emission stays the correct, cheap root-level forest seed. (Before the + /// decouple, defaulting this true forced the node-level runtime regression.) + bool order_aware_recompute = true; + + /// Emission-placement knob for external (spectator) modes, INDEPENDENT of the + /// order-aware cost model. Only meaningful with \ref batch_spectator_indices. + /// If true, the emit uses node-level placement (per-node External stamps); if + /// false (default) it uses the root-level forest seed (one global spectator + /// loop). Node-level placement is currently a net runtime REGRESSION -- ~6x + /// wall time and ~8x batch scopes on water-8, and it produces a wrong + /// residual on water-20 -- because it nests a batch scope at every carrying + /// node and the batched evaluator replays each. It stays OFF by default until + /// that is fixed; the root-seed emission is correct and cheap regardless of + /// order_aware_recompute. + bool node_level_placement = false; + /// If true, restrict batching to persistent (amplitude-independent) subtrees, /// declining to batch any subtree that contains a volatile leaf. If false /// (the default), batch ACROSS THE BOARD: slicing the batch axis shrinks any @@ -40,6 +123,36 @@ struct BatchPolicy { /// accumulator + contribution co-residency of a node that contracts a /// batchable index. double accumulation_factor = 0.0; + + /// Selects among the three runtime execution models (\ref BatchScheduler + /// above). Consulted by the + /// `sequant::evaluate(Nodes const&, BatchPolicy const&, ...)` driver + /// overload (`scope_executor.hpp`) to select the driver, and by + /// `sequant::eval::dryrun::cost_profile()` to select the matching peak + /// model: the co-residency oracle (`peak_profile_sweep` over `home_modes`) + /// for \c whole_scope, since that model is what predicts the whole-scope + /// realized peak, vs the batched-scratch replay high-watermark (models + /// forest descent) for \c forest_descent. Default \c forest_descent + /// reproduces today's behavior on every existing call site byte-for-byte. + BatchScheduler scheduler = BatchScheduler::forest_descent; + + /// Peak-memory budget in BYTES for the batched objectives. Its meaning + /// DIFFERS between them: + /// + /// - SPACE-FIRST (DenseSpaceTimeBatched): a hard feasibility gate. The + /// single-term optimizer minimizes flops among schedules whose modeled peak + /// is <= peak_threshold, falling back to min-peak (best effort) when none + /// fit. Default +infinity => every schedule feasible => min flops => no + /// batching, i.e. here a finite value is the *enable* trigger for batching. + /// + /// - TIME-FIRST (DenseTimeSpaceBatched): NOT a feasibility gate. Root + /// selection ignores it entirely (peak breaks exact flop ties only), so it + /// can neither constrain the schedule's peak nor enable CONTRACTED-axis + /// batching (which is emitted regardless). Its ONLY effect is to trigger + /// EXTERNAL (spectator) axis emission, together with + /// \c batch_spectator_indices: axes are emitted iff the selected root's + /// modeled peak exceeds this threshold. + double peak_threshold = std::numeric_limits::infinity(); }; } // namespace sequant diff --git a/SeQuant/core/binary_node.hpp b/SeQuant/core/binary_node.hpp index 5d0acb51be..c917d0c13b 100644 --- a/SeQuant/core/binary_node.hpp +++ b/SeQuant/core/binary_node.hpp @@ -463,11 +463,23 @@ class FullBinaryNode { /// \return Size of the tree rooted at this node /// [[nodiscard]] std::size_t size() const { - if (leaf()) { - return 1; + // Iterative (explicit stack) node count: recursing left().size() + + // right().size() would descend to the tree's depth and overflow the C++ + // call stack on a deep tree -- e.g. the left-folded Sum-tree binarize + // builds for a Sum with thousands of summands (mirrors the iterative + // destructor / deep_copy above; the recursive form also made this an + // O(N)-deep call on every use, e.g. each equality comparison's size check). + std::size_t n = 0; + std::vector stk; + stk.push_back(this); + while (!stk.empty()) { + FullBinaryNode const* cur = stk.back(); + stk.pop_back(); + ++n; + if (cur->left_) stk.push_back(cur->left_.get()); + if (cur->right_) stk.push_back(cur->right_.get()); } - - return left().size() + right().size() + 1; + return n; } /// diff --git a/SeQuant/core/eval/backend_array_ops.hpp b/SeQuant/core/eval/backend_array_ops.hpp new file mode 100644 index 0000000000..29bcccf0b0 --- /dev/null +++ b/SeQuant/core/eval/backend_array_ops.hpp @@ -0,0 +1,56 @@ +#ifndef SEQUANT_EVAL_BACKEND_ARRAY_OPS_HPP +#define SEQUANT_EVAL_BACKEND_ARRAY_OPS_HPP + +#include +#include +#include + +#include +#include +#include + +namespace sequant { + +/// \brief Backend-provided realizations of the two operations external-axis +/// batching needs but that are backend-specific: constructing a zero +/// destination array and chunking an axis into batches. +/// +/// \details The neutral eval layer names only INDICES (which carry their +/// spaces); the backend (the "user", e.g. mpqc) supplies these closures, so no +/// backend artifact -- a TiledArray tiling has no meaning for, say, an on-disk +/// backend -- ever leaks into the eval layer. +/// +/// This replaces the old "carrier" model, in which the batched executor +/// borrowed an axis's tiling from whichever array in the DAG happened to carry +/// it (\c Result::pre_sized_zeros_over_mode / \c Result::mode_batches) and then +/// had to reconcile that array's Result TYPE and mode ordinal against the +/// scatter destination's. Tiling is a property of the space, not of any one +/// array, so it is sourced once, backend-side, from the index alone. +struct BackendArrayOps { + /// Construct a sufficiently-initialized ZERO result shaped by \p descriptor + /// -- a FULL (unsliced) index list, e.g. a node's \c canon_indices(). The + /// backend maps each index's space to its own artifact and applies its own + /// outer/inner split for proto-bearing (nested) indices, so flat-vs-nested + /// is decided by the descriptor, not by any type reconciliation here. + /// "Sufficiently initialized" is backend-defined (TA: a World + TiledRange, + /// zero-filled; a nested result gets empty inner tiles, filled by the + /// subsequent scatter writes). + std::function const& descriptor)> + make_zeros; + + /// Enumerate the half-open [lo,hi) element ranges chunking \p axis at + /// ~\p target_batch_size. The backend owns the chunking rule (TA lands on + /// tile boundaries). Per-space: two indices of one space chunk identically. + std::function>( + Index const& axis, std::size_t target_batch_size)> + axis_batches; + + /// True iff both closures are installed (a batched run requires them). + explicit operator bool() const noexcept { + return static_cast(make_zeros) && static_cast(axis_batches); + } +}; + +} // namespace sequant + +#endif // SEQUANT_EVAL_BACKEND_ARRAY_OPS_HPP diff --git a/SeQuant/core/eval/backends/btas/result.hpp b/SeQuant/core/eval/backends/btas/result.hpp index a2031a3908..5dce784ae8 100644 --- a/SeQuant/core/eval/backends/btas/result.hpp +++ b/SeQuant/core/eval/backends/btas/result.hpp @@ -379,6 +379,12 @@ class ResultTensorBTAS final : public Result { return eval_result>(std::move(pre)); } + /// Deep copy: the backing tensor type owns its elements, so its copy + /// constructor already produces an independently owned buffer. + [[nodiscard]] ResultPtr clone() const override { + return eval_result>(get()); + } + [[nodiscard]] ResultPtr permute( std::array const& ann) const override { auto const pre_annot = std::any_cast(ann[0]); diff --git a/SeQuant/core/eval/backends/dryrun/cost_model_object.hpp b/SeQuant/core/eval/backends/dryrun/cost_model_object.hpp new file mode 100644 index 0000000000..e73e7d259d --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/cost_model_object.hpp @@ -0,0 +1,262 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_MODEL_OBJECT_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_MODEL_OBJECT_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// +/// \brief Opt-in accumulator for the REPLAY-tallied (recompute-aware) cost. +/// +/// The static per-node cost walk in \c cost_profile() reports order-/batching- +/// blind DP-model quantities (\c CostProfile::model_flops etc.): each internal +/// node is priced ONCE, so the walk never sees the per-occ-block REPLAY +/// recompute the batched evaluator incurs at runtime. This sink is the +/// replay-side counterpart: when a non-null \c CostSink is attached to the +/// \c CostModel shared by every dry-run \c Result token, each ACTUAL product-op +/// execution during the \c Trace::On replay folds its own SLICED-extent cost +/// here (see \c CostModel::tally_op and \c DryRunOps::prod). Because a sliced, +/// occ-DEPENDENT op executed N times does ~1/N work each pass, its sliced-cost +/// sum is work-neutral (~= its unsliced cost); only the occ-INDEPENDENT work +/// re-executed at full size once per block inflates -- so the totals here +/// isolate the recompute the model walk cannot. +/// +/// Mirrors \c sequant::eval::PeakSink (eval.hpp): an OPTIONAL sink, defaulting +/// off, so the production runtime path (which never constructs a dry-run \c +/// CostModel) is byte-identical. The atomics let a fold from a concurrent +/// evaluator stay correct, though \c cost_profile() itself is single-threaded. +/// +/// Per-node AVOIDABLE-recompute tally, keyed by the LABEL signature (result + +/// operand indices). Avoidable recompute is measured in FLOPs against the +/// BATCHING-FREE (unlimited-memory) ideal, where each distinct value is built +/// ONCE at full extent and reused: \c total_flops accumulates the actual +/// (possibly sliced) FLOPs over every build of this value; \c full_flops is the +/// FLOPs to build it once at FULL extent (constant per label). The rollup takes +/// avoidable = max(0, total_flops - full_flops) -- the arithmetic batching +/// repeats beyond building the value once, which is exactly the recompute +/// hoisting exists to avoid. +/// +/// FLOPs (unlike roofline exec) is LINEAR in extents, hence ADDITIVE across +/// slices: disjoint slices that tile the full value sum to exactly \c +/// full_flops +/// => 0 avoidable (tiling repeats no arithmetic), while a value rebuilt full +/// once per block sums to N*full => (N-1)*full avoidable. That additivity is +/// why no slice-context bucketing is needed and why the pathological >100% +/// roofline-spread of the exec-weighted metric cannot arise. +/// +/// NOTE: the per-DISTINCT-value avoidable rollup does NOT live here. It is kept +/// by \c CacheManager::recompute_tally(), keyed by the exact cache node +/// identity (TreeNodeHasher + TreeNodeEqualityComparator) so 64-bit hash +/// collisions are not folded; a string-keyed sink here could not reproduce that +/// identity (the node type is kept out of this header by the +/// dryrun/eval_expr.hpp -> cost_model_object.hpp include cycle). This sink +/// carries only the whole-forest scalar totals (flops/exec/n_ops via \c +/// tally_op). +struct CostSink { + std::atomic flops{0.0}; + std::atomic exec{0.0}; + std::atomic n_ops{0}; +}; + +/// Per-index extent OVERRIDE table: narrows specific indices (by identity, so +/// it survives reshaping across prod/sum/permute -- the same shared/ +/// tensor MODE POSITION (0-based, in the value's canon index order) to a +/// runtime-realized element count. Positional -- NOT keyed by Index -- because +/// a DAG value has no intrinsic labels: only an op binds labels to it, so the +/// only stable handle on a mode across ops is its position. Populated by +/// Result::slice_mode()/mode_batches() call sites (see result.hpp); empty => +/// no override, the regime's nominal extent applies. This table -- not a +/// second cost model -- is what lets a zero-data DryRun Result report the +/// REALIZED (possibly runtime-sliced) size rather than always the full +/// regime extent, which is exactly the signal Task 6's replay witnesses. +/// Consumers that operate in LABEL space (flops, keyed by the op's annotation) +/// resolve positions to labels through that annotation first. +using ExtentOverrides = container::map; + +/// +/// \brief Bundles the optimizer's own cost closures (memsize/flops/roofline) +/// behind one value type so dry-run Results report MODEL size (not an +/// allocated size), and the harness can additionally read FLOPs and +/// projected execution cost per operation. +/// +/// This is a thin wrapper: all arithmetic is delegated verbatim to +/// \c sequant::opt::detail::memsize_counter / \c flops_counter / \c +/// roofline_op_cost (see \c core/optimize/single_term_detail.hpp and \c +/// core/optimize/cost_model.hpp) -- no parallel cost model is implemented +/// here. The only thing this class adds is the ExtentOverrides indirection: +/// each query builds a fresh (cheap; no heap allocation beyond the closure +/// itself) index-to-extent callable that consults \p overrides before +/// falling back to the SizeRegime's nominal extent, then hands that callable +/// to the counter. +/// +class CostModel { + public: + explicit CostModel(SizeRegime regime, RooflineParams roofline = {}) + : regime_{std::move(regime)}, roofline_{roofline} {} + + /// + /// \brief Bytes for a tensor with these (literal, canon-order) indices, + /// honoring any per-index extent override (a runtime slice_mode()/ + /// mode_batches() narrowing). + /// + /// Delegates the extent-product / composite-moment math to \c + /// memsize_counter, invoked with \p idxset as the sole (`lhs`) operand and + /// empty `rhs`/`result` -- an empty operand's tot_indices() split + /// accumulates the starting product of 1.0, which memsize_counter itself + /// special-cases to contribute zero bytes, so this reproduces exactly the + /// single-operand byte count \c memsize_counter is designed to report per + /// operand. + /// + [[nodiscard]] std::size_t memsize( + container::svector const& idxset, + ExtentOverrides const& overrides = {}) const { + // Resolve the POSITIONAL overrides against THIS index list: override at + // mode position `pos` applies to `idxset[pos]`, whatever its label. Build a + // per-call Index->extent map so make_extent_fn's atom lookup finds it (the + // counter revisits idxset[pos] by identity, incl. as a composite proto). + // Named local: make_extent_fn captures it by reference, so it must outlive + // `ext` (a temporary here would dangle). + auto const resolved = resolve_overrides(idxset, overrides); + auto const ext = make_extent_fn(resolved); + auto const mc = + sequant::opt::detail::memsize_counter(ext, regime_.inner_pow_fn()); + double const elems = + mc(idxset, container::svector{}, container::svector{}); + return static_cast(elems * numeric_size_); + } + + /// + /// \brief Multiply-add count for a contraction whose free (result) indices + /// are \p out and whose contracted (summed-over) indices are + /// \p contracted. + /// + /// Delegates to \c flops_counter, which prices the union of its (lhs, rhs, + /// result) arguments; passing (\p out, \p contracted, {}) makes that union + /// exactly `out U contracted` -- the full index set touched by the + /// contraction, since by construction `contracted` holds precisely the + /// indices present in both operands but absent from the result. + /// + /// \p label_extents maps an ANNOTATION label (an Index appearing in \p out or + /// \p contracted) to its runtime-realized (sliced) extent. Unlike the value's + /// positional \c ExtentOverrides, this is keyed by Index because \p out / + /// \p contracted ARE labels -- the op's annotation is the sole source of + /// labels. \c DryRunOps::prod builds it from each operand's positional + /// overrides via that operand's annotation (see \c extents_by_label). + [[nodiscard]] double flops( + container::svector const& out, + container::svector const& contracted, + container::map const& label_extents = {}) const { + auto const ext = make_extent_fn(label_extents); + auto const fc = + sequant::opt::detail::flops_counter(ext, regime_.inner_pow_fn()); + return fc(out, contracted, container::svector{}); + } + + /// + /// \brief Roofline-projected execution cost of one contraction (see + /// \c sequant::opt::detail::roofline_op_cost). + /// + /// \p left_bytes / \p right_bytes are operand footprints in BYTES (as + /// reported by \c Result::size_in_bytes()); converted to elements (the + /// counter's native unit) via \c numeric_size before delegating. + /// + [[nodiscard]] double exec_cost(double flops_count, std::size_t left_bytes, + std::size_t right_bytes) const { + double const traffic_elems = + static_cast(left_bytes + right_bytes) / numeric_size_; + return sequant::opt::detail::roofline_op_cost( + flops_count, traffic_elems, roofline_.machine_balance, + roofline_.fast_mem_elems, roofline_.block_tiles, + roofline_.block_prefactor); + } + + [[nodiscard]] SizeRegime const& regime() const noexcept { return regime_; } + + /// + /// \brief Attach (or detach with nullptr) the optional replay cost sink. + /// + /// Const because the \c CostModel is shared as \c shared_ptr + /// by every dry-run \c Result token; \c cost_profile() sets this on its one + /// shared model just before the \c Trace::On replay so each product op can + /// fold into it. The pointee (a \c CostSink) is external and owns the mutable + /// state; this only records where to fold. Off by default => no fold => the + /// dry-run backend is byte-identical when unused. + /// + void set_cost_sink(CostSink* sink) const noexcept { sink_ = sink; } + + /// + /// \brief Fold one product op's SLICED-extent \p flops_count / \p exec into + /// the attached sink (no-op when none is attached). + /// + /// Called at each actual product execution in the replay, so a contraction + /// re-executed once per occ block is tallied once per block at its sliced + /// size -- exactly the recompute signal (see \c CostSink). + /// + void tally_op(double flops_count, double exec) const noexcept { + if (!sink_) return; + sink_->flops.fetch_add(flops_count, std::memory_order_relaxed); + sink_->exec.fetch_add(exec, std::memory_order_relaxed); + sink_->n_ops.fetch_add(1, std::memory_order_relaxed); + } + + private: + // Index-to-extent callable consulting `overrides` first, else the + // regime's nominal extent. The returned std::function captures `overrides` + // (and `this`) BY REFERENCE and is only ever used -- never stored -- + // within the (memsize/flops) call that constructs it, so the reference + // stays valid for its entire lifetime. Explicit (non-deduced) return type + // so this can be called from memsize()/flops(), which appear earlier in + // the class body (a deduced `auto` return type would require the + // definition to precede every use, even within the same class). + [[nodiscard]] std::function make_extent_fn( + container::map const& overrides) const { + return [this, &overrides](Index const& ix) -> std::size_t { + if (auto it = overrides.find(ix); it != overrides.end()) + return it->second; + return regime_.extent(ix); + }; + } + + // Resolve a value's POSITIONAL overrides against its own index list: override + // at mode position `pos` binds to `idxset[pos]`. Yields an Index-keyed map so + // make_extent_fn's per-atom lookup finds the sliced extent wherever that + // Index recurs in idxset (including as a composite's outer proto), exactly as + // the pre-positional Index-keyed table did -- but now the key is derived from + // THIS list, not carried from a producer's labels. + [[nodiscard]] static container::map resolve_overrides( + container::svector const& idxset, + ExtentOverrides const& overrides) { + container::map out; + for (auto const& [pos, w] : overrides) + if (pos < idxset.size()) out.emplace(idxset[pos], w); + return out; + } + + SizeRegime regime_; + RooflineParams roofline_; + // Optional replay cost sink (see set_cost_sink/tally_op). Mutable so it can + // be (de)attached on a shared_ptr; a raw non-owning pointer + // to caller-owned state. nullptr (default) => tally_op is a no-op. + mutable CostSink* sink_ = nullptr; + // sizeof(double); see doc/dev/plans/2026-07-04-dryrun-eval-backend.md Task 2 + // note on OptimizeOptions::numeric_size (hardcoded here, matching the C60 + // trace's real-only CSV-CCk path; complex CSV-CCk is out of scope, see the + // plan's carried-minor N4). + double numeric_size_ = 8.0; +}; + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_MODEL_OBJECT_HPP diff --git a/SeQuant/core/eval/backends/dryrun/cost_profile.hpp b/SeQuant/core/eval/backends/dryrun/cost_profile.hpp new file mode 100644 index 0000000000..1c1583a14d --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/cost_profile.hpp @@ -0,0 +1,634 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_PROFILE_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_PROFILE_HPP + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// Configuration for a faithful (gated) dry-run cache: the same footprint gate +/// and cross-occurrence batch-variant veto the real batched eval loop applies, +/// so a batch-variant giant (a `mu~`/`K`-carrying DF intermediate) is NOT +/// cached whole but recomputed sliced under each consumer's batch trigger. +/// +/// The element types mirror the gated \c sequant::cache_manager overload +/// (\c cache_manager.hpp): \c is_volatile is invoked on every \c TreeNode +/// (deduced as \c EvalNodeDryRun for the dry-run backend). +/// +/// This struct lives here (rather than only in the test) because Task 4's +/// \c cost_profile() entry point consumes it. +struct CacheConfig { + /// Footprint gate (bytes): a node whose result footprint exceeds this is not + /// cached. 0 (default) disables the gate. + double max_footprint = 0.; + /// Minimum non-persistent repeats to cache an internal node (CSE rule). + std::size_t min_repeats = 1; + /// `bool(EvalNodeDryRun const&)`: true if the node is intrinsically volatile + /// (typically the amplitude leaves). Empty => nothing is volatile. + std::function is_volatile; +}; + +/// Builds a gated dry-run cache from an eval-node range, a \p cfg, and a +/// \p regime that supplies the moment-aware node-size model used for the +/// footprint gate. +/// +/// The footprint functor sizes a node's result (its \c canon_indices()) with +/// the SAME moment-aware counter the DryRun \c Result uses +/// (\c memsize_counter over \c regime.idx_to_extent()/inner_pow_fn()), scaled +/// to bytes, so the gate compares like-for-like against \c cfg.max_footprint. +/// +/// Unlike the SIMPLE \c cache_manager(nodes) factory the ad-hoc dry-run test +/// sites use, this routes through the GATED overload so free-batchable-axis +/// giants are vetoed (matching the real run). The returned cache is used across +/// the WHOLE forest without a per-summand reset (matching a real solve's +/// whole-iteration cache scope; \c cost_profile relies on the lifetime mask to +/// release each value after its last cross-term use, so cross-summand-shared +/// values are reused, not rebuilt). +/// +/// \param nodes the evaluation forest (a range of \c EvalNodeDryRun). +/// \param cfg footprint/repeat/volatility/batchability configuration. +/// \param regime the size regime supplying extents and CSV moment tables. +/// \return a \c CacheManager over \c EvalNodeDryRun. +template +auto build_dryrun_cache(NodeRange const& nodes, CacheConfig const& cfg, + SizeRegime const& regime) { + auto memsize = sequant::opt::detail::memsize_counter(regime.idx_to_extent(), + regime.inner_pow_fn()); + + // Footprint (bytes) of a node's RESULT: canon_indices() fed to the + // moment-aware counter (as the counter's `result` slot; the empty lhs/rhs + // contribute nothing) times 8 bytes/element. Same arithmetic as the DryRun + // Result::size_in_bytes(), so the gate is faithful. + auto footprint_of = + [memsize = std::move(memsize)](EvalNodeDryRun const& n) -> double { + std::vector const result(n->canon_indices().begin(), + n->canon_indices().end()); + return memsize(std::vector{}, std::vector{}, result) * 8.0; + }; + + // Default the volatility predicate so the gated factory never invokes an + // empty std::function (nothing volatile leaves that gate inert, matching the + // factory's own default). + std::function is_volatile = + cfg.is_volatile ? cfg.is_volatile + : std::function( + [](EvalNodeDryRun const&) { return false; }); + + // Note: the gated sequant::cache_manager() overload below stamps the + // cross-occurrence lifetime mask on `nodes` itself before its DAG walk / + // veto (cache_manager.hpp), so this call site does not need to do so. + return sequant::cache_manager(nodes, std::move(is_volatile), cfg.min_repeats, + std::move(footprint_of), cfg.max_footprint); +} + +/// One recomputed value's avoidable-recompute breakdown (see +/// \c CostProfile::avoidable_nodes). \c label is the value's signature (a +/// `result;lhs;rhs` full-label string built in \c DryRunOps::prod). \c flops is +/// the avoidable recompute in FLOPs -- `total_flops - full_flops`, the +/// arithmetic the batched replay repeated beyond building this value ONCE at +/// full extent (the batching-free / unlimited-memory ideal). \c count is the +/// equivalent number of extra full rebuilds (`total_flops/full_flops - 1`): 0 +/// when the builds tile the value once (disjoint slices), ~N-1 when the value +/// is rebuilt full N times (an un-hoisted invariant). Only values with a +/// positive avoidable FLOP count are recorded. +struct AvoidableNode { + std::string label; + double count = 0; + double flops = 0; +}; + +/// Roll a replay's per-DISTINCT-value build tally (a \c CacheManager's +/// \c recompute_tally(), populated by \c CacheManager::tally_build from the +/// eval loop's product-build site during a \c Trace::On replay) into the +/// avoidable-recompute breakdown: per distinct value, avoidable FLOPs = +/// max(0, total_flops - full_flops), sorted by avoidable FLOPs descending, +/// keeping only values with a positive amount. The tally is keyed by the EXACT +/// cache identity (TreeNodeHasher + TreeNodeEqualityComparator = topological +/// hash bin + Bliss connectivity 3-way cmp + recursive child compare), so two +/// topologically-distinct nodes sharing a 64-bit hash are NOT folded (a +/// hash-string key folds them, inventing avoidable recompute; a space/arity +/// structural key can't separate same-shape different-connectivity nodes +/// either), and per-block / alpha-renamed builds of ONE value ARE folded. +/// Shared by \c cost_profile() (whole-forest rollup) and any caller that drives +/// its own tally-enabled replay (e.g. the schedule-dump test). \c label is the +/// value's topological hash as a string (the join key the IR and run-event +/// nodes carry). Single-threaded caller expected: the replay has finished. +template +inline std::vector avoidable_nodes_from_tally( + TallyMap const& tally) { + // Per DISTINCT value, rolled up over its SLICES (see BuildTally): for each + // slice, total += builds*cost and build_once += cost, so avoidable (the + // arithmetic the replay repeated beyond building each distinct slice once) is + // sum over slices of (builds-1)*cost. A value tiled over DISTINCT slices has + // builds==1 per slice => 0 avoidable (tiling, even non-uniform); a value + // rebuilt at the SAME slice (e.g. an invariant rebuilt every block of a loop + // it does not carry) has builds>1 there => that slice's (builds-1)*cost is + // avoidable. No full-extent denominator is used; every number is actual + // replay FLOPs. + auto roll = [](auto const& t) { + double total = 0, once = 0, extra_builds = 0; + for (auto const& [sig, bc] : t.slices) { + total += bc.count * bc.flops; + once += bc.flops; + extra_builds += static_cast(bc.count - 1); + } + return std::tuple{total, once, extra_builds}; + }; + std::vector out; + for (auto const& [node, t] : tally) { + auto const [total, once, extra_builds] = roll(t); + double const avoidable = total - once; + if (avoidable <= 0.0) continue; + out.push_back( + {std::to_string(node->hash_value()), extra_builds, avoidable}); + } + std::sort(out.begin(), out.end(), + [](AvoidableNode const& a, AvoidableNode const& b) { + return a.flops > b.flops; + }); + + // DIAGNOSTIC (SEQUANT_AVOIDABLE_DEBUG): per-value FLOP accounting, worst + // first. total == build_once => every slice built once (tiling, 0 avoidable); + // total >> build_once => some slice rebuilt (invariant recompute). + if (std::getenv("SEQUANT_AVOIDABLE_DEBUG")) { + double sum_total = 0, sum_avoid = 0; + for (auto const& [node, t] : tally) { + auto const [total, once, extra] = roll(t); + (void)extra; + sum_total += total; + if (total > once) sum_avoid += total - once; + } + std::fprintf(stderr, + "[avoidable-debug] dryrun_flops=%.6g avoidable_flops=%.6g " + "frac=%.4f n_values=%zu\n", + sum_total, sum_avoid, + sum_total > 0 ? sum_avoid / sum_total : 0, tally.size()); + // Worst offenders by avoidable flops: builds = total builds over slices, + // slices = distinct slices, so builds>>slices is genuine same-slice + // recompute (an invariant rebuilt every block), builds==slices is pure + // tiling. + std::vector> + ranked; // {avoid, builds, slices, total, once} + for (auto const& [node, t] : tally) { + auto const [total, once, extra] = roll(t); + (void)extra; + if (total <= once) continue; + std::size_t builds = 0; + for (auto const& [sig, bc] : t.slices) builds += bc.count; + ranked.emplace_back(total - once, builds, t.slices.size(), total, once); + } + std::sort(ranked.begin(), ranked.end(), [](auto const& a, auto const& b) { + return std::get<0>(a) > std::get<0>(b); + }); + std::size_t shown = 0; + for (auto const& [av, builds, nslices, total, once] : ranked) { + if (shown++ >= 20) break; + std::fprintf(stderr, + " builds=%zu slices=%zu total=%.4g build_once=%.4g " + "avoid=%.4g\n", + builds, nslices, total, once, av); + } + } + return out; +} + +/// Summary of the modeled cost of a factorized dry-run eval forest, as produced +/// by \c cost_profile(). All quantities are summed/maxed over every summand +/// tree in the forest. +struct CostProfile { + /// Predicted peak working-set (bytes). Task 6 (whole-scope batched DAG + /// execution design) makes this model SELECTED by \c + /// BatchPolicy::scheduler, since forest descent and whole-scope descent + /// realize different co-residency: + /// + /// - \p policy.scheduler != BatchScheduler::whole_scope (default, forest + /// descent): the max over summands of the batched-scratch high-watermark + /// folded by the Task-3 \c PeakSink and the outer gated cache's \c + /// working_set_hwmark() -- unchanged from before this field's Task-6 + /// selection existed. See the paragraphs below for its accounting detail. + /// - \p policy.scheduler == BatchScheduler::whole_scope (whole-scope + /// descent): the CO-RESIDENCY oracle (\c eval::peak_profile_sweep over \c + /// eval::compute_dag_path's \c home_modes-based footprints), computed + /// ONCE over the whole fused forest rather than per-summand. Per the + /// design's "paradox resolved" section, this is the model that MATCHES + /// the realized whole-scope peak (forest descent never co-resides + /// cross-tree, so the batched-scratch replay watermark below is the wrong + /// oracle once execution actually routes through \c + /// eval::evaluate_whole_scope). + /// + /// The remainder of this doc comment describes the flag-OFF (default) + /// accounting; it is unaffected by the flag. + /// + /// This ACCOUNTS FOR co-resident residency across the scope chain, rather + /// than being the max-of-independent-hwmarks lower bound it was before: each + /// per-op hwmark folded into a cache's \c working_set_hwmark_ (in eval.hpp) + /// already adds \c CacheManager::chain_residency() of that cache's + /// scope-chain ancestors at the instant of the op, so a scratch cache's + /// high-watermark is the max over its life of (scratch residency + + /// everything alive up its parent chain at that instant) -- the co-resident + /// sum when a persistent cross-term cache entry is alive at the same instant + /// as a batched-inner transient. The outer (root) cache has no parent, so + /// its own hwmark is unaffected (added term is 0); the \c max() fold here + /// (over \c peak.load() and \c cache.working_set_hwmark()) is then correct in + /// both regimes -- batched (peak.load() already carries the co-resident + /// scratch peak) and unbatched (peak.load() == 0, the outer hwmark alone is + /// the peak). + /// + /// Each live buffer is counted exactly once: the per-op operand guards skip + /// an operand whose buffer any cache on the scope chain already holds + /// (\c CacheManager::chain_holds(), pointer identity), so a value read full + /// from an ANCESTOR cache is counted only via \c chain_residency() and not + /// again as an operand, while a sliced/permuted/phase-shifted read (a + /// distinct buffer) is correctly added. + /// + /// One known deviation remains -- an UNDER-count that keeps this a lower + /// bound in exactly one place: the external-scatter accumulator (\c dest in + /// eval.hpp) is a plain local, not a cache entry, so it is invisible to + /// \c chain_residency() even though it co-resides with every inner block's + /// working set. Pre-existing, outside this field's scope. + double peak_bytes = 0; + /// STATIC per-node DP-MODEL FLOPs: summed unweighted static contraction FLOPs + /// over all internal nodes, from the order-/batching-blind static walk. NOT + /// CSE-aware across summands (a cross-term shared intermediate is walked, and + /// its FLOPs counted, once per occurrence, not once overall), and NOT + /// replay-aware: each node is priced exactly once, so this never reflects the + /// per-occ-block recompute the batched replay incurs. Compare against \c + /// dryrun_flops: the two are ~equal when no batching engages, and \c + /// dryrun_flops exceeds this by the recompute factor when it does. + double model_flops = 0; + /// STATIC per-node DP-MODEL roofline exec cost, summed over all internal + /// nodes. Same per-occurrence (not CSE-deduplicated), batching-blind caveat + /// as \c model_flops. + double model_exec = 0; + /// Number of internal (contraction) nodes across the forest (static count). + std::size_t model_n_ops = 0; + /// REPLAY-tallied (recompute-aware) FLOPs: the sum, over every ACTUAL product + /// op executed in the \c Trace::On replay, of that op's SLICED-extent flops + /// (folded via the \c CostSink attached to the shared \c CostModel; see + /// result.hpp \c DryRunOps::prod). A batched op re-executed once per occ + /// block is charged once per block at its sliced size, so occ-DEPENDENT work + /// stays ~work-neutral while occ-INDEPENDENT recompute (persistent + /// intermediates / leaf re-materializations re-run at full size per block) + /// inflates -- making this the metric that PREDICTS batched-replay + /// overcompute (e.g. occ-batching being ~Nx aux-only). Equal to \c + /// model_flops (up to the product-op-only tally) when no batching engages. + double dryrun_flops = 0; + /// REPLAY-tallied roofline exec cost (traffic-dominated, the better wall-time + /// proxy), summed per product-op execution. Same recompute semantics as \c + /// dryrun_flops. + double dryrun_exec = 0; + /// Number of product-op EXECUTIONS in the replay (counts re-executions per + /// batch block), so it grows with recompute -- unlike \c model_n_ops. + std::size_t dryrun_n_ops = 0; + + /// Per-value avoidable-recompute breakdown: one entry per DISTINCT value + /// whose batched replay repeated arithmetic beyond building it ONCE at full + /// extent + /// (`total_flops > full_flops`) -- the recompute a hoist would have avoided. + /// Sorted by avoidable FLOPs descending. Empty when batching repeats no + /// arithmetic (every value is built at most once-worth, e.g. disjoint slices + /// tiling it). See \c DryRunOps::prod (per-build tally) and the post-replay + /// rollup in \c cost_profile(). + std::vector avoidable_nodes; + /// DIAGNOSTIC: per-value (label signature) build-once FLOPs from the replay, + /// so a caller can join per node (by the SAME signature the schedule Build + /// event carries) against an independent per-node model and localize any + /// per-node flops disagreement. Populated from the CostSink's per_node map. + std::map sig_full_flops; + /// Total avoidable recompute in FLOPs (sum of \c avoidable_nodes[i].flops): + /// arithmetic the batched replay repeated beyond the build-once ideal. + /// Compare against \c dryrun_flops for the avoidable FRACTION (see \c + /// avoidable_time()). Zero when batching repeats no arithmetic. + double avoidable_flops = 0; + /// Total avoidable recompute expressed as equivalent extra full rebuilds (sum + /// of \c avoidable_nodes[i].count). + double avoidable_ops = 0; + + /// Avoidable FRACTION of replay arithmetic: \c avoidable_flops / \c + /// dryrun_flops (0 when no ops ran), in [0, 1] by construction. The + /// single-number "how much of the batched replay's arithmetic was repeated + /// recompute vs. the unlimited-memory ideal" summary. (FLOPs, not roofline + /// exec: recompute is repeated WORK, and FLOPs -- being linear in extents -- + /// makes disjoint slicing exactly free and keeps this bounded.) + [[nodiscard]] double avoidable_time() const { + return dryrun_flops > 0 ? avoidable_flops / dryrun_flops : 0.0; + } +}; + +/// Replays a factorized eval forest zero-data through the real eval loop -- +/// with a gated cache built from \p cfg (Task 2) and a \c PeakSink threaded +/// through the batched evaluator (Task 3) -- and, alongside, does a static walk +/// of the forest to accumulate FLOPs / roofline exec cost / op count. This is +/// the single reusable entry point both SeQuant tests and MPQC call. +/// +/// \par The printing gate +/// \c CacheManager::working_set_hwmark() only accumulates while +/// \c sequant::eval::log::printing() is true (the hwmark update sits on the +/// trace-printing path). This routine therefore FORCES the eval logger's level +/// > 0 around the replay -- discarding the narrow trace to a null sink when no +/// \p trace is requested -- and restores the previous logger state afterward, +/// so \c peak_bytes is non-zero even with no trace stream. +/// +/// \par Global state / threading +/// This routine mutates the process-global \c Logger::instance().eval state +/// (\c level and \c stream) for the duration of the replay (restored on every +/// exit path, including exceptions). Because that state is a singleton shared +/// by the whole process, \c cost_profile() MUST be called single-threaded -- +/// e.g. as a pre-flight step before, or a post-hoc step after, the real +/// multi-threaded eval -- never concurrently with other code that reads or +/// writes \c Logger::instance().eval (including another concurrent +/// \c cost_profile() call). +/// +/// \par FLOPs / exec accounting (model vs dryrun) +/// \c CostProfile::model_flops and \c CostProfile::model_exec are accumulated +/// by a STATIC walk that sums a contribution per BINARIZED internal node of the +/// forest; they are NOT CSE-aware across summands (a shared intermediate that +/// recurs across summand trees, or multiple times within one, is counted once +/// per occurrence) and NOT replay-aware (each node is priced exactly once, +/// blind to order/batching). \c CostProfile::dryrun_flops / \c dryrun_exec / +/// \c dryrun_n_ops are the recompute-aware counterparts, tallied from the +/// \c Trace::On replay below: every ACTUAL product-op execution folds its +/// SLICED-extent cost into a \c CostSink attached to the shared \c CostModel, +/// so an op re-executed once per batch block is counted once per block. When no +/// batching engages the two agree (up to the product-op-only dryrun tally); +/// when it does, \c dryrun_* exceeds \c model_* by the recompute factor -- a +/// TIME (arithmetic) cost that \c peak_bytes, a SPACE (co-resident working +/// set) measure, does not express. +/// +/// \param forest per-summand optimized+binarized eval forest (the real IR). +/// \param policy the batch policy driving the replay evaluator; its accept is +/// the derived role union \c policy.is_batchable_index(). +/// \param cfg gated-cache config (footprint gate, volatile, repeats). +/// \param regime the size regime supplying extents and CSV moment tables; +/// the internal \c CostModel and \c DryRunLeafEvaluator are built from +/// it. +/// \param trace optional per-op trace sink (nullptr = no trace). When +/// non-null, the eval loop's narrow trace is transcoded (UTF-8) into it. +/// \param router optional placement router (see \c placement_router.hpp), +/// attached to the replay cache right after it is built. Every current +/// caller omits this (nullptr, the default), which leaves the router +/// seam in \c evaluate() inert -- byte-identical to before this +/// parameter existed. Phase 2 wires a router only from test call sites. +/// \return the accumulated \c CostProfile. +inline CostProfile cost_profile( + std::vector const& forest, BatchPolicy const& policy, + CacheConfig const& cfg, SizeRegime const& regime, + std::wostream* trace = nullptr, + PlacementRouter const* router = nullptr, + sequant::eval::ScheduleSink* schedule_sink = nullptr) { + CostProfile profile; + + auto cm = std::make_shared(regime); + DryRunLeafEvaluator const leaf{cm}; + + // ---- static cost walk (independent of the replay) -------------------- + // For every internal node: flops = flops_counter(left, right, result); the + // roofline exec cost uses the left operand's footprint as the transferred + // bytes and the arena convention (4096) the [dryrun-costmodel] test fixes. + auto const flops_of = sequant::opt::detail::flops_counter( + regime.idx_to_extent(), regime.inner_pow_fn()); + std::function walk = + [&](EvalNodeDryRun const& n) { + if (n.leaf()) return; + profile.model_n_ops += 1; + double const node_flops = + flops_of(n.left()->canon_indices(), n.right()->canon_indices(), + n->canon_indices()); + profile.model_flops += node_flops; + container::svector const left(n.left()->canon_indices().begin(), + n.left()->canon_indices().end()); + profile.model_exec += + cm->exec_cost(node_flops, cm->memsize(left), 4096); + walk(n.left()); + walk(n.right()); + }; + for (auto const& root : forest) walk(root); + + // ---- peak replay through the real eval loop -------------------------- + // The cache's batch-variant veto is driven by the cross-occurrence lifetime + // mask (stamped inside build_dryrun_cache -> cache_manager); the replay + // EVALUATOR's accept is the derived role union, applied inside + // make_evaluator(policy) via policy.is_batchable_index(). + auto cache = build_dryrun_cache(forest, cfg, regime); + // The batched custom evaluator (make_evaluator with a batching policy) reads + // the backend array-ops off the cache chain (zero destination + axis + // chunking); without them make_batched_custom_evaluator asserts and the + // replay below throws (caught and swallowed -> a silently ZERO dryrun tally). + // Wire the DryRun array-ops so the batched replay actually runs. `aops` must + // OUTLIVE the replay loop (the cache holds a non-owning pointer). + auto const aops = make_dryrun_array_ops(cm); + cache.set_array_ops(&aops); + // Enable the per-DISTINCT-value recompute tally on the (root) cache: the eval + // loop's product-build site records each build against the node's identity + // here (CacheManager::tally_build), keyed by the exact cache identity, for + // the avoidable rollup below. Off by default so the wet eval path never + // populates it; only this costing replay opts in. + cache.set_recompute_tally_enabled(true); + // Null (default) => every existing caller's replay is unaffected, since + // set_placement_router(nullptr) is exactly the cache's own default. + cache.set_placement_router(router); + // Route this replay's SCHEDULE_RUN_EVENT records to the caller's sink (if + // any). Null (default) => no dump, byte-identical. The wet-run sets an + // equivalent sink on its own eval cache, so the two batched schedules can be + // captured and diffed for structural equivalence. + cache.set_schedule_sink(schedule_sink); + + auto& logger = Logger::instance(); + // RAII guard restoring the process-global Logger::eval state on EVERY exit + // path from this point on -- normal return, early return, or an exception + // unwinding out of the replay loop below -- not just the two trailing + // assignments a plain save/restore would rely on. Without this, a throw + // from anything in the loop OTHER than evaluate (e.g. + // std::bad_alloc from make_evaluator/set_custom_evaluator/ + // working_set_hwmark/cache.reset()) would unwind past the local + // `trace_capture` destructor while `logger.eval.stream` still points at it, + // leaving a dangling pointer in the process-global singleton with + // level == 2 still set. + struct LoggerEvalGuard { + decltype(logger.eval)& eval; + std::size_t const prev_level; + std::ostream* const prev_stream; + ~LoggerEvalGuard() { + eval.level = prev_level; + eval.stream = prev_stream; + } + } logger_eval_guard{logger.eval, logger.eval.level, logger.eval.stream}; + + // Force printing() on so working_set_hwmark() accumulates. The eval logger + // stream is narrow; capture into a narrow buffer only when a (wide) trace + // sink was requested, else discard to a null stream. + std::ostringstream trace_capture; + logger.eval.level = 2; + logger.eval.stream = trace ? &trace_capture : nullptr; + + // Attach a replay cost sink to the shared CostModel so each product op + // executed in the Trace::On replay below folds its SLICED-extent cost here + // (DryRunOps::prod -> CostModel::tally_op). Only DryRun Results built from + // this same `cm` fold in, and only while the sink is attached, so this + // records exactly the replay recompute for THIS forest. Detached on every + // exit path by the guard below (the model outlives the results, but leaving a + // dangling sink pointer set would be a latent hazard if `cm` were reused). + CostSink costsink; + cm->set_cost_sink(&costsink); + struct CostSinkGuard { + CostModel const& cm; + ~CostSinkGuard() { cm.set_cost_sink(nullptr); } + } costsink_guard{*cm}; + + std::atomic peak{0.0}; + for (auto const& root : forest) { + cache.set_custom_evaluator(sequant::make_evaluator( + policy, leaf, sequant::make_no_scope_guard{}, &peak)); + try { + (void)sequant::evaluate(root, leaf, cache); + } catch (std::exception const&) { + // A zero-data DryRun sizing throw must not mask the peak read. + } + // Fold the outer cached residency BEFORE reset() (which zeroes the + // hwmark). `peak` folds every batched scratch high-watermark across all + // summands via std::max, so its running load() is the global scratch peak. + // Only feeds profile.peak_bytes when the co-residency oracle (below) is + // NOT selected -- see CostProfile::peak_bytes's doc comment (Task 6): + // this replay watermark models forest descent, the co-residency oracle + // models whole-scope descent, and the two are mutually exclusive + // predictors, not folded together. + if (policy.scheduler != BatchScheduler::whole_scope) + profile.peak_bytes = std::max({profile.peak_bytes, peak.load(), + double(cache.working_set_hwmark())}); + // NO per-term reset. A real solve's cache spans the whole iteration (all + // summands + equations) and reuses cross-summand values, evicting only by + // the lifetime mask stamped over the whole forest. Resetting between terms + // would instead drop non-persistent scratch after each summand, REBUILDING + // a cross-summand-shared value in every summand -- over-counting recompute + // (and mis-estimating peak) vs any real run. So the replay keeps the shared + // cache across the whole forest; the lifetime mask releases each value + // after its last cross-term use. (This makes the dry-run schedule match the + // wet run's; see doc/dev/specs/2026-08-05-...schedule-equivalence.) + } + + // Task 6 (whole-scope batched DAG execution design): under + // policy.scheduler == BatchScheduler::whole_scope, replace the per-summand + // replay watermark folded above (skipped, see the guard inside the loop) + // with the + // CO-RESIDENCY oracle computed ONCE over the WHOLE fused forest -- the + // model that matches the peak sequant::eval::evaluate_whole_scope actually + // realizes (see CostProfile::peak_bytes's doc comment). block_of mirrors + // the batch-partition source the whole-scope driver itself uses (see + // sequant::evaluate(Nodes const&, BatchPolicy const&, ...), + // scope_executor.hpp): policy.batch_target_size, guarded the same way + // (empty => decline batching, size 1) so an unset policy never throws + // std::bad_function_call out of compute_dag_path. + if (policy.scheduler == BatchScheduler::whole_scope) { + std::function const block_of = + policy.batch_target_size + ? policy.batch_target_size + : std::function( + [](Index const&) -> std::size_t { return 1; }); + auto const dag = compute_dag_path(forest, *cm, block_of); + profile.peak_bytes = peak_profile_sweep(dag).peak_bytes; + } + + // Read the replay-tallied (recompute-aware) totals the sink accumulated over + // every product op of every summand's Trace::On replay. (costsink_guard + // detaches the sink from `cm` on function exit.) + profile.dryrun_flops = costsink.flops.load(std::memory_order_relaxed); + profile.dryrun_exec = costsink.exec.load(std::memory_order_relaxed); + profile.dryrun_n_ops = costsink.n_ops.load(std::memory_order_relaxed); + + // Per-value avoidable-recompute rollup (shared with the schedule-dump + // emitter): for each DISTINCT value the replay built, avoidable FLOPs = the + // actual replay FLOPs of every slice rebuilt beyond once (see + // avoidable_nodes_from_tally() and CacheManager::BuildTally). + auto const& tally = cache.recompute_tally(); + profile.avoidable_nodes = avoidable_nodes_from_tally(tally); + for (auto const& an : profile.avoidable_nodes) { + profile.avoidable_flops += an.flops; + profile.avoidable_ops += an.count; + } + // DIAGNOSTIC: per-DISTINCT-value build-once flops (sum over its DISTINCT + // slices of one build's cost), for external per-node join and the build-once + // identity check (sum == dryrun_flops - avoidable_flops). Keyed by a unique + // running index prefixed to the node hash so two nodes that share a 64-bit + // hash (the case this whole tally keying exists to separate) still get + // distinct map entries and the sum stays exact. + { + std::size_t idx = 0; + for (auto const& [node, t] : tally) { + double once = 0.0; + for (auto const& [sig, bc] : t.slices) once += bc.flops; + profile.sig_full_flops.emplace( + std::to_string(idx++) + ":" + std::to_string(node->hash_value()), + once); + } + } + + // logger_eval_guard's destructor restores logger.eval.{level,stream} at + // function exit (see above); no manual restore needed here. + + // If a wide trace sink was requested, transcode the captured narrow (UTF-8) + // eval trace into it (the eval loop writes only to the narrow logger stream; + // index labels such as mu~/K are multi-byte, so a plain widen would corrupt + // them -- decode UTF-8 to code points instead). + if (trace) { + std::string const s = trace_capture.str(); + std::wstring w; + w.reserve(s.size()); + for (std::size_t i = 0; i < s.size();) { + unsigned char const c = static_cast(s[i]); + char32_t cp; + std::size_t len; + if (c < 0x80) { + cp = c; + len = 1; + } else if ((c >> 5) == 0x6) { + cp = c & 0x1Fu; + len = 2; + } else if ((c >> 4) == 0xE) { + cp = c & 0x0Fu; + len = 3; + } else if ((c >> 3) == 0x1E) { + cp = c & 0x07u; + len = 4; + } else { + cp = c; // invalid lead byte: pass through + len = 1; + } + for (std::size_t k = 1; k < len && i + k < s.size(); ++k) + cp = (cp << 6) | (static_cast(s[i + k]) & 0x3Fu); + w.push_back(static_cast(cp)); + i += len; + } + *trace << w; + } + + return profile; +} + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_COST_PROFILE_HPP diff --git a/SeQuant/core/eval/backends/dryrun/eval_expr.hpp b/SeQuant/core/eval/backends/dryrun/eval_expr.hpp new file mode 100644 index 0000000000..a7bcad51f9 --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/eval_expr.hpp @@ -0,0 +1,87 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_EVAL_EXPR_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_EVAL_EXPR_HPP + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// +/// \brief Extends EvalExpr with an annot() method so DryRun eval nodes can be +/// evaluated. +/// +/// Unlike \c EvalExprTAPP (opaque \c int64_t hashes of index labels -- see +/// \c backends/tapp/eval_expr.hpp), DryRun's annotation IS the plain literal +/// (canon-order) index list itself: \c Result::prod/sum/permute need each +/// index's actual space/extent (via \c CostModel), not just its identity, to +/// compute a modeled size. +/// +class EvalExprDryRun final : public EvalExpr { + public: + using annot_t = dryrun::annot_t; // container::svector + + template >> + explicit EvalExprDryRun(Args&&... args) + : EvalExpr{std::forward(args)...} { + annot_ = canon_indices() | ranges::to; + } + + /// + /// \return Annotation (container::svector) for DryRun tensors. + /// + [[nodiscard]] annot_t const& annot() const noexcept { return annot_; } + + private: + annot_t annot_; +}; + +/// Type alias for DryRun evaluation nodes +using EvalNodeDryRun = EvalNode; + +static_assert(meta::eval_node); +static_assert(meta::can_evaluate); + +/// +/// \brief Leaf yielder: turns each IR leaf (a tensor/constant/variable node) +/// into a zero-data DryRun Result. This is the `F` in +/// \c evaluate(node, layout, F, cache). +/// +/// A tensor leaf's literal (canon-order) index list decides flat vs nested: +/// \c make_dryrun_result builds a flat \c ResultDryRun if none of the leaf's +/// indices are proto-indexed, or a nested \c ResultDryRunNested (a CSV/PNO +/// amplitude or coefficient) if any are -- and threads that SAME literal list +/// through as the nested result's canon-order position map, so a later +/// \c slice_mode()/\c mode_batches() call (which the batched runtime only +/// ever issues against a LEAF's result) resolves its positional `mode` +/// argument correctly regardless of the leaf's flat/nested-ness. +/// +struct DryRunLeafEvaluator { + std::shared_ptr cm; + + [[nodiscard]] ResultPtr operator()(EvalNodeDryRun const& leaf) const { + SEQUANT_ASSERT(leaf.leaf()); + if (!leaf->is_tensor()) { + // Constant / Variable leaf: a bare scalar. No real numeric value is + // ever tracked by this zero-data backend (only sizes/costs), so 1.0 is + // a placeholder never meant to be read as a physical result. + return eval_result>(1.0); + } + container::svector idx = leaf->canon_indices() | ranges::to; + return make_dryrun_result(std::move(idx), cm); + } +}; + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_EVAL_EXPR_HPP diff --git a/SeQuant/core/eval/backends/dryrun/meter.hpp b/SeQuant/core/eval/backends/dryrun/meter.hpp new file mode 100644 index 0000000000..de0355c013 --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/meter.hpp @@ -0,0 +1,403 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_METER_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_METER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// +/// \brief One value's build-vs-home fidelity witness for a \c MeterReport +/// (see \c assemble_report): how many times a distinct value was built (over +/// its whole recompute tally) versus WHERE it is homed and WHERE it is used, +/// read off the matching \c RichSchedule::ValueCell (looked up by hash). +/// +struct HomeFidelity { + std::string label; ///< value signature "idx:hash" (idx disambiguates a + ///< 64-bit hash collision between distinct nodes; + ///< see cost_profile.hpp's sig_full_flops) + std::size_t hash = 0; ///< the value's EvalExpr::hash_value() + std::size_t builds = 0; ///< total builds across slices (recompute-aware) + std::string home; ///< dag-scope of home_modes ("" == root {} -- a + ///< whole-nest invariant) + std::string uses; ///< dag-scope list of the value's occurrences +}; + +/// +/// \brief Summary of one metered dry-run (or wet) replay: the hierarchy-wide +/// peak (from a wired \c PeakMonitor), the persistent/volatile FLOPs and +/// CostModel exec-time split (rolled up from a \c CacheManager's recompute +/// tally, classified by \c compute_volatility), the total build count, and +/// the per-value build-vs-home fidelity list (see \c HomeFidelity). +/// +struct MeterReport { + double peak_bytes = 0; ///< PeakMonitor high-water (dense on dry + ///< run / sparse on wet run) + std::size_t peak_op_hash = 0; ///< location (op hash) of the peak + + double flops_persistent = 0, flops_volatile = 0; ///< dense model; dry-only + double cost_persistent = 0, cost_volatile = 0; ///< CostModel exec; dry-only + + std::size_t builds_total = 0; + + std::vector home_fidelity; ///< sorted: builds desc + + BatchScheduler scheduler = + BatchScheduler::forest_descent; ///< which executor this report + ///< describes +}; + +/// +/// \brief Bottom-up memoized volatility over an evaluation \p forest: a node +/// is volatile iff \p is_volatile flags it directly, or (for an internal +/// node) either child is volatile -- the SAME rule the gated +/// \c sequant::cache_manager factory applies while building its NV/V +/// frontier (see cache_manager.hpp's DAG walk). Keyed by +/// \c TreeNode::hash_value() (rather than the node identity itself) so a +/// caller can classify a \c CacheManager::recompute_tally() entry -- keyed by +/// the SAME node identity but not necessarily the SAME node object -- by its +/// hash. +/// +/// \param forest the evaluation forest (a range of eval nodes). +/// \param is_volatile `bool(TreeNode const&)`: true if the node is +/// intrinsically volatile. Only its value on leaves matters in +/// practice (volatility propagates up), but it is consulted on every +/// node, matching \c cache_manager's gated factory. +/// \return a map from \c hash_value() to whether that value is volatile. +/// +template +std::unordered_map compute_volatility( + Forest const& forest, IsVolatile const& is_volatile) { + using Node = std::ranges::range_value_t; + + std::unordered_map volatile_of; + + auto visit = [&](auto&& self, Node const& n) -> bool { + std::size_t const h = n->hash_value(); + if (auto it = volatile_of.find(h); it != volatile_of.end()) + return it->second; + bool v; + if (n.leaf()) { + v = is_volatile(n); + } else { + bool const vl = self(self, n.left()); + bool const vr = self(self, n.right()); + v = is_volatile(n) || vl || vr; + } + volatile_of.emplace(h, v); + return v; + }; + + for (auto const& tree : forest) visit(visit, tree); + return volatile_of; +} + +/// +/// \brief Assemble a \c MeterReport from a metered replay: a walked +/// \p cache (its \c recompute_tally() populated by \c CacheManager:: +/// tally_build over the replay), the hierarchy-wide \p mon (\c PeakMonitor), +/// the \p rich linearized schedule (\c compute_dag_boulevard over the SAME +/// \p forest, supplying each value's home/use dag-scope), and \p is_volatile +/// (fed to \c compute_volatility to classify each distinct value). +/// +/// Per distinct value (one \c cache.recompute_tally() entry): \c builds is +/// the sum, over its slices, of each slice's build count; \c node_flops / +/// \c node_exec are the sum, over its slices, of build-count times that +/// slice's actual (flops, exec). The value is classified persistent/volatile +/// by \c compute_volatility's verdict for its hash and folded into the +/// matching \c MeterReport::flops_*/cost_* accumulator. Its \c HomeFidelity +/// entry's \c home/uses are read off the \p rich cell sharing its hash (empty +/// if the value has no matching cell, e.g. a leaf never realized as its own +/// distinct product build). +/// +/// \param cache the (root) cache whose \c recompute_tally() was populated by +/// a \c Trace::On metered replay. +/// \param mon the \c PeakMonitor wired onto \p cache's scope chain during the +/// replay. +/// \param rich the linearized schedule (\c compute_dag_boulevard) over the +/// SAME forest the replay walked. +/// \param forest the evaluation forest (fed to \c compute_volatility). +/// \param is_volatile `bool(TreeNode const&)`: intrinsic volatility +/// predicate, as for \c compute_volatility. +/// \param scheduler which executor this report describes (stashed verbatim +/// into \c MeterReport::scheduler). +/// \return the assembled \c MeterReport. +/// +template +MeterReport assemble_report(Cache const& cache, PeakMonitor const& mon, + RichSchedule const& rich, Forest const& forest, + IsVolatile const& is_volatile, + BatchScheduler scheduler) { + MeterReport report; + report.scheduler = scheduler; + report.peak_bytes = static_cast(mon.hwmark_bytes); + report.peak_op_hash = mon.peak.op_hash; + + auto const volatility = compute_volatility(forest, is_volatile); + + // hash -> ValueCell* lookup, mirroring make_node_meta's map build + // (scope_executor.hpp): rich.cells is a flat vector, not keyed by hash. + std::unordered_map cell_by_hash; + cell_by_hash.reserve(rich.cells.size()); + for (auto const& cell : rich.cells) cell_by_hash.emplace(cell.hash, &cell); + + // dag-scope formatting: comma-joined IndexSpace base_keys, no trailing + // comma -- the same convention make_node_meta uses (scope_executor.hpp). + auto const dag_scope = [](auto const& modes) { + std::string s; + for (auto const& m : modes) { + if (!s.empty()) s += ","; + s += toUtf8(m.space().base_key()); + } + return s; + }; + + std::size_t idx = 0; + for (auto const& [node, tally] : cache.recompute_tally()) { + std::size_t builds = 0; + double node_flops = 0.0, node_exec = 0.0; + for (auto const& [sig, rec] : tally.slices) { + builds += rec.count; + node_flops += static_cast(rec.count) * rec.flops; + node_exec += static_cast(rec.count) * rec.exec; + } + report.builds_total += builds; + + std::size_t const hash = node->hash_value(); + bool const is_vol = [&] { + auto it = volatility.find(hash); + return it != volatility.end() && it->second; + }(); + + if (is_vol) { + report.flops_volatile += node_flops; + report.cost_volatile += node_exec; + } else { + report.flops_persistent += node_flops; + report.cost_persistent += node_exec; + } + + HomeFidelity hf; + hf.label = std::to_string(idx++) + ":" + std::to_string(hash); + hf.hash = hash; + hf.builds = builds; + if (auto it = cell_by_hash.find(hash); it != cell_by_hash.end()) { + auto const* cell = it->second; + hf.home = dag_scope(cell->home_modes); + container::svector uses_modes; + for (auto const& occ : cell->occurrences) + for (auto const& [mode, range] : occ.ectx) uses_modes.push_back(mode); + hf.uses = dag_scope(uses_modes); + } + report.home_fidelity.push_back(std::move(hf)); + } + + std::sort(report.home_fidelity.begin(), report.home_fidelity.end(), + [](HomeFidelity const& a, HomeFidelity const& b) { + return a.builds > b.builds; + }); + + return report; +} + +/// +/// \brief Runs the policy-selected executor (forest descent, whole-scope, or +/// ordered, per \p policy.scheduler) over \p forest through the +/// DryRun sizing backend, metering the replay with a fresh, \c PeakMonitor +/// -wired, build-tallying cache, and returns the assembled \c MeterReport. +/// +/// Mirrors MPQC's wet dispatch: this drives the SAME Task-6 coexistence entry +/// point (\c sequant::evaluate(Nodes const&, BatchPolicy const&, layout, F, +/// CacheManager&, mode_order, ScopeGuardFactory), \c scope_executor.hpp) a +/// real solve would use under \p policy -- all three executors are selected +/// by the SAME \c policy.scheduler, not independently maintained code paths -- +/// so the metered replay is exactly the run \p policy describes, not a +/// hand-rolled proxy of it. Non-throwing wrapper (if desired) is the +/// caller's responsibility; an exception from the replay propagates out of +/// this call, but the RAII logger-state guard still restores +/// \c Logger::instance().eval on the way out. +/// +/// \param forest the evaluation forest (a range of \c EvalNodeDryRun). +/// \param policy the batch policy driving the coexistence entry -- in +/// particular \c scheduler (executor selection) and +/// \c batch_target_size (the batch-partition source; also the source +/// of the \c block_of function this call builds its OWN \c rich +/// schedule with, for \c assemble_report -- the coexistence entry +/// builds an independent, internal \c RichSchedule of its own from +/// the SAME \p policy.batch_target_size to drive the executor). +/// \param regime the size regime supplying the DryRun \c CostModel. +/// \param cfg cache configuration (footprint gate, min repeats, volatility) +/// for the metered cache, built exactly as \c build_dryrun_cache does +/// (same footprint arithmetic, same is_volatile default) -- NOT via +/// that builder directly, since its is_volatile default (substituted +/// for an empty \c cfg.is_volatile) is internal to it and would +/// otherwise be invisible to \c assemble_report below, which also +/// needs a callable predicate (an empty \c cfg.is_volatile passed to +/// it directly throws \c std::bad_function_call from +/// \c compute_volatility). The SAME locally-defaulted predicate is +/// used for both. +/// \param router optional placement override, installed on the metered cache +/// when non-null. +/// \param trace optional sink for the eval trace; when non-null, +/// \c Logger::instance().eval.stream is redirected there for the +/// duration of the call (restored on exit, along with the elevated +/// \c eval.level and the installed \c eval.node_meta). +/// \return the assembled \c MeterReport (peak, persistent/volatile +/// FLOPs+time, build-vs-home fidelity), stamped with +/// \p policy.scheduler. +/// +inline MeterReport meter( + std::vector const& forest, BatchPolicy const& policy, + SizeRegime const& regime, CacheConfig const& cfg, + PlacementRouter const* router = nullptr, + std::ostream* trace = nullptr) { + auto cm = std::make_shared(regime); + DryRunLeafEvaluator yield{cm}; + + // Default is_volatile the SAME way build_dryrun_cache does (an empty + // cfg.is_volatile means nothing is volatile) -- but keep the defaulted + // function LOCAL rather than routing through that builder, so the exact + // same predicate can also be threaded to assemble_report below. + std::function const is_volatile = + cfg.is_volatile ? cfg.is_volatile + : std::function( + [](EvalNodeDryRun const&) { return false; }); + + // Footprint (bytes) of a node's RESULT, identical to build_dryrun_cache's + // footprint_of (cost_profile.hpp): the moment-aware memsize counter over + // canon_indices(), scaled to bytes, so cfg.max_footprint gates like-for-like. + auto memsize = sequant::opt::detail::memsize_counter(regime.idx_to_extent(), + regime.inner_pow_fn()); + auto footprint_of = + [memsize = std::move(memsize)](EvalNodeDryRun const& n) -> double { + std::vector const result(n->canon_indices().begin(), + n->canon_indices().end()); + return memsize(std::vector{}, std::vector{}, result) * 8.0; + }; + + auto cache = + sequant::cache_manager(forest, is_volatile, cfg.min_repeats, + std::move(footprint_of), cfg.max_footprint); + cache.set_recompute_tally_enabled(true); + if (router) cache.set_placement_router(router); + + PeakMonitor mon; + cache.set_peak_monitor(&mon); + + // Backend array-ops for the dry-run backend: the batched executors build a + // scatter destination / enumerate axis batches through this seam (the SAME + // one the wet run uses), so the dry replay realizes the same scatter + // footprint and batch count. Sourced from the shared CostModel -- no array in + // the DAG is consulted. Must outlive the replay below (it is a local here). + auto const dry_aops = make_dryrun_array_ops(cm); + cache.set_array_ops(&dry_aops); + + // The SAME block_of source the coexistence entry itself derives from + // policy.batch_target_size (scope_executor.hpp's evaluate(Nodes const&, + // BatchPolicy const&, ...)) -- an empty batch_target_size means "no + // batching", guarded identically so compute_dag_boulevard never invokes an + // empty std::function. + std::function const block_of = + policy.batch_target_size + ? policy.batch_target_size + : std::function( + [](Index const&) -> std::size_t { return 1; }); + RichSchedule const rich = compute_dag_boulevard(forest, *cm, block_of); + + // RAII save/restore of every Logger::eval field this call touches, so an + // exception from the replay below still leaves the process-wide Singleton + // exactly as this call found it. + auto& logger = Logger::instance(); + struct LoggerStateGuard { + Logger& l; + std::size_t prev_level; + std::ostream* prev_stream; + std::function prev_node_meta; + ~LoggerStateGuard() { + l.eval.level = prev_level; + l.eval.stream = prev_stream; + l.eval.node_meta = std::move(prev_node_meta); + } + } guard{logger, logger.eval.level, logger.eval.stream, logger.eval.node_meta}; + + // Ensure printing() so DryRunOps::prod records flops/exec (feeding + // cache.tally_build) and note_working_set() actually observes the + // PeakMonitor -- without raising the level any HIGHER than a caller who + // already wants a louder trace. + logger.eval.level = std::max(logger.eval.level, 1); + if (trace) logger.eval.stream = trace; + logger.eval.node_meta = make_node_meta(rich); + + // Forest descent (BatchScheduler::forest_descent) needs the SAME batched + // custom evaluator MPQC's wet forest path installs (cck.ipp's `else` + // branch, `cache.set_custom_evaluator(sequant::make_evaluator(ctx. + // batch_policy, yielder, make_scope_guard))`): without it, plain + // sequant::evaluate(Nodes const&, ...) ignores every node_slice_mask() + // stamp and runs an unbatched, no-schedule single pass -- an infidelity + // vs. the wet run this meter is supposed to mirror. Installed ONLY on + // this branch: evaluate_impl consults cache.custom_evaluator() on every + // non-leaf node, so installing it unconditionally would also fire on the + // whole-scope AND ordered paths' own evaluate_impl calls too. Both + // whole_scope (evaluate_whole_scope) and ordered (evaluate_ordered_ + // schedule) drive their OWN executor via the coexistence entry + // (sequant::evaluate(forest, policy, ...) below, which dispatches on + // policy.scheduler) -- installing the forest custom evaluator for ordered + // would silently reroute its builds through the forest evaluator instead + // of run_ordered_contracted_block, diverging from what the wet ordered run + // (which installs NO custom evaluator) actually does. That divergence was + // a real dry-run/wet-run fidelity bug; restricting this install to forest + // descent fixes it. + if (policy.scheduler == BatchScheduler::forest_descent) + cache.set_custom_evaluator( + sequant::make_evaluator(policy, yield, sequant::make_no_scope_guard{})); + + (void)sequant::evaluate(forest, policy, std::wstring{}, yield, + cache, {}, sequant::make_no_scope_guard{}); + + // INSTRUMENTATION (SEQUANT_RECOMPUTE_DUMP, analysis-only): genuine, + // batching-aware recompute is a SINGLE (node, slice) built more than once + // (a value tiled over distinct batch slices has count==1 per slice -- see + // BuildRecord). Dump every such slice with its hash so the recompute can be + // localized without the batch-slice / accumulation confound. + if (std::getenv("SEQUANT_RECOMPUTE_DUMP")) { + std::size_t genuine = 0, slices_gt1 = 0; + for (auto const& [node, bt] : cache.recompute_tally()) + for (auto const& [slice, rec] : bt.slices) + if (rec.count > 1) { + ++slices_gt1; + genuine += rec.count - 1; + std::cerr << "RECOMPUTE count=" << rec.count + << " hash=" << node->hash_value() << " slice=[" << slice + << "]\n"; + } + std::cerr << "GENUINE-RECOMPUTE: distinct (node,slice) built >1 = " + << slices_gt1 << " ; total avoidable rebuilds = " << genuine + << "\n"; + } + + return assemble_report(cache, mon, rich, forest, is_volatile, + policy.scheduler); +} + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_METER_HPP diff --git a/SeQuant/core/eval/backends/dryrun/result.hpp b/SeQuant/core/eval/backends/dryrun/result.hpp new file mode 100644 index 0000000000..765c135654 --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/result.hpp @@ -0,0 +1,864 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_RESULT_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_RESULT_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// +/// \brief Annotation type DryRun's Result ops decode from the eval engine's +/// std::any [l,r,res] / [pre,post] triples/pairs. +/// +/// Unlike \c EvalExprTAPP (opaque \c int64_t index-label hashes -- see +/// \c backends/tapp/eval_expr.hpp), DryRun's annotation IS the plain literal +/// (canon-order) index list itself: \c Result::prod/sum/permute need each +/// index's actual space/extent (via \c CostModel), not just an opaque +/// identity, to compute a modeled size. +/// +using annot_t = container::svector; + +/// Per-mode assembled element coverage recorded by write_into_slice(): maps an +/// outer mode position to the contiguous `[lo, hi)` element range filled so far +/// by scattered blocks. Lets a zero-data DryRun destination report the REALIZED +/// (assembled) size along a partitioned mode and detect gaps/overlaps between +/// blocks -- the assemble-side analogue of ExtentOverrides for slice_mode(). +using AssembledCoverage = + container::map>; + +class ResultDryRun; +class ResultDryRunNested; + +/// +/// \brief Builds whichever concrete DryRun Result type matches \p idx's +/// content: a nested \c ResultDryRunNested if any index in \p idx is +/// proto-indexed (a CSV/PNO composite leg, e.g. a CSV amplitude's PNO +/// domain leg `a_1`), otherwise a flat \c ResultDryRun. +/// +/// Dispatch is by CONTENT of the decoded result annotation, not by either +/// operand's concrete type -- exactly mirroring how the real eval engine +/// itself decides tensor-of-tensor-ness (\c EvalExpr::tot(), from the same +/// proto-indexed-leg criterion). This is what lets \c prod()/sum() freely +/// combine a flat operand (e.g. a bare 3-center DF integral) with a nested +/// one (e.g. a CSV/PNO coefficient), exactly as real CSV-CCSD terms do, +/// without either side needing to know the other's concrete type. +/// +[[nodiscard]] inline ResultPtr make_dryrun_result( + container::svector idx, std::shared_ptr cm, + ExtentOverrides overrides = {}, ExtentOverrides lobounds = {}); + +namespace detail { + +[[nodiscard]] inline bool has_proto(container::svector const& idx) { + return std::any_of(idx.begin(), idx.end(), + [](Index const& ix) { return ix.has_proto_indices(); }); +} + +[[nodiscard]] inline ExtentOverrides merge_overrides(ExtentOverrides const& a, + ExtentOverrides const& b) { + ExtentOverrides out = a; + for (auto const& [pos, n] : b) out[pos] = n; + return out; +} + +// Remap positional overrides from one annotation's mode positions to another's +// by matching labels: position p (labeled `from[p]`) -> the position of that +// same label in `to`. A position whose label is absent from `to` (e.g. an index +// this op contracts away) is dropped -- it has no mode in the result. This is +// how a sliced mode's width survives prod/sum/permute now that the value itself +// carries no labels: the op's annotation is the only place labels live, so two +// positional maps from different operands can only be combined after both are +// projected onto a COMMON annotation (the result's). +[[nodiscard]] inline ExtentOverrides remap_overrides_by_annot( + ExtentOverrides const& ov, annot_t const& from, annot_t const& to) { + ExtentOverrides out; + for (auto const& [pos, w] : ov) { + if (pos >= from.size()) continue; + auto const it = std::find(to.begin(), to.end(), from[pos]); + if (it != to.end()) + out.emplace(static_cast(it - to.begin()), w); + } + return out; +} + +// Project a value's positional overrides into LABEL space via its annotation: +// position p -> (annot[p] -> width). For the flops call, whose out/contracted +// index sets ARE annotation labels (see CostModel::flops). +[[nodiscard]] inline container::map extents_by_label( + ExtentOverrides const& ov, annot_t const& annot) { + container::map out; + for (auto const& [pos, w] : ov) + if (pos < annot.size()) out.emplace(annot[pos], w); + return out; +} + +// Uniform read access to a DryRun Result's (index list, overrides, cost +// model) regardless of which concrete DryRun type `r` is. `is()`/`as()` +// are public Result methods, so no friendship is needed; declared here (and +// defined below, after both concrete classes) purely because their bodies +// need the concrete classes' definitions. +[[nodiscard]] container::svector indices_of(Result const& r); +[[nodiscard]] ExtentOverrides overrides_of(Result const& r); +/// Positional lower bounds of SLICED modes (mode -> element lobound); a mode +/// absent here is whole (lobound 0). Kept parallel to \c ExtentOverrides so a +/// slice preserves its ABSOLUTE position, as the TA backend does. +[[nodiscard]] ExtentOverrides lobounds_of(Result const& r); + +/// The realized element range of position \p pos of an operand: (lo, extent) +/// -- sliced modes from the overrides, whole modes from the regime. +/// (lo, extent); extent is 0 when unknown (whole mode whose label the regime +/// does not size, e.g. a fixture hyperindex) -- callers compare only known +/// extents. +[[nodiscard]] inline std::pair range_at( + ExtentOverrides const& ov, ExtentOverrides const& lob, + std::shared_ptr const& cm, annot_t const& annot, + std::size_t pos) { + std::size_t lo = 0, ext = 0; + if (auto it = ov.find(pos); it != ov.end()) + ext = it->second; + else + try { + ext = cm->regime().extent(annot[pos]); + } catch (std::exception const&) { + ext = 0; // unknown + } + if (auto it = lob.find(pos); it != lob.end()) lo = it->second; + return {lo, ext}; +} + +/// DRY-RUN CONFORMANCE CHECK (the dry-run analogue of TA's einsum index-map +/// merge and is_range_set_congruent): every label shared by the two operand +/// annotations must realize the SAME element range (lobound AND extent) on +/// both sides. A whole operand meeting a sliced partner (extent mismatch) or +/// two operands sliced to DIFFERENT batches of one loop (lobound mismatch, +/// e.g. a stale per-batch cell reused across batches) is a schedule/runtime +/// slicing defect; the wet backend hangs or asserts on it, the dry run used to +/// pass silently. Throws with both ranges named. +inline void check_shared_ranges(char const* op, annot_t const& lannot, + ExtentOverrides const& lov, + ExtentOverrides const& llob, + annot_t const& rannot, + ExtentOverrides const& rov, + ExtentOverrides const& rlob, + std::shared_ptr const& cm) { + for (std::size_t lp = 0; lp < lannot.size(); ++lp) { + auto const rit = std::find(rannot.begin(), rannot.end(), lannot[lp]); + if (rit == rannot.end()) continue; + std::size_t const rp = static_cast(rit - rannot.begin()); + // Both whole on this label => trivially equal (no regime query needed). + if (!lov.count(lp) && !llob.count(lp) && !rov.count(rp) && !rlob.count(rp)) + continue; + auto const [llo, lext] = range_at(lov, llob, cm, lannot, lp); + auto const [rlo, rext] = range_at(rov, rlob, cm, rannot, rp); + if (llo == rlo && (lext == rext || lext == 0 || rext == 0)) continue; + auto lbls = [](annot_t const& a) { + std::string s; + for (auto const& x : a) s += toUtf8(x.full_label()) + " "; + return s; + }; + throw std::runtime_error(std::format( + "[dryrun] {}: shared label {} realizes DIFFERENT ranges on the two " + "operands: L[{}]=[{},{}) vs R[{}]=[{},{}) -- a whole operand against " + "a sliced partner (extent) or two slices of different batches " + "(lobound); L=({}) R=({})", + op, toUtf8(lannot[lp].full_label()), lp, llo, llo + lext, rp, rlo, + rlo + rext, lbls(lannot), lbls(rannot))); + } +} + +/// Accumulation (add_inplace) conformance: the accumulator and the partial +/// must realize the SAME range on every mode (same slicing, same batch) -- +/// summing a partial of one batch into a cell of another is the stale-cell +/// signature. Positional (both carry the result's own index order). +inline void check_accumulate_ranges( + container::svector const& idx, ExtentOverrides const& ov, + ExtentOverrides const& lob, Result const& other, + std::shared_ptr const& cm) { + auto const oov = overrides_of(other); + auto const olob = lobounds_of(other); + annot_t const a(idx.begin(), idx.end()); + for (std::size_t pos = 0; pos < idx.size(); ++pos) { + if (!ov.count(pos) && !lob.count(pos) && !oov.count(pos) && + !olob.count(pos)) + continue; + auto const [lo, ext] = range_at(ov, lob, cm, a, pos); + auto const [olo, oext] = range_at(oov, olob, cm, a, pos); + if (lo == olo && (ext == oext || ext == 0 || oext == 0)) continue; + throw std::runtime_error(std::format( + "[dryrun] add_inplace: mode {} ({}) accumulator range [{},{}) vs " + "partial range [{},{}) -- accumulating a partial of a different " + "slicing/batch into this cell", + pos, toUtf8(idx[pos].full_label()), lo, lo + ext, olo, olo + oext)); + } +} + +/// +/// \brief Shared op bodies for the two DryRun Result concrete types. +/// +/// Both \c ResultDryRun and \c ResultDryRunNested carry exactly an (index +/// list, ExtentOverrides, CostModel) triple and differ only in what they +/// additionally expose (\c ResultDryRunNested splits its index list into +/// outer()/inner() views for CSV-composite-aware inspection/testing). +/// Implemented once here so the two classes' prod/sum/permute/slice_mode/ +/// mode_batches bodies are one-line forwards, not near-duplicated logic. +/// +struct DryRunOps { + [[nodiscard]] static ResultPtr sum(container::svector const& idx, + ExtentOverrides const& ov, + std::shared_ptr const& cm, + Result const& other, + std::array const& annot, + ExtentOverrides const& lob = {}) { + auto const a = Annot{annot}; + auto const other_ov = overrides_of(other); + auto const other_lob = lobounds_of(other); + check_shared_ranges("sum", a.lannot, ov, lob, a.rannot, other_ov, other_lob, + cm); + // Both summands share the result's index set (possibly reordered); project + // each operand's positional slice widths onto the result annotation by + // label before merging (position k is a different mode in each operand). + auto merged = merge_overrides( + remap_overrides_by_annot(ov, a.lannot, a.this_annot), + remap_overrides_by_annot(other_ov, a.rannot, a.this_annot)); + auto merged_lob = merge_overrides( + remap_overrides_by_annot(lob, a.lannot, a.this_annot), + remap_overrides_by_annot(other_lob, a.rannot, a.this_annot)); + return make_dryrun_result( + container::svector(a.this_annot.begin(), a.this_annot.end()), cm, + std::move(merged), std::move(merged_lob)); + } + + [[nodiscard]] static ResultPtr prod( + container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, Result const& other, + std::array const& annot, ExtentOverrides const& lob = {}) { + if (other.is>()) { + // Scalar * tensor: shape (and any accumulated slicing) unchanged. + return make_dryrun_result(idx, cm, ov, lob); + } + auto const a = Annot{annot}; + auto const other_ov = overrides_of(other); + auto const other_lob = lobounds_of(other); + check_shared_ranges("prod", a.lannot, ov, lob, a.rannot, other_ov, + other_lob, cm); + auto merged_lob = merge_overrides( + remap_overrides_by_annot(lob, a.lannot, a.this_annot), + remap_overrides_by_annot(other_lob, a.rannot, a.this_annot)); + // RESULT overrides: each operand's positional slice widths are positional + // against ITS OWN annotation (== its canon index order); project both onto + // the result's annotation by label (dropping any contracted-away mode), + // then merge. Merging the two raw positional maps directly would be wrong + // -- position k means a different mode in each operand. + auto merged = merge_overrides( + remap_overrides_by_annot(ov, a.lannot, a.this_annot), + remap_overrides_by_annot(other_ov, a.rannot, a.this_annot)); + + // Emit the cost model's OWN flops / roofline exec_cost for THIS op into the + // eval trace (gated on the eval log level), interleaved right before the + // generic engine's `Eval | Product` line for the same op. This lets trace + // post-processing weight avoidable recomputation by MODELLED TIME without + // re-deriving the cost downstream (which would silently drift from the + // model). The (out, contracted) index sets and the realized (sliced) extent + // overrides feed the same CostModel closures the static cost_profile() walk + // uses, so per-op costs are consistent with the whole-forest totals. + if (Logger::instance().eval.level > 0) { + // Cost THIS op in the einsum ANNOTATION label space (lannot/rannot/ + // this_annot), NOT the operands' stored `indices_` (idx / indices_of( + // other)). A DAG VALUE HAS NO INTRINSIC LABELS -- only ops bind labels to + // it, meaningful only within that op; a value's `indices_` merely holds + // whatever labels its PRODUCER used, which say nothing about how a + // CONSUMER binds it. The two diverge for every CSE-shared value used + // under a different binding: e.g. the (g.C)(g.C) legs are the SAME cached + // value (its `indices_` reads [i_1 i_4 K_2 a_4] from its producer) yet + // THIS op binds it as lannot=[i_2 i_3 K_2 a_3] and rannot=[i_2 i_1 K_2 + // a_1]. Deriving `contracted` from the producer-labeled stored indices + // instead of this op's annotations unions modes from different label + // contexts and exploded the flops (6.65e16 vs the correct ~1.3e13). out U + // (lannot & rannot) == lannot U rannot == the real contraction volume. + container::svector out(a.this_annot.begin(), a.this_annot.end()); + container::svector contracted; + for (auto const& ix : a.lannot) + if (std::find(a.rannot.begin(), a.rannot.end(), ix) != a.rannot.end()) + contracted.push_back(ix); + // Slice widths in LABEL space for the flops call: project each operand's + // positional overrides through its annotation. A batched label shared by + // both operands (e.g. a contracted, batched aux index) carries the same + // width from either side, so the first insertion wins harmlessly. + auto label_extents = extents_by_label(ov, a.lannot); + for (auto const& [lbl, w] : extents_by_label(other_ov, a.rannot)) + label_extents.emplace(lbl, w); + double const flops = cm->flops(out, contracted, label_extents); + sequant::eval::detail::last_op_flops() = flops; // for the Build event + double const exec = cm->exec_cost(flops, cm->memsize(idx, ov), 4096); + sequant::eval::detail::last_op_exec() = exec; // for the Build event + write_log(Logger::instance(), "OpCost", std::format(" | {}", flops), + std::format(" | {}", exec), '\n'); + // Fold this op's SLICED-extent cost into the replay cost sink, if one is + // attached (cost_profile()'s recompute-aware tally). merged/ov carry the + // runtime slicing, so a contraction re-executed once per occ block is + // charged once per block at its sliced size -- the same numbers already + // logged above, now summed. No-op (byte-identical) when unattached. + cm->tally_op(flops, exec); + // last_op_flops (set above) is THIS build's ACTUAL realized-extent cost. + // The eval loop's build choke point reads it and records it against the + // node's IDENTITY, at the (value, SLICE) granularity, in the (root) + // cache's recompute tally -- so avoidable recompute is the actual FLOPs a + // slice was rebuilt beyond once, with no ill-defined "full extent" + // denominator (slicing is non-uniform). prod cannot form the node + // identity here (it has no node, and the include cycle + // dryrun/eval_expr.hpp -> cost_model_object.hpp keeps the node type out + // of the CostSink), so the rollup is done there. See + // CacheManager::tally_build / recompute_tally(). + } + + if (a.this_annot.empty()) { + // Full contraction -> scalar. No real numeric value is ever tracked by + // this zero-data backend (only sizes/costs), so the placeholder 0.0 + // is never meant to be read as a physical result. + return eval_result>(0.0); + } + return make_dryrun_result( + container::svector(a.this_annot.begin(), a.this_annot.end()), cm, + std::move(merged), std::move(merged_lob)); + } + + [[nodiscard]] static ResultPtr permute( + container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, + std::array const& ann, ExtentOverrides const& lob = {}) { + auto const post = std::any_cast(ann[1]); + // Reordering modes moves each mode's position, so the positional overrides + // must move with them: `ov` is positional against `idx` (the pre-permute + // canon order); project it onto `post` by label. + return make_dryrun_result( + container::svector(post.begin(), post.end()), cm, + remap_overrides_by_annot(ov, idx, post), + remap_overrides_by_annot(lob, idx, post)); + } + + [[nodiscard]] static ResultPtr slice_mode( + container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, std::size_t mode, + std::size_t elem_lo, std::size_t elem_hi, + ExtentOverrides const& lob = {}) { + SEQUANT_ASSERT(mode < idx.size()); + auto merged = ov; + merged[mode] = elem_hi - elem_lo; // positional: mode `mode`, any label + auto merged_lob = lob; + merged_lob[mode] = elem_lo; // ABSOLUTE position preserved (as TA does) + return make_dryrun_result(idx, cm, std::move(merged), + std::move(merged_lob)); + } + + /// Scatter \p block into the `[block_lo, block_hi)` element slice of the + /// destination's mode \p mode -- the inverse of slice_mode(). Zero-data: + /// updates only the destination's modelled size and assembled-coverage + /// bookkeeping. \p ov and \p cov are the destination's (mutated in place). + static void write_into_slice(container::svector const& idx, + ExtentOverrides& ov, AssembledCoverage& cov, + std::shared_ptr const& cm, + Result const& block, std::size_t mode, + std::size_t block_lo, std::size_t block_hi) { + SEQUANT_ASSERT(mode < idx.size()); + SEQUANT_ASSERT(block_lo < block_hi); + Index const& mix = idx[mode]; + // Tile/width consistency: the block's own modelled extent on the shared + // mode index must equal the slice width it is being written into. The + // block's overrides are positional against ITS OWN index list, so locate + // the shared index there (its mode need not equal the dest's `mode`). + auto const bov = overrides_of(block); + auto const bidx = indices_of(block); + std::size_t const block_extent = [&] { + auto const it = std::find(bidx.begin(), bidx.end(), mix); + if (it != bidx.end()) { + auto const bpos = static_cast(it - bidx.begin()); + if (auto ov_it = bov.find(bpos); ov_it != bov.end()) + return ov_it->second; + } + return cm->regime().extent(mix); + }(); + if (block_extent != block_hi - block_lo) { + std::cerr << "[scatter-mismatch] result idx=["; + for (auto const& x : idx) std::cerr << toUtf8(x.full_label()) << " "; + std::cerr << "] slice mode=" << mode << " (" << toUtf8(mix.full_label()) + << ") block idx=["; + for (auto const& x : bidx) std::cerr << toUtf8(x.full_label()) << " "; + std::cerr << "] block_extent=" << block_extent << " expected slice=[" + << block_lo << "," << block_hi << ")=" << (block_hi - block_lo) + << " regime_extent(mix)=" << cm->regime().extent(mix) + << " block_ov={"; + for (auto const& [bp, ex] : bov) std::cerr << bp << ":" << ex << " "; + std::cerr << "}" << std::endl; + } + SEQUANT_ASSERT(block_extent == block_hi - block_lo); + if (block_extent != block_hi - block_lo) + throw std::runtime_error(std::format( + "[dryrun] write_into_slice: block extent {} on mode {} ({}) != " + "destination slice [{},{})", + block_extent, mode, toUtf8(mix.full_label()), block_lo, block_hi)); + // The block's ABSOLUTE position on this mode (if it is a slice) must be + // the destination slice it is written into. + { + auto const blob = lobounds_of(block); + auto const bit = std::find(bidx.begin(), bidx.end(), mix); + if (bit != bidx.end()) { + auto const bpos = static_cast(bit - bidx.begin()); + if (auto lit = blob.find(bpos); + lit != blob.end() && lit->second != block_lo) + throw std::runtime_error(std::format( + "[dryrun] write_into_slice: block lobound {} on mode {} ({}) != " + "destination slice lobound {}", + lit->second, mode, toUtf8(mix.full_label()), block_lo)); + } + } + // Merge the block's range into the assembled coverage, requiring + // contiguity: a block that neither appends after nor prepends before the + // filled range would leave a gap or overlap another block (a + // double-count). This is what makes disjoint gap-free tiling the only + // accepted assembly. + if (auto it = cov.find(mode); it == cov.end()) { + cov.emplace(mode, + std::pair{block_lo, block_hi}); + } else { + auto& lohi = it->second; + bool const append = block_lo == lohi.second; + bool const prepend = block_hi == lohi.first; + SEQUANT_ASSERT(append || prepend); + if (append) + lohi.second = block_hi; + else + lohi.first = block_lo; + } + // Reflect the assembled element width (hi - lo, lobound preserved) as the + // realized extent of the batch mode so size_in_bytes() tracks the + // reconstructed footprint. + auto const& lohi = cov.at(mode); + ov[mode] = lohi.second - lohi.first; // positional: dest mode `mode` + } + + [[nodiscard]] static container::svector> + mode_batches(container::svector const& idx, ExtentOverrides const& ov, + std::shared_ptr const& cm, std::size_t mode, + std::size_t target_batch_size) { + SEQUANT_ASSERT(mode < idx.size()); + Index const& ix = idx[mode]; // for the regime extent / slice partition + std::size_t extent; + if (auto it = ov.find(mode); it != ov.end()) // positional override + extent = it->second; + else + extent = cm->regime().extent(ix); + + container::svector> out; + if (target_batch_size == 0 || extent == 0) { + out.push_back({0, extent}); + return out; + } + // CALLER-SUPPLIED PARTITION: if the mode's space has a recorded batch + // partition (SizeRegime::space_slice_extents), the wet backend slices this + // axis along whole TILES (mode_batches_of_trange1 reads the operand's real + // TiledRange1), so a batch boundary always falls on a tile edge and a + // (sub)range spanning N whole tiles yields N batches -- NOT extent/target + // uniform blocks. The partition slices ARE those target-grouped tile edges + // (batch_slice_extents_from_tiles applied the target once, at harvest), so + // we emit the PREFIX of partition slices that sums to `extent`: + // - outer call (ov absent, extent == full axis) => all slices, the full + // partition (e.g. aux 672 -> [168,168,168,168], 4 batches); + // - nested call (ov narrowed the axis to one outer batch, extent < full) + // => the prefix reaching that extent, so a single-tile sub-range (168) + // is ONE atomic batch, matching the wet backend, instead of being + // re-sliced into ceil(168/64)=3 uniform blocks (which then cascade). + // `target_batch_size` is not re-applied here: the partition already encodes + // it. If `extent` does not land on a partition boundary (not tile-aligned), + // fall through to uniform blocks. The dry-run stays model-agnostic -- it + // only reads slice extents; the caller decided the tiling. + auto const& slices = cm->regime().slice_extents(ix); + if (!slices.empty()) { + std::size_t lo = 0; + for (std::size_t const s : slices) { + if (lo >= extent) break; + out.push_back({lo, lo + s}); + lo += s; + } + if (lo == extent) return out; // extent tile-aligned to the partition + out.clear(); // not aligned -> uniform fallback below + } + // Fallback: uniform target_batch_size blocks (no partition recorded). + for (std::size_t lo = 0; lo < extent; lo += target_batch_size) + out.push_back({lo, std::min(extent, lo + target_batch_size)}); + return out; + } +}; + +} // namespace detail + +/// +/// \brief Flat (non-CSV) zero-data tensor token. +/// +/// Carries only its own literal outer index list (canon order -- the same +/// order \c EvalExpr::canon_indices()/annot() use, so \c slice_mode()/ +/// \c mode_batches()'s positional `mode` argument indexes it correctly), an +/// \c ExtentOverrides table recording any runtime \c slice_mode()/ +/// \c mode_batches() narrowing (keyed by Index so it survives reshaping +/// across prod/sum/permute), and a shared \c CostModel. No tensor data is +/// ever allocated or copied; every op is index-set bookkeeping plus a +/// CostModel query. Mirrors \c ResultTensorTAPP's structure +/// (backends/tapp/result.hpp) with every real-tensor line replaced by that +/// bookkeeping. +/// +class ResultDryRun final : public Result { + public: + using Result::id_t; + + ResultDryRun(container::svector idxset, + std::shared_ptr cm, + ExtentOverrides overrides = {}, ExtentOverrides lobounds = {}) + : Result{Payload{}}, + indices_{std::move(idxset)}, + cm_{std::move(cm)}, + overrides_{std::move(overrides)}, + lobounds_{std::move(lobounds)} {} + [[nodiscard]] ExtentOverrides const& lobounds() const noexcept { + return lobounds_; + } + + [[nodiscard]] container::svector const& indices() const noexcept { + return indices_; + } + [[nodiscard]] ExtentOverrides const& overrides() const noexcept { + return overrides_; + } + + /// The contiguous `[lo, hi)` element range of outer mode \p mode assembled so + /// far by write_into_slice() (empty `{0, 0}` if nothing written). + [[nodiscard]] std::pair assembled_range( + std::size_t mode) const { + if (auto it = assembled_.find(mode); it != assembled_.end()) + return it->second; + return {0, 0}; + } + + private: + struct Payload {}; + + [[nodiscard]] id_t type_id() const noexcept override { + return id_for_type(); + } + + [[nodiscard]] ResultPtr sum( + Result const& other, + std::array const& annot) const override { + return detail::DryRunOps::sum(indices_, overrides_, cm_, other, annot, + lobounds_); + } + + [[nodiscard]] ResultPtr prod(Result const& other, + std::array const& annot, + DeNest /*DeNestFlag*/) const override { + return detail::DryRunOps::prod(indices_, overrides_, cm_, other, annot, + lobounds_); + } + + [[nodiscard]] ResultPtr permute( + std::array const& ann) const override { + return detail::DryRunOps::permute(indices_, overrides_, cm_, ann, + lobounds_); + } + + [[nodiscard]] ResultPtr adjoint( + std::array const& ann) const override { + return detail::DryRunOps::permute(indices_, overrides_, cm_, ann, + lobounds_); + } + + [[nodiscard]] ResultPtr slice_mode(std::size_t mode, std::size_t elem_lo, + std::size_t elem_hi) const override { + return detail::DryRunOps::slice_mode(indices_, overrides_, cm_, mode, + elem_lo, elem_hi, lobounds_); + } + + void write_into_slice(Result const& block, std::size_t mode, + std::size_t block_lo, std::size_t block_hi) override { + detail::DryRunOps::write_into_slice(indices_, overrides_, assembled_, cm_, + block, mode, block_lo, block_hi); + } + + void add_inplace(Result const& other) override { + SEQUANT_ASSERT(other.is() || other.is()); + detail::check_accumulate_ranges(indices_, overrides_, lobounds_, other, + cm_); + overrides_ = + detail::merge_overrides(overrides_, detail::overrides_of(other)); + lobounds_ = detail::merge_overrides(lobounds_, detail::lobounds_of(other)); + } + + [[nodiscard]] ResultPtr symmetrize() const override { + return eval_result(indices_, cm_, overrides_, lobounds_); + } + + [[nodiscard]] ResultPtr antisymmetrize(size_t /*bra_rank*/) const override { + return eval_result(indices_, cm_, overrides_, lobounds_); + } + + [[nodiscard]] ResultPtr mult_by_phase(std::int8_t /*factor*/) const override { + return eval_result(indices_, cm_, overrides_, lobounds_); + } + + /// A dry-run token owns only bookkeeping (indices, extent overrides, + /// lobounds, assembled coverage), so its copy IS the deep copy. + [[nodiscard]] ResultPtr clone() const override { + return std::make_shared(*this); + } + + [[nodiscard]] std::size_t size_in_bytes() const final { + return cm_->memsize(indices_, overrides_); + } + + container::svector indices_; + std::shared_ptr cm_; + ExtentOverrides overrides_; + ExtentOverrides lobounds_; // positional lobounds of sliced modes + AssembledCoverage assembled_; +}; + +/// +/// \brief CSV/PNO tensor-of-tensor zero-data token. +/// +/// Like \c ResultDryRun, but additionally exposes an outer()/inner() split of +/// its (canon-order) index list -- inner = the proto-indexed (composite) +/// legs, e.g. a CSV amplitude's PNO domain leg `a_1`; outer = every +/// other (plain) leg, e.g. the PAO index `mu~_1`. The split is purely an +/// observability/testing convenience: \c size_in_bytes()'s arithmetic is +/// IDENTICAL to \c ResultDryRun's (\c CostModel::memsize already routes any +/// index list containing a proto-indexed entry through the moment-aware +/// `inner_pow` path internally, via \c tot_indices/inner_aware_volume -- +/// content-driven, not type-driven), so tests that want to confirm "this used +/// the k-th moment, not extent^k" can inspect inner() directly. +/// +/// Position semantics for \c slice_mode()/\c mode_batches(): the `mode` +/// argument the runtime passes is always resolved against the FULL +/// canon-order list (an optional trailing constructor argument, defaulting to +/// `outer ++ inner` when the caller does not need position accuracy, e.g. a +/// hand-built test instance); the \c DryRunLeafEvaluator (eval_expr.hpp) +/// always supplies the leaf's true \c canon_indices() order there, since only +/// LEAF-constructed instances are ever sliced by the runtime (\c slice_mode() +/// is invoked only inside the batched evaluator's leaf-wrapping closure, never +/// on a prod()/sum()-produced intermediate). +/// +class ResultDryRunNested final : public Result { + public: + using Result::id_t; + + ResultDryRunNested(container::svector outer, + container::svector inner, + std::shared_ptr cm, + ExtentOverrides overrides = {}, + container::svector canon_order = {}, + ExtentOverrides lobounds = {}) + : Result{Payload{}}, + outer_{std::move(outer)}, + inner_{std::move(inner)}, + indices_{canon_order.empty() + ? [this] { + container::svector c = outer_; + c.insert(c.end(), inner_.begin(), inner_.end()); + return c; + }() + : std::move(canon_order)}, + cm_{std::move(cm)}, + overrides_{std::move(overrides)}, + lobounds_{std::move(lobounds)} {} + [[nodiscard]] ExtentOverrides const& lobounds() const noexcept { + return lobounds_; + } + + [[nodiscard]] container::svector const& outer() const noexcept { + return outer_; + } + [[nodiscard]] container::svector const& inner() const noexcept { + return inner_; + } + [[nodiscard]] container::svector const& indices() const noexcept { + return indices_; + } + [[nodiscard]] ExtentOverrides const& overrides() const noexcept { + return overrides_; + } + + /// The contiguous `[lo, hi)` element range of outer mode \p mode assembled so + /// far by write_into_slice() (empty `{0, 0}` if nothing written). + [[nodiscard]] std::pair assembled_range( + std::size_t mode) const { + if (auto it = assembled_.find(mode); it != assembled_.end()) + return it->second; + return {0, 0}; + } + + private: + struct Payload {}; + + [[nodiscard]] id_t type_id() const noexcept override { + return id_for_type(); + } + + [[nodiscard]] ResultPtr sum( + Result const& other, + std::array const& annot) const override { + return detail::DryRunOps::sum(indices_, overrides_, cm_, other, annot, + lobounds_); + } + + [[nodiscard]] ResultPtr prod(Result const& other, + std::array const& annot, + DeNest /*DeNestFlag*/) const override { + return detail::DryRunOps::prod(indices_, overrides_, cm_, other, annot, + lobounds_); + } + + [[nodiscard]] ResultPtr permute( + std::array const& ann) const override { + return detail::DryRunOps::permute(indices_, overrides_, cm_, ann, + lobounds_); + } + + [[nodiscard]] ResultPtr adjoint( + std::array const& ann) const override { + return detail::DryRunOps::permute(indices_, overrides_, cm_, ann, + lobounds_); + } + + [[nodiscard]] ResultPtr slice_mode(std::size_t mode, std::size_t elem_lo, + std::size_t elem_hi) const override { + return detail::DryRunOps::slice_mode(indices_, overrides_, cm_, mode, + elem_lo, elem_hi, lobounds_); + } + + void write_into_slice(Result const& block, std::size_t mode, + std::size_t block_lo, std::size_t block_hi) override { + detail::DryRunOps::write_into_slice(indices_, overrides_, assembled_, cm_, + block, mode, block_lo, block_hi); + } + + void add_inplace(Result const& other) override { + SEQUANT_ASSERT(other.is() || other.is()); + detail::check_accumulate_ranges(indices_, overrides_, lobounds_, other, + cm_); + overrides_ = + detail::merge_overrides(overrides_, detail::overrides_of(other)); + lobounds_ = detail::merge_overrides(lobounds_, detail::lobounds_of(other)); + } + + [[nodiscard]] ResultPtr symmetrize() const override { + return eval_result(outer_, inner_, cm_, overrides_, + indices_, lobounds_); + } + + [[nodiscard]] ResultPtr antisymmetrize(size_t /*bra_rank*/) const override { + return eval_result(outer_, inner_, cm_, overrides_, + indices_, lobounds_); + } + + [[nodiscard]] ResultPtr mult_by_phase(std::int8_t /*factor*/) const override { + return eval_result(outer_, inner_, cm_, overrides_, + indices_, lobounds_); + } + + /// See \c ResultDryRun::clone: bookkeeping only, so the copy is deep. + [[nodiscard]] ResultPtr clone() const override { + return std::make_shared(*this); + } + + [[nodiscard]] std::size_t size_in_bytes() const final { + return cm_->memsize(indices_, overrides_); + } + + container::svector outer_; + container::svector inner_; + container::svector indices_; // canon order; outer_++inner_ content + std::shared_ptr cm_; + ExtentOverrides overrides_; + ExtentOverrides lobounds_; // positional lobounds of sliced modes + AssembledCoverage assembled_; +}; + +[[nodiscard]] inline ResultPtr make_dryrun_result( + container::svector idx, std::shared_ptr cm, + ExtentOverrides overrides, ExtentOverrides lobounds) { + if (!detail::has_proto(idx)) + return eval_result(std::move(idx), std::move(cm), + std::move(overrides), std::move(lobounds)); + container::svector outer, inner; + for (auto const& ix : idx) + (ix.has_proto_indices() ? inner : outer).push_back(ix); + return eval_result(std::move(outer), std::move(inner), + std::move(cm), std::move(overrides), + std::move(idx), std::move(lobounds)); +} + +namespace detail { + +[[nodiscard]] inline container::svector indices_of(Result const& r) { + if (r.is()) return r.as().indices(); + SEQUANT_ASSERT(r.is()); + return r.as().indices(); +} + +[[nodiscard]] inline ExtentOverrides overrides_of(Result const& r) { + if (r.is()) return r.as().overrides(); + SEQUANT_ASSERT(r.is()); + return r.as().overrides(); +} +[[nodiscard]] inline ExtentOverrides lobounds_of(Result const& r) { + if (r.is()) return r.as().lobounds(); + SEQUANT_ASSERT(r.is()); + return r.as().lobounds(); +} + +} // namespace detail + +/// \brief Build a \c BackendArrayOps for the dry-run cost backend. +/// +/// \details The dry-run analogue of \c make_ta_array_ops: \c make_zeros returns +/// a full-extent dry-run token shaped by the descriptor (every mode at its +/// space's natural CostModel extent -- the same all-full result the old +/// \c DryRunOps::pre_sized_zeros_over_mode produced once the scatter axis was +/// widened), and \c axis_batches partitions an axis's FULL space extent exactly +/// as \c DryRunOps::mode_batches does for an unsliced, override-free +/// single-mode token (so the dry run realizes the same batch COUNT the wet +/// backend does, via the shared \c SizeRegime::space_slice_extents partition). +/// Install it on the eval cache (\c CacheManager::set_array_ops) for a dry-run +/// batched eval. +[[nodiscard]] inline BackendArrayOps make_dryrun_array_ops( + std::shared_ptr cm) { + BackendArrayOps aops; + aops.axis_batches = [cm](Index const& axis, std::size_t target_batch_size) { + return detail::DryRunOps::mode_batches(container::svector{axis}, {}, + cm, /*mode=*/0, target_batch_size); + }; + aops.make_zeros = + [cm](container::vector const& descriptor) -> ResultPtr { + return make_dryrun_result( + container::svector(descriptor.begin(), descriptor.end()), cm); + }; + return aops; +} + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_RESULT_HPP diff --git a/SeQuant/core/eval/backends/dryrun/size_regime.hpp b/SeQuant/core/eval/backends/dryrun/size_regime.hpp new file mode 100644 index 0000000000..ad8657912c --- /dev/null +++ b/SeQuant/core/eval/backends/dryrun/size_regime.hpp @@ -0,0 +1,136 @@ +#ifndef SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_SIZE_REGIME_HPP +#define SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_SIZE_REGIME_HPP + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace sequant::eval::dryrun { + +/// Per-space extents and per-rank CSV moment tables that define one size +/// regime for a dry-run replay. Extents are element counts; CSV moments are +/// power means over occupied pairs (PNO) or singles (OSV). +struct SizeRegime { + std::map space_extent; + + /// OPTIONAL per-space BATCH PARTITION: the element extent of each realized + /// batch slice along the space's batch axis, keyed by space base_key, in + /// order. Empty (default) => the dry-run batches a mode into UNIFORM + /// target_batch_size blocks (backend-model-agnostic fallback). When present + /// for a batch axis, ResultDryRun::mode_batches uses THIS partition directly + /// (accumulated to [lo,hi) ranges), so the dry-run's batch COUNT -- hence its + /// recompute -- matches whatever the wet backend realizes, even when a tile + /// is coarser than target_batch_size. + /// + /// The dry-run backend deliberately does NOT know how these were derived: the + /// CALLER converts its backend's structure into slice extents and supplies + /// them here, so a new backend model is supported without touching dry-run + /// eval internals. For a TILE-based wet backend, \c batch_slice_extents_from_ + /// tiles is the ready-made converter. The extents must sum to space_extent. + std::map> space_slice_extents; + + // csv_pno_moment[k] / csv_osv_moment[k] hold the k-th POWER MEAN + // M_k = (mean_over_pairs d^k)^(1/k) of the per-pair PNO / per-orbital OSV + // domain size d, for k in [1,4] (index 0 is unused, set to 1). inner_pow() + // returns M_k so that inner_aware_volume's per-member product over a + // k-composite group is M_k^k = mean(d^k), and outer_nocc^N * M_k^k equals + // the true block-sparse volume Sum_pairs d^k. Do NOT store raw moments + // mean(d^k) here: that would over-count k-composite groups by a further + // power of k. For a constant domain d, M_k = d for all k. + std::array csv_pno_moment{1.0, 1.0, 1.0, 1.0, 1.0}; + std::array csv_osv_moment{1.0, 1.0, 1.0, 1.0, 1.0}; + + // Moment tables for CSV cluster ranks >= 3 (CSV-CCSDT triples and beyond), + // keyed by cluster rank (= number of proto indices). csv_moment_by_rank[r][k] + // is the k-th power mean of the rank-r cluster domain. A rank not present + // falls back to csv_pno_moment (the rank-2 table) in inner_pow(), preserving + // the pre-rank-general behavior where every proto-rank >= 2 used the PNO + // table. Ranks 1 and 2 are held by csv_osv_moment / csv_pno_moment above and + // are NOT expected here (an entry for 1 or 2 is ignored by inner_pow()). + std::map> csv_moment_by_rank; + + /// \return the flat extent of \p ix's space; throws \c std::out_of_range + /// if the space is not present in \c space_extent (fail loud rather + /// than silently defaulting to 1). + [[nodiscard]] std::size_t extent(Index const& ix) const { + return space_extent.at(std::wstring{ix.space().base_key()}); + } + + /// \return \p ix's space batch-slice-extent sequence, or an empty span if the + /// space has no partition recorded (=> the caller falls back to + /// uniform target_batch_size blocks). Never throws. + [[nodiscard]] container::svector const& slice_extents( + Index const& ix) const { + static const container::svector empty; + auto const it = + space_slice_extents.find(std::wstring{ix.space().base_key()}); + return it != space_slice_extents.end() ? it->second : empty; + } + + /// \return the k-th power-mean moment for a proto-indexed CSV/PNO composite + /// index (\p k clamped to 0..4), or \c pow(extent, k) for a plain + /// (non-composite) index. Rank is determined by the number of proto + /// indices: 1 => OSV (occupied single), 2 => PNO (occupied pair), + /// >= 3 => the rank-specific csv_moment_by_rank table if present, + /// else the PNO (rank-2) table. + [[nodiscard]] double inner_pow(Index const& composite, std::size_t k) const { + if (k > 4) k = 4; + auto const& protos = composite.proto_indices(); + if (protos.empty()) + return std::pow(static_cast(extent(composite)), + static_cast(k)); + auto const rank = protos.size(); + if (rank <= 1) return csv_osv_moment[k]; + if (rank == 2) return csv_pno_moment[k]; + auto const it = csv_moment_by_rank.find(rank); + return (it != csv_moment_by_rank.end()) ? it->second[k] : csv_pno_moment[k]; + } + + [[nodiscard]] std::function idx_to_extent() const { + return [this](Index const& ix) { return extent(ix); }; + } + + [[nodiscard]] std::function inner_pow_fn() + const { + return [this](Index const& ix, std::size_t k) { return inner_pow(ix, k); }; + } +}; + +/// Convert a TILE-extent sequence into a BATCH-slice-extent sequence +/// (SizeRegime::space_slice_extents) by the SAME whole-tile grouping the wet +/// batched evaluator uses (mode_batches_of_trange1, tiledarray/result.hpp): +/// accumulate consecutive tiles into a slice until appending the next would +/// push the slice over \p target_batch_size, then start a new slice; a lone +/// tile larger than the target still forms its own slice. Slice boundaries fall +/// on tile edges. This is a convenience converter for a TILE-based caller; the +/// dry-run backend never calls it -- it only READS the resulting slice extents, +/// so any other backend model can populate space_slice_extents differently +/// without touching dry-run internals. +[[nodiscard]] inline container::svector +batch_slice_extents_from_tiles( + container::svector const& tile_extents, + std::size_t target_batch_size) { + container::svector slices; + std::size_t const target = std::max(target_batch_size, 1); + std::size_t acc = 0; + for (std::size_t const tsz : tile_extents) { + if (acc > 0 && acc + tsz > target) { + slices.push_back(acc); + acc = 0; + } + acc += tsz; + } + if (acc > 0) slices.push_back(acc); + return slices; +} + +} // namespace sequant::eval::dryrun + +#endif // SEQUANT_CORE_EVAL_BACKENDS_DRYRUN_SIZE_REGIME_HPP diff --git a/SeQuant/core/eval/backends/tapp/result.hpp b/SeQuant/core/eval/backends/tapp/result.hpp index bfacaf257d..2217a56932 100644 --- a/SeQuant/core/eval/backends/tapp/result.hpp +++ b/SeQuant/core/eval/backends/tapp/result.hpp @@ -235,6 +235,12 @@ class ResultTensorTAPP final : public Result { return eval_result>(std::move(pre)); } + /// Deep copy: the backing tensor type owns its elements, so its copy + /// constructor already produces an independently owned buffer. + [[nodiscard]] ResultPtr clone() const override { + return eval_result>(get()); + } + [[nodiscard]] ResultPtr permute( std::array const& ann) const override { auto const pre_annot = std::any_cast(ann[0]); diff --git a/SeQuant/core/eval/backends/tiledarray/array_ops.hpp b/SeQuant/core/eval/backends/tiledarray/array_ops.hpp new file mode 100644 index 0000000000..74d6d6389e --- /dev/null +++ b/SeQuant/core/eval/backends/tiledarray/array_ops.hpp @@ -0,0 +1,91 @@ +#ifndef SEQUANT_EVAL_BACKENDS_TILEDARRAY_ARRAY_OPS_HPP +#define SEQUANT_EVAL_BACKENDS_TILEDARRAY_ARRAY_OPS_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace sequant { + +/// \brief Build a \c BackendArrayOps for the TiledArray backend from a +/// per-space tiling map. +/// +/// \tparam FlatArray the flat (Tensor-of-Scalars) \c TA::DistArray type; +/// \tparam ToTArray the nested (Tensor-of-Tensor) \c TA::DistArray type. +/// +/// \param tr1_of_base space base_key -> the space's FULL \c TA::TiledRange1. +/// \param world the World the zero destinations are built in; it must +/// outlive the returned closures' use. +/// +/// \details The two closures are the TA realization of external-axis batching's +/// backend needs (see \c BackendArrayOps): \c make_zeros builds a zero +/// destination -- flat or nested per the descriptor's proto structure, with a +/// nested result's inner tiles left empty for the scatter writes to fill -- and +/// \c axis_batches chunks an axis on its space's tile boundaries. Tiling is a +/// property of the space, so both are sourced from \p tr1_of_base alone; no +/// array in the DAG is consulted. Both mpqc (from its orbital/basis registries) +/// and unit tests (from the tranges they build their leaves with) supply the +/// map and call this. +template +[[nodiscard]] BackendArrayOps make_ta_array_ops( + std::map tr1_of_base, TA::World& world) { + auto map = std::make_shared>( + std::move(tr1_of_base)); + BackendArrayOps aops; + aops.axis_batches = [map](Index const& axis, std::size_t target_batch_size) { + return mode_batches_of_trange1( + map->at(std::wstring(axis.space().base_key())), target_batch_size); + }; + aops.make_zeros = + [map, &world](container::vector const& descriptor) -> ResultPtr { + using numeric_type = typename FlatArray::numeric_type; + std::vector outer; + bool nested = false; + for (auto const& ix : descriptor) { + if (ix.has_proto_indices()) { + nested = true; // an inner (nested) mode -- not an outer trange mode + continue; + } + outer.push_back(map->at(std::wstring(ix.space().base_key()))); + } + TA::TiledRange otr(outer.begin(), outer.end()); + auto make_flat = [&]() -> ResultPtr { + FlatArray dest(world, otr); + dest.fill_local(numeric_type(0)); + world.gop.fence(); + return eval_result>(std::move(dest)); + }; + if constexpr (std::is_same_v) { + // Flat-only backend (default ToTArray == FlatArray): descriptors are + // flat. + SEQUANT_ASSERT(!nested && + "flat-only TA array-ops asked for a nested zero result"); + return make_flat(); + } else { + if (!nested) return make_flat(); + // Nested: the outer trange from the non-proto indices, every outer tile + // an empty-inner tensor (tot_inner_rank() == 0) -- a valid zero ToT whose + // real inner tensors the scatter's write_into_slice fills in per batch. + using OuterT = typename ToTArray::value_type; + ToTArray dest(world, otr); + for (auto it = dest.begin(); it != dest.end(); ++it) + if (dest.is_local(it.index())) *it = OuterT{it.make_range()}; + world.gop.fence(); + return eval_result>(std::move(dest)); + } + }; + return aops; +} + +} // namespace sequant + +#endif // SEQUANT_EVAL_BACKENDS_TILEDARRAY_ARRAY_OPS_HPP diff --git a/SeQuant/core/eval/backends/tiledarray/eval_context.hpp b/SeQuant/core/eval/backends/tiledarray/eval_context.hpp index fd5310a978..ef9b1440a9 100644 --- a/SeQuant/core/eval/backends/tiledarray/eval_context.hpp +++ b/SeQuant/core/eval/backends/tiledarray/eval_context.hpp @@ -6,10 +6,13 @@ #include #include #include +#include #include +#include #include +#include #include #include diff --git a/SeQuant/core/eval/backends/tiledarray/result.hpp b/SeQuant/core/eval/backends/tiledarray/result.hpp index 6c640cf1cc..39e81e06c8 100644 --- a/SeQuant/core/eval/backends/tiledarray/result.hpp +++ b/SeQuant/core/eval/backends/tiledarray/result.hpp @@ -3,6 +3,9 @@ #ifdef SEQUANT_HAS_TILEDARRAY +#include + +#include #include #include #include @@ -13,6 +16,11 @@ #include #include +#include +#include +#include +#include +#include namespace sequant { @@ -21,6 +29,56 @@ namespace sequant { // SF.21 / "Use unnamed namespaces in headers ... no" guidance) namespace detail { +// INSTRUMENTATION (SEQUANT_SYNC_STATS, analysis-only): count gop.fence() calls +// so the ordered-executor's synchronization overhead can be localized. +inline std::atomic& fence_counter() { + static std::atomic c{0}; + return c; +} +inline void note_fence() { + fence_counter().fetch_add(1, std::memory_order_relaxed); +} +inline std::atomic& wait_counter() { + static std::atomic c{0}; + return c; +} +inline void note_wait() { + wait_counter().fetch_add(1, std::memory_order_relaxed); +} +inline std::atomic& slice_counter() { + static std::atomic c{0}; + return c; +} +inline void note_slice() { + slice_counter().fetch_add(1, std::memory_order_relaxed); +} +inline std::atomic& slice_ns() { + static std::atomic c{0}; + return c; +} +struct FenceReporter { + ~FenceReporter() { + if (std::getenv("SEQUANT_SYNC_STATS")) + std::cerr << "TOTAL gop.fence() calls = " << fence_counter().load() + << " ; wait_for_lazy_cleanup calls = " << wait_counter().load() + << " ; slice_mode calls = " << slice_counter().load() + << " ; slice_mode total = " << (slice_ns().load() / 1e9) << " s" + << "\n"; + } +}; +inline FenceReporter fence_reporter_{}; + +// Wire PhaseTimer's boundary barrier to a TA world fence so per-region phase +// timers cannot misattribute async (deferred) work across regions. Runs only +// under SEQUANT_UT_PHASE (PhaseTimer::Scope calls barrier() only when enabled). +inline const bool phase_fence_installed_ = [] { + ::sequant::eval::PhaseTimer::fence_hook() = [] { + TA::get_default_world().gop.fence(); + ::sequant::detail::note_fence(); + }; + return true; +}(); + /// Inner-tensor mode count of a tensor-of-tensor DistArray (0 for a regular, /// non-nested array). The outer trange carries no inner information, so the /// inner rank is read from the first local non-empty inner tile and reduced @@ -133,6 +191,7 @@ auto column_symmetrize_ta(TA::DistArray const& arr) { result(lannot) = nf * result(lannot); TA::DistArray::wait_for_lazy_cleanup(result.world()); + ::sequant::detail::note_wait(); return result; } @@ -204,6 +263,7 @@ auto particle_antisymmetrize_ta(TA::DistArray const& arr, result(lannot) = nf * result(lannot); TA::DistArray::wait_for_lazy_cleanup(result.world()); + ::sequant::detail::note_wait(); return result; } @@ -212,6 +272,33 @@ inline void log_ta(Args const&... args) noexcept { log_result("[TA] ", args...); } +/// Batch-step data-movement record, emitted at eval log level > 0 (the same +/// gate as the executor's `Eval | ...` records): +/// +/// Batch | |