diff --git a/src/Profiling.cpp b/src/Profiling.cpp index b2b62e7891e5..7c23f2abe2d7 100644 --- a/src/Profiling.cpp +++ b/src/Profiling.cpp @@ -356,9 +356,17 @@ class InjectCounters : public IRMutator { ParallelLoops, ParallelTasks, PointsRequiredAtRoot, - PointsComputed }; - - static constexpr int num_counters = PointsComputed + 1; + PointsComputed, + ScalarLoads, + VectorLoads, + Gathers, + BytesLoaded, + ScalarStores, + VectorStores, + Scatters, + BytesStored }; + + static constexpr int num_counters = BytesStored + 1; struct Counters { @@ -706,6 +714,10 @@ class InjectCounters : public IRMutator { bool is_real_data_buffer(const Store *op) const { return op->param.defined() || is_func(names.prefix(op->name)); } + bool is_real_data_buffer(const Load *op) const { + return op->param.defined() || op->image.defined() || + is_func(names.prefix(op->name)); + } Stmt visit(const Store *op) override { if (is_real_data_buffer(op)) { @@ -718,6 +730,17 @@ class InjectCounters : public IRMutator { names.id_for_name(f); Counters &c = counters[id]; int lanes = op->value.type().lanes(); + // Classify the store by its index: scalar, unit-stride vector, or + // scatter. Drives the vectorization performance warnings. + if (op->index.type().is_scalar()) { + c.count(ScalarStores); + } else if (const Ramp *r = op->index.as(); + r && is_const_one(r->stride)) { + c.count(VectorStores); + } else { + c.count(Scatters); + } + c.count(BytesStored, make_const(UInt(64), op->value.type().bytes() * lanes)); // Only the pure def (stage 0) contributes to "points computed"; // update-def stores are a separate kind of work and shouldn't // show up as recompute. For Tuple-valued Funcs each output @@ -737,6 +760,25 @@ class InjectCounters : public IRMutator { return IRMutator::visit(op); } + Expr visit(const Load *op) override { + // We bill these to the Func we're producing, not the Func being + // loaded. These counters are about what kinds of loads we do while + // computing a given Func. + if (producer_id >= 0 && is_real_data_buffer(op)) { + Counters &c = counters[producer_id]; + if (op->index.type().is_scalar()) { + c.count(ScalarLoads); + } else if (const Ramp *r = op->index.as(); + r && is_const_one(r->stride)) { + c.count(VectorLoads); + } else { + c.count(Gathers); + } + c.count(BytesLoaded, make_const(UInt(64), op->type.bytes() * op->type.lanes())); + } + return IRMutator::visit(op); + } + Stmt visit(const ProducerConsumer *op) override { if (op->is_producer) { // One entry per producer node, parented to the surrounding diff --git a/src/runtime/HalideRuntime.h b/src/runtime/HalideRuntime.h index a3038f3277fd..f9fc4b53d18d 100644 --- a/src/runtime/HalideRuntime.h +++ b/src/runtime/HalideRuntime.h @@ -2097,6 +2097,17 @@ struct HALIDE_ATTRIBUTE_ALIGN(8) halide_profiler_func_stats { * stores keeps update definitions from being conflated as * "recompute". */ uint64_t points_computed; + + /** The number of loads of various kinds done while computing this Func, + * classified by the index expression: scalar (a scalar index), + * vector (a unit-stride ramp), or gather (anything else). bytes_loaded + * is the total number of bytes loaded across all three. */ + uint64_t scalar_loads, vector_loads, gathers, bytes_loaded; + + /** The number of stores of various kinds done while computing this Func, + * classified the same way (scalar / unit-stride vector / scatter), plus + * the total bytes stored. */ + uint64_t scalar_stores, vector_stores, scatters, bytes_stored; }; /** Per-pipeline state tracked by the sampling profiler. These exist diff --git a/src/runtime/fake_thread_pool.cpp b/src/runtime/fake_thread_pool.cpp index e1003fd902df..fe8c852897f1 100644 --- a/src/runtime/fake_thread_pool.cpp +++ b/src/runtime/fake_thread_pool.cpp @@ -126,6 +126,11 @@ WEAK int halide_set_num_threads(int n) { return 1; } +WEAK int halide_get_num_threads() { + // This thread pool never runs work on any thread but the caller. + return 1; +} + WEAK halide_do_task_t halide_set_custom_do_task(halide_do_task_t f) { halide_do_task_t result = custom_do_task; custom_do_task = f; diff --git a/src/runtime/profiler_common.cpp b/src/runtime/profiler_common.cpp index 9bb35e5bb6f7..1a086e376634 100644 --- a/src/runtime/profiler_common.cpp +++ b/src/runtime/profiler_common.cpp @@ -490,6 +490,14 @@ enum { counter_parallel_tasks = 3, counter_points_required_at_root = 4, counter_points_computed = 5, + counter_scalar_loads = 6, + counter_vector_loads = 7, + counter_gathers = 8, + counter_bytes_loaded = 9, + counter_scalar_stores = 10, + counter_vector_stores = 11, + counter_scatters = 12, + counter_bytes_stored = 13, }; ALWAYS_INLINE bool counter_is_approximate(const halide_profiler_func_stats *fs, int counter) { @@ -687,13 +695,13 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st constexpr const char *horiz_rule = "--------------------------------------------------------------------------------------------------------\n"; constexpr const char *func_row = - "NNNNNNNNNNNNNNNNNNNNNNNNN|TTTTTTTTT PPPPPPPP|HHHHHH |LLLLLL|KKKKKK|AAAAAA|MMMMMM|VVVVVV|RRRRRRRR |"; + "NNNNNNNNNNNNNNNNNNNNNNNNN|TTTTTTTTT PPPPPPPP|HHHHHH |LLLLLL|KKKKKK|AAAAAA|MMMMMM|VVVVVV|RRRRRRRR |YYYYY|"; constexpr const char *allocation_func_row = - "NNNNNNNNNNNNNNNNNNNNNNNNN|ZZZZZZZZZZZZZZZZZZ| | | |AAAAAA|MMMMMM|VVVVVV| |"; + "NNNNNNNNNNNNNNNNNNNNNNNNN|ZZZZZZZZZZZZZZZZZZ| | | |AAAAAA|MMMMMM|VVVVVV| | |"; constexpr const char *column_legend_row_1 = - " name | time percent | active| parallel | heap | peak | avg |recompute|"; + " name | time percent | active| parallel | heap | peak | avg |recompute|notes|"; constexpr const char *column_legend_row_2 = - " | |threads| loops| tasks|allocs| mem | mem | ratio |"; + " | |threads| loops| tasks|allocs| mem | mem | ratio | |"; for (halide_profiler_pipeline_stats *p = s->pipelines; p; p = (halide_profiler_pipeline_stats *)(p->next)) { @@ -880,6 +888,403 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st f_stats[f_stats_count++] = fs; } + // ---- Per-Func rolled-up stats ----------------------------------- + // + // A Func may have multiple instances in this stats array (different + // inlining chains, or an unscheduled Func realized separately under + // each caller). The per-instance rows are useful for display, but + // for warnings we want one shot per Func, against Func-wide totals. + // Sum the counter region of every instance into a single aggregate + // keyed on canonical_id; identity fields come from the canonical + // entry; peaks take the max. + // + // Note this is a different kind of aggregation from cum_stats: that + // one sums children's stats into a parent (subtree totals). This one + // sums siblings sharing a name into a single per-Func entry. + // + // The counter region is every field from memory_total to the end of + // the struct: all uint64_t and contiguous, so we treat it as a + // uint64_t[] and blindly add it. Adding a new counter then requires + // no changes here. The non-summable identity fields (including + // counters_approximated) all sit before memory_total, so they're + // untouched by the blind add and handled explicitly. + constexpr size_t counter_offset = __builtin_offsetof(halide_profiler_func_stats, memory_total); + constexpr size_t counter_bytes = sizeof(halide_profiler_func_stats) - counter_offset; + constexpr int num_counter_words = (int)(counter_bytes / sizeof(uint64_t)); + + size_t canon_fs_size = p->num_funcs * sizeof(halide_profiler_func_stats); + size_t canon_cs_size = p->num_funcs * sizeof(CumulativeStats); + halide_profiler_func_stats *canon_fs = + (halide_profiler_func_stats *)__builtin_alloca(canon_fs_size); + CumulativeStats *canon_cs = + (CumulativeStats *)__builtin_alloca(canon_cs_size); + __builtin_memset(canon_fs, 0, canon_fs_size); + __builtin_memset(canon_cs, 0, canon_cs_size); + // canonical_id <= i for every instance, so a single forward pass + // initializes each aggregate's identity fields when it hits the + // canonical entry, then folds later non-canonical instances into it. + for (int i = 0; i < p->num_funcs; i++) { + const halide_profiler_func_stats &src = p->funcs[i]; + int c = src.canonical_id; + halide_profiler_func_stats &dst = canon_fs[c]; + if (i == c) { + // Identity / non-summable fields come from the canonical entry. + dst.name = src.name; + dst.parent = src.parent; + dst.canonical_id = c; + dst.kind = src.kind; + dst.buffer_func_id = src.buffer_func_id; + } + // If any instance's counters are approximate, so is the aggregate. + dst.counters_approximated |= src.counters_approximated; + dst.time += src.time; + if (src.memory_peak > dst.memory_peak) { + dst.memory_peak = src.memory_peak; + } + if (src.stack_peak > dst.stack_peak) { + dst.stack_peak = src.stack_peak; + } + // Blind-add the counter region (everything from memory_total + // onwards in the struct). + uint64_t *dst_counters = (uint64_t *)((char *)&dst + counter_offset); + const uint64_t *src_counters = (const uint64_t *)((const char *)&src + counter_offset); + for (int j = 0; j < num_counter_words; j++) { + dst_counters[j] += src_counters[j]; + } + + CumulativeStats &dst_cs = canon_cs[c]; + dst_cs.time += cum_stats[i].time; + dst_cs.active_threads_numerator += cum_stats[i].active_threads_numerator; + dst_cs.active_threads_denominator += cum_stats[i].active_threads_denominator; + dst_cs.parallel_tasks += cum_stats[i].parallel_tasks; + } + + // ---- Heuristic warnings ----------------------------------------- + // + // Many counters are recorded but only a few are shown in the table. + // For everything else, the `rule` function below scans each Func's + // rolled-up stats and reports a numbered warning when the schedule + // looks suspicious. Each Func's row gets a "notes" cell listing the + // numbers of the warnings that apply; the messages themselves print + // after the table. Because warnings fire per-Func (on the canonical + // aggregate), every instance of the same Func shows the same numbers. + // + // Each rule's trigger condition and message body live in the same + // case, sharing locally-computed metrics. The function is called + // with emit=false during the pre-scan to discover firings, and again + // with emit=true to render each warning's text. + struct WarningEntry { + int canonical_id; + int rule_id; + }; + constexpr int max_warnings = 256; + WarningEntry *warnings = (WarningEntry *)__builtin_alloca(max_warnings * sizeof(WarningEntry)); + int num_warnings = 0; + + int num_threads = halide_get_num_threads(); + if (num_threads < 1) { + num_threads = 1; + } + + // To add a new warning: add an entry to this enum (before + // num_warning_kinds) and a matching case in the rule() lambda. The + // position in the enum doesn't matter; the switch dispatches by name. + enum WarningKind { + warning_allocs_in_parallel_loop, + warning_poor_thread_utilization_many_loops, + warning_poor_thread_utilization_fine_tasks, + warning_too_few_parallel_tasks, + warning_not_parallelized, + warning_high_recompute, + warning_no_vector_ops, + warning_more_gathers_than_vector_loads, + warning_more_scatters_than_vector_stores, + warning_many_scalar_stores, + warning_narrow_vector_stores, + warning_approximated_counters, + warning_device_bouncing, + num_warning_kinds + }; + + // Returns true if warning `w` fires for `fs`. When `emit` is true, + // also writes the warning message to sstr (using the same metrics + // the trigger condition reads). + auto rule = [&](const halide_profiler_func_stats *fs, + const CumulativeStats *cs, + WarningKind w, + bool emit) -> bool { + float threads_avg = cs->active_threads_numerator / + (cs->active_threads_denominator + 1e-10f); + uint64_t tasks_per_run = fs->parallel_tasks / p->runs; + uint64_t loops_per_run = fs->parallel_loops / p->runs; + bool poor_thread_utilization = threads_avg < num_threads * 0.75f; + // Recompute = points-actually-computed / points-required-at-root. + float recompute = fs->points_required_at_root ? + (fs->points_computed / (float)fs->points_required_at_root) : + 0.f; + float ms_per_task = (cs->time / (fs->parallel_tasks + 1e-10f)) * 1e-6f; + uint64_t total_vector_loads = fs->vector_loads + fs->gathers; + uint64_t total_vector_stores = fs->vector_stores + fs->scatters; + uint64_t total_stores = fs->scalar_stores + fs->vector_stores + fs->scatters; + bool vector_loads_or_stores = total_vector_loads + total_vector_stores != 0; + + // Rules we want to check for *every* Func, even cheap ones. + switch (w) { + case warning_allocs_in_parallel_loop: + if (fs->parallel_loops == 0 && + cs->parallel_tasks != 0 && + fs->num_allocs > fs->parallel_tasks) { + if (emit) { + sstr << fs->name << " was realized inside " << (cs->parallel_tasks / p->runs) + << " parallel tasks, yet made " << (fs->num_allocs / p->runs) + << " heap allocations. Consider hoisting storage for the " + << "Func to the parallel loop to cut down on the number of " + << "heap allocations, or using .store_in(MemoryType::Stack)."; + } + return true; + } + return false; + case warning_approximated_counters: + if (fs->counters_approximated) { + if (emit) { + sstr << fs->name << " has counter contributions that could not be " + << "exactly accumulated (e.g. hoisted out of a GPU kernel via " + << "an upper-bound substitution, or across an IfThenElse with an " + << "impure condition). Its numerical counters are conservative " + << "upper bounds rather than exact totals."; + } + return true; + } + return false; + default: + break; + } + + // Skip anything that takes less than 1% of total runtime + // (including all children). + float frac_time = (float)cs->time / (float)(p->time + 1); + if (frac_time < 0.01) { + return false; + } + + switch (w) { + case warning_poor_thread_utilization_many_loops: + // Significant func with low thread utilization that's also + // launching many parallel loops -- thread pool overhead is + // probably eating the parallelism gains. + if (poor_thread_utilization && + loops_per_run > 1) { + if (emit) { + sstr << fs->name << " launches " << loops_per_run + << " parallel loops and shows poor utilization of the " + << "thread pool. Ensure the parallel loop is the outermost " + << "one. Fuse multiple nested parallel loops into one with " + << "Func::fuse. If this Func has multiple update stages, " + << "consider wrapping them in a single parallel outer " + << "loop with .in()."; + } + return true; + } + return false; + case warning_poor_thread_utilization_fine_tasks: + // Very high task launch rate -- the inner loop is too + // fine-grained, so per-task overhead drowns out the work. + if (poor_thread_utilization && + tasks_per_run > (uint64_t)num_threads * 4) { + if (emit) { + sstr << fs->name << " spawns " << tasks_per_run / loops_per_run + << " parallel tasks per parallel loop and shows poor utilization" + << " of the thread pool. The parallel loop may be too fine-grained." + << " Consider splitting it into a parallel outer loop and a serial" + << " inner loop. Each task currently takes " << ms_per_task << "ms."; + } + return true; + } + return false; + case warning_too_few_parallel_tasks: + // Too few parallel tasks per run to keep the thread pool + // busy -- the loop is too coarse-grained. + if (fs->parallel_tasks > 0 && + tasks_per_run < (uint64_t)num_threads) { + if (emit) { + sstr << fs->name << "'s parallel loop has only " << tasks_per_run + << " task" << (tasks_per_run == 1 ? "" : "s") + << " per run, fewer than the " << num_threads + << " available threads; the loop may be too coarse-grained. " + << "Consider either splitting it more finely or finding other " + << "loops that can be parallel too. Each task currently takes " + << ms_per_task << "ms."; + } + return true; + } + return false; + case warning_not_parallelized: + // Not parallelized at all. + if (!serial && + fs->parent == -1 && + fs->parallel_loops == 0) { + if (emit) { + sstr << fs->name << " is compute_root but not parallelized, while " + << "some other Funcs are. Consider parallelizing it."; + } + return true; + } + return false; + case warning_device_bouncing: { + // Look for copy synthetics anywhere in the pipeline whose + // buffer_func_id points at this Func. Fire only when BOTH + // directions exist -- a Func whose buffer is copied both to + // host and to device within the same pipeline run is bouncing + // between devices, regardless of which scope each copy ended + // up in. A single one-way copy is ordinary pipeline I/O and + // shouldn't trigger. + int my_canonical = fs->canonical_id; + bool has_copy_to_host = false; + bool has_copy_to_device = false; + for (int i = 0; i < p->num_funcs; i++) { + const halide_profiler_func_stats *c = p->funcs + i; + if (c->buffer_func_id != my_canonical) { + continue; + } + if (c->kind == halide_profiler_func_kind_copy_to_host) { + has_copy_to_host = true; + } else if (c->kind == halide_profiler_func_kind_copy_to_device) { + has_copy_to_device = true; + } + } + if (has_copy_to_host && has_copy_to_device) { + if (emit) { + sstr << fs->name << " has stages computing on different devices, " + << "forcing host<->device buffer transfers between stages of " + << "the same Func. This usually means an update definition " + << "was left unscheduled or scheduled on the wrong device. " + << "Schedule all update definitions of " << fs->name + << " to compute on the same device as the pure definition."; + } + return true; + } + return false; + } + default: + break; + } + + // For rules below here, we only care if the self time is more + // than 1%. + float self_time = fs->time / (float)p->time; + if (self_time < 0.01) { + return false; + } + + switch (w) { + case warning_high_recompute: + // High redundant recompute. Only fires on non-inlined Funcs: + // for inlined Funcs the textual call count is a poor proxy for + // real work, since LLVM's LICM / constant-folding routinely + // makes the visible "recompute" disappear. + if (recompute > 2.0f && + fs->parent >= 0) { + if (emit) { + sstr << fs->name << " redundantly recomputes each value " << recompute + << " times on average. Consider a store_at/compute_at location " + << "further outwards in the parent's loop nest, check whether " + << "sliding-window optimization failed, or reduce the split " + << "factors used in its schedule."; + } + return true; + } + return false; + case warning_no_vector_ops: + if (!vector_loads_or_stores) { + if (emit) { + sstr << fs->name << " performs no vector loads or stores. Ensure it is" + << " vectorized."; + } + return true; + } + return false; + case warning_more_gathers_than_vector_loads: + if (fs->gathers > fs->vector_loads) { + if (emit) { + sstr << fs->name << " performs more vector gathers than dense vector " + << "loads ("; + emit_si(fs->gathers); + sstr << " vs "; + emit_si(fs->vector_loads); + sstr << "). It may be possible to improve performance by vectorizing " + << "a different Var, precomputing boundary conditions, or by " + << "reordering the storage layout of Funcs that this one calls."; + } + return true; + } + return false; + case warning_more_scatters_than_vector_stores: + if (fs->scatters > fs->vector_stores) { + if (emit) { + sstr << fs->name << " performs more vector scatters than dense vector " + << "stores ("; + emit_si(fs->scatters); + sstr << " vs "; + emit_si(fs->vector_stores); + sstr << "). It may be possible to improve performance by vectorizing a " + << "different Var, or by reordering the storage layout of this Func."; + } + return true; + } + return false; + case warning_many_scalar_stores: + if (vector_loads_or_stores && + total_vector_stores <= fs->scalar_stores * 10) { + if (emit) { + sstr << "A significant fraction of the stores to " << fs->name << " are scalar: "; + emit_si(fs->scalar_stores); + sstr << " out of "; + emit_si(total_vector_stores + fs->scalar_stores); + sstr << ". There may be an update definition that was not vectorized."; + } + return true; + } + return false; + case warning_narrow_vector_stores: + if (total_vector_stores > fs->scalar_stores * 10 && + fs->bytes_stored < total_vector_stores * p->native_vector_bytes) { + if (emit) { + sstr << "Stores to " << fs->name << " only write an average of " + << fs->bytes_stored / total_stores << " bytes each. " + << "This is less than the machine native vector width. Consider " + << "using wider vectors."; + } + return true; + } + return false; + default: + return false; + } + }; + + for (int f = 0; f < f_stats_count; f++) { + const halide_profiler_func_stats *fs = f_stats[f]; + int idx = (int)(fs - p->funcs); + // Run each rule once per Func, on the canonical instance only. + if (fs->canonical_id != idx) { + continue; + } + // Rules only apply to real Funcs -- skip bookkeeping slots + // (overhead, thread idle, malloc, free) and synthesized + // buffer-copy timing entries. + if (fs->kind != halide_profiler_func_kind_func) { + continue; + } + const halide_profiler_func_stats *agg_fs = &canon_fs[idx]; + const CumulativeStats *agg_cs = &canon_cs[idx]; + for (int w = 0; w < num_warning_kinds; w++) { + if (rule(agg_fs, agg_cs, (WarningKind)w, /*emit=*/false) && + num_warnings < max_warnings) { + warnings[num_warnings++] = {idx, w}; + } + } + } + // Func name slot, including a tree-art indent. One column per // tree level: │ continues an ancestor's subtree; ├/└ are this // row's connector. Glyphs and ANSI escapes contribute zero @@ -993,6 +1398,29 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st } break; } + case 'Y': { + // Notes cell: comma-separated numbers of the warnings + // that apply to this Func. Warnings fire per-Func (on the + // canonical instance), so every instance with the same + // canonical_id shows the same numbers. Truncated if more + // notes than the cell can hold. + uint64_t target = sstr.size() + w; + bool first = true; + for (int wi = 0; wi < num_warnings; wi++) { + if (warnings[wi].canonical_id == fs->canonical_id) { + if (!first && sstr.size() + 1 < target) { + sstr << ","; + } + if (sstr.size() < target) { + sstr << (wi + 1); + } + first = false; + } + } + truncate_bytes_to(target); + pad_bytes_to(target); + break; + } case '|': // Column separator, dimmed so the data stands out. for (int i = 0; i < w; i++) { @@ -1069,6 +1497,87 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st print_func_row(fs, cs); } + // ---- Warning messages ------------------------------------------- + // + // The per-Func rules discovered above are rendered here, plus a few + // pipeline-wide heuristics that don't attach to a single Func. Wrap + // at the table width so the messages don't run past its right edge. + + // Warn if not enough pipeline samples. + bool too_few_samples = p->samples < 100; + + // Warn if more than 10% of the time was spent in unnamed Funcs and + // there are at least three of them. + uint64_t anon_time = 0; + int anon_funcs = 0; + for (int i = 0; i < p->num_funcs; i++) { + const char *name = p->funcs[i].name; + bool anon = name[0] == 'f'; + for (int j = 1; name[j]; j++) { + anon &= name[j] >= '0' && name[j] <= '9'; + } + if (anon) { + anon_funcs++; + anon_time += p->funcs[i].time; + } + } + bool too_many_anon_funcs = anon_funcs >= 3 && anon_time * 10 > p->time; + + // Warn if the pipeline allocates at least 100MB and spends at least + // 10% of its time freeing it. The free bookkeeping slot's time is + // what we sampled while halide_free was running. + uint64_t free_time = 0; + for (int i = 0; i < p->num_funcs; i++) { + if (p->funcs[i].kind == halide_profiler_func_kind_free) { + free_time = p->funcs[i].time; + break; + } + } + bool expensive_free = + p->memory_peak > 100 * 1000 * 1000 && + free_time * 10 > p->time; + + if (num_warnings || too_few_samples || too_many_anon_funcs || expensive_free) { + halide_print(user_context, " Performance warnings:\n"); + int max_cols = (int)strlen(func_row); + // print_wrapped doesn't understand non-printing characters. + bool old = support_colors; + support_colors = false; + + if (too_many_anon_funcs) { + sstr.clear(); + sstr << " - " << anon_funcs << " Funcs have auto-generated names and " + << "collectively take up a significant fraction of the total runtime. " + << "Consider giving them explicit names by passing a string to the " + << "Func constructor. This will make this profile easier to read.\n"; + print_wrapped(user_context, 4, max_cols, sstr.str()); + } + if (too_few_samples) { + sstr.clear(); + sstr << " - Only " << p->samples + << " profiling samples taken. Consider running the " + << "pipeline more times in a loop for more accurate results.\n"; + print_wrapped(user_context, 4, max_cols, sstr.str()); + } + if (expensive_free) { + sstr.clear(); + sstr << " - The pipeline allocates a significant amount of memory, and a " + << "lot of time is spent freeing it. Either fuse stages more aggressively " + << "to use less memory, or consider a using caching allocator with " + << "retention enabled to make freeing it cheaper.\n"; + print_wrapped(user_context, 4, max_cols, sstr.str()); + } + for (int w = 0; w < num_warnings; w++) { + sstr.clear(); + sstr << " " << (w + 1) << ") "; + int cid = warnings[w].canonical_id; + rule(&canon_fs[cid], &canon_cs[cid], (WarningKind)warnings[w].rule_id, /*emit=*/true); + sstr << "\n"; + print_wrapped(user_context, 5, max_cols, sstr.str()); + } + support_colors = old; + } + sstr.clear(); emit_dim(horiz_rule); halide_print(user_context, sstr.str()); @@ -1165,7 +1674,15 @@ WEAK void halide_profiler_report_unlocked(void *user_context, halide_profiler_st field_u64(" ", "parallel_loops", fs->parallel_loops); field_u64(" ", "parallel_tasks", fs->parallel_tasks); field_u64(" ", "points_required_at_root", fs->points_required_at_root); - field_u64(" ", "points_computed", fs->points_computed, true); + field_u64(" ", "points_computed", fs->points_computed); + field_u64(" ", "scalar_loads", fs->scalar_loads); + field_u64(" ", "vector_loads", fs->vector_loads); + field_u64(" ", "gathers", fs->gathers); + field_u64(" ", "bytes_loaded", fs->bytes_loaded); + field_u64(" ", "scalar_stores", fs->scalar_stores); + field_u64(" ", "vector_stores", fs->vector_stores); + field_u64(" ", "scatters", fs->scatters); + field_u64(" ", "bytes_stored", fs->bytes_stored, true); json << " }"; // Flush periodically so we don't overflow the buffer for diff --git a/src/runtime/profiler_inlined.cpp b/src/runtime/profiler_inlined.cpp index 3c59557f9e3f..e7b1b75a12b4 100644 --- a/src/runtime/profiler_inlined.cpp +++ b/src/runtime/profiler_inlined.cpp @@ -68,7 +68,15 @@ WEAK_INLINE int halide_profiler_update_counters(struct halide_profiler_instance_ uint64_t parallel_loops, uint64_t parallel_tasks, uint64_t points_required_at_root, - uint64_t points_computed) { + uint64_t points_computed, + uint64_t scalar_loads, + uint64_t vector_loads, + uint64_t gathers, + uint64_t bytes_loaded, + uint64_t scalar_stores, + uint64_t vector_stores, + uint64_t scatters, + uint64_t bytes_stored) { using namespace Halide::Runtime::Internal::Synchronization; halide_profiler_func_stats &stats = instance->funcs[id]; @@ -87,6 +95,14 @@ WEAK_INLINE int halide_profiler_update_counters(struct halide_profiler_instance_ UPDATE_COUNTER(parallel_tasks); UPDATE_COUNTER(points_required_at_root); UPDATE_COUNTER(points_computed); + UPDATE_COUNTER(scalar_loads); + UPDATE_COUNTER(vector_loads); + UPDATE_COUNTER(gathers); + UPDATE_COUNTER(bytes_loaded); + UPDATE_COUNTER(scalar_stores); + UPDATE_COUNTER(vector_stores); + UPDATE_COUNTER(scatters); + UPDATE_COUNTER(bytes_stored); #undef UPDATE_COUNTER diff --git a/test/generator/profiler_instances_aottest.cpp b/test/generator/profiler_instances_aottest.cpp index 816f367b4f69..2236e89db1fb 100644 --- a/test/generator/profiler_instances_aottest.cpp +++ b/test/generator/profiler_instances_aottest.cpp @@ -101,6 +101,45 @@ void check_unrolled_pure_update(const halide_profiler_pipeline_stats *p) { auto fs = entries_of(p, "unrolled_pu"); REQUIRE(fs.size() == 1); REQUIRE(fs[0]->points_computed == 4); + // Fully unrolled, so every store is scalar: pure-def + update-def = 8. + REQUIRE(fs[0]->scalar_stores == 8); + REQUIRE(fs[0]->vector_stores == 0); + REQUIRE(fs[0]->scatters == 0); +} + +// The load/store op-profile counters classify each access by its index: +// scalar, dense unit-stride vector (ls_n/8 = 8 wide ops over the 64-wide +// domain), or gather/scatter. The generator's ls_* Funcs each exercise one +// classification with exactly-known counts. Every element is a 4-byte int. +void check_load_store_counters(const halide_profiler_pipeline_stats *p) { + struct Expected { + const char *name; + uint64_t scalar_loads, vector_loads, gathers, bytes_loaded; + uint64_t scalar_stores, vector_stores, scatters, bytes_stored; + }; + const Expected expected[] = { + // Dense and vectorized: 8 unit-stride vector loads and stores. + {"ls_vec", 0, 8, 0, 256, 0, 8, 0, 256}, + // Unvectorized: 64 scalar loads and stores. + {"ls_scalar", 64, 0, 0, 256, 64, 0, 0, 256}, + // Data-dependent load index: 8 gathers, plus the dense inner load. + {"ls_gather", 0, 8, 8, 512, 0, 8, 0, 256}, + // Non-unit-stride (strided) vector store: 32 scatters. + {"ls_scatter", 0, 0, 0, 0, 0, 0, 32, 1024}, + }; + for (const auto &e : expected) { + auto fs = entries_of(p, e.name); + REQUIRE(fs.size() == 1); + const halide_profiler_func_stats *f = fs[0]; + REQUIRE(f->scalar_loads == e.scalar_loads); + REQUIRE(f->vector_loads == e.vector_loads); + REQUIRE(f->gathers == e.gathers); + REQUIRE(f->bytes_loaded == e.bytes_loaded); + REQUIRE(f->scalar_stores == e.scalar_stores); + REQUIRE(f->vector_stores == e.vector_stores); + REQUIRE(f->scatters == e.scatters); + REQUIRE(f->bytes_stored == e.bytes_stored); + } } // compute_with: two Funcs share a loop nest, so their stage stores @@ -377,6 +416,7 @@ int main(int argc, char **argv) { check_roundup_overstores_bytes(target); check_guardwithif_no_overstore(target); check_unrolled_pure_update(target); + check_load_store_counters(target); check_compute_with(target); // Only present when the pipeline was built with a GPU feature — the diff --git a/test/generator/profiler_instances_generator.cpp b/test/generator/profiler_instances_generator.cpp index 10a6cc39dd13..4d199a83e0b1 100644 --- a/test/generator/profiler_instances_generator.cpp +++ b/test/generator/profiler_instances_generator.cpp @@ -279,6 +279,44 @@ class ProfilerInstances : public Generator { inwards_red_at_y(x) = 0; inwards_red_at_y(x) += inwards_f_at_y(r_inwards, x); + // Load/store op-profile counters. Each Func exercises one access + // classification over a 64-wide domain (a multiple of the vector + // width, so no tail), giving exactly-known counts. ls_sum samples + // them at x % ls_n so the output's extent doesn't constrain theirs. + const int ls_n = 64; + Var lsx("lsx"); + Func ls_data("ls_data"); + ls_data(lsx) = lsx * 2 + 1; + ls_data.compute_root(); + + // Dense and vectorized: ls_n/8 vector loads (of ls_data) and stores. + Func ls_vec("ls_vec"); + ls_vec(lsx) = ls_data(lsx) + 3; + ls_vec.compute_root().vectorize(lsx, 8); + + // Unvectorized: ls_n scalar loads and scalar stores. + Func ls_scalar("ls_scalar"); + ls_scalar(lsx) = ls_data(lsx) * 5; + ls_scalar.compute_root(); + + // Data-dependent load index in a vectorized Func: gathers. + Func ls_gather("ls_gather"); + ls_gather(lsx) = ls_data(clamp(ls_data(lsx), 0, ls_n - 1)); + ls_gather.compute_root().vectorize(lsx, 8); + + // Non-unit-stride vector store: scatters. Vectorizing the outer + // storage dim makes each vector store stride by the inner extent, + // so its index is a non-unit-stride ramp rather than a dense one. + Func ls_scatter("ls_scatter"); + Var ssx("ssx"), ssy("ssy"); + ls_scatter(ssx, ssy) = ssx + ssy; + ls_scatter.compute_root().vectorize(ssy, 8); + + Func ls_sum("ls_sum"); + ls_sum(x) = ls_vec(x % ls_n) + ls_scalar(x % ls_n) + + ls_gather(x % ls_n) + ls_scatter(x % 4, x % ls_n); + ls_sum.compute_root(); + Func caller_g("caller_g"), caller_h("caller_h"); caller_g(x) = update_f(x); caller_h(x) = update_f(x) * 2; @@ -287,7 +325,7 @@ class ProfilerInstances : public Generator { roundup_outer(x) + guard_outer(x) + stencil_out(x) + unrolled_pu(x % 4) + cw_a(x) + cw_b(x) + slide_out(x) + slide_fail_f(x) + extern_stage_e(x) + - inwards_red_root(x) + inwards_red_at_y(x); + inwards_red_root(x) + inwards_red_at_y(x) + ls_sum(x); if (get_target().has_gpu_feature()) { out_value = out_value + approx_out(x) + xfer_out(x) + mixed_sched(x) + shared_out(x) + dev_only_out(x);