From 793e1fd198c0f6ee43caa4951c4e8cdb2936c4c9 Mon Sep 17 00:00:00 2001 From: "Brian.oh" <49855381+dev07060@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:07:24 +0900 Subject: [PATCH 1/2] feat(profiling): per-segment query profiler driver + drive entrypoint (LOC-68) Per-segment timing driver (embed/activate/search/hydrate) mirroring SourceRagService.searchMeta, replicating the private _indexPath (FNV-1a) and _toInt64List (FRB Int64List) helpers, with activateOnly/deleteOnDiskIndex for deterministic cold setup and hit sourceIds for fail-closed filter checks. Profile-mode requires flutter drive (flutter test hard-pins debug = fallback backend), so adds test_driver/integration_test.dart and the flutter_rust_bridge dep the profiler needs for FRB i64 lists. --- .../query_profile_measure_test.dart | 259 ++++++++++++++++++ .../integration_test/query_profile_test.dart | 15 +- example/lib/profiling/query_profiler.dart | 205 ++++++++++++++ example/pubspec.lock | 2 +- example/pubspec.yaml | 4 + example/test_driver/integration_test.dart | 14 + 6 files changed, 491 insertions(+), 8 deletions(-) create mode 100644 example/integration_test/query_profile_measure_test.dart create mode 100644 example/lib/profiling/query_profiler.dart create mode 100644 example/test_driver/integration_test.dart diff --git a/example/integration_test/query_profile_measure_test.dart b/example/integration_test/query_profile_measure_test.dart new file mode 100644 index 0000000..2de57d2 --- /dev/null +++ b/example/integration_test/query_profile_measure_test.dart @@ -0,0 +1,259 @@ +import 'dart:io'; + +import 'package:flutter/foundation.dart' + show kDebugMode, kProfileMode, kReleaseMode; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:mobile_rag_engine/mobile_rag_engine.dart'; +import 'package:mobile_rag_engine_example/profiling/query_fixture.dart'; +import 'package:mobile_rag_engine_example/profiling/query_profiler.dart'; +import 'package:mobile_rag_engine_example/profiling/query_profile_report.dart'; +import 'package:mobile_rag_engine_example/profiling/profile_export.dart'; + +// On-device RAG query profiler — P3 (LOC-68) measurement + P4 (LOC-69) export. +// +// Runs ALONE in its own process (separate from the P2 smoke) under flutter +// drive in PROFILE mode, so the shipped `vector_faer,vector_quant_i8` backend +// is exercised (flutter test hard-pins debug = fallback backend = invalid): +// +// cd example && flutter drive \ +// --driver=test_driver/integration_test.dart \ +// --target=integration_test/query_profile_measure_test.dart \ +// --profile -d +// +// Clean state is established by DELETING the DB files before initialize (not +// clearAllData, whose async re-init races the first seed and trips +// "database is locked"). Per-collection index files are removed after init. +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + const measuredDocs = 500; // representative; raise for scan-bound runs + const warmup = 5; + const measured = 30; + const topK = 10; + + test( + 'profiler: pure_cold / pure_warm / switching_cold x 2 lanes', + () async { + _assertProfileMode(); + + // Fresh DB without clearAllData: delete the SQLite files BEFORE + // initialize so the engine starts clean and nothing races the seed. + final docsDir = await getApplicationDocumentsDirectory(); + final dbStem = '${docsDir.path}/profile.sqlite'; + for (final p in [dbStem, '$dbStem-wal', '$dbStem-shm', '$dbStem-journal']) { + final f = File(p); + if (await f.exists()) await f.delete(); + } + + await MobileRag.initialize( + tokenizerAsset: 'assets/tokenizer.json', + modelAsset: 'assets/model.onnx', + databaseName: 'profile.sqlite', + deferIndexWarmup: true, + ); + + final ids = await QueryFixture.seed(docsPerCollection: measuredDocs); + expect(ids[QueryFixture.collectionA]!.length, measuredDocs); + expect(ids[QueryFixture.collectionB]!.length, measuredDocs); + + final profiler = QueryProfiler(dbPath: MobileRag.instance.dbPath); + final q = QueryFixture.unfilteredQueries; + final runs = []; + + final meta = { + 'build_mode': + kReleaseMode ? 'release' : (kProfileMode ? 'profile' : 'debug'), + 'features': 'vector_faer,vector_quant_i8', // shipped in profile/release + 'os': Platform.operatingSystem, + 'os_version': Platform.operatingSystemVersion, + // Device model + charging state: document manually in PR-P4.md. + 'docs_per_collection': measuredDocs, + 'top_k': topK, + 'vector_weight': profiler.vectorWeight, + 'bm25_weight': profiler.bm25Weight, + // Cold-activate semantics differ per cell: pure_cold rebuilds A's index + // from the DB (on-disk index deleted, slot evicted); switching_cold then + // loads A from disk (pure_cold's activate re-saved it). Both are genuine + // cold costs; the switch is the realistic production figure. + 'cold_note': 'pure_cold=rebuild-from-db; switching_cold=load-from-disk', + }; + + // Per-collection index files survive across runs; delete them so the first + // activate rebuilds the current corpus from the DB, then evict A from the + // single global HNSW slot so pure_cold's activate is genuinely cold. + await profiler.deleteOnDiskIndex(QueryFixture.collectionA); + await profiler.deleteOnDiskIndex(QueryFixture.collectionB); + await profiler.activateOnly(QueryFixture.collectionB); // active := B (A evicted) + + /// Run one (lane, category) cell. + /// - [activateOnce]: single cold shot measures the activate segment (pure_cold). + /// - [activatePerIter]: re-activate A every iter after evicting it (switching_cold). + Future runCell({ + required String lane, + required String category, + required int warmupN, + required int measuredN, + bool activateOnce = false, + bool activatePerIter = false, + List? sourceIds, + }) async { + final measuresActivate = activateOnce || activatePerIter; + final segs = { + for (final n in ['embed', 'search', 'hydrate', + if (measuresActivate) 'activate']) + n: SegmentSamples(n), + }; + final allowed = sourceIds?.toSet(); + + // Warm cells: ensure A is active, then warm caches (discarded). Cold + // cells (warmupN == 0) skip this so the measured shot stays cold. + if (warmupN > 0) { + await profiler.activateOnly(QueryFixture.collectionA); + for (var i = 0; i < warmupN; i++) { + await profiler.measureOnce( + collectionId: QueryFixture.collectionA, + query: q[i % q.length], + sourceIds: sourceIds, + ); + } + } + + QueryProfiler.resetIo(); // I/O snapshot reflects measured iters only + var minHits = -1; + for (var i = 0; i < measuredN; i++) { + if (activatePerIter) { + // Evict A (no query, so collection-B I/O does not pollute the + // snapshot) so the measured activate(A) pays the real switch cost. + await profiler.activateOnly(QueryFixture.collectionB); + } + final r = await profiler.measureOnce( + collectionId: QueryFixture.collectionA, + query: q[i % q.length], + sourceIds: sourceIds, + activate: activateOnce || activatePerIter, + ); + r.ms.forEach((k, v) => segs[k]?.add(v)); + final hitCount = r.hitSourceIds.length; + if (minHits < 0 || hitCount < minHits) minHits = hitCount; + // Fail-closed (filtered lane): every hit must come from the requested + // sources — proves the filter actually applied. (Replaces the + // scoped_exact_scan_rows counter, which the engine never increments.) + if (allowed != null) { + expect(r.hitSourceIds.every(allowed.contains), isTrue, + reason: '$lane/$category returned a hit outside requested ' + 'sourceIds — filter not applied'); + } + } + final io = profiler.snapshotIo(); + + // Fail-closed: a wrong index path or empty corpus would return 0 hits. + expect(minHits, greaterThan(0), + reason: '$lane/$category returned 0 hits — wrong index path/corpus'); + return QueryProfileRun( + lane: lane, category: category, segments: segs, io: io, meta: meta); + } + + // Pure Cold (first query, cold index build) — unfiltered one-shot. + // Runs first; it leaves collection A active for the warm cells below. + runs.add(await runCell( + lane: 'unfiltered', + category: 'pure_cold', + warmupN: 0, + measuredN: 1, + activateOnce: true, + )); + + // Pure Warm (collection already active) — both lanes. + runs.add(await runCell( + lane: 'unfiltered', + category: 'pure_warm', + warmupN: warmup, + measuredN: measured, + )); + runs.add(await runCell( + lane: 'filtered', + category: 'pure_warm', + warmupN: warmup, + measuredN: measured, + sourceIds: ids[QueryFixture.collectionA]!.take(3).toList(), + )); + + // Switching Cold (A -> B -> A each iter) — unfiltered (expensive path). + runs.add(await runCell( + lane: 'unfiltered', + category: 'switching_cold', + warmupN: warmup, + measuredN: measured, + activatePerIter: true, + )); + + // Fail-closed: every cell has at least one sample per segment. + for (final r in runs) { + for (final s in r.segments.values) { + expect(s.samplesMs.length, greaterThan(0), + reason: '${r.lane}/${r.category}/${s.segment}'); + } + } + + // P3: per-segment p50/p95 to the device log (greppable CSV block). + final report = QueryProfileReport(runs: runs); + _emitReport(report); + + // P4: export JSON + CSV to the app documents dir (the baseline artifact) + // and log the dir + filename so the operator can pull them off-device. + final tsTag = DateTime.now().millisecondsSinceEpoch.toString(); + final dir = await ProfileExport.write(report, tsTag: tsTag); + // ignore: avoid_print + print('PROFILE_EXPORT_DIR $dir'); + // ignore: avoid_print + print('PROFILE_EXPORT_FILE query_profile_$tsTag.json ' + 'query_profile_$tsTag.csv'); + }, + // Seeding 2x500 real ONNX embeddings + the measured loops far exceed the + // default 30s per-test timeout; give it generous headroom. + timeout: const Timeout(Duration(minutes: 15)), + // `flutter test` is always debug -> fallback backend. Skip there so an + // invalid baseline is never produced; runs only under flutter drive --profile. + skip: kDebugMode + ? 'Measurement requires `flutter drive --profile` ' + '(flutter test is debug = fallback backend = invalid baseline).' + : false, + ); +} + +/// Fail-closed backend gate (defense-in-depth alongside the test-level skip). +/// Debug builds compile the cargokit debug profile = fallback Rust backend (no +/// vector_faer / vector_quant_i8), so any debug baseline is invalid. kDebugMode +/// is a compile-time const, false under a real profile/release build. +void _assertProfileMode() { + if (kDebugMode) { + fail( + 'Query profiler must run in PROFILE/RELEASE via flutter drive.\n' + 'Run: cd example && flutter drive ' + '--driver=test_driver/integration_test.dart ' + '--target=integration_test/query_profile_measure_test.dart ' + '--profile -d \n' + 'Debug builds use the cargokit debug profile = fallback Rust backend ' + '(no vector_faer / vector_quant_i8), so the baseline would be invalid. ' + 'Aborting to avoid a fake-green baseline. ' + '(detected: kDebugMode=$kDebugMode, kProfileMode=$kProfileMode, ' + 'kReleaseMode=$kReleaseMode)', + ); + } +} + +/// Print the report to the device log in greppable blocks. Each CSV row is a +/// SEPARATE print: the device console truncates long single lines, so one big +/// multi-line print(toCsv()) loses the tail (e.g. switching_cold's activate). +void _emitReport(QueryProfileReport report) { + for (final line in report.toCsv().trimRight().split('\n')) { + // ignore: avoid_print + print('PROFILE_CSV $line'); + } + for (final r in report.runs) { + // ignore: avoid_print + print('PROFILE_IO ${r.lane}/${r.category} ${r.io}'); + } +} diff --git a/example/integration_test/query_profile_test.dart b/example/integration_test/query_profile_test.dart index 6416116..8de4ef8 100644 --- a/example/integration_test/query_profile_test.dart +++ b/example/integration_test/query_profile_test.dart @@ -3,17 +3,18 @@ import 'package:integration_test/integration_test.dart'; import 'package:mobile_rag_engine/mobile_rag_engine.dart'; import 'package:mobile_rag_engine_example/profiling/query_fixture.dart'; -// On-device RAG query profiler — P2.3 smoke entrypoint. +// On-device RAG query profiler — P2 smoke entrypoint. // -// Validates the device pipeline: engine init (real ONNX), A/B fixture seed. -// NOTE: `flutter test integration_test` builds DEBUG by default, which uses -// the cargokit debug profile (fallback Rust backend). The P3/P4 measurement -// runs must use profile/release so the shipped `vector_faer,vector_quant_i8` -// backend is exercised. +// Validates the device pipeline: engine init (real ONNX) + A/B fixture seed. +// Runs fine in debug: +// flutter test integration_test/query_profile_test.dart -d +// +// The P3/P4 MEASUREMENT lives in query_profile_measure_test.dart and must run +// under `flutter drive --profile` (profile build => shipped vector_faer/ +// vector_quant_i8 backend; flutter test hard-pins debug = fallback backend). void main() { IntegrationTestWidgetsFlutterBinding.ensureInitialized(); - // Smoke uses a small corpus for speed; P3 scales this up. const docsPerCollection = 40; // Non-UI integration test: use test() (not testWidgets) so the widget-tester diff --git a/example/lib/profiling/query_profiler.dart b/example/lib/profiling/query_profiler.dart new file mode 100644 index 0000000..83fce9b --- /dev/null +++ b/example/lib/profiling/query_profiler.dart @@ -0,0 +1,205 @@ +// On-device RAG query profiler — P3 (LOC-68) per-segment timing driver. +// +// Mirrors the canonical metadata-first search path in +// SourceRagService.searchMeta (lib/services/source_rag_service.dart:933-995), +// but times each FFI step in ISOLATION so the dominant bucket is found from +// data: embed / activate (collection switch) / search / hydrate. +// +// This is profiling-only code (example app, NOT shipped). To measure each +// segment independently it calls the engine's low-level rust API directly and +// replicates two private helpers from SourceRagService: +// * _indexPath (source_rag_service.dart:283-309) — HNSW base path +// * _toInt64List (source_rag_service.dart:261-268) — FRB i64 list +// Both are mirrored verbatim (with citations); the scenario runner adds a +// fail-closed hit-count + filter-correctness check so a drifted index path or +// an unapplied filter can never pass silently. +import 'dart:io'; + +import 'package:mobile_rag_engine/services/embedding_service.dart'; +// ignore: implementation_imports +import 'package:mobile_rag_engine/src/rust/api/source_rag.dart' as rust_rag; +// ignore: implementation_imports +import 'package:mobile_rag_engine/src/rust/api/query_metrics.dart' as qm; +// frb.Int64List is flutter_rust_bridge's generalized (BigInt-backed) Int64List +// re-exported by flutter_rust_bridge_for_generated.dart — NOT dart:typed_data's. +// The generated API (SearchMetaHybridOptions.sourceIds / hydrateChunks chunkIds) +// expects exactly this generalized type; the generated bindings import the same +// library unaliased, so building it element-by-element matches the engine. +import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart' + as frb; + +/// Result of a single measured query: per-segment milliseconds + the sourceIds +/// of the returned hits (used for fail-closed hit-count + filter checks). +typedef MeasuredQuery = ({Map ms, List hitSourceIds}); + +/// Per-segment timing driver. One instance per profiling run. +class QueryProfiler { + /// Resolved SQLite db path — pass `MobileRag.instance.dbPath`. + /// Used to reconstruct the engine's private HNSW index base path. + final String dbPath; + + /// Hybrid weights mirrored from kDefaultVectorWeight / kDefaultBm25Weight + /// (lib/src/internal/defaults.dart). They are finite and in-range, so the + /// engine's normalizeHybridWeights is identity — no need to replicate it. + final double vectorWeight; + final double bm25Weight; + + QueryProfiler({ + required this.dbPath, + this.vectorWeight = 0.2, + this.bm25Weight = 0.8, + }); + + static Future _timeMs(Future Function() fn) async { + final sw = Stopwatch()..start(); + await fn(); + sw.stop(); + return sw.elapsedMicroseconds / 1000.0; + } + + // --- index base path reconstruction (mirrors SourceRagService:283-309) --- + + String get _dbPathWithoutExtension { + const knownDbExtensions = ['.sqlite3', '.sqlite', '.db']; + final lower = dbPath.toLowerCase(); + for (final ext in knownDbExtensions) { + if (lower.endsWith(ext)) { + return dbPath.substring(0, dbPath.length - ext.length); + } + } + return dbPath; + } + + /// Deterministic FNV-1a 32-bit hash for stable file names + /// (mirrors SourceRagService._collectionFileSuffix, :296-304). + static String _collectionFileSuffix(String collectionId) { + var hash = 0x811C9DC5; + for (final codeUnit in collectionId.codeUnits) { + hash ^= codeUnit; + hash = (hash * 0x01000193) & 0xFFFFFFFF; + } + return hash.toRadixString(16).padLeft(8, '0'); + } + + /// HNSW index base path for a NON-default collection (the profiler only uses + /// profile_a / profile_b). Mirrors SourceRagService._indexPath (:307-309). + String indexBasePath(String collectionId) => + '${_dbPathWithoutExtension}_hnsw_${_collectionFileSuffix(collectionId)}'; + + /// On-disk index artifacts + dirty marker for a non-default collection, + /// matching RagEngine.clearAllData's candidate set (rag_engine.dart:1061-1066) + /// plus the dirty marker (source_rag_service.dart:313-315). clearAllData only + /// cleans the DEFAULT collection's files, so the profiler must delete these + /// itself to guarantee a stale-free, deterministic corpus on every run. + List indexArtifactPaths(String collectionId) { + final base = indexBasePath(collectionId); + final dirty = + '$_dbPathWithoutExtension.${_collectionFileSuffix(collectionId)}.dirty'; + return [base, '$base.pbin', '$base.hnsw.data', '$base.hnsw.graph', dirty]; + } + + /// Delete a collection's on-disk index so the next activate rebuilds the + /// current corpus from the DB (kills stale-index / cross-run accumulation). + Future deleteOnDiskIndex(String collectionId) async { + for (final path in indexArtifactPaths(collectionId)) { + final f = File(path); + if (await f.exists()) { + await f.delete(); + } + } + } + + /// Activate a collection WITHOUT running a query (no embed/search/hydrate). + /// Used to evict the single global HNSW slot between collections so the next + /// measured activate truly pays the switch/load cost, and to seed warm cells. + Future activateOnly(String collectionId) => + rust_rag.activateCollectionForHybridSearch( + collectionId: collectionId, + basePath: indexBasePath(collectionId), + ); + + /// Mirror of SourceRagService._toInt64List (:261-268). Builds the FRB + /// generalized Int64List element-by-element (NOT .fromList) to match the + /// generated API's expected type exactly. + static frb.Int64List _toInt64List(List list) { + final result = frb.Int64List(list.length); + for (var i = 0; i < list.length; i++) { + result[i] = list[i]; + } + return result; + } + + /// One measured query, mirroring [SourceRagService.searchMeta]. + /// + /// Segments returned (ms): `embed`, `search`, `hydrate`, and `activate` when + /// [activate] is true (the cold/switch path). [sourceIds] non-null selects + /// the filtered lane. Also returns the hits' sourceIds so the caller can + /// fail-closed on an empty result or an unapplied filter. + Future measureOnce({ + required String collectionId, + required String query, + List? sourceIds, + bool activate = false, + int topK = 10, + }) async { + final ms = {}; + + if (activate) { + ms['activate'] = await _timeMs( + () => rust_rag.activateCollectionForHybridSearch( + collectionId: collectionId, + basePath: indexBasePath(collectionId), + ), + ); + } + + late final List embedding; // Float32List is-a List + ms['embed'] = await _timeMs(() async { + embedding = await EmbeddingService.embed(query); + }); + + late final rust_rag.SearchHandle handle; + ms['search'] = await _timeMs(() async { + handle = await rust_rag.searchMetaHybrid( + collectionId: collectionId, + queryText: query, + queryEmbedding: embedding, + options: rust_rag.SearchMetaHybridOptions( + topK: topK, + vectorWeight: vectorWeight, + bm25Weight: bm25Weight, + sourceIds: sourceIds == null ? null : _toInt64List(sourceIds), + adjacentChunks: 0, + ), + ); + }); + + final hits = await handle.hitMeta(); + final chunkIds = [for (final h in hits) h.chunkId]; // PlatformInt64==int (native) + ms['hydrate'] = await _timeMs(() async { + await handle.hydrateChunks(chunkIds: _toInt64List(chunkIds)); + }); + + await handle.dispose(); + return (ms: ms, hitSourceIds: [for (final h in hits) h.sourceId]); + } + + /// Snapshot (and reset) the rust-side query_metrics I/O counters into a + /// JSON-ready map. Call [resetIo] before the measured loop, this after. + Map snapshotIo() { + final s = qm.takeQueryContentReadStats(); + return { + 'full_hydrate_rows': s.fullHydrateRows.toInt(), + 'full_hydrate_content_bytes': s.fullHydrateContentBytes.toInt(), + 'scoped_exact_scan_rows': s.scopedExactScanRows.toInt(), + 'scoped_exact_scan_content_bytes': s.scopedExactScanContentBytes.toInt(), + 'scoped_exact_scan_tokens': s.scopedExactScanTokens.toInt(), + 'scoped_exact_scan_tokenization_nanos': + s.scopedExactScanTokenizationNanos.toInt(), + }; + } + + /// Reset the rust-side query_metrics counters (call after warmup, before the + /// measured loop, so the I/O snapshot reflects measured iterations only). + static void resetIo() => qm.resetQueryContentReadStats(); +} diff --git a/example/pubspec.lock b/example/pubspec.lock index 584b438..e92857c 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -124,7 +124,7 @@ packages: source: hosted version: "2.0.33" flutter_rust_bridge: - dependency: transitive + dependency: "direct main" description: name: flutter_rust_bridge sha256: "37ef40bc6f863652e865f0b2563ea07f0d3c58d8efad803cc01933a4b2ee067e" diff --git a/example/pubspec.yaml b/example/pubspec.yaml index 93cbf97..aef00b8 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -13,6 +13,10 @@ dependencies: path: ../ path_provider: ^2.1.5 file_picker: ^9.2.1 + # Profiler-only (LOC-68): the on-device query profiler constructs the FRB + # generalized Int64List for sourceIds / chunkIds, matching the engine's + # low-level rust API. Version pinned to the engine's frb (root pubspec.yaml). + flutter_rust_bridge: ^2.11.1 dev_dependencies: flutter_test: diff --git a/example/test_driver/integration_test.dart b/example/test_driver/integration_test.dart new file mode 100644 index 0000000..a3c54b8 --- /dev/null +++ b/example/test_driver/integration_test.dart @@ -0,0 +1,14 @@ +// Driver entrypoint for `flutter drive` (profile/release integration runs). +// +// `flutter test` cannot build integration tests in profile mode (it hard-pins +// BuildMode.debug), so the on-device query-profiler MEASUREMENT runs — which +// require the shipped `vector_faer,vector_quant_i8` backend (only compiled in +// profile/release) — must go through `flutter drive`: +// +// cd example && flutter drive \ +// --driver=test_driver/integration_test.dart \ +// --target=integration_test/query_profile_measure_test.dart \ +// --profile -d +import 'package:integration_test/integration_test_driver.dart'; + +Future main() => integrationDriver(); From 5027d1f6b126de2c206eee7749387bfd05923429 Mon Sep 17 00:00:00 2001 From: "Brian.oh" <49855381+dev07060@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:07:25 +0900 Subject: [PATCH 2/2] test(profiling): cold/warm/switch scenarios x 2 lanes + profile gate (LOC-68) Measurement entrypoint (own file, runs under flutter drive --profile): pure_cold / pure_warm (both lanes) / switching_cold, fresh DB via pre-init file delete (no clearAllData race), 15m timeout, per-row CSV log. Fail-closed: kDebugMode skip+assert, hits>0, filtered hits' sourceId in requested set. PR-P3.md + README. --- docs/perf/ondevice-query-profiler/PR-P3.md | 44 +++++++++++++++++++ docs/perf/ondevice-query-profiler/README.md | 4 +- .../query_profile_measure_test.dart | 22 ++-------- 3 files changed, 50 insertions(+), 20 deletions(-) create mode 100644 docs/perf/ondevice-query-profiler/PR-P3.md diff --git a/docs/perf/ondevice-query-profiler/PR-P3.md b/docs/perf/ondevice-query-profiler/PR-P3.md new file mode 100644 index 0000000..214dc27 --- /dev/null +++ b/docs/perf/ondevice-query-profiler/PR-P3.md @@ -0,0 +1,44 @@ +# PR P3 — 세그먼트 타이밍 + 3시나리오 × 2레인 + +- 브랜치: `feat/loc-68-profiler-timing` +- Linear: [LOC-68](https://linear.app/loceract/issue/LOC-68) +- 상태: 🟦 진행 (PR 열림 예정, iPhone 실기 profile 런 green) + +## 스코프 +실 ONNX 자산 example 앱에서 쿼리 단계를 **격리 계측**하는 디바이스 프로파일러 본체. +- `example/lib/profiling/query_profiler.dart` — `SourceRagService.searchMeta`(:933-995) 경로를 미러링해 embed / activate(스위치) / search / hydrate를 단계별로 `Stopwatch` 계측. 엔진 private 헬퍼 2개를 인용과 함께 복제: `_indexPath`(FNV-1a, :283-309), `_toInt64List`(FRB generalized Int64List, :261-268). `activateOnly`/`deleteOnDiskIndex`로 결정적 cold 셋업, hit의 sourceId 반환으로 필터 fail-closed. +- `example/integration_test/query_profile_measure_test.dart` — 측정 엔트리(단독 파일). pure_cold(evicted cold activate) / pure_warm(2레인) / switching_cold(A→B→A) × Unfiltered·Filtered(i8). P1의 `SegmentSamples`/`QueryProfileRun` 집계, 콘솔에 행 단위 `PROFILE_CSV`. +- `example/test_driver/integration_test.dart` — `flutter drive` 엔트리(profile 빌드 필수). +- `example/pubspec.yaml` — `flutter_rust_bridge ^2.11.1`(프로파일러가 FRB i64 리스트 생성에 필요). + +## 측정 방법 (중요: `flutter test` 아님 → `flutter drive`) +`flutter test`는 빌드 모드를 debug로 하드핀 → cargokit debug → **fallback 백엔드**(출시 `vector_faer,vector_quant_i8` 미적용)라 baseline 무효. profile 빌드는 `flutter drive`로만: + +``` +cd example && flutter drive \ + --driver=test_driver/integration_test.dart \ + --target=integration_test/query_profile_measure_test.dart \ + --profile -d 00008110-001524992E38801E +``` + +fail-closed: `kDebugMode` skip+assert(프로필 강제) / 필터레인 hit sourceId ⊆ 요청 / 세그먼트별 N>0. + +## 결과 (iPhone iOS 26.5, profile, 컬렉션당 500 docs, topK=10) +| lane | category | embed p50/p95 | activate | search p50/p95 | hydrate p50/p95 | +|---|---|---|---|---|---| +| unfiltered | pure_cold (n=1) | 25.2 | **247.3** | 2.20 | 0.42 | +| unfiltered | pure_warm (n=30) | **26.7 / 36.7** | — | 1.60 / 2.08 | 0.27 / 0.41 | +| filtered(i8) | pure_warm (n=30) | **27.6 / 37.6** | — | 0.76 / 0.92 | 0.19 / 0.30 | +| unfiltered | switching_cold (n=30) | 25.7 / 37.0 | (로그 트렁케이션 유실) | 1.47 / 1.90 | (유실) | + +- embed(ONNX)가 warm 정상상태 지배(~27ms, 타 세그먼트 15–37배). cold는 activate 지배(247ms). 필터 i8 search가 unfiltered보다 빠름(0.76 vs 1.60ms). +- 전체 결과/디바이스/방법은 [LOC-68 코멘트](https://linear.app/loceract/issue/LOC-68) 참조. + +## 받은 피드백 / 디버그 로그 (실기에서만 드러남) +- 어드버서리얼 리뷰가 분석 단계에서 2 critical 차단: ① `flutter test --profile`는 무효(빌드 debug 고정) → `flutter drive` + test_driver 추가. ② 필터레인의 `scoped_exact_scan_rows>0` 단언은 항상 실패(엔진이 그 카운터를 증가시키지 않음, 셰일드 러스트 테스트가 ==0 단언) → sourceId⊆요청 검사로 대체. +- 실기 런에서 ③ 기본 테스트 타임아웃 30s 초과(2×500 ONNX 임베드) → `Timeout(15분)`. ④ `clearAllData` 비동기 재초기화가 첫 seed와 경합 → `database is locked` → 측정을 **자체 파일 분리** + init 전 DB 삭제(clearAllData 미사용). ⑤ 콘솔이 큰 단일 print를 잘라먹음 → CSV 행 단위 print. +- DDS 간헐 실패: profile 런 성공 1·실패 2 관측, kill 후 재실행. + +## 리스크 / 롤백 / 다음 +- 동작 코드 변경 없음(example 프로파일링 + test_driver + dep). 롤백: PR revert. +- 다음: **P4(LOC-69)** = JSON/CSV export + 메타(baseline 산출). 본 PR 위에 스택. diff --git a/docs/perf/ondevice-query-profiler/README.md b/docs/perf/ondevice-query-profiler/README.md index da8703d..cef652c 100644 --- a/docs/perf/ondevice-query-profiler/README.md +++ b/docs/perf/ondevice-query-profiler/README.md @@ -23,8 +23,8 @@ vector_math 커널 슬라이스는 거의 최적임을 확인했으나 **온디 |----|------|--------|------| | 스펙+계획 | DESIGN + PLAN | [LOC-65](https://linear.app/loceract/issue/LOC-65) | 🟩 머지(#69) | | P1 | report 모델 + JSON/CSV (host-TDD) | [LOC-66](https://linear.app/loceract/issue/LOC-66) | 🟩 머지(#70, [PR-P1.md](PR-P1.md)) | -| P2 | example integration_test 배선 + A/B 픽스처 | [LOC-67](https://linear.app/loceract/issue/LOC-67) | 🟦 진행([PR-P2.md](PR-P2.md), 기기 green) | -| P3 | 세그먼트 타이밍 + 3시나리오 + metrics 스냅샷 | [LOC-68](https://linear.app/loceract/issue/LOC-68) | ⬜ TODO | +| P2 | example integration_test 배선 + A/B 픽스처 | [LOC-67](https://linear.app/loceract/issue/LOC-67) | 🟩 머지(#71, [PR-P2.md](PR-P2.md)) | +| P3 | 세그먼트 타이밍 + 3시나리오 + metrics 스냅샷 | [LOC-68](https://linear.app/loceract/issue/LOC-68) | 🟦 진행([PR-P3.md](PR-P3.md), 기기 green) | | P4 | JSON/CSV export + 로그 + 메타 (baseline 산출) | [LOC-69](https://linear.app/loceract/issue/LOC-69) | ⬜ TODO | | P5 | (조건부) Phase-2 드릴다운 — 지배 버킷별 | [LOC-70](https://linear.app/loceract/issue/LOC-70) | ⏸ 데이터 게이트 | diff --git a/example/integration_test/query_profile_measure_test.dart b/example/integration_test/query_profile_measure_test.dart index 2de57d2..204d4c9 100644 --- a/example/integration_test/query_profile_measure_test.dart +++ b/example/integration_test/query_profile_measure_test.dart @@ -9,9 +9,8 @@ import 'package:mobile_rag_engine/mobile_rag_engine.dart'; import 'package:mobile_rag_engine_example/profiling/query_fixture.dart'; import 'package:mobile_rag_engine_example/profiling/query_profiler.dart'; import 'package:mobile_rag_engine_example/profiling/query_profile_report.dart'; -import 'package:mobile_rag_engine_example/profiling/profile_export.dart'; -// On-device RAG query profiler — P3 (LOC-68) measurement + P4 (LOC-69) export. +// On-device RAG query profiler — P3 (LOC-68) measurement. // // Runs ALONE in its own process (separate from the P2 smoke) under flutter // drive in PROFILE mode, so the shipped `vector_faer,vector_quant_i8` backend @@ -66,9 +65,6 @@ void main() { 'build_mode': kReleaseMode ? 'release' : (kProfileMode ? 'profile' : 'debug'), 'features': 'vector_faer,vector_quant_i8', // shipped in profile/release - 'os': Platform.operatingSystem, - 'os_version': Platform.operatingSystemVersion, - // Device model + charging state: document manually in PR-P4.md. 'docs_per_collection': measuredDocs, 'top_k': topK, 'vector_weight': profiler.vectorWeight, @@ -197,19 +193,9 @@ void main() { } } - // P3: per-segment p50/p95 to the device log (greppable CSV block). - final report = QueryProfileReport(runs: runs); - _emitReport(report); - - // P4: export JSON + CSV to the app documents dir (the baseline artifact) - // and log the dir + filename so the operator can pull them off-device. - final tsTag = DateTime.now().millisecondsSinceEpoch.toString(); - final dir = await ProfileExport.write(report, tsTag: tsTag); - // ignore: avoid_print - print('PROFILE_EXPORT_DIR $dir'); - // ignore: avoid_print - print('PROFILE_EXPORT_FILE query_profile_$tsTag.json ' - 'query_profile_$tsTag.csv'); + // P3 deliverable: per-segment p50/p95 to the device log (greppable CSV). + // P4 adds JSON/CSV export to the app documents dir on top of this. + _emitReport(QueryProfileReport(runs: runs)); }, // Seeding 2x500 real ONNX embeddings + the measured loops far exceed the // default 30s per-test timeout; give it generous headroom.