From bf4137937d76c0de96f0ee5352dabe66a108e749 Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Mon, 27 Jul 2026 16:16:23 +0200 Subject: [PATCH 01/32] fix(desktop): app bundle structure --- desktop/src-tauri/src/lib.rs | 153 ++++++++++++++++++++++++++++------- 1 file changed, 125 insertions(+), 28 deletions(-) diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index ce155a0db..fa0be0a15 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -1480,33 +1480,51 @@ async fn download_synkronus_app_bundle_zip( Ok(buf) } -const DEV_MIRROR_BUNDLE_TOP_DIRS: [&str; 3] = ["app", "forms", "renderers"]; const SHARED_CHOICE_REF_PREFIX: &str = "forms/shared-choice-defs.schema.json#/$defs/"; -fn publish_form_bundle_rel_path(rel: &str) -> bool { +/// Normalize a path relative to the bundle root to forward-slash zip entry names. +fn zip_entry_path_normalized(rel: &Path) -> String { + rel.components() + .filter_map(|c| match c { + std::path::Component::Normal(s) => Some(s.to_string_lossy()), + _ => None, + }) + .collect::>() + .join("/") +} + +fn publish_path_has_skipped_segment(rel: &str) -> bool { + rel.split('/').any(|s| { + matches!( + s, + ".DS_Store" | "Thumbs.db" | "desktop.ini" | ".git" | "node_modules" + ) + }) +} + +/// Top-level sibling `forms/{form}/{schema,ui}.json` only (Synkronus is strict here). +fn publish_top_level_form_rel_path(rel: &str) -> bool { let parts: Vec<&str> = rel.split('/').collect(); - if parts.len() == 3 && parts[0] == "forms" { - return parts[2] == "schema.json" || parts[2] == "ui.json"; - } - if parts.len() == 4 && parts[0] == "app" && parts[1] == "forms" { - return parts[3] == "schema.json" || parts[3] == "ui.json"; - } - false + parts.len() == 3 && parts[0] == "forms" && (parts[2] == "schema.json" || parts[2] == "ui.json") } -/// Synkronus only accepts `forms/{form}/{schema,ui}.json` (and the `app/forms/…` variant). -fn publish_bundle_zip_entry_allowed(rel: &str) -> bool { - if rel.is_empty() { +/// Whether a relative zip entry may be published. +/// +/// - Full `app/` trees are allowed (including `app/forms/ext.json` and other nested artifacts). +/// - Top-level `forms/` is only for the legacy sibling layout and is schema/ui-only. +/// - When `omit_top_level_forms` is true (nested `app/forms/` present), skip top-level `forms/`. +fn publish_bundle_zip_entry_allowed(rel: &str, omit_top_level_forms: bool) -> bool { + if rel.is_empty() || publish_path_has_skipped_segment(rel) { return false; } if rel.starts_with("forms/") { - return publish_form_bundle_rel_path(rel); - } - if rel.starts_with("app/forms/") { - return publish_form_bundle_rel_path(rel); + if omit_top_level_forms { + return false; + } + return publish_top_level_form_rel_path(rel); } let top = rel.split('/').next().unwrap_or(""); - DEV_MIRROR_BUNDLE_TOP_DIRS.contains(&top) + matches!(top, "app" | "renderers") } fn forms_root_for_publish_schema(dev_local: &Path, rel: &str) -> Option { @@ -1660,6 +1678,8 @@ fn read_publish_schema_bytes( } /// Zips `bundles/dev-local/` into a temp file with Synkronus-compatible paths (`app/`, `forms/`, …). +/// +/// Prefer app-only layout when `app/forms/` exists (omit duplicate top-level `forms/`). fn zip_dev_mirror_bundle(ws: &Path) -> Result { let dev_local = ws.join("bundles/dev-local"); let index = dev_local.join("app/index.html"); @@ -1668,6 +1688,7 @@ fn zip_dev_mirror_bundle(ws: &Path) -> Result { "developer mirror missing app/index.html — use Refresh app first".to_string(), )); } + let omit_top_level_forms = dev_local.join("app/forms").is_dir(); let zip_path = std::env::temp_dir().join(format!("ode-dev-bundle-{}.zip", Uuid::new_v4())); let file = fs::File::create(&zip_path)?; let mut zip = ZipWriter::new(BufWriter::new(file)); @@ -1683,8 +1704,8 @@ fn zip_dev_mirror_bundle(ws: &Path) -> Result { let rel = path .strip_prefix(&dev_local) .map_err(|e| CustodianError::Message(e.to_string()))?; - let name = rel.to_string_lossy(); - if !publish_bundle_zip_entry_allowed(&name) { + let name = zip_entry_path_normalized(rel); + if !publish_bundle_zip_entry_allowed(&name, omit_top_level_forms) { continue; } let bytes = if name.ends_with("schema.json") { @@ -1696,7 +1717,7 @@ fn zip_dev_mirror_bundle(ws: &Path) -> Result { } else { fs::read(path)? }; - zip.start_file(name.as_ref(), options) + zip.start_file(name.as_str(), options) .map_err(|e| CustodianError::Message(e.to_string()))?; zip.write_all(&bytes) .map_err(|e| CustodianError::Message(e.to_string()))?; @@ -5772,15 +5793,33 @@ mod tests { #[test] fn zip_dev_mirror_bundle_produces_valid_layout() { + // Nested app/forms (CI style) plus a duplicate top-level forms/ copy from the mirror. let base = std::env::temp_dir().join(format!("ode_dev_zip_test_{}", std::process::id())); let _ = fs::remove_dir_all(&base); - fs::create_dir_all(base.join("bundles/dev-local/app")).unwrap(); + fs::create_dir_all(base.join("bundles/dev-local/app/forms/demo")).unwrap(); fs::create_dir_all(base.join("bundles/dev-local/forms/demo")).unwrap(); fs::write( base.join("bundles/dev-local/app/index.html"), b"", ) .unwrap(); + fs::write( + base.join("bundles/dev-local/app/forms/ext.json"), + br#"{"version":"1","renderers":{}}"#, + ) + .unwrap(); + fs::write( + base.join("bundles/dev-local/app/forms/forms-manifest.json"), + br#"[]"#, + ) + .unwrap(); + fs::write( + base.join("bundles/dev-local/app/forms/demo/schema.json"), + b"{}", + ) + .unwrap(); + fs::write(base.join("bundles/dev-local/app/forms/demo/ui.json"), b"{}").unwrap(); + // Duplicate sibling forms/ (mirror side-effect) — must be omitted from the zip. fs::write(base.join("bundles/dev-local/forms/demo/schema.json"), b"{}").unwrap(); fs::write(base.join("bundles/dev-local/forms/demo/ui.json"), b"{}").unwrap(); fs::write( @@ -5795,6 +5834,44 @@ mod tests { ) .unwrap(); + let zip_path = zip_dev_mirror_bundle(Path::new(&base)).unwrap(); + let file = fs::File::open(&zip_path).unwrap(); + let mut archive = ZipArchive::new(file).unwrap(); + let mut names: Vec = (0..archive.len()) + .map(|i| archive.by_index(i).unwrap().name().to_string()) + .collect(); + names.sort(); + assert!(names.contains(&"app/index.html".to_string())); + assert!(names.contains(&"app/forms/ext.json".to_string())); + assert!(names.contains(&"app/forms/forms-manifest.json".to_string())); + assert!(names.contains(&"app/forms/demo/schema.json".to_string())); + assert!(names.contains(&"app/forms/demo/ui.json".to_string())); + assert!(!names.iter().any(|n| n.starts_with("forms/"))); + let _ = fs::remove_file(&zip_path); + let _ = fs::remove_dir_all(&base); + } + + #[test] + fn zip_dev_mirror_bundle_sibling_forms_strict_filter() { + // Legacy sibling forms/ only (no app/forms/) — schema/ui only, strip authoring junk. + let base = std::env::temp_dir().join(format!("ode_dev_zip_sibling_{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(base.join("bundles/dev-local/app")).unwrap(); + fs::create_dir_all(base.join("bundles/dev-local/forms/demo")).unwrap(); + fs::write( + base.join("bundles/dev-local/app/index.html"), + b"", + ) + .unwrap(); + fs::write(base.join("bundles/dev-local/forms/demo/schema.json"), b"{}").unwrap(); + fs::write(base.join("bundles/dev-local/forms/demo/ui.json"), b"{}").unwrap(); + fs::write( + base.join("bundles/dev-local/forms/shared-choice-defs.schema.json"), + br#"{"$defs":{"yesno":{"type":"string"}}}"#, + ) + .unwrap(); + fs::write(base.join("bundles/dev-local/forms/ext.json"), b"{}").unwrap(); + let zip_path = zip_dev_mirror_bundle(Path::new(&base)).unwrap(); let file = fs::File::open(&zip_path).unwrap(); let mut archive = ZipArchive::new(file).unwrap(); @@ -5806,7 +5883,7 @@ mod tests { assert!(names.contains(&"forms/demo/schema.json".to_string())); assert!(names.contains(&"forms/demo/ui.json".to_string())); assert!(!names.iter().any(|n| n.contains("shared-choice-defs"))); - assert!(!names.iter().any(|n| n.contains("extensions/"))); + assert!(!names.iter().any(|n| n == "forms/ext.json")); let _ = fs::remove_file(&zip_path); let _ = fs::remove_dir_all(&base); } @@ -5853,19 +5930,39 @@ mod tests { #[test] fn publish_bundle_zip_entry_allowed_filters_authoring_artifacts() { - assert!(publish_bundle_zip_entry_allowed("app/index.html")); + // Nested app/forms: allow full tree extras. + assert!(publish_bundle_zip_entry_allowed("app/index.html", true)); assert!(publish_bundle_zip_entry_allowed( - "forms/household/schema.json" + "app/forms/household/ui.json", + true )); + assert!(publish_bundle_zip_entry_allowed("app/forms/ext.json", true)); assert!(publish_bundle_zip_entry_allowed( - "app/forms/household/ui.json" + "app/forms/forms-manifest.json", + true + )); + assert!(!publish_bundle_zip_entry_allowed( + "forms/household/schema.json", + true + )); + + // Sibling forms layout: schema/ui only at top-level forms/. + assert!(publish_bundle_zip_entry_allowed( + "forms/household/schema.json", + false + )); + assert!(!publish_bundle_zip_entry_allowed( + "forms/shared-choice-defs.schema.json", + false )); assert!(!publish_bundle_zip_entry_allowed( - "forms/shared-choice-defs.schema.json" + "forms/extensions/helpers/queryHelpers.js", + false )); + assert!(!publish_bundle_zip_entry_allowed("forms/ext.json", false)); assert!(!publish_bundle_zip_entry_allowed( - "forms/extensions/helpers/queryHelpers.js" + "app/node_modules/pkg/index.js", + false )); - assert!(!publish_bundle_zip_entry_allowed("forms/ext.json")); } } From 57e0875392f971990c7a53f21e6df9c27fb28a07 Mon Sep 17 00:00:00 2001 From: Najuna Brian Date: Fri, 31 Jul 2026 00:09:07 +0300 Subject: [PATCH 02/32] fix numeric fields exporting as null in parquet export casts to double precision instead of numeric, since lib/pq doesn't decode numeric and was returning raw bytes, which failed the float64 type assertion and got silently nulled. fixes #707 --- synkronus/pkg/dataexport/postgres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synkronus/pkg/dataexport/postgres.go b/synkronus/pkg/dataexport/postgres.go index 53532c33e..ff51d3f41 100644 --- a/synkronus/pkg/dataexport/postgres.go +++ b/synkronus/pkg/dataexport/postgres.go @@ -148,7 +148,7 @@ func (p *postgresDB) GetObservationsForFormType(ctx context.Context, formType st for _, col := range schema.Columns { switch col.SQLType { case "numeric": - selectParts = append(selectParts, fmt.Sprintf("(data ->> '%s')::numeric AS data_%s", col.Key, col.Key)) + selectParts = append(selectParts, fmt.Sprintf("(data ->> '%s')::double precision AS data_%s", col.Key, col.Key)) case "boolean": selectParts = append(selectParts, fmt.Sprintf("(data ->> '%s')::boolean AS data_%s", col.Key, col.Key)) case "adate": From d9bc95c1fe3a32fd0eab3dbd6df5d51b841fa158 Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Fri, 31 Jul 2026 14:20:54 +0200 Subject: [PATCH 03/32] feat(synkronus): Add charts and API stats route --- synkronus-portal/eslint.config.js | 2 +- synkronus-portal/package.json | 3 +- synkronus-portal/pnpm-lock.yaml | 308 ++++++++++++++++++ .../generated/.openapi-generator/FILES | 5 + .../src/api/synkronus/generated/api.ts | 173 ++++++++++ .../docs/ObservationFormTypeCount.md | 22 ++ .../docs/ObservationStatsResponse.md | 26 ++ .../generated/docs/ObservationTimeline.md | 26 ++ .../docs/ObservationTimelineBucket.md | 24 ++ .../api/synkronus/generated/docs/StatsApi.md | 62 ++++ synkronus-portal/src/components/HomePanel.tsx | 126 +++++++ .../charts/ObservationFormTypeChart.tsx | 104 ++++++ .../charts/ObservationTimelineChart.tsx | 99 ++++++ .../components/charts/OverviewChartPanel.tsx | 27 ++ synkronus-portal/src/contexts/AuthContext.tsx | 54 +-- .../src/lib/observationStatsCharts.ts | 71 ++++ synkronus-portal/src/pages/Dashboard.css | 201 ++++++++++++ synkronus-portal/src/pages/Dashboard.tsx | 102 +++++- synkronus-portal/src/services/api.ts | 14 + synkronus/cmd/synkronus/main.go | 6 + synkronus/internal/api/api.go | 5 + .../internal/api/api_integration_test.go | 1 + synkronus/internal/api/api_test.go | 1 + .../internal/handlers/appbundle_push_test.go | 6 +- .../handlers/attachment_manifest_test.go | 1 + .../attachment_sync_integration_test.go | 1 + synkronus/internal/handlers/handlers.go | 4 + synkronus/internal/handlers/mocks/stats.go | 36 ++ synkronus/internal/handlers/stats.go | 28 ++ synkronus/internal/handlers/sync_e2e_test.go | 1 + synkronus/internal/handlers/test_helpers.go | 2 + synkronus/internal/handlers/user_test.go | 1 + synkronus/openapi/synkronus.yaml | 98 ++++++ ...000_observations_created_at_live_index.sql | 7 + synkronus/pkg/stats/database.go | 91 ++++++ synkronus/pkg/stats/service.go | 45 +++ synkronus/pkg/stats/timeline.go | 90 +++++ synkronus/pkg/stats/timeline_test.go | 96 ++++++ synkronus/pkg/stats/types.go | 38 +++ 39 files changed, 1962 insertions(+), 45 deletions(-) create mode 100644 synkronus-portal/src/api/synkronus/generated/docs/ObservationFormTypeCount.md create mode 100644 synkronus-portal/src/api/synkronus/generated/docs/ObservationStatsResponse.md create mode 100644 synkronus-portal/src/api/synkronus/generated/docs/ObservationTimeline.md create mode 100644 synkronus-portal/src/api/synkronus/generated/docs/ObservationTimelineBucket.md create mode 100644 synkronus-portal/src/api/synkronus/generated/docs/StatsApi.md create mode 100644 synkronus-portal/src/components/HomePanel.tsx create mode 100644 synkronus-portal/src/components/charts/ObservationFormTypeChart.tsx create mode 100644 synkronus-portal/src/components/charts/ObservationTimelineChart.tsx create mode 100644 synkronus-portal/src/components/charts/OverviewChartPanel.tsx create mode 100644 synkronus-portal/src/lib/observationStatsCharts.ts create mode 100644 synkronus/internal/handlers/mocks/stats.go create mode 100644 synkronus/internal/handlers/stats.go create mode 100644 synkronus/pkg/migrations/sql/20260731120000_observations_created_at_live_index.sql create mode 100644 synkronus/pkg/stats/database.go create mode 100644 synkronus/pkg/stats/service.go create mode 100644 synkronus/pkg/stats/timeline.go create mode 100644 synkronus/pkg/stats/timeline_test.go create mode 100644 synkronus/pkg/stats/types.go diff --git a/synkronus-portal/eslint.config.js b/synkronus-portal/eslint.config.js index 4cb6e2ce7..b1fada31c 100644 --- a/synkronus-portal/eslint.config.js +++ b/synkronus-portal/eslint.config.js @@ -9,7 +9,7 @@ import prettierPlugin from 'eslint-plugin-prettier'; import { defineConfig, globalIgnores } from 'eslint/config'; export default defineConfig([ - globalIgnores(['dist']), + globalIgnores(['dist', 'src/api/synkronus/generated/**']), { files: ['**/*.{ts,tsx}'], extends: [ diff --git a/synkronus-portal/package.json b/synkronus-portal/package.json index e8591f8a9..289336de9 100644 --- a/synkronus-portal/package.json +++ b/synkronus-portal/package.json @@ -23,7 +23,8 @@ "qrcode": "^1.5.4", "react": "^19.2.4", "react-dom": "^19.2.4", - "react-icons": "^5.5.0" + "react-icons": "^5.5.0", + "recharts": "^3.10.1" }, "devDependencies": { "@eslint/js": "^9.39.2", diff --git a/synkronus-portal/pnpm-lock.yaml b/synkronus-portal/pnpm-lock.yaml index d6d78444d..d8e7f440b 100644 --- a/synkronus-portal/pnpm-lock.yaml +++ b/synkronus-portal/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: react-icons: specifier: ^5.5.0 version: 5.6.0(react@19.2.6) + recharts: + specifier: ^3.10.1 + version: 3.10.1(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react-is@18.3.1)(react@19.2.6)(redux@5.0.1) devDependencies: '@eslint/js': specifier: ^9.39.2 @@ -564,6 +567,17 @@ packages: '@types/react': optional: true + '@reduxjs/toolkit@2.12.0': + resolution: {integrity: sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw==} + peerDependencies: + react: ^16.9.0 || ^17.0.0 || ^18 || ^19 + react-redux: ^7.2.1 || ^8.1.3 || ^9.0.0 + peerDependenciesMeta: + react: + optional: true + react-redux: + optional: true + '@rolldown/pluginutils@1.0.0-rc.3': resolution: {integrity: sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==} @@ -708,6 +722,12 @@ packages: '@sinclair/typebox@0.27.10': resolution: {integrity: sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==} + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@standard-schema/utils@0.3.0': + resolution: {integrity: sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==} + '@tokenizer/inflate@0.4.1': resolution: {integrity: sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==} engines: {node: '>=18'} @@ -730,6 +750,33 @@ packages: '@types/babel__traverse@7.28.0': resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} + '@types/d3-array@3.2.2': + resolution: {integrity: sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==} + + '@types/d3-color@3.1.3': + resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} + + '@types/d3-ease@3.0.2': + resolution: {integrity: sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==} + + '@types/d3-interpolate@3.0.4': + resolution: {integrity: sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==} + + '@types/d3-path@3.1.1': + resolution: {integrity: sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==} + + '@types/d3-scale@4.0.9': + resolution: {integrity: sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==} + + '@types/d3-shape@3.1.8': + resolution: {integrity: sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==} + + '@types/d3-time@3.0.4': + resolution: {integrity: sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==} + + '@types/d3-timer@3.0.2': + resolution: {integrity: sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==} + '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} @@ -768,6 +815,9 @@ packages: '@types/react@19.2.14': resolution: {integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==} + '@types/use-sync-external-store@0.0.6': + resolution: {integrity: sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==} + '@types/wrap-ansi@3.0.0': resolution: {integrity: sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==} @@ -1057,6 +1107,10 @@ packages: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} + clsx@2.1.1: + resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} + engines: {node: '>=6'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -1126,6 +1180,50 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + d3-array@3.2.4: + resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} + engines: {node: '>=12'} + + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-format@3.1.2: + resolution: {integrity: sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-path@3.1.0: + resolution: {integrity: sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==} + engines: {node: '>=12'} + + d3-scale@4.0.2: + resolution: {integrity: sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==} + engines: {node: '>=12'} + + d3-shape@3.2.0: + resolution: {integrity: sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==} + engines: {node: '>=12'} + + d3-time-format@4.1.0: + resolution: {integrity: sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==} + engines: {node: '>=12'} + + d3-time@3.1.0: + resolution: {integrity: sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + data-uri-to-buffer@6.0.2: resolution: {integrity: sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==} engines: {node: '>= 14'} @@ -1163,6 +1261,9 @@ packages: resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} engines: {node: '>=0.10.0'} + decimal.js-light@2.5.1: + resolution: {integrity: sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==} + deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -1276,6 +1377,9 @@ packages: resolution: {integrity: sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==} engines: {node: '>= 0.4'} + es-toolkit@1.50.0: + resolution: {integrity: sha512-OyZKhUVvEep9ITEiwHn8GKnMRQIVqoSIX7WnRbkWgJkllCujilqP2rD0u979tkl8wqyc8ICwlc1UBVv/Sl1G6w==} + esbuild@0.27.7: resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} engines: {node: '>=18'} @@ -1397,6 +1501,9 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} + eventemitter3@5.0.4: + resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + exponential-backoff@3.1.3: resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==} @@ -1645,6 +1752,9 @@ packages: engines: {node: '>=16.x'} hasBin: true + immer@11.1.15: + resolution: {integrity: sha512-VrNANlmnWQnh5COXIIOQXM9oOJw7naGKlBT74ZOOR6lpVXc3gFEu9FJLDFcpCJ2j+NWr8TIwtWD//T6ZX6TKiQ==} + import-fresh@3.3.1: resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} engines: {node: '>=6'} @@ -1660,6 +1770,10 @@ packages: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} + internmap@2.0.3: + resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} + engines: {node: '>=12'} + invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} @@ -2280,6 +2394,18 @@ packages: '@types/react': optional: true + react-redux@9.3.0: + resolution: {integrity: sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g==} + peerDependencies: + '@types/react': ^18.2.25 || ^19 + react: ^18.0 || ^19 + redux: ^5.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + redux: + optional: true + react-refresh@0.14.2: resolution: {integrity: sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA==} engines: {node: '>=0.10.0'} @@ -2292,6 +2418,22 @@ packages: resolution: {integrity: sha512-sfWGGfavi0xr8Pg0sVsyHMAOziVYKgPLNrS7ig+ivMNb3wbCBw3KxtflsGBAwD3gYQlE/AEZsTLgToRrSCjb0Q==} engines: {node: '>=0.10.0'} + recharts@3.10.1: + resolution: {integrity: sha512-QXFrvt6IVcw7eeZCoyXTwkIJAX3Dv1nyVhMicXJ47GsGDDpcN8z6o644DibE9XjpBTThtsomLKnTV6lc+cVFUA==} + engines: {node: '>=18'} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-is: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + redux-thunk@3.1.0: + resolution: {integrity: sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==} + peerDependencies: + redux: ^5.0.0 + + redux@5.0.1: + resolution: {integrity: sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==} + reflect-metadata@0.2.2: resolution: {integrity: sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==} @@ -2313,6 +2455,9 @@ packages: require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} + reselect@5.2.0: + resolution: {integrity: sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw==} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -2525,6 +2670,9 @@ packages: throat@5.0.0: resolution: {integrity: sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==} + tiny-invariant@1.3.3: + resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} + tinyglobby@0.2.16: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} @@ -2635,10 +2783,18 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + use-sync-external-store@1.6.0: + resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + utils-merge@1.0.1: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} engines: {node: '>= 0.4.0'} + victory-vendor@37.3.6: + resolution: {integrity: sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==} + vite@7.3.3: resolution: {integrity: sha512-/4XH147Ui7OGTjg3HbdWe5arnZQSbfuRzdr9Ec7TQi5I7R+ir0Rlc9GIvD4v0XZurELqA035KVXJXpR61xhiTA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3270,6 +3426,18 @@ snapshots: optionalDependencies: '@types/react': 19.2.14 + '@reduxjs/toolkit@2.12.0(react-redux@9.3.0(@types/react@19.2.14)(react@19.2.6)(redux@5.0.1))(react@19.2.6)': + dependencies: + '@standard-schema/spec': 1.1.0 + '@standard-schema/utils': 0.3.0 + immer: 11.1.15 + redux: 5.0.1 + redux-thunk: 3.1.0(redux@5.0.1) + reselect: 5.2.0 + optionalDependencies: + react: 19.2.6 + react-redux: 9.3.0(@types/react@19.2.14)(react@19.2.6)(redux@5.0.1) + '@rolldown/pluginutils@1.0.0-rc.3': {} '@rollup/rollup-android-arm-eabi@4.60.4': @@ -3349,6 +3517,10 @@ snapshots: '@sinclair/typebox@0.27.10': {} + '@standard-schema/spec@1.1.0': {} + + '@standard-schema/utils@0.3.0': {} + '@tokenizer/inflate@0.4.1': dependencies: debug: 4.4.3 @@ -3381,6 +3553,30 @@ snapshots: dependencies: '@babel/types': 7.29.0 + '@types/d3-array@3.2.2': {} + + '@types/d3-color@3.1.3': {} + + '@types/d3-ease@3.0.2': {} + + '@types/d3-interpolate@3.0.4': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-path@3.1.1': {} + + '@types/d3-scale@4.0.9': + dependencies: + '@types/d3-time': 3.0.4 + + '@types/d3-shape@3.1.8': + dependencies: + '@types/d3-path': 3.1.1 + + '@types/d3-time@3.0.4': {} + + '@types/d3-timer@3.0.2': {} + '@types/estree@1.0.8': {} '@types/estree@1.0.9': {} @@ -3421,6 +3617,8 @@ snapshots: dependencies: csstype: 3.2.3 + '@types/use-sync-external-store@0.0.6': {} + '@types/wrap-ansi@3.0.0': {} '@types/yargs-parser@21.0.3': {} @@ -3778,6 +3976,8 @@ snapshots: clone@1.0.4: optional: true + clsx@2.1.1: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -3849,6 +4049,44 @@ snapshots: csstype@3.2.3: {} + d3-array@3.2.4: + dependencies: + internmap: 2.0.3 + + d3-color@3.1.0: {} + + d3-ease@3.0.1: {} + + d3-format@3.1.2: {} + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-path@3.1.0: {} + + d3-scale@4.0.2: + dependencies: + d3-array: 3.2.4 + d3-format: 3.1.2 + d3-interpolate: 3.0.1 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + + d3-shape@3.2.0: + dependencies: + d3-path: 3.1.0 + + d3-time-format@4.1.0: + dependencies: + d3-time: 3.1.0 + + d3-time@3.1.0: + dependencies: + d3-array: 3.2.4 + + d3-timer@3.0.1: {} + data-uri-to-buffer@6.0.2: {} data-view-buffer@1.0.2: @@ -3879,6 +4117,8 @@ snapshots: decamelize@1.2.0: {} + decimal.js-light@2.5.1: {} + deep-is@0.1.4: {} defaults@1.0.4: @@ -4061,6 +4301,8 @@ snapshots: is-date-object: 1.1.0 is-symbol: 1.1.1 + es-toolkit@1.50.0: {} + esbuild@0.27.7: optionalDependencies: '@esbuild/aix-ppc64': 0.27.7 @@ -4230,6 +4472,8 @@ snapshots: event-target-shim@5.0.1: {} + eventemitter3@5.0.4: {} + exponential-backoff@3.1.3: {} fast-deep-equal@3.1.3: {} @@ -4484,6 +4728,8 @@ snapshots: dependencies: queue: 6.0.2 + immer@11.1.15: {} + import-fresh@3.3.1: dependencies: parent-module: 1.0.1 @@ -4499,6 +4745,8 @@ snapshots: hasown: 2.0.3 side-channel: 1.1.0 + internmap@2.0.3: {} + invariant@2.2.4: dependencies: loose-envify: 1.4.0 @@ -5259,12 +5507,47 @@ snapshots: - supports-color - utf-8-validate + react-redux@9.3.0(@types/react@19.2.14)(react@19.2.6)(redux@5.0.1): + dependencies: + '@types/use-sync-external-store': 0.0.6 + react: 19.2.6 + use-sync-external-store: 1.6.0(react@19.2.6) + optionalDependencies: + '@types/react': 19.2.14 + redux: 5.0.1 + react-refresh@0.14.2: {} react-refresh@0.18.0: {} react@19.2.6: {} + recharts@3.10.1(@types/react@19.2.14)(react-dom@19.2.6(react@19.2.6))(react-is@18.3.1)(react@19.2.6)(redux@5.0.1): + dependencies: + '@reduxjs/toolkit': 2.12.0(react-redux@9.3.0(@types/react@19.2.14)(react@19.2.6)(redux@5.0.1))(react@19.2.6) + clsx: 2.1.1 + decimal.js-light: 2.5.1 + es-toolkit: 1.50.0 + eventemitter3: 5.0.4 + immer: 11.1.15 + react: 19.2.6 + react-dom: 19.2.6(react@19.2.6) + react-is: 18.3.1 + react-redux: 9.3.0(@types/react@19.2.14)(react@19.2.6)(redux@5.0.1) + reselect: 5.2.0 + tiny-invariant: 1.3.3 + use-sync-external-store: 1.6.0(react@19.2.6) + victory-vendor: 37.3.6 + transitivePeerDependencies: + - '@types/react' + - redux + + redux-thunk@3.1.0(redux@5.0.1): + dependencies: + redux: 5.0.1 + + redux@5.0.1: {} + reflect-metadata@0.2.2: {} reflect.getprototypeof@1.0.10: @@ -5293,6 +5576,8 @@ snapshots: require-main-filename@2.0.0: {} + reselect@5.2.0: {} + resolve-from@4.0.0: {} resolve@2.0.0-next.7: @@ -5583,6 +5868,8 @@ snapshots: throat@5.0.0: {} + tiny-invariant@1.3.3: {} + tinyglobby@0.2.16: dependencies: fdir: 6.5.0(picomatch@4.0.4) @@ -5697,8 +5984,29 @@ snapshots: dependencies: punycode: 2.3.1 + use-sync-external-store@1.6.0(react@19.2.6): + dependencies: + react: 19.2.6 + utils-merge@1.0.1: {} + victory-vendor@37.3.6: + dependencies: + '@types/d3-array': 3.2.2 + '@types/d3-ease': 3.0.2 + '@types/d3-interpolate': 3.0.4 + '@types/d3-scale': 4.0.9 + '@types/d3-shape': 3.1.8 + '@types/d3-time': 3.0.4 + '@types/d3-timer': 3.0.2 + d3-array: 3.2.4 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-scale: 4.0.2 + d3-shape: 3.2.0 + d3-time: 3.1.0 + d3-timer: 3.0.1 + vite@7.3.3(@types/node@24.12.4)(terser@5.47.1)(yaml@2.9.0): dependencies: esbuild: 0.27.7 diff --git a/synkronus-portal/src/api/synkronus/generated/.openapi-generator/FILES b/synkronus-portal/src/api/synkronus/generated/.openapi-generator/FILES index 03184907e..a9761a534 100644 --- a/synkronus-portal/src/api/synkronus/generated/.openapi-generator/FILES +++ b/synkronus-portal/src/api/synkronus/generated/.openapi-generator/FILES @@ -35,7 +35,11 @@ docs/GetHealth503Response.md docs/HealthApi.md docs/LoginRequest.md docs/Observation.md +docs/ObservationFormTypeCount.md docs/ObservationGeolocation.md +docs/ObservationStatsResponse.md +docs/ObservationTimeline.md +docs/ObservationTimelineBucket.md docs/ProblemDetail.md docs/ProblemDetailErrorsInner.md docs/RefreshTokenRequest.md @@ -44,6 +48,7 @@ docs/RepositoryResetResponse.md docs/ResetUserPassword200Response.md docs/ResetUserPasswordRequest.md docs/ServerInfo.md +docs/StatsApi.md docs/SwitchAppBundleVersion200Response.md docs/SyncPullRequest.md docs/SyncPullRequestSince.md diff --git a/synkronus-portal/src/api/synkronus/generated/api.ts b/synkronus-portal/src/api/synkronus/generated/api.ts index f0e406027..8dde1b5a0 100644 --- a/synkronus-portal/src/api/synkronus/generated/api.ts +++ b/synkronus-portal/src/api/synkronus/generated/api.ts @@ -299,6 +299,13 @@ export interface Observation { */ 'tags'?: Array | null; } +export interface ObservationFormTypeCount { + /** + * Form type id; empty/whitespace stored values become \"(no form type)\" + */ + 'formType': string; + 'count': number; +} /** * Optional geolocation data for the observation */ @@ -328,6 +335,55 @@ export interface ObservationGeolocation { */ 'timestamp'?: string; } +export interface ObservationStatsResponse { + /** + * Total non-deleted observations + */ + 'totalCount': number; + 'byFormType': Array; + 'timeline': ObservationTimeline; + /** + * UTC timestamp when this aggregate was computed + */ + 'computedAt': string; +} +export interface ObservationTimeline { + /** + * Bucket granularity (week when span of dated observations is >= 365 days) + */ + 'bucketUnit': ObservationTimelineBucketUnitEnum; + /** + * First bucket start date (YYYY-MM-DD), empty when there are no observations + */ + 'rangeStart': string; + /** + * Last bucket start date (YYYY-MM-DD), empty when there are no observations + */ + 'rangeEnd': string; + /** + * Dense zero-filled buckets from rangeStart through rangeEnd + */ + 'buckets': Array; +} + +export const ObservationTimelineBucketUnitEnum = { + Day: 'day', + Week: 'week', +} as const; + +export type ObservationTimelineBucketUnitEnum = typeof ObservationTimelineBucketUnitEnum[keyof typeof ObservationTimelineBucketUnitEnum]; + +export interface ObservationTimelineBucket { + /** + * Bucket start date (YYYY-MM-DD, UTC) + */ + 'bucketStart': string; + /** + * Human-readable label (e.g. \"Jan 1\") + */ + 'label': string; + 'count': number; +} export interface ProblemDetail { 'type': string; 'title': string; @@ -3208,3 +3264,120 @@ export class HealthApi extends BaseAPI { +/** + * StatsApi - axios parameter creator + */ +export const StatsApiAxiosParamCreator = function (configuration?: Configuration) { + return { + /** + * Returns non-deleted observation totals grouped by form type and a dense activity timeline bucketed by UTC calendar day (or week when the span is at least 365 days). Soft-deleted observations are excluded. Timeline dates use `created_at` interpreted in UTC. Intended for portal/desktop overview charts; not a general-purpose query API. + * @summary Observation aggregate stats for dashboard charts + * @param {string} xOdeVersion Client semantic version; the major segment must match the server. Optional leading v/V and semver pre-release/build suffixes are accepted (same rules as Synkronus). + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getObservationStats: async (xOdeVersion: string, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'xOdeVersion' is not null or undefined + assertParamExists('getObservationStats', 'xOdeVersion', xOdeVersion) + const localVarPath = `/api/stats/observations`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication bearerAuth required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + localVarHeaderParameter['Accept'] = 'application/json'; + + if (xOdeVersion != null) { + localVarHeaderParameter['x-ode-version'] = String(xOdeVersion); + } + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + } +}; + +/** + * StatsApi - functional programming interface + */ +export const StatsApiFp = function(configuration?: Configuration) { + const localVarAxiosParamCreator = StatsApiAxiosParamCreator(configuration) + return { + /** + * Returns non-deleted observation totals grouped by form type and a dense activity timeline bucketed by UTC calendar day (or week when the span is at least 365 days). Soft-deleted observations are excluded. Timeline dates use `created_at` interpreted in UTC. Intended for portal/desktop overview charts; not a general-purpose query API. + * @summary Observation aggregate stats for dashboard charts + * @param {string} xOdeVersion Client semantic version; the major segment must match the server. Optional leading v/V and semver pre-release/build suffixes are accepted (same rules as Synkronus). + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getObservationStats(xOdeVersion: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getObservationStats(xOdeVersion, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['StatsApi.getObservationStats']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, + } +}; + +/** + * StatsApi - factory interface + */ +export const StatsApiFactory = function (configuration?: Configuration, basePath?: string, axios?: AxiosInstance) { + const localVarFp = StatsApiFp(configuration) + return { + /** + * Returns non-deleted observation totals grouped by form type and a dense activity timeline bucketed by UTC calendar day (or week when the span is at least 365 days). Soft-deleted observations are excluded. Timeline dates use `created_at` interpreted in UTC. Intended for portal/desktop overview charts; not a general-purpose query API. + * @summary Observation aggregate stats for dashboard charts + * @param {StatsApiGetObservationStatsRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getObservationStats(requestParameters: StatsApiGetObservationStatsRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.getObservationStats(requestParameters.xOdeVersion, options).then((request) => request(axios, basePath)); + }, + }; +}; + +/** + * Request parameters for getObservationStats operation in StatsApi. + */ +export interface StatsApiGetObservationStatsRequest { + /** + * Client semantic version; the major segment must match the server. Optional leading v/V and semver pre-release/build suffixes are accepted (same rules as Synkronus). + */ + readonly xOdeVersion: string +} + +/** + * StatsApi - object-oriented interface + */ +export class StatsApi extends BaseAPI { + /** + * Returns non-deleted observation totals grouped by form type and a dense activity timeline bucketed by UTC calendar day (or week when the span is at least 365 days). Soft-deleted observations are excluded. Timeline dates use `created_at` interpreted in UTC. Intended for portal/desktop overview charts; not a general-purpose query API. + * @summary Observation aggregate stats for dashboard charts + * @param {StatsApiGetObservationStatsRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + public getObservationStats(requestParameters: StatsApiGetObservationStatsRequest, options?: RawAxiosRequestConfig) { + return StatsApiFp(this.configuration).getObservationStats(requestParameters.xOdeVersion, options).then((request) => request(this.axios, this.basePath)); + } +} + + + diff --git a/synkronus-portal/src/api/synkronus/generated/docs/ObservationFormTypeCount.md b/synkronus-portal/src/api/synkronus/generated/docs/ObservationFormTypeCount.md new file mode 100644 index 000000000..90eb22afa --- /dev/null +++ b/synkronus-portal/src/api/synkronus/generated/docs/ObservationFormTypeCount.md @@ -0,0 +1,22 @@ +# ObservationFormTypeCount + + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**formType** | **string** | Form type id; empty/whitespace stored values become \"(no form type)\" | [default to undefined] +**count** | **number** | | [default to undefined] + +## Example + +```typescript +import { ObservationFormTypeCount } from './api'; + +const instance: ObservationFormTypeCount = { + formType, + count, +}; +``` + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/synkronus-portal/src/api/synkronus/generated/docs/ObservationStatsResponse.md b/synkronus-portal/src/api/synkronus/generated/docs/ObservationStatsResponse.md new file mode 100644 index 000000000..1d3df849b --- /dev/null +++ b/synkronus-portal/src/api/synkronus/generated/docs/ObservationStatsResponse.md @@ -0,0 +1,26 @@ +# ObservationStatsResponse + + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**totalCount** | **number** | Total non-deleted observations | [default to undefined] +**byFormType** | [**Array<ObservationFormTypeCount>**](ObservationFormTypeCount.md) | | [default to undefined] +**timeline** | [**ObservationTimeline**](ObservationTimeline.md) | | [default to undefined] +**computedAt** | **string** | UTC timestamp when this aggregate was computed | [default to undefined] + +## Example + +```typescript +import { ObservationStatsResponse } from './api'; + +const instance: ObservationStatsResponse = { + totalCount, + byFormType, + timeline, + computedAt, +}; +``` + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/synkronus-portal/src/api/synkronus/generated/docs/ObservationTimeline.md b/synkronus-portal/src/api/synkronus/generated/docs/ObservationTimeline.md new file mode 100644 index 000000000..a6f87d58c --- /dev/null +++ b/synkronus-portal/src/api/synkronus/generated/docs/ObservationTimeline.md @@ -0,0 +1,26 @@ +# ObservationTimeline + + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**bucketUnit** | **string** | Bucket granularity (week when span of dated observations is >= 365 days) | [default to undefined] +**rangeStart** | **string** | First bucket start date (YYYY-MM-DD), empty when there are no observations | [default to undefined] +**rangeEnd** | **string** | Last bucket start date (YYYY-MM-DD), empty when there are no observations | [default to undefined] +**buckets** | [**Array<ObservationTimelineBucket>**](ObservationTimelineBucket.md) | Dense zero-filled buckets from rangeStart through rangeEnd | [default to undefined] + +## Example + +```typescript +import { ObservationTimeline } from './api'; + +const instance: ObservationTimeline = { + bucketUnit, + rangeStart, + rangeEnd, + buckets, +}; +``` + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/synkronus-portal/src/api/synkronus/generated/docs/ObservationTimelineBucket.md b/synkronus-portal/src/api/synkronus/generated/docs/ObservationTimelineBucket.md new file mode 100644 index 000000000..ad40928ed --- /dev/null +++ b/synkronus-portal/src/api/synkronus/generated/docs/ObservationTimelineBucket.md @@ -0,0 +1,24 @@ +# ObservationTimelineBucket + + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**bucketStart** | **string** | Bucket start date (YYYY-MM-DD, UTC) | [default to undefined] +**label** | **string** | Human-readable label (e.g. \"Jan 1\") | [default to undefined] +**count** | **number** | | [default to undefined] + +## Example + +```typescript +import { ObservationTimelineBucket } from './api'; + +const instance: ObservationTimelineBucket = { + bucketStart, + label, + count, +}; +``` + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/synkronus-portal/src/api/synkronus/generated/docs/StatsApi.md b/synkronus-portal/src/api/synkronus/generated/docs/StatsApi.md new file mode 100644 index 000000000..bedb39d2c --- /dev/null +++ b/synkronus-portal/src/api/synkronus/generated/docs/StatsApi.md @@ -0,0 +1,62 @@ +# StatsApi + +All URIs are relative to *http://localhost* + +|Method | HTTP request | Description| +|------------- | ------------- | -------------| +|[**getObservationStats**](#getobservationstats) | **GET** /api/stats/observations | Observation aggregate stats for dashboard charts| + +# **getObservationStats** +> ObservationStatsResponse getObservationStats() + +Returns non-deleted observation totals grouped by form type and a dense activity timeline bucketed by UTC calendar day (or week when the span is at least 365 days). Soft-deleted observations are excluded. Timeline dates use `created_at` interpreted in UTC. Intended for portal/desktop overview charts; not a general-purpose query API. + +### Example + +```typescript +import { + StatsApi, + Configuration +} from './api'; + +const configuration = new Configuration(); +const apiInstance = new StatsApi(configuration); + +let xOdeVersion: string; //Client semantic version; the major segment must match the server. Optional leading v/V and semver pre-release/build suffixes are accepted (same rules as Synkronus). (default to undefined) + +const { status, data } = await apiInstance.getObservationStats( + xOdeVersion +); +``` + +### Parameters + +|Name | Type | Description | Notes| +|------------- | ------------- | ------------- | -------------| +| **xOdeVersion** | [**string**] | Client semantic version; the major segment must match the server. Optional leading v/V and semver pre-release/build suffixes are accepted (same rules as Synkronus). | defaults to undefined| + + +### Return type + +**ObservationStatsResponse** + +### Authorization + +[bearerAuth](../README.md#bearerAuth) + +### HTTP request headers + + - **Content-Type**: Not defined + - **Accept**: application/json + + +### HTTP response details +| Status code | Description | Response headers | +|-------------|-------------|------------------| +|**200** | Observation aggregate stats | - | +|**401** | Unauthorized | - | +|**403** | Forbidden | - | +|**500** | Internal server error | - | + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + diff --git a/synkronus-portal/src/components/HomePanel.tsx b/synkronus-portal/src/components/HomePanel.tsx new file mode 100644 index 000000000..dc0f74578 --- /dev/null +++ b/synkronus-portal/src/components/HomePanel.tsx @@ -0,0 +1,126 @@ +import { useCallback, useEffect, useState } from 'react'; +import { Button } from '@ode/components/react-web'; +import { HiArrowPath } from 'react-icons/hi2'; +import type { ObservationStatsResponse } from '../api/synkronus/generated'; +import { api } from '../services/api'; +import { formatOverviewCount } from '../lib/observationStatsCharts'; +import { ObservationFormTypeChart } from './charts/ObservationFormTypeChart'; +import { ObservationTimelineChart } from './charts/ObservationTimelineChart'; + +export function HomePanel() { + const [stats, setStats] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const loadStats = useCallback(async () => { + setLoading(true); + setError(null); + try { + const result = await api.getObservationStats(); + setStats(result); + } catch (err) { + setError( + err instanceof Error ? err.message : 'Failed to load observation stats', + ); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + let cancelled = false; + + void api + .getObservationStats() + .then(result => { + if (cancelled) return; + setStats(result); + setError(null); + }) + .catch(err => { + if (cancelled) return; + setError( + err instanceof Error + ? err.message + : 'Failed to load observation stats', + ); + }) + .finally(() => { + if (!cancelled) { + setLoading(false); + } + }); + + return () => { + cancelled = true; + }; + }, []); + + const total = stats?.totalCount ?? 0; + + return ( +
+
+
+

Home

+

+ Observation activity across this Synkronus server +

+
+
+ +
+
+ + {error && ( +
+ {error} + +
+ )} + + {loading && !stats && ( +
+ Loading observation stats… +
+ )} + + {stats && !error && total === 0 && ( +
+ No observations yet. Once devices sync data, charts will appear here. +
+ )} + + {stats && ( + <> +

+ {formatOverviewCount(total)} observation + {total === 1 ? '' : 's'} + {stats.computedAt + ? ` · updated ${new Date(stats.computedAt).toLocaleString()}` + : ''} +

+
+ + +
+ + )} +
+ ); +} diff --git a/synkronus-portal/src/components/charts/ObservationFormTypeChart.tsx b/synkronus-portal/src/components/charts/ObservationFormTypeChart.tsx new file mode 100644 index 000000000..3ebeb0698 --- /dev/null +++ b/synkronus-portal/src/components/charts/ObservationFormTypeChart.tsx @@ -0,0 +1,104 @@ +import { useMemo } from 'react'; +import { Cell, Pie, PieChart, ResponsiveContainer, Tooltip } from 'recharts'; +import type { ObservationFormTypeCount } from '../../api/synkronus/generated'; +import { + buildFormTypeChartSlices, + formatOverviewCount, +} from '../../lib/observationStatsCharts'; +import { OverviewChartPanel } from './OverviewChartPanel'; + +export interface ObservationFormTypeChartProps { + rows: ObservationFormTypeCount[]; + totalObservations: number; +} + +export function ObservationFormTypeChart({ + rows, + totalObservations, +}: ObservationFormTypeChartProps) { + const slices = useMemo(() => buildFormTypeChartSlices(rows), [rows]); + + if (totalObservations === 0 || slices.length === 0) { + return ( + +

+ No form types to display. +

+
+ ); + } + + return ( + +
+
+ + + + {slices.map(slice => ( + + ))} + + { + const count = + typeof value === 'number' ? value : Number(value ?? 0); + const pct = + totalObservations > 0 + ? ((count / totalObservations) * 100).toFixed(1) + : '0'; + const formType = + item && typeof item === 'object' && 'payload' in item + ? String( + (item.payload as { formType?: string }).formType ?? + '', + ) + : ''; + return [`${formatOverviewCount(count)} (${pct}%)`, formType]; + }} + /> + + +
+ + {formatOverviewCount(totalObservations)} + + total +
+
+
    + {slices.map(slice => ( +
  • + + + {slice.formType} + + + {formatOverviewCount(slice.count)} + +
  • + ))} +
+
+
+ ); +} diff --git a/synkronus-portal/src/components/charts/ObservationTimelineChart.tsx b/synkronus-portal/src/components/charts/ObservationTimelineChart.tsx new file mode 100644 index 000000000..4b4466c35 --- /dev/null +++ b/synkronus-portal/src/components/charts/ObservationTimelineChart.tsx @@ -0,0 +1,99 @@ +import { useMemo } from 'react'; +import { + Bar, + BarChart, + CartesianGrid, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from 'recharts'; +import type { ObservationTimeline } from '../../api/synkronus/generated'; +import { formatOverviewCount } from '../../lib/observationStatsCharts'; +import { OverviewChartPanel } from './OverviewChartPanel'; + +export interface ObservationTimelineChartProps { + timeline: ObservationTimeline; + totalObservations: number; +} + +export function ObservationTimelineChart({ + timeline, + totalObservations, +}: ObservationTimelineChartProps) { + const data = useMemo( + () => + timeline.buckets.map(b => ({ + label: b.label, + count: b.count, + })), + [timeline.buckets], + ); + + const subtitle = useMemo(() => { + if (!timeline.rangeStart || !timeline.rangeEnd) { + return undefined; + } + return `${timeline.rangeStart} — ${timeline.rangeEnd} · ${timeline.bucketUnit === 'week' ? 'weekly' : 'daily'} buckets`; + }, [timeline]); + + if (totalObservations === 0 || data.length === 0) { + return ( + +

+ Sync observations to see a timeline. +

+
+ ); + } + + return ( + + + + + + + { + const count = + typeof value === 'number' ? value : Number(value ?? 0); + return [formatOverviewCount(count), 'Observations']; + }} + labelFormatter={label => `Period: ${label}`} + /> + + + + + ); +} diff --git a/synkronus-portal/src/components/charts/OverviewChartPanel.tsx b/synkronus-portal/src/components/charts/OverviewChartPanel.tsx new file mode 100644 index 000000000..2ef4a2ce0 --- /dev/null +++ b/synkronus-portal/src/components/charts/OverviewChartPanel.tsx @@ -0,0 +1,27 @@ +import type { ReactNode } from 'react'; + +export interface OverviewChartPanelProps { + title: string; + subtitle?: string; + children: ReactNode; + className?: string; +} + +export function OverviewChartPanel({ + title, + subtitle, + children, + className, +}: OverviewChartPanelProps) { + return ( +
+
+

{title}

+ {subtitle ? ( +

{subtitle}

+ ) : null} +
+
{children}
+
+ ); +} diff --git a/synkronus-portal/src/contexts/AuthContext.tsx b/synkronus-portal/src/contexts/AuthContext.tsx index 2b9233f1d..f86e81a7a 100644 --- a/synkronus-portal/src/contexts/AuthContext.tsx +++ b/synkronus-portal/src/contexts/AuthContext.tsx @@ -1,4 +1,4 @@ -import { createContext, useContext, useState, useEffect } from 'react'; +import { createContext, useContext, useState } from 'react'; import type { ReactNode } from 'react'; import { api } from '../services/api'; import type { LoginRequest, User, AuthState } from '../types/auth'; @@ -11,37 +11,37 @@ interface AuthContextType extends AuthState { const AuthContext = createContext(undefined); -export function AuthProvider({ children }: { children: ReactNode }) { - const [authState, setAuthState] = useState({ +function readStoredAuthState(): AuthState { + const token = localStorage.getItem('token'); + const refreshToken = localStorage.getItem('refreshToken'); + const userStr = localStorage.getItem('user'); + + if (token && refreshToken && userStr) { + try { + const user = JSON.parse(userStr) as User; + return { + user, + token, + refreshToken, + isAuthenticated: true, + }; + } catch { + localStorage.removeItem('token'); + localStorage.removeItem('refreshToken'); + localStorage.removeItem('user'); + } + } + + return { user: null, token: null, refreshToken: null, isAuthenticated: false, - }); + }; +} - // Load auth state from localStorage on mount - useEffect(() => { - const token = localStorage.getItem('token'); - const refreshToken = localStorage.getItem('refreshToken'); - const userStr = localStorage.getItem('user'); - - if (token && refreshToken && userStr) { - try { - const user = JSON.parse(userStr); - setAuthState({ - user, - token, - refreshToken, - isAuthenticated: true, - }); - } catch { - // Invalid stored data, clear it - localStorage.removeItem('token'); - localStorage.removeItem('refreshToken'); - localStorage.removeItem('user'); - } - } - }, []); +export function AuthProvider({ children }: { children: ReactNode }) { + const [authState, setAuthState] = useState(readStoredAuthState); const login = async (credentials: LoginRequest) => { const response = await api.login(credentials); diff --git a/synkronus-portal/src/lib/observationStatsCharts.ts b/synkronus-portal/src/lib/observationStatsCharts.ts new file mode 100644 index 000000000..4bcd9ac3b --- /dev/null +++ b/synkronus-portal/src/lib/observationStatsCharts.ts @@ -0,0 +1,71 @@ +import type { ObservationFormTypeCount } from '../api/synkronus/generated'; + +/** Series colors tuned for the portal light/dark themes. */ +export const OVERVIEW_CHART_COLORS = [ + '#6b8ae8', + '#5cb88a', + '#e0a86a', + '#c77dff', + '#56cfe1', + '#ff8f8f', + '#90a3cb', + '#ffb95f', +] as const; + +export const OVERVIEW_CHART_OTHER_COLOR = '#5a6478'; + +const MAX_PIE_SLICES = 8; + +export interface FormTypeChartSlice { + formType: string; + count: number; + color: string; +} + +export function colorForFormType(formType: string, index: number): string { + if (formType.startsWith('Other')) { + return OVERVIEW_CHART_OTHER_COLOR; + } + return OVERVIEW_CHART_COLORS[index % OVERVIEW_CHART_COLORS.length]; +} + +/** Collapse form-type rows to at most 8 pie slices (top 7 + Other). */ +export function buildFormTypeChartSlices( + rows: ObservationFormTypeCount[], +): FormTypeChartSlice[] { + if (rows.length === 0) { + return []; + } + + const sorted = [...rows].sort((a, b) => b.count - a.count); + + if (sorted.length <= MAX_PIE_SLICES) { + return sorted.map((row, i) => ({ + formType: row.formType, + count: row.count, + color: colorForFormType(row.formType, i), + })); + } + + const head = sorted.slice(0, MAX_PIE_SLICES - 1); + const tail = sorted.slice(MAX_PIE_SLICES - 1); + const otherCount = tail.reduce((sum, row) => sum + row.count, 0); + + const slices: FormTypeChartSlice[] = head.map((row, i) => ({ + formType: row.formType, + count: row.count, + color: colorForFormType(row.formType, i), + })); + + slices.push({ + formType: `Other (${tail.length} types)`, + count: otherCount, + color: OVERVIEW_CHART_OTHER_COLOR, + }); + + return slices; +} + +export function formatOverviewCount(n: number): string { + return n.toLocaleString(); +} diff --git a/synkronus-portal/src/pages/Dashboard.css b/synkronus-portal/src/pages/Dashboard.css index bb298718b..582e9812c 100644 --- a/synkronus-portal/src/pages/Dashboard.css +++ b/synkronus-portal/src/pages/Dashboard.css @@ -169,6 +169,22 @@ html::-webkit-scrollbar-thumb:hover { position: relative; } +button.logo-home-link { + background: none; + border: none; + padding: 0; + cursor: pointer; + font: inherit; + color: inherit; + text-align: left; +} + +button.logo-home-link:focus-visible { + outline: 2px solid var(--color-brand-primary-500); + outline-offset: 4px; + border-radius: 8px; +} + .logo-icon { width: var(--icon-size-2xl); height: var(--icon-size-2xl); @@ -4656,3 +4672,188 @@ html::-webkit-scrollbar-thumb:hover { [data-theme='light'] .info-card:nth-child(even) .info-icon svg { color: var(--color-brand-primary-500); } + +/* ---- Home overview charts + footer ---- */ +.dashboard-footer { + text-align: center; + padding: var(--spacing-4) var(--spacing-6) var(--spacing-6); + margin-top: auto; +} + +.dashboard-footer .version-text { + font-size: var(--font-size-xs); + color: var(--color-text-tertiary); + font-weight: var(--font-weight-regular); + opacity: var(--opacity-7); + letter-spacing: var(--letter-spacing-wide); + text-transform: uppercase; +} + +[data-theme='light'] .dashboard-footer .version-text { + color: var(--color-text-tertiary-light); +} + +.home-section { + --home-chart-tick: #90a3cb; + --home-chart-axis: #2d3449; + --home-chart-tooltip-bg: #131b2e; + --home-chart-tooltip-border: #2d3449; + --home-chart-tooltip-fg: #e8edf8; + --home-chart-donut-stroke: var(--color-background-card, #131b2e); +} + +[data-theme='light'] .home-section { + --home-chart-tick: #5a6a85; + --home-chart-axis: #c5cddb; + --home-chart-tooltip-bg: #ffffff; + --home-chart-tooltip-border: #d0d7e2; + --home-chart-tooltip-fg: #1a2333; + --home-chart-donut-stroke: #ffffff; +} + +.home-stats-loading, +.home-stats-empty { + padding: var(--spacing-8) 0; + text-align: center; +} + +.home-stats-error { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--spacing-3); + margin-bottom: var(--spacing-4); +} + +.home-stats-summary { + margin: 0 0 var(--spacing-4); + font-size: var(--font-size-sm); + color: var(--color-text-secondary); +} + +.home-overview-charts-row { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--spacing-4); + align-items: stretch; +} + +@media (max-width: 900px) { + .home-overview-charts-row { + grid-template-columns: 1fr; + } +} + +.home-overview-chart { + display: flex; + flex-direction: column; + min-height: 0; + margin: 0; + padding: var(--spacing-4); + border-radius: var(--border-radius-lg, 12px); + background: var(--color-background-card); + border: 1px solid var(--color-border-subtle, rgba(255, 255, 255, 0.08)); +} + +[data-theme='light'] .home-overview-chart { + border-color: var(--color-border-default); + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.04); +} + +.home-overview-chart-header h3 { + margin: 0; + font-size: var(--font-size-sm); + font-weight: var(--font-weight-semibold); + color: var(--color-text-primary); +} + +.home-overview-chart-subtitle { + margin: 0.35rem 0 0; + font-size: var(--font-size-sm); +} + +.home-overview-chart-body { + margin-top: var(--spacing-3); + min-height: 0; +} + +.home-overview-chart-empty { + margin: 0; + padding: var(--spacing-6) 0; + text-align: center; +} + +.home-overview-donut-layout { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: var(--spacing-3); + align-items: center; +} + +@media (max-width: 640px) { + .home-overview-donut-layout { + grid-template-columns: 1fr; + } +} + +.home-overview-donut-wrap { + position: relative; + min-height: 240px; +} + +.home-overview-donut-center { + position: absolute; + inset: 0; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + pointer-events: none; +} + +.home-overview-donut-total { + font-size: 1.35rem; + font-weight: 700; + color: var(--color-text-primary); +} + +.home-overview-legend { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 0.45rem; + max-height: 240px; + overflow-y: auto; +} + +.home-overview-legend li { + display: grid; + grid-template-columns: 0.75rem 1fr auto; + gap: 0.5rem; + align-items: center; + font-size: var(--font-size-sm); +} + +.home-overview-legend-swatch { + width: 0.75rem; + height: 0.75rem; + border-radius: 999px; + flex-shrink: 0; +} + +.home-overview-legend-label { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.home-overview-legend-count { + font-variant-numeric: tabular-nums; +} + +.muted { + color: var(--color-text-tertiary); +} diff --git a/synkronus-portal/src/pages/Dashboard.tsx b/synkronus-portal/src/pages/Dashboard.tsx index 88613d404..a6ae1b590 100644 --- a/synkronus-portal/src/pages/Dashboard.tsx +++ b/synkronus-portal/src/pages/Dashboard.tsx @@ -16,6 +16,8 @@ import { HiCube, HiOutlineUsers, HiUsers, + HiOutlineHome, + HiHome, HiCheckCircle, HiExclamationTriangle, HiArrowUpTray, @@ -36,6 +38,7 @@ import { ColorBrandPrimary500 } from '@ode/tokens'; import portalLogo from '../assets/portal.png'; import dashboardBackgroundDark from '../assets/dashboard-background.png'; import dashboardBackgroundLight from '../assets/dashboard-background-light.png'; +import { HomePanel } from '../components/HomePanel'; import './Dashboard.css'; import type { UserListItem } from '../api/synkronus/generated'; @@ -49,7 +52,7 @@ function userMatchesSearch(u: UserListItem, query: string): boolean { ); } -type TabType = 'users' | 'app-bundles' | 'data-export'; +type TabType = 'home' | 'users' | 'app-bundles' | 'data-export'; interface AppBundleVersion { version: string; @@ -84,12 +87,13 @@ interface AppBundleVersionsResponse { export function Dashboard() { const { user, logout } = useAuth(); const { resolvedTheme } = useTheme(); - const [activeTab, setActiveTab] = useState('users'); + const [activeTab, setActiveTab] = useState('home'); const [appBundles, setAppBundles] = useState([]); const [users, setUsers] = useState([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const [success, setSuccess] = useState(null); + const [serverVersion, setServerVersion] = useState(null); const [uploadProgress, setUploadProgress] = useState(0); const fileInputRef = useRef(null); /** Prevents overlapping listUsers calls without putting `loading` in loadUsers deps (which would retrigger the initial-load effect). */ @@ -362,15 +366,37 @@ export function Dashboard() { } }; - // Initial load: after login, the "users" tab is already active, but - // handleTabChange() won't run again. So explicitly fetch users when the - // logged-in user becomes an admin and the Users tab is active. + // Users load when switching to the Users tab (see handleTabChange). + // Home is the default tab, so no initial users fetch on mount. + useEffect(() => { - if (activeTab !== 'users') return; - if (user?.role !== 'admin') return; - if (users.length !== 0) return; - loadUsers(); - }, [activeTab, user?.role, users.length, loadUsers]); + const fetchVersion = async () => { + try { + const data = await api.getVersion(); + const version = data.server?.version || data.version; + if (version) { + setServerVersion(version); + return; + } + } catch { + // Continue to health fallback below. + } + + try { + const health = await api.getHealth(); + if (typeof health?.version === 'string') { + setServerVersion(health.version); + return; + } + } catch (err) { + console.debug('Failed to fetch server version:', err); + } + + setServerVersion('Unknown'); + }; + + void fetchVersion(); + }, []); const handleUploadClick = () => { if (loading) return; @@ -645,14 +671,18 @@ export function Dashboard() { }>
-
+
+
Welcome back: @@ -740,6 +770,44 @@ export function Dashboard() {
- {skipFinalize && isLastContentPage && validationErrorCount > 0 && ( - - {validationAlertMessage} - - )} + {skipFinalize && + isLastContentPage && + showValidationErrors && + validationErrorCount > 0 && ( + + {validationAlertMessage} + + )} {snackbarOpen && typeof document !== 'undefined' && From 611244f4757fbbc62fa9d312e074a6244ed89924 Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Thu, 6 Aug 2026 13:02:30 +0200 Subject: [PATCH 17/32] fix(formplayer): Improve validation error handling in SwipeLayoutRenderer --- formulus-formplayer/src/App.tsx | 9 +- .../src/DynamicEnumControl.tsx | 12 +- formulus-formplayer/src/locales/en.json | 9 ++ formulus-formplayer/src/locales/fr.json | 9 ++ formulus-formplayer/src/locales/pt.json | 9 ++ .../src/renderers/FileQuestionRenderer.tsx | 4 +- .../src/renderers/PhotoQuestionRenderer.tsx | 104 +++++++++++------- .../src/theme/material-wrappers.tsx | 9 +- .../utils/additionalErrorsForDisplay.test.ts | 31 ++++++ .../src/utils/additionalErrorsForDisplay.ts | 14 +++ 10 files changed, 157 insertions(+), 53 deletions(-) create mode 100644 formulus-formplayer/src/utils/additionalErrorsForDisplay.test.ts create mode 100644 formulus-formplayer/src/utils/additionalErrorsForDisplay.ts diff --git a/formulus-formplayer/src/App.tsx b/formulus-formplayer/src/App.tsx index 07da65418..df5ee50ba 100644 --- a/formulus-formplayer/src/App.tsx +++ b/formulus-formplayer/src/App.tsx @@ -115,6 +115,7 @@ import ShellInputControl, { } from './jsonforms/ShellInputControl'; import { applyClearOnHideToRenderers } from './jsonforms/applyClearOnHideToRenderers'; import type { KeyboardPrimaryEnterKeyHint } from './utils/keyboardEnterKeyHint'; +import { additionalErrorsForDisplay } from './utils/additionalErrorsForDisplay'; import ErrorBoundary from './components/ErrorBoundary'; import { draftService } from './services/DraftService'; @@ -460,9 +461,10 @@ function App() { // Deferred validation: new forms start hidden (no red errors on first paint), // then switch to ValidateAndShow on first forward navigation / finalize. Edits // and draft resumes start shown. Host can override via params.validationMode. + // Default Hide so pre-init flash cannot paint Show before FormInitData arrives. const [validationMode, setValidationMode] = useState< 'ValidateAndShow' | 'ValidateAndHide' | 'NoValidation' - >('ValidateAndShow'); + >('ValidateAndHide'); const [uiLocale, setUiLocale] = useState('en'); const uiLocaleRef = useRef(uiLocale); uiLocaleRef.current = uiLocale; @@ -1572,7 +1574,10 @@ function App() { onChange={handleDataChange} validationMode={validationMode} ajv={ajv} - additionalErrors={customValidatorErrors} + additionalErrors={additionalErrorsForDisplay( + validationMode, + customValidatorErrors, + )} /> {/* Success Snackbar */} diff --git a/formulus-formplayer/src/DynamicEnumControl.tsx b/formulus-formplayer/src/DynamicEnumControl.tsx index 29e6ae91d..e87f44a49 100644 --- a/formulus-formplayer/src/DynamicEnumControl.tsx +++ b/formulus-formplayer/src/DynamicEnumControl.tsx @@ -158,9 +158,11 @@ const DynamicEnumControl: React.FC = ({ [handleChange, path], ); - // Find selected option based on current data value - must be before early returns + // Find selected option based on current data value - must be before early returns. + // Coerce so number/string mismatches from saved observations still resolve. const selectedOption = useMemo(() => { - return choices.find(opt => opt.const === data) || null; + if (data == null || data === '') return null; + return choices.find(opt => String(opt.const) === String(data)) || null; }, [choices, data]); // Get display label from schema or uischema - computed before early returns @@ -352,14 +354,16 @@ const DynamicEnumControl: React.FC = ({ onChange={handleValueChange} options={choices} getOptionLabel={option => option.title || String(option.const)} - isOptionEqualToValue={(option, value) => option.const === value.const} + isOptionEqualToValue={(option, value) => + String(option.const) === String(value.const) + } disabled={!enabled} sx={{ mt: 1 }} renderInput={params => ( )} /> diff --git a/formulus-formplayer/src/locales/en.json b/formulus-formplayer/src/locales/en.json index cacb2df77..533a9b7e4 100644 --- a/formulus-formplayer/src/locales/en.json +++ b/formulus-formplayer/src/locales/en.json @@ -82,6 +82,15 @@ "media.qrCode": "QR Code", "media.approximateDate": "Approximate Date", "media.takePhoto": "Take photo", + "media.openingCamera": "Opening camera...", + "media.retakePhoto": "Retake photo", + "media.deletePhoto": "Delete photo", + "media.captureHint": "Capture a clear photo.", + "media.capturedPhotoAlt": "Captured photo", + "media.invalidFilename": "Invalid photo filename from camera.", + "media.cameraError": "Camera error occurred", + "media.unknownCameraError": "Unknown camera error", + "media.captureFailed": "Failed to capture photo. Please try again.", "media.recording": "Recording...", "cqt.errorTitle": "Custom Question Type Error", "cqt.errorBody": "The custom question type \"{{format}}\" encountered an error and could not be rendered.", diff --git a/formulus-formplayer/src/locales/fr.json b/formulus-formplayer/src/locales/fr.json index 44afd1d1f..7f553586a 100644 --- a/formulus-formplayer/src/locales/fr.json +++ b/formulus-formplayer/src/locales/fr.json @@ -82,6 +82,15 @@ "media.qrCode": "Code QR", "media.approximateDate": "Date approximative", "media.takePhoto": "Prendre une photo", + "media.openingCamera": "Ouverture de l'appareil photo...", + "media.retakePhoto": "Reprendre la photo", + "media.deletePhoto": "Supprimer la photo", + "media.captureHint": "Prenez une photo nette.", + "media.capturedPhotoAlt": "Photo capturée", + "media.invalidFilename": "Nom de fichier photo invalide provenant de l'appareil photo.", + "media.cameraError": "Une erreur de l'appareil photo s'est produite", + "media.unknownCameraError": "Erreur inconnue de l'appareil photo", + "media.captureFailed": "Échec de la capture de la photo. Veuillez réessayer.", "media.recording": "Enregistrement...", "cqt.errorTitle": "Erreur du type de question personnalisé", "cqt.errorBody": "Le type de question personnalisé « {{format}} » a rencontré une erreur et n'a pas pu être affiché.", diff --git a/formulus-formplayer/src/locales/pt.json b/formulus-formplayer/src/locales/pt.json index f81439b8e..9a34c4463 100644 --- a/formulus-formplayer/src/locales/pt.json +++ b/formulus-formplayer/src/locales/pt.json @@ -82,6 +82,15 @@ "media.qrCode": "Código QR", "media.approximateDate": "Data aproximada", "media.takePhoto": "Tirar fotografia", + "media.openingCamera": "A abrir a câmara...", + "media.retakePhoto": "Repetir fotografia", + "media.deletePhoto": "Eliminar fotografia", + "media.captureHint": "Capture uma fotografia nítida.", + "media.capturedPhotoAlt": "Fotografia capturada", + "media.invalidFilename": "Nome de ficheiro de fotografia inválido da câmara.", + "media.cameraError": "Ocorreu um erro da câmara", + "media.unknownCameraError": "Erro desconhecido da câmara", + "media.captureFailed": "Falha ao capturar a fotografia. Tente novamente.", "media.recording": "A gravar...", "cqt.errorTitle": "Erro no tipo de pergunta personalizado", "cqt.errorBody": "O tipo de pergunta personalizado \"{{format}}\" encontrou um erro e não pôde ser apresentado.", diff --git a/formulus-formplayer/src/renderers/FileQuestionRenderer.tsx b/formulus-formplayer/src/renderers/FileQuestionRenderer.tsx index 8dce9f985..56465b64b 100644 --- a/formulus-formplayer/src/renderers/FileQuestionRenderer.tsx +++ b/formulus-formplayer/src/renderers/FileQuestionRenderer.tsx @@ -25,6 +25,7 @@ import { attachmentBasenameFromObservation, } from '../utils/attachmentBasename'; import FormulusClient from '../services/FormulusInterface'; +import { formatControlErrors } from '../utils/formatControlErrors'; const parsePx = (value: string): number => parseInt(value.replace('px', ''), 10); @@ -183,8 +184,7 @@ const FileQuestionRenderer: React.FC = ({ const obs = fileObservationRecord(data); const hasData = obs !== null; const displayName = displayFilenameForFileObservation(obs); - const validationError = - errors && errors.length > 0 ? String(errors[0]) : null; + const validationError = formatControlErrors(errors); const label = (uischema as { label?: string }).label ?? schema.title; const description = schema.description; diff --git a/formulus-formplayer/src/renderers/PhotoQuestionRenderer.tsx b/formulus-formplayer/src/renderers/PhotoQuestionRenderer.tsx index c999c45f8..39658a864 100644 --- a/formulus-formplayer/src/renderers/PhotoQuestionRenderer.tsx +++ b/formulus-formplayer/src/renderers/PhotoQuestionRenderer.tsx @@ -28,6 +28,12 @@ import { attachmentBasenameFromFilename, attachmentBasenameFromObservation, } from '../utils/attachmentBasename'; +import { formatControlErrors } from '../utils/formatControlErrors'; +import { + resolveControlDescription, + resolveControlLabel, +} from '../utils/controlDisplayText'; +import { useOdeT } from '../i18n/useOdeT'; // Helper to parse pixel values from tokens const parsePx = (value: string): number => { @@ -68,36 +74,41 @@ interface PhotoQuestionProps extends ControlProps { // Additional props specific to photo questions can be added here } -const PhotoQuestionRenderer: React.FC = ({ - data, - handleChange, - path, - errors, - schema, - uischema, - enabled = true, - visible = true, -}) => { +const PhotoQuestionRenderer: React.FC = props => { + const { + data, + handleChange, + path, + errors, + schema, + enabled = true, + visible = true, + } = props; + + const t = useOdeT(); const [isLoading, setIsLoading] = useState(false); const [photoUrl, setPhotoUrl] = useState(null); const [error, setError] = useState(null); // Safe error setter to prevent corruption - const setSafeError = useCallback((errorMessage: string | null) => { - if (errorMessage === null || errorMessage === undefined) { - setError(null); - } else if (typeof errorMessage === 'string' && errorMessage.length > 0) { - setError(errorMessage); - } else { - console.warn( - 'Invalid error message detected:', - errorMessage, - 'Type:', - typeof errorMessage, - ); - setError('An unknown error occurred'); - } - }, []); + const setSafeError = useCallback( + (errorMessage: string | null) => { + if (errorMessage === null || errorMessage === undefined) { + setError(null); + } else if (typeof errorMessage === 'string' && errorMessage.length > 0) { + setError(errorMessage); + } else { + console.warn( + 'Invalid error message detected:', + errorMessage, + 'Type:', + typeof errorMessage, + ); + setError(t('cqt.unknownError', 'Unknown error')); + } + }, + [t], + ); const formulusClient = useRef(FormulusClient.getInstance()); // Extract field ID from the path for use with the camera interface @@ -152,7 +163,9 @@ const PhotoQuestionRenderer: React.FC = ({ cameraResult.data.filename, ); if (!storedBasename) { - setSafeError('Invalid photo filename from camera.'); + setSafeError( + t('media.invalidFilename', 'Invalid photo filename from camera.'), + ); return; } @@ -196,24 +209,29 @@ const PhotoQuestionRenderer: React.FC = ({ console.log('Camera operation cancelled by user'); setSafeError(null); } else if (cameraError.status === 'error') { - const errorMessage = cameraError.message || 'Camera error occurred'; + const errorMessage = + cameraError.message || + t('media.cameraError', 'Camera error occurred'); console.log('Setting camera error message:', errorMessage); setSafeError(errorMessage); } else { - setSafeError('Unknown camera error'); + setSafeError(t('media.unknownCameraError', 'Unknown camera error')); } } else { const errorMessage = err?.message || err?.toString() || - 'Failed to capture photo. Please try again.'; + t( + 'media.captureFailed', + 'Failed to capture photo. Please try again.', + ); console.log('Setting error message:', errorMessage); setSafeError(errorMessage); } } finally { setIsLoading(false); } - }, [fieldId, enabled, handleChange, path, setSafeError]); + }, [fieldId, enabled, handleChange, path, setSafeError, t]); // Handle photo deletion const handleDeletePhoto = useCallback(() => { @@ -225,17 +243,15 @@ const PhotoQuestionRenderer: React.FC = ({ console.log('Photo deleted for field:', fieldId); }, [fieldId, handleChange, path, enabled, setSafeError]); - // Get display label from schema or uischema - const label = (uischema as any)?.label || schema.title || 'Photo'; - const description = schema.description; + const label = resolveControlLabel(props) || t('media.photo', 'Photo'); + const description = resolveControlDescription(props) ?? schema.description; const isRequired = Boolean( - (uischema as any)?.options?.required ?? + (props.uischema as any)?.options?.required ?? (schema as any)?.options?.required ?? false, ); - const validationError = - errors && errors.length > 0 ? String(errors[0]) : null; + const validationError = formatControlErrors(errors); const displayBasename = attachmentBasenameFromObservation( currentPhotoData as Record | null, @@ -252,7 +268,9 @@ const PhotoQuestionRenderer: React.FC = ({ required={isRequired} error={error || validationError} helperText={ - displayBasename ? `File: ${displayBasename}` : 'Capture a clear photo.' + displayBasename + ? undefined + : t('media.captureHint', 'Capture a clear photo.') } metadata={ process.env.NODE_ENV === 'development' ? ( @@ -301,7 +319,7 @@ const PhotoQuestionRenderer: React.FC = ({ component="img" height="200" image={photoUrl} - alt="Captured photo" + alt={t('media.capturedPhotoAlt', 'Captured photo')} sx={{ objectFit: 'cover' }} /> @@ -323,7 +341,7 @@ const PhotoQuestionRenderer: React.FC = ({ disabled={!enabled || isLoading} color="primary" size="small" - aria-label="Retake photo"> + aria-label={t('media.retakePhoto', 'Retake photo')}> = ({ disabled={!enabled} color="error" size="small" - aria-label="Delete photo"> + aria-label={t('media.deletePhoto', 'Delete photo')}> @@ -366,14 +384,16 @@ const PhotoQuestionRenderer: React.FC = ({ color: 'action.disabled', }, }} - aria-label="Take photo"> + aria-label={t('media.takePhoto', 'Take photo')}> - {isLoading ? 'Opening camera...' : 'Tap to capture photo'} + {isLoading + ? t('media.openingCamera', 'Opening camera...') + : t('media.photoTap', 'Tap to capture photo')} )} diff --git a/formulus-formplayer/src/theme/material-wrappers.tsx b/formulus-formplayer/src/theme/material-wrappers.tsx index fc8f256aa..e72419039 100644 --- a/formulus-formplayer/src/theme/material-wrappers.tsx +++ b/formulus-formplayer/src/theme/material-wrappers.tsx @@ -221,6 +221,8 @@ const SelectOneOfEnumControl = (props: ControlProps & OwnPropsOfEnum) => { typeof (uischema as any)?.options?.placeholder === 'string' ? (uischema as any).options.placeholder : '—'; + // Coerce so option values (always strings) match saved data (number/boolean). + const selectValue = data == null || data === '' ? '' : String(data); return ( { disabled={!enabled}> setIncludePending(e.target.checked)} + /> + Include pending observations + + + + +
+

+ Hint: Prefix basename references in observation data with this + folder to resolve files in the live workspace: +

+ + {workspaceAttachmentsPath || '—'} + +
+ +
+ +
+ + +
+
+ +
+ + {lastResult?.exportDir ? ( + + ) : null} +
+ + {error ? ( +
+ {error} +
+ ) : null} +
+ +
+

Load in analysis tools

+

+ {lastResult?.parquetFiles && + Object.keys(lastResult.parquetFiles).length > 0 + ? 'Paths match the last export. Scripts are also written under snippets/ in that folder.' + : 'Replace [INSERT PATH TO PARQUET FILE] after exporting, or copy again once paths are known.'} +

+
+ {EXPORT_SNIPPET_LANGS.map(lang => ( + + ))} +
+ +
+
+
+          {snippetCode}
+        
+
+ + ); +} diff --git a/desktop/src/store/useCustodianStore.ts b/desktop/src/store/useCustodianStore.ts index 7a31fcd58..9671f8647 100644 --- a/desktop/src/store/useCustodianStore.ts +++ b/desktop/src/store/useCustodianStore.ts @@ -217,6 +217,11 @@ interface CustodianState { done: number; total: number; } | null; + exportActivity: { + statusText: string; + done: number; + total: number; + } | null; /** Persisted Rust sync job awaiting resume (transient stall, auth, or cold start). */ syncPausedJob: SyncJobRowOut | null; /** Bumped after each successful `refresh_custom_app_dev_mirror` (Workbench embeds). */ @@ -230,6 +235,8 @@ interface CustodianState { setDevError: (message: string | null) => void; setBundleActivity: (activity: CustodianState['bundleActivity']) => void; clearBundleActivity: () => void; + setExportActivity: (activity: CustodianState['exportActivity']) => void; + clearExportActivity: () => void; dismissObservationIndexPrompt: () => void; createPendingObservationIndexes: () => Promise; refreshSettings: () => Promise; @@ -365,6 +372,7 @@ export const useCustodianStore = create((set, get) => ({ syncMessage: null, syncActivity: null, bundleActivity: null, + exportActivity: null, syncPausedJob: null, devMirrorGeneration: 0, devBusy: false, @@ -377,6 +385,8 @@ export const useCustodianStore = create((set, get) => ({ setBundleActivity: activity => set({ bundleActivity: activity }), clearBundleActivity: () => set({ bundleActivity: null }), + setExportActivity: activity => set({ exportActivity: activity }), + clearExportActivity: () => set({ exportActivity: null }), ensureActiveProfileAuth: async () => { if (selectAuthSessionForActiveProfile(get())?.token) { @@ -952,6 +962,10 @@ export function selectBundleActivity(state: CustodianState) { return state.bundleActivity; } +export function selectExportActivity(state: CustodianState) { + return state.exportActivity; +} + export function selectPausedSyncJob(state: CustodianState) { return state.syncPausedJob; } diff --git a/desktop/src/store/useExportPageStore.ts b/desktop/src/store/useExportPageStore.ts new file mode 100644 index 000000000..49da97448 --- /dev/null +++ b/desktop/src/store/useExportPageStore.ts @@ -0,0 +1,85 @@ +import { create } from 'zustand'; +import type { ExportParquetResult, ServerProfile } from '../types/domain'; +import type { ExportSnippetLang } from '../lib/exportLoadSnippets'; + +interface ExportPageState { + /** Profile whose export prefs are currently loaded into this store. */ + hydratedProfileId: string | null; + /** Parent folder chosen for export (`YYYYMMDD` leaf is created underneath). */ + destinationParent: string | null; + includePending: boolean; + includeAttachments: boolean; + snippetLang: ExportSnippetLang; + lastResult: ExportParquetResult | null; + /** ISO timestamp of last successful export (persisted on the profile). */ + lastExportAt: string | null; + error: string | null; + busy: boolean; + hydrateFromProfile: (profile: ServerProfile) => void; + clearHydration: () => void; + setDestinationParent: (path: string | null) => void; + setIncludePending: (v: boolean) => void; + setIncludeAttachments: (v: boolean) => void; + setSnippetLang: (lang: ExportSnippetLang) => void; + setLastResult: (result: ExportParquetResult | null) => void; + setLastExportAt: (iso: string | null) => void; + setError: (error: string | null) => void; + setBusy: (busy: boolean) => void; + recordSuccessfulExport: (result: ExportParquetResult) => string; +} + +function trimPath(path: string | null | undefined): string | null { + const t = path?.trim(); + return t ? t : null; +} + +export const useExportPageStore = create(set => ({ + hydratedProfileId: null, + destinationParent: null, + includePending: false, + includeAttachments: false, + snippetLang: 'r', + lastResult: null, + lastExportAt: null, + error: null, + busy: false, + + hydrateFromProfile: profile => + set({ + hydratedProfileId: profile.id, + destinationParent: trimPath(profile.exportDestinationParent), + lastExportAt: trimPath(profile.lastExportAt), + lastResult: profile.lastExport ?? null, + error: null, + busy: false, + }), + clearHydration: () => + set({ + hydratedProfileId: null, + destinationParent: null, + lastResult: null, + lastExportAt: null, + error: null, + busy: false, + }), + setDestinationParent: path => + set({ + destinationParent: trimPath(path), + }), + setIncludePending: includePending => set({ includePending }), + setIncludeAttachments: includeAttachments => set({ includeAttachments }), + setSnippetLang: snippetLang => set({ snippetLang }), + setLastResult: lastResult => set({ lastResult }), + setLastExportAt: lastExportAt => set({ lastExportAt }), + setError: error => set({ error }), + setBusy: busy => set({ busy }), + recordSuccessfulExport: result => { + const lastExportAt = new Date().toISOString(); + set({ + lastResult: result, + lastExportAt, + error: null, + }); + return lastExportAt; + }, +})); diff --git a/desktop/src/store/useToastStore.ts b/desktop/src/store/useToastStore.ts index 3cdfb952d..804ffe75a 100644 --- a/desktop/src/store/useToastStore.ts +++ b/desktop/src/store/useToastStore.ts @@ -6,35 +6,52 @@ export interface ToastItem { id: string; message: string; variant: ToastVariant; + /** Optional secondary lines (e.g. per–form-type counts); shown in a scrollable list. */ + detailLines?: string[]; } const MAX_TOASTS = 3; let nextId = 0; -function autoDismissMs(variant: ToastVariant): number { +function autoDismissMs(variant: ToastVariant, hasDetails: boolean): number { + if (hasDetails) { + return 14000; + } if (variant === 'error' || variant === 'warn') { return 8000; } return 5000; } +export type PushToastInput = { + message: string; + variant?: ToastVariant; + detailLines?: string[]; +}; + interface ToastState { toasts: ToastItem[]; - pushToast: (input: { message: string; variant?: ToastVariant }) => void; + pushToast: (input: PushToastInput) => void; dismissToast: (id: string) => void; } export const useToastStore = create((set, get) => ({ toasts: [], - pushToast: ({ message, variant = 'info' }) => { + pushToast: ({ message, variant = 'info', detailLines }) => { const id = `toast-${++nextId}`; - const item: ToastItem = { id, message, variant }; + const lines = detailLines?.filter(l => l.trim().length > 0); + const item: ToastItem = { + id, + message, + variant, + ...(lines && lines.length > 0 ? { detailLines: lines } : {}), + }; set(state => ({ toasts: [item, ...state.toasts].slice(0, MAX_TOASTS), })); - const ms = autoDismissMs(variant); + const ms = autoDismissMs(variant, Boolean(item.detailLines?.length)); window.setTimeout(() => { if (get().toasts.some(t => t.id === id)) { get().dismissToast(id); diff --git a/desktop/src/types/domain.ts b/desktop/src/types/domain.ts index 4f29a5eaa..488dbef3a 100644 --- a/desktop/src/types/domain.ts +++ b/desktop/src/types/domain.ts @@ -230,6 +230,12 @@ export interface ServerProfile { customAppDeveloperMode?: boolean | null; /** Absolute path to a folder containing `index.html` (e.g. custom app `dist/`). */ customAppLocalFolder?: string | null; + /** Parent folder last chosen on the Export page (survives restart). */ + exportDestinationParent?: string | null; + /** ISO timestamp of the last successful Parquet export for this profile. */ + lastExportAt?: string | null; + /** Summary of the last successful export (folder, counts, parquet paths). */ + lastExport?: ExportParquetResult | null; } /** Result of mirroring a local custom app folder into the profile workspace. */ @@ -435,3 +441,27 @@ export interface WorkspaceDomainItem { path: string; kind: 'directory' | 'file'; } + +/** Result of local Parquet export (`export_observations_parquet`). */ +export interface ExportParquetResult { + exportDir: string; + formTypeCounts: Record; + /** Absolute `.parquet` paths keyed by form type. */ + parquetFiles: Record; + totalRows: number; + attachmentsCopied: number; + attachmentsMissing: number; + includePending: boolean; + includeAttachments: boolean; + workspaceAttachmentsPath: string; + exportAttachmentsPath?: string | null; + manifestPath: string; +} + +export interface ExportParquetRequest { + parentDir: string; + includePending?: boolean; + includeAttachments?: boolean; + overwrite?: boolean; + profileLabel?: string | null; +} From 148763d6e40de681260f884d74a996f952ae7f59 Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Sun, 16 Aug 2026 09:39:19 +0200 Subject: [PATCH 25/32] fix(formulus): resume interrupted observation pulls instead of restarting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The observation pull cursor was persisted only after every page had been applied and the attachment manifest had been processed. Any interruption — lost connection, crash, cancel, or an attachment failure — therefore discarded the whole pull, and the next attempt started again from version zero. On a large repository over a field connection that can mean sync never completes, and because push runs after pull, local records are never uploaded either. Persist the cursor after each page instead. This cannot skip records: change_cutoff is the same watermark the loop already uses to request the next page, so a resumed run asks the server exactly what this run would have asked next, and applyServerChanges upserts. Also move the cursor write ahead of processAttachmentManifest, since attachments carry their own cursor and should not be able to force a full observation re-pull; refuse to loop when the server reports more pages without a cutoff that advances past `since`, which previously re-requested the same page indefinitely; and log page counts rather than whole pages of observation records, which were being written to logcat. Co-authored-by: Cursor --- .../synkronus/__tests__/pullCursor.test.ts | 104 ++++++++++++++++++ formulus/src/api/synkronus/index.ts | 82 +++++++++----- formulus/src/api/synkronus/pullCursor.ts | 75 +++++++++++++ 3 files changed, 233 insertions(+), 28 deletions(-) create mode 100644 formulus/src/api/synkronus/__tests__/pullCursor.test.ts create mode 100644 formulus/src/api/synkronus/pullCursor.ts diff --git a/formulus/src/api/synkronus/__tests__/pullCursor.test.ts b/formulus/src/api/synkronus/__tests__/pullCursor.test.ts new file mode 100644 index 000000000..75749ffac --- /dev/null +++ b/formulus/src/api/synkronus/__tests__/pullCursor.test.ts @@ -0,0 +1,104 @@ +import { pullPageOutcome } from '../pullCursor'; + +describe('pullPageOutcome', () => { + it('finishes on the last page using current_version', () => { + expect( + pullPageOutcome( + { current_version: 420, change_cutoff: 410, has_more: false }, + 300, + ), + ).toEqual({ kind: 'done', version: 420 }); + }); + + it('treats a missing has_more as the last page', () => { + expect( + pullPageOutcome({ current_version: 7, change_cutoff: 7 }, 0), + ).toEqual({ kind: 'done', version: 7 }); + }); + + it('continues from change_cutoff while more pages follow', () => { + expect( + pullPageOutcome( + { current_version: 900, change_cutoff: 500, has_more: true }, + 300, + ), + ).toEqual({ kind: 'continue', nextSince: 500 }); + }); + + it('accepts a first page starting from version 0', () => { + expect( + pullPageOutcome( + { current_version: 900, change_cutoff: 100, has_more: true }, + 0, + ), + ).toEqual({ kind: 'continue', nextSince: 100 }); + }); + + it('refuses to loop when change_cutoff does not advance past since', () => { + const outcome = pullPageOutcome( + { current_version: 900, change_cutoff: 300, has_more: true }, + 300, + ); + expect(outcome.kind).toBe('unusable'); + expect(outcome).toMatchObject({ + reason: expect.stringContaining('does not advance past since'), + }); + }); + + it('refuses to loop when change_cutoff moves backwards', () => { + expect( + pullPageOutcome( + { current_version: 900, change_cutoff: 120, has_more: true }, + 300, + ).kind, + ).toBe('unusable'); + }); + + it.each([ + ['missing', undefined], + ['null', null], + ['negative', -1], + ['not finite', Number.NaN], + ])('rejects a %s change_cutoff when more pages follow', (_label, cutoff) => { + expect( + pullPageOutcome( + { + current_version: 900, + change_cutoff: cutoff as number | null | undefined, + has_more: true, + }, + 10, + ).kind, + ).toBe('unusable'); + }); + + it.each([ + ['missing', undefined], + ['null', null], + ['negative', -5], + ['not finite', Number.NaN], + ])( + 'rejects a %s current_version on the final page', + (_label, currentVersion) => { + expect( + pullPageOutcome( + { + current_version: currentVersion as number | null | undefined, + change_cutoff: 10, + has_more: false, + }, + 10, + ).kind, + ).toBe('unusable'); + }, + ); + + it('allows a repository that is still empty to finish at version 0', () => { + expect( + pullPageOutcome( + { current_version: 0, change_cutoff: 0, has_more: false }, + 0, + ), + ).toEqual({ kind: 'done', version: 0 }); + }); +}); diff --git a/formulus/src/api/synkronus/index.ts b/formulus/src/api/synkronus/index.ts index 618cc5733..a0b44cdbb 100644 --- a/formulus/src/api/synkronus/index.ts +++ b/formulus/src/api/synkronus/index.ts @@ -31,6 +31,7 @@ import { } from '../../errors/RepositoryResetRequiredError'; import type { AxiosError, AxiosResponse } from 'axios'; import { effectiveRepositoryGenerationForRequest } from './repositoryGenerationRequest'; +import { pullPageOutcome } from './pullCursor'; import { formatCountProgress, type SynkronusSyncOptions, @@ -1097,6 +1098,8 @@ class SynkronusApi { let currentSince = since; let totalServerRecordsThisPull = 0; let pullPage = 0; + let hasMorePages = true; + let finalVersion = since; reportSyncProgress(report, { phase: 'pull_observations', @@ -1148,7 +1151,11 @@ class SynkronusApi { note: 'repository_generation = server epoch (resets); current_version = observation stream cursor — a 4 and a 1 here are not a mismatch.', }); - console.debug('Pull response: ', res.data); + console.debug( + `Pull response: page ${pullPage}, ${ + res.data.records?.length ?? 0 + } record(s), has_more=${String(res.data.has_more)}`, + ); this.ensureRepoGenResponseMatchesSent( 'syncPull', @@ -1186,35 +1193,40 @@ class SynkronusApi { : i18n.t('sync.progress.downloading'), }); - console.debug('Pulled observations: ', domainObservations); - - // Update since version for next iteration using change_cutoff - if (res.data.has_more && res.data.change_cutoff) { - currentSince = res.data.change_cutoff; - console.debug(`Continuing pagination from version ${currentSince}`); + // 3. Advance the cursor and persist it before fetching the next page, so + // an interrupted pull resumes here instead of restarting from zero. + // See pullCursor.ts for why this cannot skip records. + const pageOutcome = pullPageOutcome(res.data, currentSince); + if (pageOutcome.kind === 'unusable') { + throw new Error( + `Sync pull stopped after page ${pullPage}: ${pageOutcome.reason}`, + ); } - } while (res.data.has_more); - if (includeAttachments) { - await this.processAttachmentManifest(options); - } + hasMorePages = pageOutcome.kind === 'continue'; + const cursor = + pageOutcome.kind === 'continue' + ? pageOutcome.nextSince + : pageOutcome.version; + if (pageOutcome.kind === 'continue') { + currentSince = cursor; + } else { + finalVersion = cursor; + } - reportSyncProgress(report, { - phase: 'pull_observations', - current: 1, - total: 1, - details: - totalServerRecordsThisPull > 0 - ? i18n.t('sync.progress.recordsSummary', { - count: totalServerRecordsThisPull, - }) - : i18n.t('sync.progress.upToDate'), - }); + await AsyncStorage.setItem('@last_seen_version', String(cursor)); + console.debug( + `Pull cursor persisted at version ${cursor}${ + hasMorePages ? ' (more pages follow)' : '' + }`, + ); + } while (hasMorePages); logRepositoryGenerationSync('syncPull all pages done', { totalServerRecordsReceived: totalServerRecordsThisPull, finalBodyCurrentVersion: res.data.current_version, finalBodyRepositoryGeneration: res.data.repository_generation, + persistedObservationCursor: finalVersion, }); if (totalServerRecordsThisPull === 0) { @@ -1231,12 +1243,26 @@ class SynkronusApi { } } - // Only when all observations are pulled and ingested by WatermelonDB, update the last seen version - await AsyncStorage.setItem( - '@last_seen_version', - res.data.current_version.toString(), - ); - return res.data.current_version; + // The observation cursor is already persisted (per page, above). Attachments + // carry their own cursor, so a failure here no longer forces a full + // re-pull of every observation on the next attempt. + if (includeAttachments) { + await this.processAttachmentManifest(options); + } + + reportSyncProgress(report, { + phase: 'pull_observations', + current: 1, + total: 1, + details: + totalServerRecordsThisPull > 0 + ? i18n.t('sync.progress.recordsSummary', { + count: totalServerRecordsThisPull, + }) + : i18n.t('sync.progress.upToDate'), + }); + + return finalVersion; } /** diff --git a/formulus/src/api/synkronus/pullCursor.ts b/formulus/src/api/synkronus/pullCursor.ts new file mode 100644 index 000000000..810512868 --- /dev/null +++ b/formulus/src/api/synkronus/pullCursor.ts @@ -0,0 +1,75 @@ +/** + * Decides how the observation pull loop proceeds after a page has been applied + * to the local database. + * + * The pull cursor (`@last_seen_version`) is persisted **per page** rather than + * once at the end of the whole pull. That is what makes an interrupted pull + * resumable: without it, losing the connection on the last page of a large + * repository throws away every page before it and the next attempt starts from + * zero again, which on a field connection can mean sync never completes at all. + * + * Persisting `change_cutoff` cannot skip records. It is the same watermark the + * loop already uses to request the next page, so a resumed run asks the server + * exactly what this run would have asked for next, and `applyServerChanges` + * upserts — re-applying a page that was already applied changes nothing. + */ + +export type PullPageOutcome = + /** More pages follow. Request the next one from `nextSince` and persist it. */ + | { kind: 'continue'; nextSince: number } + /** Final page. Persist `version` as the observation stream cursor. */ + | { kind: 'done'; version: number } + /** + * The response cannot be used to make progress. Continuing would re-request + * the same page forever, so the caller must fail loudly instead of looping. + */ + | { kind: 'unusable'; reason: string }; + +export interface PullPageCursorFields { + current_version?: number | null; + change_cutoff?: number | null; + has_more?: boolean; +} + +function asVersion(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + return null; + } + return value; +} + +export function pullPageOutcome( + page: PullPageCursorFields, + since: number, +): PullPageOutcome { + if (!page.has_more) { + const version = asVersion(page.current_version); + if (version == null) { + return { + kind: 'unusable', + reason: `final page reported current_version=${JSON.stringify( + page.current_version, + )}, which is not a usable version number`, + }; + } + return { kind: 'done', version }; + } + + const cutoff = asVersion(page.change_cutoff); + if (cutoff == null) { + return { + kind: 'unusable', + reason: `server reported more pages but change_cutoff=${JSON.stringify( + page.change_cutoff, + )}, which is not a usable version number`, + }; + } + if (cutoff <= since) { + return { + kind: 'unusable', + reason: `server reported more pages but change_cutoff (${cutoff}) does not advance past since (${since})`, + }; + } + + return { kind: 'continue', nextSince: cutoff }; +} From c63ab696595d2a2b0d2dc0690f051cc927e0235d Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Sun, 16 Aug 2026 10:46:52 +0200 Subject: [PATCH 26/32] fix: making index rebuilding idempotent and more robust sync --- desktop/docs/UI_FEEDBACK.md | 2 +- desktop/docs/screens/import.md | 3 + desktop/src-tauri/src/observation_query.rs | 17 + desktop/src/lib/importSummary.test.ts | 87 ++++++ desktop/src/lib/importSummary.ts | 59 ++++ desktop/src/pages/ImportPage.tsx | 60 +++- .../__tests__/observationIndexGuards.test.ts | 179 +++++++++++ formulus/src/database/database.ts | 12 + .../database/repositories/WatermelonDBRepo.ts | 22 +- .../__tests__/WatermelonDBRepo.test.ts | 48 +++ formulus/src/database/schema.ts | 7 +- formulus/src/locales/en.json | 1 + formulus/src/locales/fr.json | 1 + formulus/src/locales/pt.json | 1 + formulus/src/services/AppConfigService.ts | 26 +- .../src/services/ObservationIndexService.ts | 294 +++++++++++++++--- .../src/services/RepositoryRecoveryService.ts | 5 + formulus/src/services/ServerSwitchService.ts | 5 + formulus/src/services/SyncService.ts | 31 +- .../__tests__/ServerSwitchService.test.ts | 10 + .../__tests__/SyncService.autoLogin.test.ts | 8 + formulus/src/sync/syncProgress.ts | 6 +- .../undeclared_numeric_fallback_guard.json | 20 ++ packages/observation-query/package.json | 9 +- packages/observation-query/pnpm-lock.yaml | 101 +++--- packages/observation-query/src/compiler.ts | 40 ++- .../src/compilerDifferential.test.ts | 214 +++++++++++++ 27 files changed, 1166 insertions(+), 102 deletions(-) create mode 100644 formulus/src/database/__tests__/observationIndexGuards.test.ts create mode 100644 packages/observation-query/fixtures/undeclared_numeric_fallback_guard.json create mode 100644 packages/observation-query/src/compilerDifferential.test.ts diff --git a/desktop/docs/UI_FEEDBACK.md b/desktop/docs/UI_FEEDBACK.md index d18330a53..56269352b 100644 --- a/desktop/docs/UI_FEEDBACK.md +++ b/desktop/docs/UI_FEEDBACK.md @@ -32,7 +32,7 @@ Single source of truth for how the shell surfaces status, errors, and confirmati ## Native confirm (Tauri) -**Use for:** destructive actions, import-with-issues, closing unsaved observation tabs. +**Use for:** destructive actions, import-with-issues, skip already-synced import rows, closing unsaved observation tabs. - `confirmDestructiveAction()` for destructive flows (clear, profile-scoped wording). - `confirm()` from `@tauri-apps/plugin-dialog` for save-anyway / import-anyway / tab discard (Save / Don't save / Cancel via separate flows). diff --git a/desktop/docs/screens/import.md b/desktop/docs/screens/import.md index 5fe1ff55e..762569aee 100644 --- a/desktop/docs/screens/import.md +++ b/desktop/docs/screens/import.md @@ -14,6 +14,7 @@ Bring external JSON observation files into the active profile’s local reposito - Pre-flight summary (counts, form types, attachment hints) - Per-file parse/normalization issues - Import action and clear/reset +- Optional skip of Formulus-export rows that already appear synced (`syncedAt` ≥ `updatedAt`) — confirm dialog before write ## What to exclude @@ -24,11 +25,13 @@ Bring external JSON observation files into the active profile’s local reposito ## Key actions - Stage files, review summary, import, clear +- When import JSON carries Formulus `syncedAt` metadata, confirm whether to skip already-synced observations and write only unsynced ones ## Data dependencies - Store: `import` flow via `tauriClient.importObservations`, refresh `loadObservations` / `loadHealth` after import - Active profile determines target SQLite repository +- Sync appearance: Formulus hail-mary export includes `syncedAt`; Desktop treats a row as already synced when `syncedAt` is meaningful and `updatedAt <= syncedAt` (same rule as Formulus pending detection) ## Observation indexes diff --git a/desktop/src-tauri/src/observation_query.rs b/desktop/src-tauri/src/observation_query.rs index c2b15aadd..a4fc2a7fa 100644 --- a/desktop/src-tauri/src/observation_query.rs +++ b/desktop/src-tauri/src/observation_query.rs @@ -291,6 +291,23 @@ fn compile_json_extract( } }; let p = push_param(params, value); + + // A numeric comparison has to match the indexed path, which compares against + // `value_num` and therefore only ever considers values that are JSON numbers. + // SQLite orders by storage class rather than coercing, so `'abc' > 5` is true + // and every text value would otherwise satisfy a numeric filter here. This + // fallback is what `query_observations` switches to when the index is + // unusable, so the two paths have to return the same rows. + // + // json_type rather than typeof(json_extract(...)): JSON `true` extracts as + // the integer 1, so typeof would admit booleans that the index stores as text. + if value.is_number() { + let type_expr = format!("json_type(o.payload, '{json_path}')"); + return Ok(format!( + "({type_expr} IN ('integer','real') AND {expr} {sql_op} {p})" + )); + } + Ok(format!("{expr} {sql_op} {p}")) } diff --git a/desktop/src/lib/importSummary.test.ts b/desktop/src/lib/importSummary.test.ts index 4beab50da..a71342f16 100644 --- a/desktop/src/lib/importSummary.test.ts +++ b/desktop/src/lib/importSummary.test.ts @@ -1,9 +1,12 @@ import { describe, expect, it } from 'vitest'; import { extractObservationsFromJson, + isImportObservationApparentlySynced, + partitionImportObservationsBySyncAppearance, summarizeImportFiles, type ParsedObservationFile, } from './importSummary'; +import type { ApiObservation } from '../types/domain'; describe('extractObservationsFromJson', () => { it('parses Synkronus-style snake_case object', () => { @@ -106,3 +109,87 @@ describe('summarizeImportFiles', () => { expect(s.attachmentHintCount).toBeGreaterThanOrEqual(2); }); }); + +describe('isImportObservationApparentlySynced', () => { + const base: ApiObservation = { + observationId: 'o1', + data: {}, + updatedAt: '2026-08-15T12:00:00.000Z', + }; + + it('is false when syncedAt is missing or null', () => { + expect(isImportObservationApparentlySynced(base)).toBe(false); + expect( + isImportObservationApparentlySynced({ + ...base, + extras: { syncedAt: null }, + }), + ).toBe(false); + }); + + it('is true when updatedAt is at or before syncedAt', () => { + expect( + isImportObservationApparentlySynced({ + ...base, + extras: { syncedAt: '2026-08-15T12:00:00.000Z' }, + }), + ).toBe(true); + expect( + isImportObservationApparentlySynced({ + ...base, + updatedAt: '2026-08-15T11:00:00.000Z', + extras: { syncedAt: '2026-08-15T12:00:00.000Z' }, + }), + ).toBe(true); + }); + + it('is false when updated after syncedAt (pending local edit)', () => { + expect( + isImportObservationApparentlySynced({ + ...base, + updatedAt: '2026-08-15T13:00:00.000Z', + extras: { syncedAt: '2026-08-15T12:00:00.000Z' }, + }), + ).toBe(false); + }); + + it('ignores placeholder syncedAt before 1980', () => { + expect( + isImportObservationApparentlySynced({ + ...base, + extras: { syncedAt: '1970-01-01T00:00:00.000Z' }, + }), + ).toBe(false); + }); +}); + +describe('partitionImportObservationsBySyncAppearance', () => { + it('splits synced vs unsynced rows', () => { + const rows: ApiObservation[] = [ + { + observationId: 'synced', + data: {}, + updatedAt: '2026-01-01T00:00:00Z', + extras: { syncedAt: '2026-01-02T00:00:00Z' }, + }, + { + observationId: 'pending', + data: {}, + updatedAt: '2026-01-03T00:00:00Z', + extras: { syncedAt: '2026-01-02T00:00:00Z' }, + }, + { + observationId: 'never', + data: {}, + updatedAt: '2026-01-01T00:00:00Z', + }, + ]; + const part = partitionImportObservationsBySyncAppearance(rows); + expect(part.total).toBe(3); + expect(part.apparentlySynced.map(o => o.observationId)).toEqual(['synced']); + expect(part.unsynced.map(o => o.observationId)).toEqual([ + 'pending', + 'never', + ]); + }); +}); diff --git a/desktop/src/lib/importSummary.ts b/desktop/src/lib/importSummary.ts index d75c7102b..d9d7088cc 100644 --- a/desktop/src/lib/importSummary.ts +++ b/desktop/src/lib/importSummary.ts @@ -368,3 +368,62 @@ export function flattenObservations( } return out; } + +/** + * Ignore null / placeholder syncedAt values (same floor as Formulus + * `MIN_VALID_SYNCED_AT_MS` in observationSyncStatus.ts). + */ +export const MIN_VALID_IMPORT_SYNCED_AT_MS = new Date('1980-01-01').getTime(); + +function parseImportTimestampMs(raw: string | null | undefined): number | null { + if (raw == null || !raw.trim()) { + return null; + } + const ms = Date.parse(raw); + return Number.isFinite(ms) ? ms : null; +} + +/** + * True when Formulus-style export metadata says the observation was fully + * synced (`syncedAt` set and `updatedAt <= syncedAt`). Used to offer skipping + * already-synced rows during hail-mary device exports. + */ +export function isImportObservationApparentlySynced( + observation: ApiObservation, +): boolean { + const syncedMs = parseImportTimestampMs(observation.extras?.syncedAt ?? null); + if (syncedMs == null || syncedMs <= MIN_VALID_IMPORT_SYNCED_AT_MS) { + return false; + } + const updatedMs = parseImportTimestampMs(observation.updatedAt ?? null); + if (updatedMs == null) { + return true; + } + return updatedMs <= syncedMs; +} + +export interface ImportSyncAppearancePartition { + total: number; + apparentlySynced: ApiObservation[]; + unsynced: ApiObservation[]; +} + +/** Split flattened import rows by Formulus sync appearance (`syncedAt`). */ +export function partitionImportObservationsBySyncAppearance( + observations: readonly ApiObservation[], +): ImportSyncAppearancePartition { + const apparentlySynced: ApiObservation[] = []; + const unsynced: ApiObservation[] = []; + for (const obs of observations) { + if (isImportObservationApparentlySynced(obs)) { + apparentlySynced.push(obs); + } else { + unsynced.push(obs); + } + } + return { + total: observations.length, + apparentlySynced, + unsynced, + }; +} diff --git a/desktop/src/pages/ImportPage.tsx b/desktop/src/pages/ImportPage.tsx index 12e3963df..4a77326d0 100644 --- a/desktop/src/pages/ImportPage.tsx +++ b/desktop/src/pages/ImportPage.tsx @@ -5,16 +5,18 @@ import { tauriClient } from '../lib/tauriClient'; import { groupIssuesBySeverityAndCategory, normalizeBasename, + referencedNamesForObservation, runImportValidation, type ImportIssue, type ImportIssueCategory, type ImportValidationReport, } from '../lib/importValidation'; -import type { BundleFormSpec } from '../types/domain'; +import type { ApiObservation, BundleFormSpec } from '../types/domain'; import { flattenObservations, mapPool, parseObservationJsonPathsViaRust, + partitionImportObservationsBySyncAppearance, summarizeImportFiles, } from '../lib/importSummary'; import { @@ -55,6 +57,22 @@ function formatBytes(n: number) { return `${(n / (1024 * 1024)).toFixed(1)} MB`; } +/** Attachment basenames referenced by the observations actually being written. */ +function referencedAttachmentNamesForObservations( + observations: readonly ApiObservation[], + formSpecsByType: Map, +): string[] { + const refs = new Set(); + for (const obs of observations) { + const ft = obs.formType?.trim(); + const schema = ft ? formSpecsByType.get(ft)?.formSchema : undefined; + for (const name of referencedNamesForObservation(schema, obs.data)) { + refs.add(name); + } + } + return [...refs]; +} + function normalizeDialogPaths( selected: string | string[] | null | undefined, ): string[] { @@ -425,7 +443,37 @@ export function ImportPage() { } } - const observations = flattenObservations(report.parsedFiles); + const allObservations = flattenObservations(report.parsedFiles); + const syncPartition = + partitionImportObservationsBySyncAppearance(allObservations); + let observations = allObservations; + let skippedSyncedCount = 0; + + if (syncPartition.apparentlySynced.length > 0) { + const skipSynced = await confirm( + `${syncPartition.total} observations were found. ${syncPartition.apparentlySynced.length} already appear to be synced — skip those and import only the ${syncPartition.unsynced.length} new observations?`, + { + title: 'Skip already-synced observations?', + kind: 'info', + okLabel: 'Skip synced', + cancelLabel: 'Import all', + }, + ); + if (skipSynced) { + observations = syncPartition.unsynced; + skippedSyncedCount = syncPartition.apparentlySynced.length; + } + } + + if (observations.length === 0) { + setMessage( + skippedSyncedCount > 0 + ? `Nothing to import — all ${skippedSyncedCount} observations already appear to be synced.` + : 'Nothing to import — no observations found in staged JSON.', + ); + return; + } + const writeTotal = observations.length; let imported = 0; let conflicts = 0; @@ -464,7 +512,7 @@ export function ImportPage() { } const refNorm = new Set( - report.referencedAttachmentNames + referencedAttachmentNamesForObservations(observations, formSpecsByType) .map(n => normalizeBasename(n)) .filter(Boolean), ); @@ -545,7 +593,11 @@ export function ImportPage() { await loadObservations(); await loadHealth(); - const baseMsg = `Imported ${result.imported} observations (${result.conflicts} conflicts).`; + const skipMsg = + skippedSyncedCount > 0 + ? ` Skipped ${skippedSyncedCount} already-synced observation(s).` + : ''; + const baseMsg = `Imported ${result.imported} observations (${result.conflicts} conflicts).${skipMsg}`; const indexMsg = result.indexRebuildScheduled ? ' Rebuilding observation indexes in the background (see activity banner).' : ''; diff --git a/formulus/src/database/__tests__/observationIndexGuards.test.ts b/formulus/src/database/__tests__/observationIndexGuards.test.ts new file mode 100644 index 000000000..8f3bef530 --- /dev/null +++ b/formulus/src/database/__tests__/observationIndexGuards.test.ts @@ -0,0 +1,179 @@ +/** + * Guards that keep an unusable index from silently answering queries. + * + * The failure this protects against is not an error but an empty result set: + * a declared key routes a query into `observation_index`, and if the index was + * never built, was emptied by a database wipe, or cannot represent the value, + * every predicate simply matches nothing. + */ +const configIndexes: Array> = []; + +jest.mock('../../database/database', () => ({ + database: { + write: jest.fn(async (fn: () => Promise) => fn()), + adapter: { unsafeExecute: jest.fn(async () => undefined) }, + get: jest.fn(() => ({ + query: jest.fn(() => ({ unsafeFetchRaw: jest.fn(async () => []) })), + })), + }, +})); + +jest.mock('../../webview/FormulusMessageHandlers', () => ({ + appEvents: { + addListener: jest.fn(), + removeListener: jest.fn(), + emit: jest.fn(), + }, +})); + +jest.mock('../../services/AppConfigService', () => ({ + __esModule: true, + default: { + getInstance: jest.fn(() => ({ + loadConfig: jest.fn(async () => undefined), + getConfig: jest.fn(() => ({ observationIndexes: configIndexes })), + })), + }, +})); + +import ObservationIndexService, { + computeDefsSignature, +} from '../../services/ObservationIndexService'; + +const EMPTY_SIGNATURE = computeDefsSignature([]); + +/** Rows handed to successive `unsafeFetchRaw` calls, in order. */ +const rawResults: unknown[][] = []; + +const mockDb = { + write: jest.fn(async (fn: () => Promise) => fn()), + adapter: { unsafeExecute: jest.fn(async () => undefined) }, + get: jest.fn(() => ({ + query: jest.fn(() => ({ + unsafeFetchRaw: jest.fn(async () => rawResults.shift() ?? []), + })), + })), +}; + +describe('ObservationIndexService guards', () => { + let service: ObservationIndexService; + let warn: jest.SpyInstance; + + beforeAll(async () => { + service = ObservationIndexService.getInstance(mockDb as never); + // The constructor kicks off a bootstrap rebuild; let it settle so it does + // not consume the rows queued by individual tests. + await service.ensureInitialRebuild(); + }); + + beforeEach(() => { + rawResults.length = 0; + configIndexes.length = 0; + service.reset(); + warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + warn.mockRestore(); + }); + + describe('getIndexDefs', () => { + it('keeps definitions with a top-level key and path', () => { + configIndexes.push({ key: 'hh_id', path: '$.hh_id' }); + expect(service.getIndexDefs()).toEqual([ + { key: 'hh_id', path: '$.hh_id' }, + ]); + }); + + it('drops definitions missing a key or a path', () => { + configIndexes.push( + { key: '', path: '$.skip' }, + { key: 'hh_id', path: '' }, + ); + expect(service.getIndexDefs()).toEqual([]); + }); + + it('drops a nested path, which the extractor cannot read', () => { + // Extraction reads data['a.b'] rather than data.a.b, so a nested + // definition writes no rows while still routing queries to the index. + configIndexes.push({ key: 'a.b', path: '$.a.b' }); + expect(service.getIndexDefs()).toEqual([]); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining('declares nested path'), + ); + }); + + it('warns about a nested path only once', () => { + configIndexes.push({ key: 'a.b', path: '$.a.b' }); + service.getIndexDefs(); + service.getIndexDefs(); + expect(warn).toHaveBeenCalledTimes(1); + }); + }); + + describe('isIndexUsable', () => { + function queueUsableIndex(): void { + rawResults.push( + [{ active_generation: 1 }], + [{ present: 1 }], + [ + { + active_generation: 1, + last_rebuild_at: '2026-01-01', + defs_signature: EMPTY_SIGNATURE, + }, + ], + ); + } + + it('reports true when rows exist and the stored signature matches', async () => { + queueUsableIndex(); + await expect(service.isIndexUsable()).resolves.toBe(true); + }); + + it('reports false when the active generation is empty', async () => { + rawResults.push([{ active_generation: 1 }], [{ present: 0 }]); + await expect(service.isIndexUsable()).resolves.toBe(false); + }); + + it('reports false when rows exist but were built from different definitions', async () => { + rawResults.push( + [{ active_generation: 1 }], + [{ present: 1 }], + [ + { + active_generation: 1, + last_rebuild_at: '2026-01-01', + defs_signature: 'stale-bundle', + }, + ], + ); + await expect(service.isIndexUsable()).resolves.toBe(false); + }); + + it('re-checks after a negative answer so recovery is picked up', async () => { + rawResults.push([{ active_generation: 1 }], [{ present: 0 }]); + await expect(service.isIndexUsable()).resolves.toBe(false); + + queueUsableIndex(); + await expect(service.isIndexUsable()).resolves.toBe(true); + }); + + it('caches a positive answer instead of querying on every call', async () => { + queueUsableIndex(); + await expect(service.isIndexUsable()).resolves.toBe(true); + + // No rows queued: a second query would read the empty queue and report + // false, so answering true proves the result was cached. + await expect(service.isIndexUsable()).resolves.toBe(true); + }); + + it('forgets the cached answer after reset, which follows a database wipe', async () => { + queueUsableIndex(); + await expect(service.isIndexUsable()).resolves.toBe(true); + + service.reset(); + await expect(service.isIndexUsable()).resolves.toBe(false); + }); + }); +}); diff --git a/formulus/src/database/database.ts b/formulus/src/database/database.ts index 974b5b6f8..cc4472eb2 100644 --- a/formulus/src/database/database.ts +++ b/formulus/src/database/database.ts @@ -87,6 +87,18 @@ const migrations = schemaMigrations({ `), ], }, + { + toVersion: 7, + steps: [ + // Records which index definitions the current rows were built from, so + // an interrupted rebuild is detected on the next launch instead of + // looking complete forever. Left NULL for existing installs, which + // forces exactly one rebuild after upgrading. + unsafeExecuteSql(` + ALTER TABLE observation_index_meta ADD COLUMN defs_signature TEXT; + `), + ], + }, ], }); diff --git a/formulus/src/database/repositories/WatermelonDBRepo.ts b/formulus/src/database/repositories/WatermelonDBRepo.ts index fd6492c7c..32b87e3ea 100644 --- a/formulus/src/database/repositories/WatermelonDBRepo.ts +++ b/formulus/src/database/repositories/WatermelonDBRepo.ts @@ -257,7 +257,17 @@ export class WatermelonDBRepo implements LocalRepoInterface { const indexService = ObservationIndexService.getInstance(this.database); await indexService.ensureInitialRebuild(); - const indexKeys = indexKeysFromConfig(indexService.getIndexDefs()); + let indexKeys = indexKeysFromConfig(indexService.getIndexDefs()); + if (indexKeys.size > 0 && !(await indexService.isIndexUsable())) { + // ensureInitialRebuild() reports success even when it gave up, so this + // is the last point at which an unusable index can be caught. Querying + // an empty index returns no rows, and one built from a previous + // bundle's definitions returns wrong ones — neither raises an error. + console.warn( + '[queryObservations] observation_index is empty or stale for the active generation; using json_extract so results stay correct', + ); + indexKeys = new Set(); + } const compiled = compileObservationQuery({ dialect: 'formulus', jsonColumn: 'data', @@ -522,6 +532,12 @@ export class WatermelonDBRepo implements LocalRepoInterface { return 0; } + // Only the changes whose data actually landed in the database may be + // indexed. A locally dirty record keeps its local data and is merely tagged, + // so indexing the server payload for it would leave the index describing + // values the stored record does not have — invisible until a full rebuild. + const applied: Observation[] = []; + const count = await this.database.write(async () => { const existingRecords = await this.observationsCollection .query( @@ -549,6 +565,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { record.tags = serializeTagsColumn(merged); }); } + applied.push(change); return existing.prepareUpdate(record => { record.formType = change.formType || record.formType; record.formVersion = change.formVersion || record.formVersion; @@ -576,6 +593,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { console.debug( `Preparing create for new observation: ${change.observationId}`, ); + applied.push(change); return this.observationsCollection.prepareCreate(record => { record._raw.id = change.observationId; record.observationId = change.observationId; @@ -603,7 +621,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { }); const indexService = ObservationIndexService.getInstance(this.database); - const indexRows = changes.map(change => ({ + const indexRows = applied.map(change => ({ observationId: change.observationId, formType: change.formType, dataJson: diff --git a/formulus/src/database/repositories/__tests__/WatermelonDBRepo.test.ts b/formulus/src/database/repositories/__tests__/WatermelonDBRepo.test.ts index 08d161c95..2c5a917a8 100644 --- a/formulus/src/database/repositories/__tests__/WatermelonDBRepo.test.ts +++ b/formulus/src/database/repositories/__tests__/WatermelonDBRepo.test.ts @@ -20,6 +20,7 @@ jest.mock('../../../services/ObservationIndexService', () => { const incrementalReindexMany = jest.fn(async () => undefined); const getIndexDefs = jest.fn(() => []); const ensureInitialRebuild = jest.fn(async () => undefined); + const isIndexUsable = jest.fn(async () => true); const rebuildAllIndexes = jest.fn(async () => ({ generation: 1, lastRebuildAt: null, @@ -32,6 +33,7 @@ jest.mock('../../../services/ObservationIndexService', () => { incrementalReindexMany, getIndexDefs, ensureInitialRebuild, + isIndexUsable, rebuildAllIndexes, })), }, @@ -48,6 +50,7 @@ import { Observation } from '../LocalRepoInterface'; import { ObservationMapper } from '../../../mappers/ObservationMapper'; import { LAST_WRITE_WON_TAG } from '../../../sync/syncConstants'; import { Q } from '@nozbe/watermelondb'; +import ObservationIndexService from '../../../services/ObservationIndexService'; // Create a test database with in-memory LokiJS adapter function createTestDatabase() { @@ -720,4 +723,49 @@ describe('WatermelonDBRepo', () => { const appliedAgain = await repo.applyServerChanges([serverObservation]); expect(appliedAgain).toBe(0); }); + + /** + * A locally dirty row keeps its local data and is only tagged, so feeding the + * server payload to the index would leave the index describing values the + * stored record does not have. Nothing surfaces that divergence until a full + * rebuild, and in the meantime an indexed query matches on the wrong values. + */ + test('applyServerChanges only indexes the changes it actually stored', async () => { + const localId = await repo.saveObservation({ + formType: 'form_lww', + data: { source: 'local' }, + }); + + const conflicting: Observation = { + observationId: localId, + formType: 'form_lww', + formVersion: '1.0', + createdAt: new Date('2025-01-02T10:00:00.000Z'), + updatedAt: new Date('2025-01-02T12:00:00.000Z'), + syncedAt: null, + deleted: false, + data: { source: 'server' }, + geolocation: null, + }; + const fresh: Observation = { + ...conflicting, + observationId: 'obs_server_only_3003', + }; + + const indexService = ObservationIndexService.getInstance( + database, + ) as unknown as { + incrementalReindexMany: jest.Mock; + }; + indexService.incrementalReindexMany.mockClear(); + + await repo.applyServerChanges([conflicting, fresh]); + + const indexed = indexService.incrementalReindexMany.mock.calls[0][0] as + | Array<{ observationId: string }> + | undefined; + expect(indexed?.map(row => row.observationId)).toEqual([ + 'obs_server_only_3003', + ]); + }); }); diff --git a/formulus/src/database/schema.ts b/formulus/src/database/schema.ts index 540cb6750..31c0f6af0 100644 --- a/formulus/src/database/schema.ts +++ b/formulus/src/database/schema.ts @@ -2,7 +2,7 @@ import { appSchema, tableSchema } from '@nozbe/watermelondb'; // Define the database schema export const schemas = appSchema({ - version: 6, + version: 7, tables: [ tableSchema({ name: 'observations', @@ -27,6 +27,11 @@ export const schemas = appSchema({ { name: 'active_generation', type: 'number' }, { name: 'building_generation', type: 'number', isOptional: true }, { name: 'last_rebuild_at', type: 'string', isOptional: true }, + // Identifies the index definitions the current rows were built from. + // `last_rebuild_at` only records that some rebuild happened; without + // this, a rebuild interrupted by a crash is indistinguishable from a + // completed one and never reruns. + { name: 'defs_signature', type: 'string', isOptional: true }, ], }), tableSchema({ diff --git a/formulus/src/locales/en.json b/formulus/src/locales/en.json index c66fb7489..06ef01bc8 100644 --- a/formulus/src/locales/en.json +++ b/formulus/src/locales/en.json @@ -135,6 +135,7 @@ "sync.progress.phase.push_attachments": "Uploading attachments", "sync.progress.phase.push_observations": "Uploading changes", "sync.progress.phase.app_bundle": "Updating forms", + "sync.progress.phase.index_rebuild": "Preparing data for search", "sync.progress.phase.default": "Syncing", "sync.progress.syncingAndUpdatingForms": "Syncing & updating forms", "sync.progress.syncingAndUpdating": "Syncing & updating", diff --git a/formulus/src/locales/fr.json b/formulus/src/locales/fr.json index c3f96127f..941a9ec51 100644 --- a/formulus/src/locales/fr.json +++ b/formulus/src/locales/fr.json @@ -135,6 +135,7 @@ "sync.progress.phase.push_attachments": "Envoi des pièces jointes", "sync.progress.phase.push_observations": "Envoi des modifications", "sync.progress.phase.app_bundle": "Mise à jour des formulaires", + "sync.progress.phase.index_rebuild": "Préparation des données pour la recherche", "sync.progress.phase.default": "Synchronisation", "sync.progress.syncingAndUpdatingForms": "Synchronisation et mise à jour des formulaires", "sync.progress.syncingAndUpdating": "Synchronisation et mise à jour", diff --git a/formulus/src/locales/pt.json b/formulus/src/locales/pt.json index 4dc6a991f..1aae6947a 100644 --- a/formulus/src/locales/pt.json +++ b/formulus/src/locales/pt.json @@ -135,6 +135,7 @@ "sync.progress.phase.push_attachments": "A enviar anexos", "sync.progress.phase.push_observations": "A enviar alterações", "sync.progress.phase.app_bundle": "A atualizar formulários", + "sync.progress.phase.index_rebuild": "A preparar dados para pesquisa", "sync.progress.phase.default": "A sincronizar", "sync.progress.syncingAndUpdatingForms": "A sincronizar e atualizar formulários", "sync.progress.syncingAndUpdating": "A sincronizar e atualizar", diff --git a/formulus/src/services/AppConfigService.ts b/formulus/src/services/AppConfigService.ts index 315abd6bc..cbe3f0a91 100644 --- a/formulus/src/services/AppConfigService.ts +++ b/formulus/src/services/AppConfigService.ts @@ -94,21 +94,26 @@ class AppConfigService { const raw = await RNFS.readFile(APP_CONFIG_PATH, 'utf8'); const parsed: AppConfig = JSON.parse(raw); - // Basic validation + // A missing or malformed theme only disables theming. It used to discard + // the whole config, which also silently dropped observationIndexes and + // left every declared query unindexed for no visible reason. if (!parsed.theme?.light || !parsed.theme?.dark) { console.warn( - '[AppConfigService] app.config.json is missing theme.light or theme.dark — ignoring.', - ); - this.config = null; - } else { - this.config = parsed; - console.log( - `[AppConfigService] Loaded config for "${parsed.name}" v${parsed.version}`, + '[AppConfigService] app.config.json is missing theme.light or theme.dark — falling back to ODE colors.', ); } + + this.config = parsed; + console.log( + `[AppConfigService] Loaded config for "${parsed.name}" v${parsed.version}`, + ); } catch (err) { + // Deliberately not latching `loaded` here. A read that throws mid-bundle + // extraction is transient, and latching would pin the config to null — + // and with it the index definitions — for the rest of the process. console.warn('[AppConfigService] Failed to load app.config.json:', err); this.config = null; + return; } this.loaded = true; @@ -134,10 +139,7 @@ class AppConfigService { * Returns the custom app colors if available, otherwise ODE defaults. */ getThemeColors(mode: 'light' | 'dark'): ThemeColors { - if (this.config) { - return this.config.theme[mode]; - } - return getOdeFallbackColors(mode); + return this.config?.theme?.[mode] ?? getOdeFallbackColors(mode); } /** diff --git a/formulus/src/services/ObservationIndexService.ts b/formulus/src/services/ObservationIndexService.ts index a9bbb793b..6df6eab2b 100644 --- a/formulus/src/services/ObservationIndexService.ts +++ b/formulus/src/services/ObservationIndexService.ts @@ -15,7 +15,6 @@ import { Database, Q } from '@nozbe/watermelondb'; import { database } from '../database/database'; import { ObservationModel } from '../database/models/ObservationModel'; import AppConfigService from './AppConfigService'; -import { appEvents } from '../webview/FormulusMessageHandlers'; import type { ObservationIndexDef } from '../types/AppConfig'; type SqlArg = string | number | boolean | null; @@ -33,11 +32,23 @@ function jsonPathToKey(path: string): string { return path.startsWith('$.') ? path.slice(2) : path; } +/** + * Map a JSON scalar onto the index columns, or null when it should not be + * indexed at all. + * + * Numbers go to `value_num` and other scalars to `value_text`; the query + * compiler encodes the same split, so changing it here silently changes query + * results. Arrays and objects are deliberately excluded: stringifying them + * produced rows like `"[object Object],[object Object]"` that no query could + * ever match, since a scalar filter compares against the whole blob and an + * `any` filter goes through `json_each` instead. The row was pure write cost + * with a misleading appearance of coverage. + */ function scalarToColumns( val: unknown, valueType?: string, -): { valueText: string | null; valueNum: number | null } { - if (val == null) return { valueText: null, valueNum: null }; +): { valueText: string | null; valueNum: number | null } | null { + if (val == null) return null; if (valueType === 'number' || typeof val === 'number') { const n = Number(val); if (!Number.isNaN(n)) return { valueText: null, valueNum: n }; @@ -45,7 +56,67 @@ function scalarToColumns( if (typeof val === 'string') return { valueText: val, valueNum: null }; if (typeof val === 'boolean') return { valueText: String(val), valueNum: null }; - return { valueText: String(val), valueNum: null }; + return null; +} + +/** + * Only top-level keys can be indexed. + * + * Extraction reads `data[key]`, so a nested path yields undefined and writes no + * rows — while the declared key still routes queries to the index, where they + * match nothing. Rejecting the definition instead keeps those queries on the + * json_extract path, which handles nesting correctly. + */ +function isSupportedIndexPath(path: string): boolean { + const key = jsonPathToKey(path); + return key.length > 0 && !key.includes('.') && !key.includes('['); +} + +/** + * A stable fingerprint of the index definitions a rebuild was run against. + * + * Stored alongside the rows so `last_rebuild_at` stops being the only evidence + * that the index is current. A timestamp records that *a* rebuild happened; it + * cannot distinguish an index built from today's definitions from one built + * from the previous bundle's, nor a completed rebuild from one the app died + * halfway through. Comparing signatures makes both cases self-correcting: the + * next launch sees a mismatch and rebuilds. + * + * Order-independent, so merely reordering entries in app.config.json does not + * trigger a rebuild of every observation on every device. + */ +export function computeDefsSignature(defs: ObservationIndexDef[]): string { + const normalized = defs + .map(def => ({ + key: def.key, + path: def.path, + formTypes: [...(def.formTypes ?? [])].sort(), + valueType: def.valueType ?? null, + expressionIndex: def.enableExpressionIndex !== false, + })) + .sort((a, b) => a.key.localeCompare(b.key)); + return JSON.stringify(normalized); +} + +/** + * Yield to the event loop so React Native can paint. + * + * A rebuild walks every observation and parses its JSON once per definition. + * Left as one uninterrupted loop that work blocks the JS thread, which freezes + * the UI — including whatever spinner is meant to show the rebuild running. + */ +function yieldToUi(): Promise { + return new Promise(resolve => setTimeout(resolve, 0)); +} + +/** Observations read per batch between yields. */ +const REBUILD_BATCH_SIZE = 200; + +export interface IndexRebuildProgress { + /** Observations processed so far. */ + current: number; + /** Total observations to process. */ + total: number; } function extractScalar(dataJson: string, path: string): unknown { @@ -62,22 +133,17 @@ export class ObservationIndexService { private readonly db: Database; private initialRebuildPromise: Promise | null = null; private initialRebuildFinished = false; + private indexUsable = false; + private readonly warnedOnce = new Set(); private constructor(db: Database) { this.db = db; - appEvents.addListener('bundleUpdated', () => { - void (async () => { - try { - await AppConfigService.getInstance().loadConfig(/* force */ true); - await this.rebuildAllIndexes(); - } catch (err) { - console.warn( - '[ObservationIndexService] rebuild after bundle failed:', - err, - ); - } - })(); - }); + // The rebuild that follows a bundle update is driven by SyncService via + // `rebuildForBundleUpdate`, which awaits it and reports progress. It used + // to be a fire-and-forget listener here: the sync reported "complete" while + // the rebuild was still running, and any failure went to a console warning + // that nothing acted on. `bundleUpdated` still fires for the other + // listeners; it just no longer owns the rebuild. void this.ensureInitialRebuild(); } @@ -90,7 +156,83 @@ export class ObservationIndexService { getIndexDefs(): ObservationIndexDef[] { const cfg = AppConfigService.getInstance().getConfig(); - return (cfg?.observationIndexes ?? []).filter(d => d.key && d.path); + return (cfg?.observationIndexes ?? []).filter(d => { + if (!d.key || !d.path) return false; + if (!isSupportedIndexPath(d.path)) { + this.warnOnce( + `path:${d.key}`, + `[ObservationIndexService] index "${d.key}" declares nested path "${d.path}"; only top-level keys can be indexed, so queries on it will use json_extract`, + ); + return false; + } + return true; + }); + } + + /** + * Whether queries may trust `observation_index` for the active generation. + * + * A declared key routes a query through the index, so an index that cannot be + * trusted turns every predicate on it into a silent wrong answer rather than + * an error. Two ways that happens: + * + * - **No rows.** Never built, emptied by a database wipe, or abandoned by a + * failed rebuild. Every indexed predicate matches nothing. + * - **Rows built from different definitions.** A bundle changed the declared + * indexes and the rebuild did not finish. The rows look healthy and are + * quietly answering for the previous bundle's schema. + * + * Either way the caller drops back to json_extract, which is slower but reads + * the observation JSON directly and is therefore always right. + * + * Only the negative answer is re-checked. A usable index stays usable until a + * rebuild or a wipe, and both clear this flag. + */ + async isIndexUsable(): Promise { + if (this.indexUsable) return true; + + const generation = await this.readActiveGeneration(); + const rows = await this.query<{ present: number }>( + 'SELECT EXISTS(SELECT 1 FROM observation_index WHERE index_generation = ?) AS present', + [generation], + ); + if ((rows[0]?.present ?? 0) !== 1) { + this.indexUsable = false; + return false; + } + + const expected = computeDefsSignature(this.getIndexDefs()); + const stored = (await this.getStatus()).defsSignature; + if (stored !== expected) { + this.warnOnce( + 'signature-mismatch', + '[ObservationIndexService] index rows were built from different index definitions — falling back to json_extract until a rebuild completes', + ); + this.indexUsable = false; + return false; + } + + this.indexUsable = true; + return true; + } + + /** + * Forget everything cached about the index. + * + * `unsafeResetDatabase` recreates the index tables empty, but this singleton + * survives it and would otherwise report a rebuild it no longer has. + */ + reset(): void { + this.initialRebuildPromise = null; + this.initialRebuildFinished = false; + this.indexUsable = false; + this.warnedOnce.clear(); + } + + private warnOnce(token: string, message: string): void { + if (this.warnedOnce.has(token)) return; + this.warnedOnce.add(token); + console.warn(message); } getInitialRebuildFinished(): boolean { @@ -100,18 +242,21 @@ export class ObservationIndexService { async getStatus(): Promise<{ activeGeneration: number; lastRebuildAt: string | null; + defsSignature: string | null; }> { const rows = await this.query<{ active_generation: number; last_rebuild_at: string | null; + defs_signature: string | null; }>( - 'SELECT active_generation, last_rebuild_at FROM observation_index_meta WHERE id = ?', + 'SELECT active_generation, last_rebuild_at, defs_signature FROM observation_index_meta WHERE id = ?', ['meta'], ); const row = rows[0]; return { activeGeneration: row?.active_generation ?? 1, lastRebuildAt: row?.last_rebuild_at ?? null, + defsSignature: row?.defs_signature ?? null, }; } @@ -145,10 +290,19 @@ export class ObservationIndexService { ); const observationCount = obsCountRows[0]?.cnt ?? 0; + // A populated table is not evidence that it is *current*. If the stored + // signature does not match the definitions in force, the rows were + // built from a previous bundle or by a rebuild that never finished, and + // skipping here would leave that state in place permanently. + const signatureMatches = + status.defsSignature === computeDefsSignature(this.getIndexDefs()); + const skipBecausePopulated = - Boolean(status.lastRebuildAt) && indexCount > 0; + Boolean(status.lastRebuildAt) && indexCount > 0 && signatureMatches; const skipBecauseEmptyInstall = - observationCount === 0 && Boolean(status.lastRebuildAt); + observationCount === 0 && + Boolean(status.lastRebuildAt) && + signatureMatches; if (skipBecausePopulated) { this.initialRebuildFinished = true; @@ -160,6 +314,12 @@ export class ObservationIndexService { return; } + if (!signatureMatches && Boolean(status.lastRebuildAt)) { + console.log( + '[ObservationIndexService] index definitions changed or a previous rebuild did not complete — rebuilding', + ); + } + await this.rebuildAllIndexes(); this.initialRebuildFinished = true; } catch (err) { @@ -171,15 +331,43 @@ export class ObservationIndexService { return this.initialRebuildPromise; } - async rebuildAllIndexes(): Promise<{ + /** + * Reload the app config and rebuild the index against the definitions the + * newly installed bundle declares. + * + * Callers are expected to await this and show progress. A bundle can add, + * remove or retarget index definitions, and until the rebuild finishes every + * query on a changed key falls back to json_extract. + */ + async rebuildForBundleUpdate( + onProgress?: (progress: IndexRebuildProgress) => void, + ): Promise { + await AppConfigService.getInstance().loadConfig(/* force */ true); + try { + await this.rebuildAllIndexes({ onProgress }); + } catch (err) { + // Leave no memoised "rebuild finished" behind: the next query re-runs + // the check, sees the signature mismatch, and tries again. + this.reset(); + throw err; + } + } + + async rebuildAllIndexes(options?: { + onProgress?: (progress: IndexRebuildProgress) => void; + }): Promise<{ generation: number; lastRebuildAt: string | null; }> { const defs = this.getIndexDefs(); + const signature = computeDefsSignature(defs); // Always rebuild in-place on generation 1. The previous 1↔2 swap wrote // index rows to the new generation but the meta active_generation UPDATE // did not persist on device (runtime logs: gen-2 rows, activeGeneration=1). const gen = 1; + // The rebuild empties the table before refilling it, and may legitimately + // end with no rows at all, so any cached "index is healthy" answer is void. + this.indexUsable = false; return this.db.write(async () => { await this.db.adapter.unsafeExecute({ sqls: [ @@ -190,24 +378,44 @@ export class ObservationIndexService { ], }); - const observations = await this.query<{ - id: string; - form_type: string; - data: string; - }>('SELECT id, form_type, data FROM observations'); + const totalRows = await this.query<{ cnt: number }>( + 'SELECT COUNT(*) AS cnt FROM observations', + ); + const total = totalRows[0]?.cnt ?? 0; const sqls: SqlStatement[] = []; sqls.push(['DELETE FROM observation_index', []]); - for (const obs of observations) { - this.collectReindexStatements( - obs.id, - obs.form_type ?? '', - obs.data, - defs, - gen, - sqls, + // Read in batches and yield between them. The scan parses each + // observation's JSON once per definition, which on a full repository is + // long enough to freeze the UI if run as a single loop. + let processed = 0; + options?.onProgress?.({ current: 0, total }); + for (let offset = 0; offset < total; offset += REBUILD_BATCH_SIZE) { + const batch = await this.query<{ + id: string; + form_type: string; + data: string; + }>( + 'SELECT id, form_type, data FROM observations ORDER BY id LIMIT ? OFFSET ?', + [REBUILD_BATCH_SIZE, offset], ); + if (!batch.length) break; + + for (const obs of batch) { + this.collectReindexStatements( + obs.id, + obs.form_type ?? '', + obs.data, + defs, + gen, + sqls, + ); + } + + processed += batch.length; + options?.onProgress?.({ current: processed, total }); + await yieldToUi(); } this.collectSqliteIndexStatements(defs, sqls); @@ -215,13 +423,15 @@ export class ObservationIndexService { await this.flush(sqls); // Stamp meta in a separate execute; batched meta UPDATE was not sticking. + // Written only after the rows land, so a rebuild interrupted before this + // point leaves a signature that no longer matches and reruns next launch. await this.db.adapter.unsafeExecute({ sqls: [ [ `UPDATE observation_index_meta - SET active_generation = ?, building_generation = NULL, last_rebuild_at = datetime('now') + SET active_generation = ?, building_generation = NULL, last_rebuild_at = datetime('now'), defs_signature = ? WHERE id = ?`, - [gen, 'meta'], + [gen, signature, 'meta'], ], ], }); @@ -312,7 +522,15 @@ export class ObservationIndexService { if (!formTypeMatches(formType, def.formTypes)) continue; const val = extractScalar(dataJson, def.path); if (val == null) continue; - const { valueText, valueNum } = scalarToColumns(val, def.valueType); + const columns = scalarToColumns(val, def.valueType); + if (!columns) { + this.warnOnce( + `nonscalar:${def.key}`, + `[ObservationIndexService] index "${def.key}" holds a non-scalar value; it is not indexed, so only any() filters can match it`, + ); + continue; + } + const { valueText, valueNum } = columns; const rowId = `${observationId}:${def.key}:${generation}`; out.push([ `INSERT OR REPLACE INTO observation_index diff --git a/formulus/src/services/RepositoryRecoveryService.ts b/formulus/src/services/RepositoryRecoveryService.ts index af3ab348f..3068d1ed8 100644 --- a/formulus/src/services/RepositoryRecoveryService.ts +++ b/formulus/src/services/RepositoryRecoveryService.ts @@ -2,6 +2,7 @@ import AsyncStorage from '@react-native-async-storage/async-storage'; import RNFS from 'react-native-fs'; import { database } from '../database/database'; import { synkronusApi } from '../api/synkronus'; +import ObservationIndexService from './ObservationIndexService'; const REPOSITORY_GENERATION_KEY = '@repository_generation'; @@ -34,6 +35,10 @@ class RepositoryRecoveryService { await database.unsafeResetDatabase(); }); + // The index tables come back empty, but the service is a singleton and + // still believes it has a completed rebuild behind it. + ObservationIndexService.getInstance(database).reset(); + await AsyncStorage.multiRemove([ '@last_seen_version', '@last_attachment_version', diff --git a/formulus/src/services/ServerSwitchService.ts b/formulus/src/services/ServerSwitchService.ts index fefbe6eef..c4795b5ab 100644 --- a/formulus/src/services/ServerSwitchService.ts +++ b/formulus/src/services/ServerSwitchService.ts @@ -6,6 +6,7 @@ import { synkronusApi } from '../api/synkronus'; import { logout } from '../api/synkronus/Auth'; import { serverConfigService } from './ServerConfigService'; import { invalidateSettingsHydrationCache } from './SettingsHydrationCache'; +import ObservationIndexService from './ObservationIndexService'; /** * Handles cleanup when switching Synkronus servers to avoid cross-server data. @@ -53,6 +54,10 @@ class ServerSwitchService { await database.unsafeResetDatabase(); }); + // The index tables come back empty, but the service is a singleton and + // still believes it has a completed rebuild behind it. + ObservationIndexService.getInstance(database).reset(); + // 4) Clear sync/app metadata + tokens await AsyncStorage.multiRemove([ '@last_seen_version', diff --git a/formulus/src/services/SyncService.ts b/formulus/src/services/SyncService.ts index a4b79af5b..140380372 100644 --- a/formulus/src/services/SyncService.ts +++ b/formulus/src/services/SyncService.ts @@ -1,12 +1,16 @@ import { synkronusApi } from '../api/synkronus'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { appEvents } from '../webview/FormulusMessageHandlers'; -import type { SyncProgress } from '../sync/syncProgress'; -import type { SynkronusSyncOptions } from '../sync/syncProgress'; +import { + formatCountProgress, + type SyncProgress, + type SynkronusSyncOptions, +} from '../sync/syncProgress'; import { notificationService } from './NotificationService'; import { getUserFacingAppBundleUpdateErrorMessage } from './appBundleUpdateErrors'; import { FormService } from './FormService'; import { formLocaleIndexService } from './FormLocaleIndexService'; +import ObservationIndexService from './ObservationIndexService'; import { autoLogin, getUserFacingSyncErrorMessage, @@ -400,6 +404,29 @@ export class SyncService { await formLocaleIndexService.refreshIndex(); + // The bundle is the only way new index definitions arrive. Await the + // rebuild here rather than firing it from `bundleUpdated`: that event + // used to start a fire-and-forget rebuild, so sync reported "complete" + // while the index was still being written, and a crash left rows built + // from the previous bundle with no record that they were stale. + this.updateStatus(i18n.t('sync.progress.phase.index_rebuild')); + this.updateProgress({ + current: 0, + total: 0, + phase: 'index_rebuild', + indeterminate: true, + }); + await ObservationIndexService.getInstance().rebuildForBundleUpdate( + ({ current, total }) => { + this.updateProgress({ + current, + total, + phase: 'index_rebuild', + details: formatCountProgress(current, total), + }); + }, + ); + const syncTime = new Date().toLocaleTimeString(); await AsyncStorage.setItem('@lastSync', syncTime); this.updateStatus('App bundle sync completed'); diff --git a/formulus/src/services/__tests__/ServerSwitchService.test.ts b/formulus/src/services/__tests__/ServerSwitchService.test.ts index 5d49e6e8c..779bd9286 100644 --- a/formulus/src/services/__tests__/ServerSwitchService.test.ts +++ b/formulus/src/services/__tests__/ServerSwitchService.test.ts @@ -56,6 +56,12 @@ jest.mock('../ServerConfigService', () => ({ serverConfigService: mockServerConfigService, })); +const mockIndexReset = jest.fn(); +jest.mock('../ObservationIndexService', () => ({ + __esModule: true, + default: { getInstance: jest.fn(() => ({ reset: mockIndexReset })) }, +})); + const { serverSwitchService } = require('../ServerSwitchService'); describe('ServerSwitchService', () => { @@ -82,6 +88,10 @@ describe('ServerSwitchService', () => { expect(mockDatabase.write).toHaveBeenCalled(); expect(mockDatabase.unsafeResetDatabase).toHaveBeenCalled(); + // The index service is a singleton and survives the database reset, so it + // has to be told that the rebuild it remembers is gone. + expect(mockIndexReset).toHaveBeenCalled(); + expect(mockAsyncStorage.multiRemove).toHaveBeenCalledWith([ '@last_seen_version', '@last_attachment_version', diff --git a/formulus/src/services/__tests__/SyncService.autoLogin.test.ts b/formulus/src/services/__tests__/SyncService.autoLogin.test.ts index aa4bc292a..9b45dbc6a 100644 --- a/formulus/src/services/__tests__/SyncService.autoLogin.test.ts +++ b/formulus/src/services/__tests__/SyncService.autoLogin.test.ts @@ -118,6 +118,14 @@ jest.mock('../FormLocaleIndexService', () => ({ refreshIndex: jest.fn().mockResolvedValue([]), }, })); +jest.mock('../ObservationIndexService', () => ({ + __esModule: true, + default: { + getInstance: jest.fn(() => ({ + rebuildForBundleUpdate: jest.fn().mockResolvedValue(undefined), + })), + }, +})); import { jest, diff --git a/formulus/src/sync/syncProgress.ts b/formulus/src/sync/syncProgress.ts index 6bee97a51..c63f1d394 100644 --- a/formulus/src/sync/syncProgress.ts +++ b/formulus/src/sync/syncProgress.ts @@ -10,7 +10,9 @@ export type SyncProgressPhase = | 'push_attachments' | 'push_observations' /** Form definitions / custom app ZIP (not observation attachments). */ - | 'app_bundle'; + | 'app_bundle' + /** Rebuilding local query indexes after a bundle changed their definitions. */ + | 'index_rebuild'; export interface SyncProgress { current: number; @@ -62,6 +64,8 @@ export function syncProgressPhaseTitle(phase: SyncProgressPhase): string { return i18n.t('sync.progress.phase.push_observations'); case 'app_bundle': return i18n.t('sync.progress.phase.app_bundle'); + case 'index_rebuild': + return i18n.t('sync.progress.phase.index_rebuild'); default: return i18n.t('sync.progress.phase.default'); } diff --git a/packages/observation-query/fixtures/undeclared_numeric_fallback_guard.json b/packages/observation-query/fixtures/undeclared_numeric_fallback_guard.json new file mode 100644 index 000000000..870891ba1 --- /dev/null +++ b/packages/observation-query/fixtures/undeclared_numeric_fallback_guard.json @@ -0,0 +1,20 @@ +{ + "name": "undeclared_numeric_fallback_guard", + "jsonColumn": "payload", + "formType": "hh_person", + "includeDeleted": false, + "indexKeys": ["hh_id"], + "filter": { + "field": "data.age_years", + "op": "gte", + "value": 18 + }, + "expectedSqlFragmentsByDialect": { + "desktop": ["json_type(o.payload, '$.age_years') IN ('integer','real')"], + "formulus": [ + "json_type(observations.data, '$.age_years') IN ('integer','real')" + ] + }, + "expectWarning": true, + "expectError": false +} diff --git a/packages/observation-query/package.json b/packages/observation-query/package.json index ff697b4f1..3bc4931f8 100644 --- a/packages/observation-query/package.json +++ b/packages/observation-query/package.json @@ -9,7 +9,11 @@ ".": "./src/index.ts", "./fixtures/*": "./fixtures/*" }, - "files": ["src", "fixtures", "schema"], + "files": [ + "src", + "fixtures", + "schema" + ], "scripts": { "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js", "build": "tsc" @@ -20,7 +24,10 @@ }, "devDependencies": { "@types/jest": "^29.5.0", + "@types/node": "^26.2.0", + "@types/sql.js": "^1.4.11", "jest": "^29.7.0", + "sql.js": "^1.14.2", "ts-jest": "^29.2.0", "typescript": "^5.0.4" } diff --git a/packages/observation-query/pnpm-lock.yaml b/packages/observation-query/pnpm-lock.yaml index 1b356c4a7..1a0efedd6 100644 --- a/packages/observation-query/pnpm-lock.yaml +++ b/packages/observation-query/pnpm-lock.yaml @@ -15,12 +15,21 @@ importers: '@types/jest': specifier: ^29.5.0 version: 29.5.14 + '@types/node': + specifier: ^26.2.0 + version: 26.2.0 + '@types/sql.js': + specifier: ^1.4.11 + version: 1.4.11 jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@25.8.0) + version: 29.7.0(@types/node@26.2.0) + sql.js: + specifier: ^1.14.2 + version: 1.14.2 ts-jest: specifier: ^29.2.0 - version: 29.4.9(@babel/core@7.29.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.0))(jest-util@29.7.0)(jest@29.7.0(@types/node@25.8.0))(typescript@5.9.3) + version: 29.4.9(@babel/core@7.29.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.0))(jest-util@29.7.0)(jest@29.7.0(@types/node@26.2.0))(typescript@5.9.3) typescript: specifier: ^5.0.4 version: 5.9.3 @@ -307,6 +316,9 @@ packages: '@types/babel__traverse@7.28.0': resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} + '@types/emscripten@1.41.5': + resolution: {integrity: sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==} + '@types/graceful-fs@4.1.9': resolution: {integrity: sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==} @@ -322,8 +334,11 @@ packages: '@types/jest@29.5.14': resolution: {integrity: sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==} - '@types/node@25.8.0': - resolution: {integrity: sha512-TCFSk8IZh+iLX1xtksoBVtdmgL+1IX0fC9BeU4QqFSuNdN/K+HUlhqOzEmSYYpZUVsLYcPqc9KX+60iDuninSQ==} + '@types/node@26.2.0': + resolution: {integrity: sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==} + + '@types/sql.js@1.4.11': + resolution: {integrity: sha512-QXIx38p2ZThJaK9vP5ZdqdlRe1FG9I8SmCZOS7FHfB/2qPAjZwkL7/vlfPg6N/oWHuuOaGg/P/IRwfP2W0kWVQ==} '@types/stack-utils@2.0.3': resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} @@ -1029,6 +1044,9 @@ packages: sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + sql.js@1.14.2: + resolution: {integrity: sha512-3ZGPovObMFrdw79zrUHbfdE/DLIsy8jdNdssmMSQuRAymedU6q84asPt0kgiqrdMYlPegDItiIMfmIXzZnYFcw==} + stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -1129,8 +1147,8 @@ packages: engines: {node: '>=0.8.0'} hasBin: true - undici-types@7.24.6: - resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici-types@8.3.0: + resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} update-browserslist-db@1.2.3: resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} @@ -1389,7 +1407,7 @@ snapshots: '@jest/console@29.7.0': dependencies: '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 chalk: 4.1.2 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -1402,14 +1420,14 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 3.9.0 exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@25.8.0) + jest-config: 29.7.0(@types/node@26.2.0) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -1434,7 +1452,7 @@ snapshots: dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 jest-mock: 29.7.0 '@jest/expect-utils@29.7.0': @@ -1452,7 +1470,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 25.8.0 + '@types/node': 26.2.0 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -1474,7 +1492,7 @@ snapshots: '@jest/transform': 29.7.0 '@jest/types': 29.6.3 '@jridgewell/trace-mapping': 0.3.31 - '@types/node': 25.8.0 + '@types/node': 26.2.0 chalk: 4.1.2 collect-v8-coverage: 1.0.3 exit: 0.1.2 @@ -1544,7 +1562,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 25.8.0 + '@types/node': 26.2.0 '@types/yargs': 17.0.35 chalk: 4.1.2 @@ -1598,9 +1616,11 @@ snapshots: dependencies: '@babel/types': 7.29.0 + '@types/emscripten@1.41.5': {} + '@types/graceful-fs@4.1.9': dependencies: - '@types/node': 25.8.0 + '@types/node': 26.2.0 '@types/istanbul-lib-coverage@2.0.6': {} @@ -1617,9 +1637,14 @@ snapshots: expect: 29.7.0 pretty-format: 29.7.0 - '@types/node@25.8.0': + '@types/node@26.2.0': + dependencies: + undici-types: 8.3.0 + + '@types/sql.js@1.4.11': dependencies: - undici-types: 7.24.6 + '@types/emscripten': 1.41.5 + '@types/node': 26.2.0 '@types/stack-utils@2.0.3': {} @@ -1775,13 +1800,13 @@ snapshots: convert-source-map@2.0.0: {} - create-jest@29.7.0(@types/node@25.8.0): + create-jest@29.7.0(@types/node@26.2.0): dependencies: '@jest/types': 29.6.3 chalk: 4.1.2 exit: 0.1.2 graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@25.8.0) + jest-config: 29.7.0(@types/node@26.2.0) jest-util: 29.7.0 prompts: 2.4.2 transitivePeerDependencies: @@ -1991,7 +2016,7 @@ snapshots: '@jest/expect': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 chalk: 4.1.2 co: 4.6.0 dedent: 1.7.2 @@ -2011,16 +2036,16 @@ snapshots: - babel-plugin-macros - supports-color - jest-cli@29.7.0(@types/node@25.8.0): + jest-cli@29.7.0(@types/node@26.2.0): dependencies: '@jest/core': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 chalk: 4.1.2 - create-jest: 29.7.0(@types/node@25.8.0) + create-jest: 29.7.0(@types/node@26.2.0) exit: 0.1.2 import-local: 3.2.0 - jest-config: 29.7.0(@types/node@25.8.0) + jest-config: 29.7.0(@types/node@26.2.0) jest-util: 29.7.0 jest-validate: 29.7.0 yargs: 17.7.2 @@ -2030,7 +2055,7 @@ snapshots: - supports-color - ts-node - jest-config@29.7.0(@types/node@25.8.0): + jest-config@29.7.0(@types/node@26.2.0): dependencies: '@babel/core': 7.29.0 '@jest/test-sequencer': 29.7.0 @@ -2055,7 +2080,7 @@ snapshots: slash: 3.0.0 strip-json-comments: 3.1.1 optionalDependencies: - '@types/node': 25.8.0 + '@types/node': 26.2.0 transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -2084,7 +2109,7 @@ snapshots: '@jest/environment': 29.7.0 '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -2094,7 +2119,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@types/graceful-fs': 4.1.9 - '@types/node': 25.8.0 + '@types/node': 26.2.0 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -2133,7 +2158,7 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 jest-util: 29.7.0 jest-pnp-resolver@1.2.3(jest-resolve@29.7.0): @@ -2168,7 +2193,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 chalk: 4.1.2 emittery: 0.13.1 graceful-fs: 4.2.11 @@ -2196,7 +2221,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 chalk: 4.1.2 cjs-module-lexer: 1.4.3 collect-v8-coverage: 1.0.3 @@ -2242,7 +2267,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -2261,7 +2286,7 @@ snapshots: dependencies: '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 25.8.0 + '@types/node': 26.2.0 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.13.1 @@ -2270,17 +2295,17 @@ snapshots: jest-worker@29.7.0: dependencies: - '@types/node': 25.8.0 + '@types/node': 26.2.0 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 - jest@29.7.0(@types/node@25.8.0): + jest@29.7.0(@types/node@26.2.0): dependencies: '@jest/core': 29.7.0 '@jest/types': 29.6.3 import-local: 3.2.0 - jest-cli: 29.7.0(@types/node@25.8.0) + jest-cli: 29.7.0(@types/node@26.2.0) transitivePeerDependencies: - '@types/node' - babel-plugin-macros @@ -2461,6 +2486,8 @@ snapshots: sprintf-js@1.0.3: {} + sql.js@1.14.2: {} + stack-utils@2.0.6: dependencies: escape-string-regexp: 2.0.0 @@ -2508,12 +2535,12 @@ snapshots: dependencies: is-number: 7.0.0 - ts-jest@29.4.9(@babel/core@7.29.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.0))(jest-util@29.7.0)(jest@29.7.0(@types/node@25.8.0))(typescript@5.9.3): + ts-jest@29.4.9(@babel/core@7.29.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.0))(jest-util@29.7.0)(jest@29.7.0(@types/node@26.2.0))(typescript@5.9.3): dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 handlebars: 4.7.9 - jest: 29.7.0(@types/node@25.8.0) + jest: 29.7.0(@types/node@26.2.0) json5: 2.2.3 lodash.memoize: 4.1.2 make-error: 1.3.6 @@ -2539,7 +2566,7 @@ snapshots: uglify-js@3.19.3: optional: true - undici-types@7.24.6: {} + undici-types@8.3.0: {} update-browserslist-db@1.2.3(browserslist@4.28.2): dependencies: diff --git a/packages/observation-query/src/compiler.ts b/packages/observation-query/src/compiler.ts index 4c6ad038a..0aebea4eb 100644 --- a/packages/observation-query/src/compiler.ts +++ b/packages/observation-query/src/compiler.ts @@ -60,6 +60,24 @@ function pushParam(params: Array, value: unknown): strin return '?'; } +/** + * Whether an operand should be compared numerically. + * + * This mirrors how the index stores values — numbers in `value_num`, + * everything else in `value_text` — so both compilation paths have to consult + * it to stay in agreement. `eq` is excluded deliberately, which keeps equality + * type-strict: `eq "42"` matches the string and `eq 42` matches the number. + */ +function isNumericOperand(value: unknown, op: string): boolean { + if (typeof value === 'number') return true; + return ( + typeof value === 'string' && + value !== '' && + !Number.isNaN(Number(value)) && + op !== 'eq' + ); +} + export function compileFilter( filter: ObservationFilter, options: CompileOptions, @@ -168,9 +186,7 @@ function compileCondition( } const val = cond.value; - const isNum = - typeof val === 'number' || - (typeof val === 'string' && val !== '' && !Number.isNaN(Number(val)) && cond.op !== 'eq'); + const isNum = isNumericOperand(val, cond.op); if (isNum && ['gt', 'gte', 'lt', 'lte', 'eq', 'neq'].includes(cond.op)) { const p = pushParam(params, Number(val)); @@ -246,6 +262,24 @@ function compileJsonExtractCondition( if (!(cond.op in opMap)) { return { code: 'UNSUPPORTED_OP', message: `Unsupported op ${cond.op}` }; } + + // A numeric comparison has to match the indexed path, which compares against + // `value_num` and therefore only ever considers values that are JSON numbers. + // SQLite orders by storage class rather than coercing, so an unguarded + // json_extract comparison disagrees in both directions: `18 >= '18'` is false, + // so a numeric-looking string operand matches nothing, and `'abc' > 5` is + // true, so every text value matches a numeric filter. This fallback runs + // precisely when the index is unusable, and a safety net that quietly returns + // different rows is worse than no safety net at all. + // + // json_type rather than typeof(json_extract(...)): JSON `true` extracts as + // the integer 1, so typeof would admit booleans that the index stores as text. + if (isNumericOperand(cond.value, cond.op)) { + const p = pushParam(params, Number(cond.value)); + const typeExpr = `json_type(${tableAlias}.${jsonCol}, '${jsonPath}')`; + return `(${typeExpr} IN ('integer','real') AND ${expr} ${opMap[cond.op]} ${p})`; + } + const p = pushParam(params, cond.value as string | number | null); return `${expr} ${opMap[cond.op]} ${p}`; } diff --git a/packages/observation-query/src/compilerDifferential.test.ts b/packages/observation-query/src/compilerDifferential.test.ts new file mode 100644 index 000000000..9db29af00 --- /dev/null +++ b/packages/observation-query/src/compilerDifferential.test.ts @@ -0,0 +1,214 @@ +/** + * The indexed path and the json_extract fallback must return the same rows. + * + * Callers drop to json_extract whenever a key is undeclared, and Formulus and + * Desktop both drop to it wholesale when the index turns out to be unusable. + * That safety net is only worth having if the two paths agree: a fallback that + * quietly returns a different result set is worse than one that fails loudly. + * + * SQLite makes agreement easy to get wrong, because it orders values by storage + * class instead of coercing them. `18 >= '18'` is false, so a numeric-looking + * string operand used to match nothing on the fallback while matching on the + * index; and `'abc' > 5` is true, so every text value used to match a numeric + * filter on the fallback while matching none on the index. + * + * Each case below runs twice against the same database — once with the key + * declared (indexed EXISTS) and once undeclared (json_extract) — and the two + * result sets have to be identical. + */ +import * as path from 'path'; +import initSqlJs, { type Database } from 'sql.js'; +import { compileObservationQuery } from './compiler'; +import type { ObservationFilter } from './types'; + +/** + * Mirrors `ObservationIndexService.scalarToColumns` in Formulus and the Rust + * indexer in Desktop: JSON numbers go to `value_num`, other scalars to + * `value_text`, and absent, null and non-scalar values are not indexed at all. + */ +function indexColumns(value: unknown): { + valueText: string | null; + valueNum: number | null; +} | null { + if (value == null) return null; + if (typeof value === 'number') return { valueText: null, valueNum: value }; + if (typeof value === 'string') return { valueText: value, valueNum: null }; + if (typeof value === 'boolean') + return { valueText: String(value), valueNum: null }; + return null; +} + +const INDEX_KEYS = ['age', 'name']; + +const OBSERVATIONS: Array<{ id: string; data: Record }> = [ + { id: 'p01', data: { age: 18, name: 'ann' } }, + { id: 'p02', data: { age: 20, name: 'bob' } }, + { id: 'p03', data: { age: 5, name: 'cid' } }, + { id: 'p04', data: { age: '18', name: 'dee' } }, // numeric string + { id: 'p05', data: { age: 'abc', name: 'eve' } }, // text sorts above numbers + { id: 'p06', data: { name: 'fay' } }, // key absent + { id: 'p07', data: { age: null, name: 'gil' } }, // explicit null + { id: 'p08', data: { age: 18.5, name: 'hal' } }, // real + { id: 'p09', data: { age: true, name: 'ivy' } }, // extracts as integer 1 + { id: 'p10', data: { age: [1, 2], name: 'jan' } }, // non-scalar +]; + +async function buildDatabase(): Promise { + const SQL = await initSqlJs({ + locateFile: file => path.join(path.dirname(require.resolve('sql.js')), file), + }); + const db = new SQL.Database(); + + db.run(` + CREATE TABLE observations ( + id TEXT PRIMARY KEY NOT NULL, + observation_id TEXT NOT NULL, + form_type TEXT NOT NULL, + data TEXT NOT NULL, + deleted INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE observation_index ( + observation_id TEXT NOT NULL, + index_key TEXT NOT NULL, + index_generation INTEGER NOT NULL, + value_text TEXT, + value_num REAL + ); + CREATE TABLE observation_index_meta ( + id TEXT PRIMARY KEY NOT NULL, + active_generation INTEGER NOT NULL + ); + INSERT INTO observation_index_meta(id, active_generation) VALUES ('meta', 1); + `); + + for (const obs of OBSERVATIONS) { + db.run( + 'INSERT INTO observations(id, observation_id, form_type, data, deleted) VALUES (?, ?, ?, ?, 0)', + [obs.id, obs.id, 'person', JSON.stringify(obs.data)], + ); + for (const key of INDEX_KEYS) { + const columns = indexColumns(obs.data[key]); + if (!columns) continue; + db.run( + `INSERT INTO observation_index + (observation_id, index_key, index_generation, value_text, value_num) + VALUES (?, ?, 1, ?, ?)`, + [obs.id, key, columns.valueText, columns.valueNum], + ); + } + } + + return db; +} + +function run( + db: Database, + filter: ObservationFilter, + declaredKeys: string[], +): string[] { + const compiled = compileObservationQuery({ + dialect: 'formulus', + jsonColumn: 'data', + tableAlias: 'observations', + observationsTable: 'observations', + formType: 'person', + includeDeleted: false, + indexKeys: new Set(declaredKeys), + filter, + }); + if ('code' in compiled) { + throw new Error(`${compiled.code}: ${compiled.message}`); + } + + const statement = db.prepare(compiled.sql); + statement.bind(compiled.params as (string | number | null)[]); + const ids: string[] = []; + while (statement.step()) { + ids.push(statement.getAsObject().id as string); + } + statement.free(); + return ids.sort(); +} + +describe('indexed and json_extract paths agree', () => { + let db: Database; + + beforeAll(async () => { + db = await buildDatabase(); + }); + + afterAll(() => { + db?.close(); + }); + + const cases: Array<[string, ObservationFilter]> = [ + ['eq on a string', { field: 'data.name', op: 'eq', value: 'bob' }], + ['eq on a number', { field: 'data.age', op: 'eq', value: 18 }], + ['eq on a numeric string', { field: 'data.age', op: 'eq', value: '18' }], + ['gte with a number operand', { field: 'data.age', op: 'gte', value: 18 }], + [ + 'gte with a numeric string operand', + { field: 'data.age', op: 'gte', value: '18' }, + ], + ['gt with a number operand', { field: 'data.age', op: 'gt', value: 5 }], + ['gt with a low bound', { field: 'data.age', op: 'gt', value: 0 }], + ['lt with a number operand', { field: 'data.age', op: 'lt', value: 19 }], + ['lte with a numeric string', { field: 'data.age', op: 'lte', value: '18' }], + ['neq with a number operand', { field: 'data.age', op: 'neq', value: 18 }], + [ + 'neq with a numeric string operand', + { field: 'data.age', op: 'neq', value: '18' }, + ], + ['in with numbers', { field: 'data.age', op: 'in', value: [18, 20] }], + ['in with strings', { field: 'data.age', op: 'in', value: ['18', 'abc'] }], + [ + 'and across two fields', + { + op: 'and', + conditions: [ + { field: 'data.age', op: 'gte', value: '18' }, + { field: 'data.name', op: 'neq', value: 'hal' }, + ], + }, + ], + [ + 'or across two fields', + { + op: 'or', + conditions: [ + { field: 'data.age', op: 'lt', value: 10 }, + { field: 'data.name', op: 'eq', value: 'bob' }, + ], + }, + ], + ]; + + it.each(cases)('%s', (_name, filter) => { + expect(run(db, filter, INDEX_KEYS)).toEqual(run(db, filter, [])); + }); + + it('matches numbers when the operand arrives as a string', () => { + // The regression that motivated the guard: an age range typed into a + // whereClause reaches the compiler as a string, and the fallback used to + // return nothing at all for it. + const filter: ObservationFilter = { + field: 'data.age', + op: 'gte', + value: '18', + }; + expect(run(db, filter, [])).toEqual(['p01', 'p02', 'p08']); + }); + + it('does not let text values satisfy a numeric comparison', () => { + // 'abc' > 5 is true in SQLite, so p05 would match without the type guard. + const filter: ObservationFilter = { field: 'data.age', op: 'gt', value: 5 }; + expect(run(db, filter, [])).not.toContain('p05'); + }); + + it('does not let booleans satisfy a numeric comparison', () => { + // JSON true extracts as the integer 1, so p09 would match `gt 0` under a + // typeof() guard; json_type() reports it as 'true' and excludes it. + const filter: ObservationFilter = { field: 'data.age', op: 'gt', value: 0 }; + expect(run(db, filter, [])).not.toContain('p09'); + }); +}); From ace454adbf4ce7ffd0bdc0238a9bf22c099b2b60 Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Sun, 16 Aug 2026 11:06:36 +0200 Subject: [PATCH 27/32] feat(formulus): batching sync writes, updated progress UX --- formulus/src/api/synkronus/index.ts | 26 ++- .../__tests__/observationIndexGuards.test.ts | 34 ++++ .../repositories/LocalRepoInterface.ts | 7 +- .../database/repositories/WatermelonDBRepo.ts | 12 +- formulus/src/locales/en.json | 2 + formulus/src/locales/fr.json | 2 + formulus/src/locales/pt.json | 2 + .../src/services/ObservationIndexService.ts | 191 ++++++++++++------ .../src/sync/__tests__/syncProgressUi.test.ts | 28 +++ formulus/src/sync/syncProgressUi.ts | 6 + 10 files changed, 239 insertions(+), 71 deletions(-) diff --git a/formulus/src/api/synkronus/index.ts b/formulus/src/api/synkronus/index.ts index a0b44cdbb..626f4bac6 100644 --- a/formulus/src/api/synkronus/index.ts +++ b/formulus/src/api/synkronus/index.ts @@ -1175,7 +1175,31 @@ class SynkronusApi { // 2. Apply to local db (local dirty records will not be applied = last update wins). // Skipped rows get a `last_write_won` tag (see syncConstants / WatermelonDBRepo). - const pulledChanges = await repo.applyServerChanges(domainObservations); // ingest observations into WatermelonDB + // Report before and during this step: on a first-time pull the page + // can be thousands of rows, and indexing them used to leave the + // progress card sitting on "Connecting…" until the flush returned. + if (domainObservations.length > 0) { + reportSyncProgress(report, { + phase: 'pull_observations', + current: pullPage, + total: 0, + indeterminate: true, + details: i18n.t('sync.progress.savingRecords', { + count: domainObservations.length, + }), + }); + } + const pulledChanges = await repo.applyServerChanges(domainObservations, { + onIndexProgress: ({ current, total }) => { + if (total <= 0) return; + reportSyncProgress(report, { + phase: 'index_rebuild', + current, + total, + details: formatCountProgress(current, total), + }); + }, + }); console.debug(`Applied ${pulledChanges} changes to local database`); reportSyncProgress(report, { diff --git a/formulus/src/database/__tests__/observationIndexGuards.test.ts b/formulus/src/database/__tests__/observationIndexGuards.test.ts index 8f3bef530..99a19e91d 100644 --- a/formulus/src/database/__tests__/observationIndexGuards.test.ts +++ b/formulus/src/database/__tests__/observationIndexGuards.test.ts @@ -38,6 +38,7 @@ jest.mock('../../services/AppConfigService', () => ({ import ObservationIndexService, { computeDefsSignature, + INDEX_WRITE_BATCH_SIZE, } from '../../services/ObservationIndexService'; const EMPTY_SIGNATURE = computeDefsSignature([]); @@ -176,4 +177,37 @@ describe('ObservationIndexService guards', () => { await expect(service.isIndexUsable()).resolves.toBe(false); }); }); + + describe('incrementalReindexMany', () => { + it('flushes in bounded writes instead of one statement list for the whole page', async () => { + configIndexes.push({ key: 'hh_id', path: '$.hh_id' }); + const rows = Array.from( + { length: INDEX_WRITE_BATCH_SIZE + 50 }, + (_, i) => ({ + observationId: `obs-${i}`, + formType: 'household', + dataJson: JSON.stringify({ hh_id: `HH-${i}` }), + }), + ); + rawResults.push([{ active_generation: 1 }], [{ active_generation: 1 }]); + mockDb.write.mockClear(); + const onProgress = jest.fn(); + + await service.incrementalReindexMany(rows, onProgress); + + expect(mockDb.write).toHaveBeenCalledTimes(2); + expect(onProgress).toHaveBeenCalledWith({ + current: 0, + total: INDEX_WRITE_BATCH_SIZE + 50, + }); + expect(onProgress).toHaveBeenCalledWith({ + current: INDEX_WRITE_BATCH_SIZE, + total: INDEX_WRITE_BATCH_SIZE + 50, + }); + expect(onProgress).toHaveBeenLastCalledWith({ + current: INDEX_WRITE_BATCH_SIZE + 50, + total: INDEX_WRITE_BATCH_SIZE + 50, + }); + }); + }); }); diff --git a/formulus/src/database/repositories/LocalRepoInterface.ts b/formulus/src/database/repositories/LocalRepoInterface.ts index 11e622b01..1ebc12196 100644 --- a/formulus/src/database/repositories/LocalRepoInterface.ts +++ b/formulus/src/database/repositories/LocalRepoInterface.ts @@ -70,7 +70,12 @@ export interface LocalRepoInterface { * @param changes Array of changes to apply * @returns Promise resolving to the number of changes applied */ - applyServerChanges(changes: Observation[]): Promise; + applyServerChanges( + changes: Observation[], + options?: { + onIndexProgress?: (progress: { current: number; total: number }) => void; + }, + ): Promise; /** * Get pending changes from the local database diff --git a/formulus/src/database/repositories/WatermelonDBRepo.ts b/formulus/src/database/repositories/WatermelonDBRepo.ts index 32b87e3ea..a82aa2a83 100644 --- a/formulus/src/database/repositories/WatermelonDBRepo.ts +++ b/formulus/src/database/repositories/WatermelonDBRepo.ts @@ -527,7 +527,12 @@ export class WatermelonDBRepo implements LocalRepoInterface { * Apply changes from the server to the local database * @param changes Array of changes to apply */ - async applyServerChanges(changes: Observation[]): Promise { + async applyServerChanges( + changes: Observation[], + options?: { + onIndexProgress?: (progress: { current: number; total: number }) => void; + }, + ): Promise { if (!changes.length) { return 0; } @@ -629,7 +634,10 @@ export class WatermelonDBRepo implements LocalRepoInterface { ? change.data : JSON.stringify(change.data), })); - await indexService.incrementalReindexMany(indexRows); + await indexService.incrementalReindexMany( + indexRows, + options?.onIndexProgress, + ); return count; } diff --git a/formulus/src/locales/en.json b/formulus/src/locales/en.json index 06ef01bc8..cbbcaf572 100644 --- a/formulus/src/locales/en.json +++ b/formulus/src/locales/en.json @@ -149,6 +149,8 @@ "sync.progress.downloadingPage": "Downloading (page {{page}})…", "sync.progress.recordsDownloaded_one": "{{count}} observation downloaded", "sync.progress.recordsDownloaded_other": "{{count}} observations downloaded", + "sync.progress.savingRecords_one": "Saving {{count}} observation…", + "sync.progress.savingRecords_other": "Saving {{count}} observations…", "sync.progress.recordsSummary_one": "{{count}} observation", "sync.progress.recordsSummary_other": "{{count}} observations", "sync.progress.nothingToUpload": "Nothing to upload", diff --git a/formulus/src/locales/fr.json b/formulus/src/locales/fr.json index 941a9ec51..9e72251d5 100644 --- a/formulus/src/locales/fr.json +++ b/formulus/src/locales/fr.json @@ -149,6 +149,8 @@ "sync.progress.downloadingPage": "Téléchargement (page {{page}})…", "sync.progress.recordsDownloaded_one": "{{count}} observation téléchargée", "sync.progress.recordsDownloaded_other": "{{count}} observations téléchargées", + "sync.progress.savingRecords_one": "Enregistrement de {{count}} observation…", + "sync.progress.savingRecords_other": "Enregistrement de {{count}} observations…", "sync.progress.recordsSummary_one": "{{count}} observation", "sync.progress.recordsSummary_other": "{{count}} observations", "sync.progress.nothingToUpload": "Rien à envoyer", diff --git a/formulus/src/locales/pt.json b/formulus/src/locales/pt.json index 1aae6947a..f1657c588 100644 --- a/formulus/src/locales/pt.json +++ b/formulus/src/locales/pt.json @@ -149,6 +149,8 @@ "sync.progress.downloadingPage": "A descarregar (página {{page}})…", "sync.progress.recordsDownloaded_one": "{{count}} observação descarregada", "sync.progress.recordsDownloaded_other": "{{count}} observações descarregadas", + "sync.progress.savingRecords_one": "A guardar {{count}} observação…", + "sync.progress.savingRecords_other": "A guardar {{count}} observações…", "sync.progress.recordsSummary_one": "{{count}} observação", "sync.progress.recordsSummary_other": "{{count}} observações", "sync.progress.nothingToUpload": "Nada para enviar", diff --git a/formulus/src/services/ObservationIndexService.ts b/formulus/src/services/ObservationIndexService.ts index 6df6eab2b..a52ed56c8 100644 --- a/formulus/src/services/ObservationIndexService.ts +++ b/formulus/src/services/ObservationIndexService.ts @@ -3,10 +3,14 @@ * Rebuild uses snapshot generation swap; incremental updates on save/sync. * * Implementation notes: - * - All write paths collect SQL into an array and flush once via + * - Incremental write paths collect SQL into an array and flush once via * `db.adapter.unsafeExecute({ sqls })` inside a single `db.write(...)` * block. This avoids nested `db.write` calls, which deadlock under * WatermelonDB's serial WorkQueue. + * - A full rebuild writes in bounded batches so the INSERT list cannot grow + * with the whole repository. The signature is cleared first and stamped + * last, so a crash mid-rebuild is indistinguishable from "not current" + * and the next launch reruns. * - `ensureInitialRebuild()` runs on first instantiation to populate the * index for users who already have synced rows from before indexing * landed. @@ -109,8 +113,18 @@ function yieldToUi(): Promise { return new Promise(resolve => setTimeout(resolve, 0)); } -/** Observations read per batch between yields. */ -const REBUILD_BATCH_SIZE = 200; +/** + * Observations processed per index write. + * + * Used by a full rebuild and by `incrementalReindexMany` (sync pull). Each + * observation produces a DELETE plus one INSERT per matching definition, and + * none of that is released until `unsafeExecute` returns. Two hundred rows + * keeps the statement list in the low thousands even with a generous + * `observationIndexes` config — small enough for a Blackview-class tablet, + * large enough that a first-time pull of a few thousand observations is + * tens of writes rather than one giant flush. + */ +export const INDEX_WRITE_BATCH_SIZE = 200; export interface IndexRebuildProgress { /** Observations processed so far. */ @@ -368,63 +382,82 @@ export class ObservationIndexService { // The rebuild empties the table before refilling it, and may legitimately // end with no rows at all, so any cached "index is healthy" answer is void. this.indexUsable = false; - return this.db.write(async () => { + + // Invalidate the signature *before* touching rows. A crash after this + // point leaves `isIndexUsable` false even if some batches have landed, + // and `ensureInitialRebuild` will rerun instead of treating the partial + // table as current. OFFSET pagination is unsafe once we release the + // write lock between batches (a concurrent insert shifts the window), + // so the scan walks `id > cursor` instead. + await this.db.write(async () => { await this.db.adapter.unsafeExecute({ sqls: [ [ `INSERT OR IGNORE INTO observation_index_meta(id, active_generation) VALUES (?, ?)`, ['meta', gen], ], + [ + `UPDATE observation_index_meta + SET defs_signature = NULL, building_generation = ? + WHERE id = ?`, + [gen, 'meta'], + ], + ['DELETE FROM observation_index', []], ], }); + }); - const totalRows = await this.query<{ cnt: number }>( - 'SELECT COUNT(*) AS cnt FROM observations', + const totalRows = await this.query<{ cnt: number }>( + 'SELECT COUNT(*) AS cnt FROM observations', + ); + const total = totalRows[0]?.cnt ?? 0; + + let processed = 0; + let cursor: string | null = null; + options?.onProgress?.({ current: 0, total }); + + while (true) { + const batch = await this.query<{ + id: string; + form_type: string; + data: string; + }>( + cursor == null + ? 'SELECT id, form_type, data FROM observations ORDER BY id LIMIT ?' + : 'SELECT id, form_type, data FROM observations WHERE id > ? ORDER BY id LIMIT ?', + cursor == null + ? [INDEX_WRITE_BATCH_SIZE] + : [cursor, INDEX_WRITE_BATCH_SIZE], ); - const total = totalRows[0]?.cnt ?? 0; + if (!batch.length) break; const sqls: SqlStatement[] = []; - sqls.push(['DELETE FROM observation_index', []]); - - // Read in batches and yield between them. The scan parses each - // observation's JSON once per definition, which on a full repository is - // long enough to freeze the UI if run as a single loop. - let processed = 0; - options?.onProgress?.({ current: 0, total }); - for (let offset = 0; offset < total; offset += REBUILD_BATCH_SIZE) { - const batch = await this.query<{ - id: string; - form_type: string; - data: string; - }>( - 'SELECT id, form_type, data FROM observations ORDER BY id LIMIT ? OFFSET ?', - [REBUILD_BATCH_SIZE, offset], + for (const obs of batch) { + this.collectReindexStatements( + obs.id, + obs.form_type ?? '', + obs.data, + defs, + gen, + sqls, ); - if (!batch.length) break; - - for (const obs of batch) { - this.collectReindexStatements( - obs.id, - obs.form_type ?? '', - obs.data, - defs, - gen, - sqls, - ); - } - - processed += batch.length; - options?.onProgress?.({ current: processed, total }); - await yieldToUi(); } + await this.db.write(async () => { + await this.flush(sqls); + }); - this.collectSqliteIndexStatements(defs, sqls); - - await this.flush(sqls); + cursor = batch[batch.length - 1].id; + processed += batch.length; + options?.onProgress?.({ current: processed, total }); + await yieldToUi(); + } - // Stamp meta in a separate execute; batched meta UPDATE was not sticking. - // Written only after the rows land, so a rebuild interrupted before this - // point leaves a signature that no longer matches and reruns next launch. + const indexSqls: SqlStatement[] = []; + this.collectSqliteIndexStatements(defs, indexSqls); + await this.db.write(async () => { + await this.flush(indexSqls); + // Stamp only after every batch has landed. Batched meta UPDATE was not + // sticking on device, so this stays a separate execute. await this.db.adapter.unsafeExecute({ sqls: [ [ @@ -435,14 +468,13 @@ export class ObservationIndexService { ], ], }); - - const metaAfter = await this.getStatus(); - - return { - generation: gen, - lastRebuildAt: metaAfter.lastRebuildAt, - }; }); + + const metaAfter = await this.getStatus(); + return { + generation: gen, + lastRebuildAt: metaAfter.lastRebuildAt, + }; } async incrementalReindex( @@ -468,29 +500,54 @@ export class ObservationIndexService { } /** - * Reindex many observations in a single batched write. Used by sync paths - * that ingest large numbers of rows. + * Reindex many observations in bounded writes. Used by sync pull, where the + * first page on a new device can be thousands of rows — not a full rebuild, + * but the same shape of OOM if every INSERT is held until one flush. + * + * The signature is left alone: this is additive work against the current + * definitions. A crash mid-loop is safe because the pull cursor is persisted + * only after this returns, so the page is re-applied and these statements + * are INSERT OR REPLACE. */ async incrementalReindexMany( rows: Array<{ observationId: string; formType: string; dataJson: string }>, + onProgress?: (progress: IndexRebuildProgress) => void, ): Promise { const defs = this.getIndexDefs(); if (!defs.length || rows.length === 0) return; - await this.db.write(async () => { - const generation = await this.readActiveGeneration(); - const sqls: SqlStatement[] = []; - for (const r of rows) { - this.collectReindexStatements( - r.observationId, - r.formType, - r.dataJson, - defs, - generation, - sqls, - ); + + const total = rows.length; + onProgress?.({ current: 0, total }); + + for ( + let offset = 0; + offset < rows.length; + offset += INDEX_WRITE_BATCH_SIZE + ) { + const batch = rows.slice(offset, offset + INDEX_WRITE_BATCH_SIZE); + await this.db.write(async () => { + const generation = await this.readActiveGeneration(); + const sqls: SqlStatement[] = []; + for (const r of batch) { + this.collectReindexStatements( + r.observationId, + r.formType, + r.dataJson, + defs, + generation, + sqls, + ); + } + await this.flush(sqls); + }); + onProgress?.({ + current: Math.min(offset + batch.length, total), + total, + }); + if (offset + INDEX_WRITE_BATCH_SIZE < rows.length) { + await yieldToUi(); } - await this.flush(sqls); - }); + } } /** diff --git a/formulus/src/sync/__tests__/syncProgressUi.test.ts b/formulus/src/sync/__tests__/syncProgressUi.test.ts index 33d3ad392..83cd07933 100644 --- a/formulus/src/sync/__tests__/syncProgressUi.test.ts +++ b/formulus/src/sync/__tests__/syncProgressUi.test.ts @@ -1,4 +1,5 @@ import { + getSyncProgressCardTitle, getSyncProgressDetailsForDisplay, shouldShowSyncProgressCurrentItem, shouldShowSyncProgressPercent, @@ -54,4 +55,31 @@ describe('syncProgressUi', () => { }), ).toBe(false); }); + + it('uses the index-rebuild title even during a bundle update', () => { + expect( + getSyncProgressCardTitle( + { + phase: 'index_rebuild', + current: 200, + total: 1500, + details: '200 of 1500', + }, + 'update', + ), + ).toBe('Preparing data for search'); + }); + + it('uses the index-rebuild title during an observation pull', () => { + expect( + getSyncProgressCardTitle( + { + phase: 'index_rebuild', + current: 200, + total: 800, + }, + 'sync', + ), + ).toBe('Preparing data for search'); + }); }); diff --git a/formulus/src/sync/syncProgressUi.ts b/formulus/src/sync/syncProgressUi.ts index 9eb4dc7b3..24f521a20 100644 --- a/formulus/src/sync/syncProgressUi.ts +++ b/formulus/src/sync/syncProgressUi.ts @@ -45,6 +45,12 @@ export function getSyncProgressCardTitle( progress: SyncProgress, activeOperation: 'sync' | 'update' | 'sync_then_update' | null, ): string { + // Index work is its own phase and can run after a bundle download or + // during a first-time pull. The operation wrapper would otherwise keep + // saying "Updating forms" / "Syncing observations" while the bar moves. + if (progress.phase === 'index_rebuild') { + return syncProgressPhaseTitle('index_rebuild'); + } if (activeOperation === 'sync_then_update') { if (progress.phase === 'app_bundle') { return i18n.t('sync.progress.syncingAndUpdatingForms'); From c3ef11947c3a620e36768ff7d9388ed6e3da1934 Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Sun, 16 Aug 2026 11:34:55 +0200 Subject: [PATCH 28/32] fix(desktop): improved import functionality --- desktop/docs/screens/import.md | 4 + desktop/docs/screens/profiles.md | 1 + desktop/src-tauri/src/lib.rs | 271 ++++++++++++++++++- desktop/src/App.test.tsx | 9 + desktop/src/hooks/useProfileAutoSynkAuth.ts | 30 +- desktop/src/lib/importSummary.ts | 45 +-- desktop/src/lib/tauriClient.ts | 6 + desktop/src/pages/ImportPage.tsx | 153 ++++++++--- desktop/src/pages/ProfilesPage.tsx | 24 +- desktop/src/pages/SyncPage.tsx | 6 +- desktop/src/pages/WorkbenchCustomAppPage.tsx | 6 +- desktop/src/store/useCustodianStore.ts | 37 ++- desktop/src/store/useImportStagingStore.ts | 10 + desktop/src/types/domain.ts | 11 + 14 files changed, 530 insertions(+), 83 deletions(-) diff --git a/desktop/docs/screens/import.md b/desktop/docs/screens/import.md index 762569aee..b549330ec 100644 --- a/desktop/docs/screens/import.md +++ b/desktop/docs/screens/import.md @@ -40,3 +40,7 @@ When the active app bundle declares `observationIndexes` in `app.config.json`: 1. Import writes observations in batches (default 2000 rows per IPC call); intermediate batches skip index work. 2. After the final batch commits, Rust schedules **one** coalesced background full index rebuild (`bundle/index-rebuild` progress events). Overlapping rebuild requests while one is running are merged into a single follow-up pass. 3. Sync pull uses incremental indexing per page instead (no full rebuild after each pull). + +## Large Formulus exports + +After staging JSON (folder / drop / Add JSON), Desktop runs a lightweight host scan of `syncedAt` / `updatedAt` and may offer to drop already-synced files **before** full parse + schema validation. Staging lists truncate after ~50 rows so tens of thousands of files do not freeze the UI. diff --git a/desktop/docs/screens/profiles.md b/desktop/docs/screens/profiles.md index 307124a0e..f7916b9fb 100644 --- a/desktop/docs/screens/profiles.md +++ b/desktop/docs/screens/profiles.md @@ -14,6 +14,7 @@ Define units of custody: each profile has its own Synkronus server, credentials, - Edit: display name, server URL, username, password (OS keyring when available) - Local repository file picker, workspace folder, optional attachments folder - Reload, save profile, clear saved password +- Authenticate button: auto-recovers session (refresh token / saved password) on load and profile switch; shows **Authenticated** only after a successful check, otherwise **Authenticate** - Warnings when secure storage is unavailable (password not persisted) ## What to exclude diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 12e2337c5..76f7d717f 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -3920,7 +3920,7 @@ fn read_host_text_file_inner(path: &Path) -> Result { fs::read_to_string(path).map_err(|e| e.to_string()) } -const MAX_HOST_TEXT_BATCH_PATHS: usize = 128; +const MAX_HOST_TEXT_BATCH_PATHS: usize = 512; #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -4175,6 +4175,194 @@ fn parse_import_observation_json_paths( Ok(indexed.into_iter().map(|(_, r)| r).collect()) } +/// Same floor as Formulus / Desktop TS (`1980-01-01`). +const MIN_VALID_IMPORT_SYNCED_AT_MS: i64 = 315_532_800_000; + +fn parse_import_timestamp_ms(raw: &str) -> Option { + let t = raw.trim(); + if t.is_empty() { + return None; + } + if let Ok(dt) = DateTime::parse_from_rfc3339(t) { + return Some(dt.timestamp_millis()); + } + if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(t, "%Y-%m-%dT%H:%M:%S%.f") { + return Some(dt.and_utc().timestamp_millis()); + } + if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(t, "%Y-%m-%dT%H:%M:%S") { + return Some(dt.and_utc().timestamp_millis()); + } + None +} + +fn import_observation_apparently_synced(obj: &serde_json::Map) -> bool { + let synced_raw = optional_import_str(obj, "synced_at", "syncedAt"); + let Some(synced_ms) = synced_raw.as_deref().and_then(parse_import_timestamp_ms) else { + return false; + }; + if synced_ms <= MIN_VALID_IMPORT_SYNCED_AT_MS { + return false; + } + let updated_raw = optional_import_str(obj, "updated_at", "updatedAt"); + let Some(updated_ms) = updated_raw.as_deref().and_then(parse_import_timestamp_ms) else { + return true; + }; + updated_ms <= synced_ms +} + +fn import_json_root_observation_maps(root: &Value) -> Vec<&serde_json::Map> { + match root { + Value::Array(a) => a.iter().filter_map(|v| v.as_object()).collect(), + Value::Object(map) => { + if let Some(Value::Array(inner)) = map.get("observations") { + inner.iter().filter_map(|v| v.as_object()).collect() + } else { + vec![map] + } + } + _ => Vec::new(), + } +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ImportSyncAppearanceScanResult { + /// JSON files scanned. + file_count: usize, + observation_count: usize, + apparently_synced_count: usize, + unsynced_count: usize, + parse_error_count: usize, + /// Absolute paths to keep when skipping already-synced rows (pending + parse errors). + unsynced_paths: Vec, +} + +#[derive(Debug)] +struct ImportFileSyncScanRow { + path: String, + observation_count: usize, + apparently_synced_count: usize, + unsynced_count: usize, + parse_error: bool, +} + +fn scan_one_import_json_sync_appearance(path: &Path) -> ImportFileSyncScanRow { + let path_str = path.to_string_lossy().to_string(); + match read_host_text_file_inner(path) { + Err(_) => ImportFileSyncScanRow { + path: path_str, + observation_count: 0, + apparently_synced_count: 0, + unsynced_count: 0, + parse_error: true, + }, + Ok(text) => match serde_json::from_str::(&text) { + Err(_) => ImportFileSyncScanRow { + path: path_str, + observation_count: 0, + apparently_synced_count: 0, + unsynced_count: 0, + parse_error: true, + }, + Ok(root) => { + let maps = import_json_root_observation_maps(&root); + if maps.is_empty() { + return ImportFileSyncScanRow { + path: path_str, + observation_count: 0, + apparently_synced_count: 0, + unsynced_count: 0, + parse_error: true, + }; + } + let mut synced = 0usize; + let mut unsynced = 0usize; + for obj in maps { + if observation_id_from_obj(obj).is_none() { + unsynced += 1; + continue; + } + if import_observation_apparently_synced(obj) { + synced += 1; + } else { + unsynced += 1; + } + } + ImportFileSyncScanRow { + path: path_str, + observation_count: synced + unsynced, + apparently_synced_count: synced, + unsynced_count: unsynced, + parse_error: false, + } + } + }, + } +} + +/// Lightweight parallel scan of Formulus-style `syncedAt` / `updatedAt` without shipping payloads. +#[tauri::command] +fn scan_import_json_sync_appearance( + paths: Vec, +) -> Result { + if paths.is_empty() { + return Ok(ImportSyncAppearanceScanResult { + file_count: 0, + observation_count: 0, + apparently_synced_count: 0, + unsynced_count: 0, + parse_error_count: 0, + unsynced_paths: Vec::new(), + }); + } + if paths.len() > MAX_IMPORT_SCAN_ENTRIES { + return Err(format!( + "Too many JSON paths to scan (max {MAX_IMPORT_SCAN_ENTRIES})" + )); + } + + let file_count = paths.len(); + let rows: Vec = paths + .into_par_iter() + .map(|raw| { + let p = Path::new(raw.trim()); + scan_one_import_json_sync_appearance(p) + }) + .collect(); + + let mut observation_count = 0usize; + let mut apparently_synced_count = 0usize; + let mut unsynced_obs = 0usize; + let mut parse_error_count = 0usize; + let mut unsynced_paths = Vec::new(); + + for row in rows { + observation_count += row.observation_count; + apparently_synced_count += row.apparently_synced_count; + unsynced_obs += row.unsynced_count; + if row.parse_error { + parse_error_count += 1; + unsynced_paths.push(row.path); + } else if row.unsynced_count > 0 { + // Keep the file if any observation still needs import. + unsynced_paths.push(row.path); + } else if row.apparently_synced_count == 0 { + // Empty / unexpected — keep for validation to report. + unsynced_paths.push(row.path); + } + // else: all observations apparently synced → omit from unsynced_paths + } + + Ok(ImportSyncAppearanceScanResult { + file_count, + observation_count, + apparently_synced_count, + unsynced_count: unsynced_obs + parse_error_count, + parse_error_count, + unsynced_paths, + }) +} + #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] struct AttachmentCopyPair { @@ -5484,6 +5672,7 @@ pub fn run() { copy_workspace_attachment_from_path, expand_import_staging_paths, parse_import_observation_json_paths, + scan_import_json_sync_appearance, copy_workspace_attachments_batch, read_host_text_file, host_path_is_directory, @@ -5533,9 +5722,10 @@ mod tests { ATTACHMENT_COPY_PROGRESS_MIN_INTERVAL, ApiObservation, CompressionMethod, ObservationExtras, SimpleFileOptions, ZipArchive, ZipWriter, apply_app_bundle_zip_at_workspace, attachment_copy_progress_step, bind_query_params, - build_observation_overview, extract_observations_from_json_value, init_db, - mirror_custom_app_dev_folder, parse_observation_extras, parse_time, - publish_bundle_zip_entry_allowed, resolve_attachment_path, + build_observation_overview, extract_observations_from_json_value, + import_observation_apparently_synced, init_db, mirror_custom_app_dev_folder, + parse_observation_extras, parse_time, publish_bundle_zip_entry_allowed, + resolve_attachment_path, scan_import_json_sync_appearance, should_emit_attachment_copy_progress, should_mark_conflict, strip_ode_desktop_injection, upsert_observation_from_local_import, validate_custom_app_dev_source_folder, zip_dev_mirror_bundle, @@ -5783,6 +5973,79 @@ mod tests { assert!(extras.geolocation.is_some()); } + #[test] + fn import_observation_apparently_synced_matches_formulus_rule() { + let synced: serde_json::Map = serde_json::from_value(serde_json::json!({ + "observationId": "a", + "updatedAt": "2026-08-15T11:00:00.000Z", + "syncedAt": "2026-08-15T12:00:00.000Z", + })) + .unwrap(); + assert!(import_observation_apparently_synced(&synced)); + + let pending: serde_json::Map = serde_json::from_value(serde_json::json!({ + "observationId": "b", + "updatedAt": "2026-08-15T13:00:00.000Z", + "syncedAt": "2026-08-15T12:00:00.000Z", + })) + .unwrap(); + assert!(!import_observation_apparently_synced(&pending)); + + let never: serde_json::Map = serde_json::from_value(serde_json::json!({ + "observationId": "c", + "updatedAt": "2026-08-15T13:00:00.000Z", + "syncedAt": null, + })) + .unwrap(); + assert!(!import_observation_apparently_synced(&never)); + } + + #[test] + fn scan_import_json_sync_appearance_keeps_unsynced_paths() { + let base = std::env::temp_dir().join(format!( + "ode_import_sync_scan_{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + let synced_path = base.join("synced.json"); + let pending_path = base.join("pending.json"); + fs::write( + &synced_path, + r#"{ + "observationId": "s1", + "formType": "t", + "updatedAt": "2026-01-01T00:00:00.000Z", + "syncedAt": "2026-01-02T00:00:00.000Z", + "data": {} + }"#, + ) + .unwrap(); + fs::write( + &pending_path, + r#"{ + "observationId": "p1", + "formType": "t", + "updatedAt": "2026-01-03T00:00:00.000Z", + "syncedAt": "2026-01-02T00:00:00.000Z", + "data": {} + }"#, + ) + .unwrap(); + let result = scan_import_json_sync_appearance(vec![ + synced_path.to_string_lossy().to_string(), + pending_path.to_string_lossy().to_string(), + ]) + .unwrap(); + assert_eq!(result.file_count, 2); + assert_eq!(result.observation_count, 2); + assert_eq!(result.apparently_synced_count, 1); + assert_eq!(result.unsynced_count, 1); + assert_eq!(result.unsynced_paths.len(), 1); + assert!(result.unsynced_paths[0].ends_with("pending.json")); + let _ = fs::remove_dir_all(&base); + } + #[test] fn upsert_observation_from_local_import_persists_extras() { let conn = Connection::open_in_memory().unwrap(); diff --git a/desktop/src/App.test.tsx b/desktop/src/App.test.tsx index 73d7a120b..5eeb1827e 100644 --- a/desktop/src/App.test.tsx +++ b/desktop/src/App.test.tsx @@ -60,6 +60,15 @@ vi.mock('./lib/tauriClient', () => ({ writeWorkspaceAttachment: vi.fn(), copyWorkspaceAttachmentFromPath: vi.fn(), expandImportStagingPaths: vi.fn().mockResolvedValue([]), + parseImportObservationJsonPaths: vi.fn().mockResolvedValue([]), + scanImportJsonSyncAppearance: vi.fn().mockResolvedValue({ + fileCount: 0, + observationCount: 0, + apparentlySyncedCount: 0, + unsyncedCount: 0, + parseErrorCount: 0, + unsyncedPaths: [], + }), readHostTextFile: vi.fn().mockResolvedValue('{}'), readHostTextFilesBatch: vi.fn().mockImplementation((paths: string[]) => Promise.resolve( diff --git a/desktop/src/hooks/useProfileAutoSynkAuth.ts b/desktop/src/hooks/useProfileAutoSynkAuth.ts index bedff3091..e45fcac0c 100644 --- a/desktop/src/hooks/useProfileAutoSynkAuth.ts +++ b/desktop/src/hooks/useProfileAutoSynkAuth.ts @@ -7,6 +7,9 @@ import { /** * Silent sign-in on profile change (refresh token or keyring password), matching Sync page behavior. * Uses {@link recoverActiveProfileAuth} so expired stored tokens are renewed, not reused blindly. + * + * `authBlocked` is true when recovery failed — even if a stale session remains in localStorage — + * so UI must not claim "Authenticated" from session presence alone. */ export function useProfileAutoSynkAuth(activeProfileId: string | undefined) { const authSession = useCustodianStore(selectAuthSessionForActiveProfile); @@ -14,22 +17,37 @@ export function useProfileAutoSynkAuth(activeProfileId: string | undefined) { s => s.recoverActiveProfileAuth, ); const [authBlocked, setAuthBlocked] = useState(false); + /** False until the first recover attempt for the current profile finishes. */ + const [authReady, setAuthReady] = useState(false); const refreshAuth = useCallback(async (): Promise => { + setAuthReady(false); const ok = await recoverActiveProfileAuth(); - setAuthBlocked( - !ok && !selectAuthSessionForActiveProfile(useCustodianStore.getState()), - ); + setAuthBlocked(!ok); + setAuthReady(true); return ok; }, [recoverActiveProfileAuth]); useEffect(() => { - void refreshAuth(); - }, [activeProfileId, refreshAuth]); + let cancelled = false; + setAuthReady(false); + setAuthBlocked(false); + void (async () => { + const ok = await recoverActiveProfileAuth(); + if (cancelled) { + return; + } + setAuthBlocked(!ok); + setAuthReady(true); + })(); + return () => { + cancelled = true; + }; + }, [activeProfileId, recoverActiveProfileAuth]); const ensureAuth = useCallback(async (): Promise => { return refreshAuth(); }, [refreshAuth]); - return { authSession, authBlocked, ensureAuth, refreshAuth }; + return { authSession, authBlocked, authReady, ensureAuth, refreshAuth }; } diff --git a/desktop/src/lib/importSummary.ts b/desktop/src/lib/importSummary.ts index d9d7088cc..1ed6b9246 100644 --- a/desktop/src/lib/importSummary.ts +++ b/desktop/src/lib/importSummary.ts @@ -323,7 +323,9 @@ export async function parseObservationJsonFromPaths( return out; } -const RUST_PARSE_CHUNK = 128; +const RUST_PARSE_CHUNK = 512; +/** Overlapping IPC batches while each batch still parses in parallel on the host. */ +const RUST_PARSE_CONCURRENCY = 3; /** * Read and parse observation JSON via Rust (parallel per chunk). Preserves file order. @@ -336,27 +338,34 @@ export async function parseObservationJsonPathsViaRust( if (total === 0) { return []; } - const out: ParsedObservationFile[] = []; + const chunks: { name: string; nativePath: string }[][] = []; for (let i = 0; i < total; i += RUST_PARSE_CHUNK) { - const chunk = items.slice(i, i + RUST_PARSE_CHUNK); - const paths = chunk.map(c => c.nativePath); - const rows = await tauriClient.parseImportObservationJsonPaths(paths); - if (rows.length !== chunk.length) { - throw new Error( - `parseImportObservationJsonPaths returned ${rows.length} rows, expected ${chunk.length}`, - ); - } - for (let j = 0; j < chunk.length; j++) { - const r = rows[j]!; - out.push({ + chunks.push(items.slice(i, i + RUST_PARSE_CHUNK)); + } + + let done = 0; + const chunkResults = await mapPool( + chunks, + RUST_PARSE_CONCURRENCY, + async chunk => { + const paths = chunk.map(c => c.nativePath); + const rows = await tauriClient.parseImportObservationJsonPaths(paths); + if (rows.length !== chunk.length) { + throw new Error( + `parseImportObservationJsonPaths returned ${rows.length} rows, expected ${chunk.length}`, + ); + } + done += chunk.length; + onBatchProgress?.(Math.min(done, total), total); + return rows.map(r => ({ fileName: r.fileName, observations: r.observations, error: r.error, - }); - } - onBatchProgress?.(Math.min(i + chunk.length, total), total); - } - return out; + })); + }, + ); + + return chunkResults.flat(); } export function flattenObservations( diff --git a/desktop/src/lib/tauriClient.ts b/desktop/src/lib/tauriClient.ts index 97787a645..6a9d63ee2 100644 --- a/desktop/src/lib/tauriClient.ts +++ b/desktop/src/lib/tauriClient.ts @@ -9,6 +9,7 @@ import type { CredentialSetResult, ImportResult, ImportStagingScanEntry, + ImportSyncAppearanceScanResult, ParsedImportFileResult, AttachmentCopyBatchResult, HostTextReadResult, @@ -155,6 +156,11 @@ export const tauriClient = { 'parse_import_observation_json_paths', { paths }, ), + scanImportJsonSyncAppearance: (paths: string[]) => + invokeSafe( + 'scan_import_json_sync_appearance', + { paths }, + ), copyWorkspaceAttachmentsBatch: ( items: { sourcePath: string; attachmentId: string }[], ) => diff --git a/desktop/src/pages/ImportPage.tsx b/desktop/src/pages/ImportPage.tsx index 4a77326d0..9d1cb53bd 100644 --- a/desktop/src/pages/ImportPage.tsx +++ b/desktop/src/pages/ImportPage.tsx @@ -33,6 +33,9 @@ import { const MAX_INDIVIDUAL_FILES = 20; +/** Cap DOM rows — rendering tens of thousands of staging rows freezes the UI. */ +const STAGING_LIST_PREVIEW = 50; + /** Host copy batch size — keeps IPC payloads and UI updates manageable. */ const ATTACHMENT_COPY_CHUNK_SIZE = 400; @@ -216,6 +219,9 @@ export function ImportPage() { const busy = importActivity !== null; const addScanEntries = useImportStagingStore(s => s.addScanEntries); + const retainStagedJsonPaths = useImportStagingStore( + s => s.retainStagedJsonPaths, + ); const removeStagedJson = useImportStagingStore(s => s.removeStagedJson); const removeStagedAttachment = useImportStagingStore( s => s.removeStagedAttachment, @@ -226,6 +232,10 @@ export function ImportPage() { const setError = useImportStagingStore(s => s.setError); const setImportActivity = useImportStagingStore(s => s.setImportActivity); + /** After staging skip dialog: silently drop remaining synced rows at import. */ + const [preferSkipSynced, setPreferSkipSynced] = useState(false); + const [skippedSyncedAtStaging, setSkippedSyncedAtStaging] = useState(0); + const stagingSummary = useMemo(() => { const jsonCount = stagedJson.length; const attCount = stagedAttachments.length; @@ -235,6 +245,69 @@ export function ImportPage() { return { jsonCount, attCount, bytes }; }, [stagedJson, stagedAttachments]); + /** + * After JSON lands in staging, scan Formulus `syncedAt` metadata and optionally + * drop already-synced files before the heavy parse/validate pass. + */ + const offerSkipAlreadySynced = useCallback(async () => { + const jsonPaths = useImportStagingStore + .getState() + .stagedJson.map(s => s.nativePath); + if (jsonPaths.length === 0) { + return; + } + setImportActivity({ + statusText: `Checking sync status (${jsonPaths.length} JSON files)…`, + }); + try { + const scan = await tauriClient.scanImportJsonSyncAppearance(jsonPaths); + if (scan.apparentlySyncedCount <= 0) { + return; + } + const skipSynced = await confirm( + `${scan.observationCount} observations were found. ${scan.apparentlySyncedCount} already appear to be synced — skip those and import only the ${scan.unsyncedCount} new observations?`, + { + title: 'Skip already-synced observations?', + kind: 'info', + okLabel: 'Skip synced', + cancelLabel: 'Keep all', + }, + ); + if (skipSynced) { + retainStagedJsonPaths(scan.unsyncedPaths); + setPreferSkipSynced(true); + setSkippedSyncedAtStaging(scan.apparentlySyncedCount); + setMessage( + `Staging updated — kept ${scan.unsyncedPaths.length} JSON file(s); skipped ${scan.apparentlySyncedCount} already-synced observation(s).`, + ); + } else { + setPreferSkipSynced(false); + setSkippedSyncedAtStaging(0); + } + } catch (e) { + setError( + messageFromUnknown(e, 'Could not check already-synced observations'), + ); + } + }, [retainStagedJsonPaths, setError, setImportActivity, setMessage]); + + const stageExpandedEntries = useCallback( + async ( + expanded: Awaited< + ReturnType + >, + ) => { + if (!expanded.length) { + return; + } + addScanEntries(expanded); + if (expanded.some(e => e.isJson)) { + await offerSkipAlreadySynced(); + } + }, + [addScanEntries, offerSkipAlreadySynced], + ); + useEffect(() => { if (!isTauri()) { return undefined; @@ -267,9 +340,7 @@ export function ImportPage() { paths, MAX_INDIVIDUAL_FILES, ); - if (expanded.length) { - addScanEntries(expanded); - } + await stageExpandedEntries(expanded); } catch (e) { setError( messageFromUnknown(e, 'Could not stage dropped files'), @@ -288,7 +359,7 @@ export function ImportPage() { alive = false; unlisten?.(); }; - }, [addScanEntries, setError, setImportActivity]); + }, [setError, setImportActivity, stageExpandedEntries]); const pickImportFolder = useCallback(async () => { try { @@ -307,15 +378,13 @@ export function ImportPage() { selected, null, ); - if (expanded.length) { - addScanEntries(expanded); - } + await stageExpandedEntries(expanded); } catch (e) { setError(messageFromUnknown(e, 'Folder selection failed')); } finally { setImportActivity(null); } - }, [addScanEntries, setError, setImportActivity]); + }, [setError, setImportActivity, stageExpandedEntries]); const pickJsonFiles = useCallback(async () => { try { @@ -333,15 +402,13 @@ export function ImportPage() { paths, MAX_INDIVIDUAL_FILES, ); - if (expanded.length) { - addScanEntries(expanded); - } + await stageExpandedEntries(expanded); } catch (e) { setError(messageFromUnknown(e, 'File selection failed')); } finally { setImportActivity(null); } - }, [addScanEntries, setError, setImportActivity]); + }, [setError, setImportActivity, stageExpandedEntries]); const pickAttachmentFiles = useCallback(async () => { try { @@ -422,8 +489,11 @@ export function ImportPage() { parsedFiles: parsed, formSpecsByType, stagedAttachmentBasenames: basenames, - onFileValidated: (fi, tot, name) => - statusCtl.push(`Validating (${fi + 1}/${tot}) ${name}…`), + onFileValidated: (fi, tot, name) => { + if (tot <= 40 || fi === tot - 1 || (fi + 1) % 50 === 0) { + statusCtl.push(`Validating (${fi + 1}/${tot}) ${name}…`); + } + }, }); if (report.issues.length > 0) { @@ -446,24 +516,17 @@ export function ImportPage() { const allObservations = flattenObservations(report.parsedFiles); const syncPartition = partitionImportObservationsBySyncAppearance(allObservations); - let observations = allObservations; - let skippedSyncedCount = 0; - - if (syncPartition.apparentlySynced.length > 0) { - const skipSynced = await confirm( - `${syncPartition.total} observations were found. ${syncPartition.apparentlySynced.length} already appear to be synced — skip those and import only the ${syncPartition.unsynced.length} new observations?`, - { - title: 'Skip already-synced observations?', - kind: 'info', - okLabel: 'Skip synced', - cancelLabel: 'Import all', - }, - ); - if (skipSynced) { - observations = syncPartition.unsynced; - skippedSyncedCount = syncPartition.apparentlySynced.length; - } - } + // Staging already offered skip/keep; when skip was chosen, drop any + // remaining apparently-synced rows (e.g. mixed files) without re-prompting. + const observations = preferSkipSynced + ? syncPartition.unsynced + : allObservations; + const skippedSyncedCount = preferSkipSynced + ? Math.max( + skippedSyncedAtStaging, + syncPartition.apparentlySynced.length, + ) + : 0; if (observations.length === 0) { setMessage( @@ -607,6 +670,8 @@ export function ImportPage() { : ''; setMessage(`${baseMsg}${indexMsg}${attMsg}`); clearStagedFiles(); + setPreferSkipSynced(false); + setSkippedSyncedAtStaging(0); setPreviewReport(null); } catch (e) { setError(messageFromUnknown(e, 'Import failed')); @@ -617,6 +682,8 @@ export function ImportPage() { }, [ stagedJson, stagedAttachments, + preferSkipSynced, + skippedSyncedAtStaging, setMessage, setError, setImportActivity, @@ -689,7 +756,11 @@ export function ImportPage() { type="button" className="linkish" disabled={busy} - onClick={() => clearStagingLists()}> + onClick={() => { + clearStagingLists(); + setPreferSkipSynced(false); + setSkippedSyncedAtStaging(0); + }}> Clear staging ) : null} @@ -700,7 +771,7 @@ export function ImportPage() { {stagedJson.length > 0 ? (

JSON ({stagedJson.length})

- {stagedJson.map(s => ( + {stagedJson.slice(0, STAGING_LIST_PREVIEW).map(s => (
))} + {stagedJson.length > STAGING_LIST_PREVIEW ? ( +

+ … and {stagedJson.length - STAGING_LIST_PREVIEW} more (list + truncated for performance) +

+ ) : null}
) : null} {stagedAttachments.length > 0 ? (

Attachments ({stagedAttachments.length})

- {stagedAttachments.map(s => ( + {stagedAttachments.slice(0, STAGING_LIST_PREVIEW).map(s => (
))} + {stagedAttachments.length > STAGING_LIST_PREVIEW ? ( +

+ … and {stagedAttachments.length - STAGING_LIST_PREVIEW} more + (list truncated for performance) +

+ ) : null}
) : null}
diff --git a/desktop/src/pages/ProfilesPage.tsx b/desktop/src/pages/ProfilesPage.tsx index 0dcc2c495..ad985de1d 100644 --- a/desktop/src/pages/ProfilesPage.tsx +++ b/desktop/src/pages/ProfilesPage.tsx @@ -19,9 +19,9 @@ import { import type { ServerProfile } from '../types/domain'; import { selectActiveProfileState, - selectAuthSessionForActiveProfile, useCustodianStore, } from '../store/useCustodianStore'; +import { useProfileAutoSynkAuth } from '../hooks/useProfileAutoSynkAuth'; import { useProfileDraftGuardStore } from '../store/useProfileDraftGuardStore'; import { confirmDestructiveAction } from '../lib/destructivePolicy'; @@ -89,7 +89,8 @@ export function ProfilesPage() { error, } = useCustodianStore(); const active = useCustodianStore(selectActiveProfileState); - const authSession = useCustodianStore(selectAuthSessionForActiveProfile); + const { authSession, authBlocked, authReady, refreshAuth } = + useProfileAutoSynkAuth(active?.id); const [label, setLabel] = useState(''); const [serverUrl, setServerUrl] = useState(''); @@ -406,6 +407,7 @@ export function ProfilesPage() { username: username.trim(), password: pwd, }); + await refreshAuth(); } catch { // synkLogin reports via store `error` } @@ -415,8 +417,18 @@ export function ProfilesPage() { const derivedDb = ws ? workspaceSqlitePath(ws) : ''; const derivedAttachments = ws ? workspaceAttachmentsDir(ws) : ''; - const authIcon = authSession ? 'verified_user' : 'lock_open'; - const authLabel = authSession ? 'Authenticated' : 'Authenticate'; + const isAuthenticated = authReady && Boolean(authSession) && !authBlocked; + const authChecking = Boolean(active) && !authReady; + const authIcon = isAuthenticated + ? 'verified_user' + : authChecking + ? 'hourglass_empty' + : 'lock_open'; + const authLabel = isAuthenticated + ? 'Authenticated' + : authChecking + ? 'Checking…' + : 'Authenticate'; return (
@@ -524,8 +536,8 @@ export function ProfilesPage() {
- {authBlocked && !authSession ? ( + {authBlocked ? (

Not authenticated. Open Profiles to sign in. diff --git a/desktop/src/pages/WorkbenchCustomAppPage.tsx b/desktop/src/pages/WorkbenchCustomAppPage.tsx index 23ec22038..1cc9405e7 100644 --- a/desktop/src/pages/WorkbenchCustomAppPage.tsx +++ b/desktop/src/pages/WorkbenchCustomAppPage.tsx @@ -25,9 +25,7 @@ import type { AppBundleState } from '../types/domain'; export function WorkbenchCustomAppPage() { const navigate = useNavigate(); const activeProfile = useCustodianStore(selectActiveProfileState); - const { authSession, authBlocked, ensureAuth } = useProfileAutoSynkAuth( - activeProfile?.id, - ); + const { authBlocked, ensureAuth } = useProfileAutoSynkAuth(activeProfile?.id); const recoverActiveProfileAuth = useCustodianStore( s => s.recoverActiveProfileAuth, ); @@ -261,7 +259,7 @@ export function WorkbenchCustomAppPage() { {devError ?

{devError}

: null} {bundleError ?

{bundleError}

: null} {uploadError ?

{uploadError}

: null} - {developerMode && baseUrl && authBlocked && !authSession ? ( + {developerMode && baseUrl && authBlocked ? (

Not authenticated. Open Profiles to sign in. diff --git a/desktop/src/store/useCustodianStore.ts b/desktop/src/store/useCustodianStore.ts index 9671f8647..745544287 100644 --- a/desktop/src/store/useCustodianStore.ts +++ b/desktop/src/store/useCustodianStore.ts @@ -9,6 +9,7 @@ import { } from '../lib/syncTauriEvents'; import { partitionPendingPushObservations } from '../lib/pushAttachmentAudit'; import { getOrCreateClientId, syncGateway } from '../services/synk'; +import { isSyncHttpUnauthorized } from '../services/synk/syncErrors'; import type { AppHealth, AuthSession, @@ -117,7 +118,14 @@ async function reauthenticateActiveProfile( persistAuthMap(merged); set({ authSessionsByProfileId: merged }); return; - } catch { + } catch (refreshError) { + const cred = await tauriClient.credentialGet(id); + const password = cred.password ?? ''; + if (!password.trim()) { + // No password fallback: surface the refresh failure (401 clears session + // in recover; network errors keep the stored refresh token). + throw refreshError; + } // Fall through to password login. } } @@ -139,6 +147,20 @@ async function reauthenticateActiveProfile( set({ authSessionsByProfileId: merged }); } +function clearActiveProfileAuthSession( + set: (partial: Partial) => void, + get: () => CustodianState, +): void { + const id = get().activeProfileId; + if (!id || !get().authSessionsByProfileId[id]) { + return; + } + const auth = { ...get().authSessionsByProfileId }; + delete auth[id]; + persistAuthMap(auth); + set({ authSessionsByProfileId: auth }); +} + async function awaitSyncJobTerminal( jobId: string, set: (partial: Partial) => void, @@ -389,13 +411,13 @@ export const useCustodianStore = create((set, get) => ({ clearExportActivity: () => set({ exportActivity: null }), ensureActiveProfileAuth: async () => { - if (selectAuthSessionForActiveProfile(get())?.token) { - return true; - } try { await reauthenticateActiveProfile(set, get); return true; - } catch { + } catch (e) { + if (isSyncHttpUnauthorized(e)) { + clearActiveProfileAuthSession(set, get); + } return false; } }, @@ -404,7 +426,10 @@ export const useCustodianStore = create((set, get) => ({ try { await reauthenticateActiveProfile(set, get); return true; - } catch { + } catch (e) { + if (isSyncHttpUnauthorized(e)) { + clearActiveProfileAuthSession(set, get); + } return false; } }, diff --git a/desktop/src/store/useImportStagingStore.ts b/desktop/src/store/useImportStagingStore.ts index ed2951cff..796c4308b 100644 --- a/desktop/src/store/useImportStagingStore.ts +++ b/desktop/src/store/useImportStagingStore.ts @@ -18,6 +18,8 @@ interface ImportStagingState { error: string | null; importActivity: { statusText: string } | null; addScanEntries: (entries: ImportStagingScanEntry[]) => void; + /** Keep only JSON files whose absolute path is in `nativePaths`. */ + retainStagedJsonPaths: (nativePaths: readonly string[]) => void; removeStagedJson: (nativePath: string) => void; removeStagedAttachment: (nativePath: string) => void; /** Clears staged JSON + attachment lists (keeps message/error). */ @@ -86,6 +88,14 @@ export const useImportStagingStore = create(set => ({ }; }), + retainStagedJsonPaths: nativePaths => + set(s => { + const keep = new Set(nativePaths); + return { + stagedJson: s.stagedJson.filter(f => keep.has(f.nativePath)), + }; + }), + removeStagedJson: nativePath => set(s => ({ stagedJson: s.stagedJson.filter(f => f.nativePath !== nativePath), diff --git a/desktop/src/types/domain.ts b/desktop/src/types/domain.ts index 488dbef3a..1f008823a 100644 --- a/desktop/src/types/domain.ts +++ b/desktop/src/types/domain.ts @@ -84,6 +84,17 @@ export interface ParsedImportFileResult { error?: string; } +/** Lightweight sync-appearance scan over staged Formulus export JSON. */ +export interface ImportSyncAppearanceScanResult { + fileCount: number; + observationCount: number; + apparentlySyncedCount: number; + unsyncedCount: number; + parseErrorCount: number; + /** Absolute paths to retain when skipping already-synced observations. */ + unsyncedPaths: string[]; +} + export interface AttachmentCopyBatchResult { copied: number; failed: number; From 562bb322ee57032ff89a741609dd7d0cd9fd1c3d Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Sun, 16 Aug 2026 12:15:01 +0200 Subject: [PATCH 29/32] feat(desktop): Improved import and sync UX --- desktop/docs/UI_FEEDBACK.md | 6 +- desktop/docs/screens/import.md | 18 +- desktop/docs/screens/sync.md | 2 + desktop/src-tauri/Cargo.lock | 182 +++++- desktop/src-tauri/Cargo.toml | 1 + .../src/import_validate/attachments.rs | 483 ++++++++++++++++ desktop/src-tauri/src/import_validate/mod.rs | 532 ++++++++++++++++++ desktop/src-tauri/src/lib.rs | 167 +++++- desktop/src-tauri/src/observation_index.rs | 180 ++++-- desktop/src/App.css | 10 + desktop/src/App.test.tsx | 9 + desktop/src/App.tsx | 56 +- .../ForcePushMissingAttachmentsDialog.tsx | 4 +- .../src/components/ResetLocalDataDialog.tsx | 84 +++ desktop/src/lib/importSummary.ts | 36 ++ desktop/src/lib/pushAttachmentAudit.test.ts | 43 ++ desktop/src/lib/pushAttachmentAudit.ts | 48 ++ desktop/src/lib/tauriClient.ts | 23 +- desktop/src/pages/ImportPage.tsx | 411 ++++++++------ desktop/src/pages/SyncPage.tsx | 26 +- desktop/src/store/useCustodianStore.ts | 91 +-- desktop/src/types/domain.ts | 21 + 22 files changed, 2126 insertions(+), 307 deletions(-) create mode 100644 desktop/src-tauri/src/import_validate/attachments.rs create mode 100644 desktop/src-tauri/src/import_validate/mod.rs create mode 100644 desktop/src/components/ResetLocalDataDialog.tsx create mode 100644 desktop/src/lib/pushAttachmentAudit.test.ts diff --git a/desktop/docs/UI_FEEDBACK.md b/desktop/docs/UI_FEEDBACK.md index 56269352b..5842451b3 100644 --- a/desktop/docs/UI_FEEDBACK.md +++ b/desktop/docs/UI_FEEDBACK.md @@ -10,6 +10,7 @@ Single source of truth for how the shell surfaces status, errors, and confirmati - Shows spinner + status text; optional determinate progress bar when `bundleActivity.total > 0`. - Dismissible but reappears on next activity. - Bundle apply: Rust emits `bundle/apply-progress` and `bundle/index-rebuild`; `bundleActivity` in `useCustodianStore` drives the banner. +- Sync completion: keep the banner for short **summary** lines when a downloadable `syncDetailReport` is attached (e.g. missing-attachment push details). Offer **Save report** instead of dumping long ID lists into the banner. - Do not use for quick actions (save, auth, dev mirror refresh). ## Toasts (bottom-right stack) @@ -32,7 +33,7 @@ Single source of truth for how the shell surfaces status, errors, and confirmati ## Native confirm (Tauri) -**Use for:** destructive actions, import-with-issues, skip already-synced import rows, closing unsaved observation tabs. +**Use for:** destructive actions, skip already-synced import rows, closing unsaved observation tabs. - `confirmDestructiveAction()` for destructive flows (clear, profile-scoped wording). - `confirm()` from `@tauri-apps/plugin-dialog` for save-anyway / import-anyway / tab discard (Save / Don't save / Cancel via separate flows). @@ -41,7 +42,8 @@ Single source of truth for how the shell surfaces status, errors, and confirmati - `window.alert` — replace with toast or inline notice. - `window.confirm` — replace with Tauri `confirm`. -- Duplicate sync success in both banner and toast — prefer toast for short messages; keep banner only for multi-line sync summaries. +- Dumping long per-observation lists into the sync banner — use a short highlight + **Save report** (`syncDetailReport`). +- Duplicate sync success in both banner and toast — prefer toast for short messages; keep banner when a detail report is available or the message is multi-line. ## Dev mode bar diff --git a/desktop/docs/screens/import.md b/desktop/docs/screens/import.md index b549330ec..183abc3c6 100644 --- a/desktop/docs/screens/import.md +++ b/desktop/docs/screens/import.md @@ -11,10 +11,10 @@ Bring external JSON observation files into the active profile’s local reposito ## What to include - Drag-and-drop and multi-file picker for `.json` -- Pre-flight summary (counts, form types, attachment hints) -- Per-file parse/normalization issues -- Import action and clear/reset -- Optional skip of Formulus-export rows that already appear synced (`syncedAt` ≥ `updatedAt`) — confirm dialog before write +- **Validate** then review results in-page (no “import anyway?” popup) +- Import action on the validation results panel (available with or without errors) +- Clear label when validation finds no errors +- Optional skip of Formulus-export rows that already appear synced (`syncedAt` ≥ `updatedAt`) — confirm dialog at staging time ## What to exclude @@ -24,7 +24,7 @@ Bring external JSON observation files into the active profile’s local reposito ## Key actions -- Stage files, review summary, import, clear +- Stage files → **Validate** → review results → **Import into local store** (or clear) - When import JSON carries Formulus `syncedAt` metadata, confirm whether to skip already-synced observations and write only unsynced ones ## Data dependencies @@ -35,12 +35,10 @@ Bring external JSON observation files into the active profile’s local reposito ## Observation indexes -When the active app bundle declares `observationIndexes` in `app.config.json`: - -1. Import writes observations in batches (default 2000 rows per IPC call); intermediate batches skip index work. -2. After the final batch commits, Rust schedules **one** coalesced background full index rebuild (`bundle/index-rebuild` progress events). Overlapping rebuild requests while one is running are merged into a single follow-up pass. -3. Sync pull uses incremental indexing per page instead (no full rebuild after each pull). +When the active app bundle declares `observationIndexes` in `app.config.json`, local file import updates indexes **incrementally** for the written rows (same as sync pull). A full background rebuild is reserved for bundle apply / empty index / explicit rebuild — not for import. ## Large Formulus exports After staging JSON (folder / drop / Add JSON), Desktop runs a lightweight host scan of `syncedAt` / `updatedAt` and may offer to drop already-synced files **before** full parse + schema validation. Staging lists truncate after ~50 rows so tens of thousands of files do not freeze the UI. + +Import parse + JSON Schema validation + attachment reference checks run in **Rust (Rayon)** in one pass (`parse_and_validate_import_json_paths`), with form schemas loaded once from the active bundle. diff --git a/desktop/docs/screens/sync.md b/desktop/docs/screens/sync.md index f84f96ebf..f7a829938 100644 --- a/desktop/docs/screens/sync.md +++ b/desktop/docs/screens/sync.md @@ -18,6 +18,7 @@ Operational console to authenticate with Synkronus and exchange data: pull into - Pull and push actions - Recent operation log (per-session, on this screen) - Store-level `syncMessage` / `error` after operations +- Danger zone: reset local data (full, or **pending-only** so synced rows + sync offsets stay for a continued pull), re-create index, reset server repository ## What to exclude @@ -28,6 +29,7 @@ Operational console to authenticate with Synkronus and exchange data: pull into ## Key actions - Login, pull, push +- Reset local data (optional: pending observations only) ## Data dependencies diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 7ce2189c6..a7fb8b707 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -18,6 +18,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -129,7 +130,7 @@ checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] @@ -498,6 +499,12 @@ dependencies = [ "piper", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "brotli" version = "8.0.2" @@ -525,6 +532,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.0" @@ -1153,6 +1166,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "embed-resource" version = "3.0.8" @@ -1269,6 +1291,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1320,6 +1353,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fluent-uri" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1389,6 +1433,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "futf" version = "0.1.5" @@ -1406,6 +1460,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -2367,6 +2422,33 @@ dependencies = [ "serde_json", ] +[[package]] +name = "jsonschema" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1b46a0365a611fbf1d2143104dcf910aada96fafd295bab16c60b802bf6fa1d" +dependencies = [ + "ahash", + "base64 0.22.1", + "bytecount", + "email_address", + "fancy-regex", + "fraction", + "idna", + "itoa", + "num-cmp", + "num-traits", + "once_cell", + "percent-encoding", + "referencing", + "regex", + "regex-syntax", + "reqwest 0.12.28", + "serde", + "serde_json", + "uuid-simd", +] + [[package]] name = "keyboard-types" version = "0.7.0" @@ -2405,6 +2487,12 @@ dependencies = [ "selectors 0.24.0", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -2756,6 +2844,30 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint 0.4.8", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.5.1" @@ -2766,6 +2878,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + [[package]] name = "num-complex" version = "0.4.6" @@ -2790,6 +2908,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2952,6 +3091,7 @@ dependencies = [ "arrow", "chrono", "futures-util", + "jsonschema", "keyring", "parquet", "rayon", @@ -3050,6 +3190,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "pango" version = "0.18.3" @@ -3122,7 +3268,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.17.1", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "seq-macro", @@ -3785,6 +3931,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "referencing" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8eff4fa778b5c2a57e85c5f2fe3a709c52f0e60d23146e2151cbef5893f420e" +dependencies = [ + "ahash", + "fluent-uri", + "once_cell", + "parking_lot", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" version = "1.12.3" @@ -3823,6 +3983,7 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", @@ -5554,6 +5715,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "uuid", + "vsimd", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -5572,6 +5744,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vswhom" version = "0.1.0" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index d21d75122..625ba7b35 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -38,4 +38,5 @@ urlencoding = "2" tokio = { version = "1", features = ["sync", "time", "macros"] } arrow = { version = "59", default-features = false, features = ["chrono-tz"] } parquet = { version = "59", default-features = false, features = ["arrow", "snap"] } +jsonschema = "0.30" diff --git a/desktop/src-tauri/src/import_validate/attachments.rs b/desktop/src-tauri/src/import_validate/attachments.rs new file mode 100644 index 000000000..d197acc5c --- /dev/null +++ b/desktop/src-tauri/src/import_validate/attachments.rs @@ -0,0 +1,483 @@ +//! Attachment basename extraction for import validation (TS parity). + +use std::collections::HashSet; + +use serde_json::Value; + +const ATTACHMENT_SCHEMA_FORMATS: &[&str] = &["photo", "select_file", "signature", "audio", "video"]; + +const ATTACHMENT_BASENAME_EXT: &[&str] = &[ + ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".heic", ".tif", ".tiff", ".pdf", ".doc", + ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".csv", ".txt", ".mp3", ".mp4", ".m4a", ".wav", + ".aac", ".flac", ".webm", ".mov", ".mkv", ".svg", +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum SchemaPathSegment { + Key(String), + Each, +} + +fn is_object(v: &Value) -> bool { + v.is_object() +} + +fn resolve_json_pointer<'a>(root: &'a Value, ref_str: &str) -> Option<&'a Value> { + if !ref_str.starts_with("#/") { + return None; + } + let mut cur = root; + for raw in ref_str[2..].split('/') { + let p = raw.replace("~1", "/").replace("~0", "~"); + cur = cur.as_object()?.get(&p)?; + } + Some(cur) +} + +fn has_attachment_format(schema: &Value) -> bool { + schema + .get("format") + .and_then(|f| f.as_str()) + .is_some_and(|fmt| ATTACHMENT_SCHEMA_FORMATS.contains(&fmt)) +} + +fn path_sig(path: &[SchemaPathSegment]) -> String { + path.iter() + .map(|s| match s { + SchemaPathSegment::Key(k) => k.as_str(), + SchemaPathSegment::Each => "*", + }) + .collect::>() + .join("\0") +} + +/// Walk JSON Schema (draft-07 style) and collect property paths with attachment formats. +fn collect_attachment_paths_from_schema(schema_root: &Value) -> Vec> { + let mut out = Vec::new(); + let mut seen = HashSet::new(); + let mut stack: HashSet<*const Value> = HashSet::new(); + + fn visit( + schema_root: &Value, + schema: &Value, + path_prefix: &[SchemaPathSegment], + stack: &mut HashSet<*const Value>, + seen: &mut HashSet, + out: &mut Vec>, + ) { + if !schema.is_object() { + return; + } + let ptr = schema as *const Value; + if !stack.insert(ptr) { + return; + } + + if let Some(Value::String(r)) = schema.get("$ref") { + if let Some(resolved) = resolve_json_pointer(schema_root, r) { + visit(schema_root, resolved, path_prefix, stack, seen, out); + } + stack.remove(&ptr); + return; + } + + for combiner in ["allOf", "anyOf", "oneOf"] { + if let Some(Value::Array(arr)) = schema.get(combiner) { + for branch in arr { + visit(schema_root, branch, path_prefix, stack, seen, out); + } + } + } + if let Some(then_schema) = schema.get("then") + && then_schema.is_object() + { + visit(schema_root, then_schema, path_prefix, stack, seen, out); + } + if let Some(else_schema) = schema.get("else") + && else_schema.is_object() + { + visit(schema_root, else_schema, path_prefix, stack, seen, out); + } + + if has_attachment_format(schema) && !path_prefix.is_empty() { + let sig = path_sig(path_prefix); + if seen.insert(sig) { + out.push(path_prefix.to_vec()); + } + stack.remove(&ptr); + return; + } + + if let Some(Value::Object(props)) = schema.get("properties") { + for (key, sub) in props { + if !sub.is_object() { + continue; + } + let mut next_path = path_prefix.to_vec(); + next_path.push(SchemaPathSegment::Key(key.clone())); + + if has_attachment_format(sub) { + let sig = path_sig(&next_path); + if seen.insert(sig) { + out.push(next_path); + } + } else { + let is_array = match sub.get("type") { + Some(Value::String(t)) => t == "array", + Some(Value::Array(arr)) => arr.iter().any(|x| x.as_str() == Some("array")), + _ => false, + }; + if is_array { + if let Some(items) = sub.get("items") + && items.is_object() + { + if has_attachment_format(items) { + let mut segs = next_path; + segs.push(SchemaPathSegment::Each); + let sig = path_sig(&segs); + if seen.insert(sig) { + out.push(segs); + } + } else { + let mut segs = next_path; + segs.push(SchemaPathSegment::Each); + visit(schema_root, items, &segs, stack, seen, out); + } + } + } else { + visit(schema_root, sub, &next_path, stack, seen, out); + } + } + } + } + + match schema.get("additionalProperties") { + Some(Value::Bool(true)) => { + visit( + schema_root, + &Value::Object(Default::default()), + path_prefix, + stack, + seen, + out, + ); + } + Some(addl) if addl.is_object() => { + visit(schema_root, addl, path_prefix, stack, seen, out); + } + _ => {} + } + + stack.remove(&ptr); + } + + visit( + schema_root, + schema_root, + &[], + &mut stack, + &mut seen, + &mut out, + ); + out +} + +fn basename_only(raw: &str) -> String { + let t = raw.trim().replace('\\', "/"); + t.rsplit('/').next().unwrap_or("").trim().to_string() +} + +fn has_attachment_ext(basename: &str) -> bool { + let lower = basename.to_lowercase(); + ATTACHMENT_BASENAME_EXT + .iter() + .any(|ext| lower.ends_with(ext)) +} + +fn string_looks_like_attachment_ref(raw: &str) -> bool { + let t = raw.trim(); + if t.is_empty() || t == "*" { + return false; + } + if regex_is_mime(t) { + return false; + } + let b = basename_only(t); + if b.is_empty() || b == "*" || b.contains("..") { + return false; + } + if has_attachment_ext(&b) { + return true; + } + if regex_is_uuid(&b) { + return true; + } + if b.len() >= 12 + && b.chars().any(|c| c == '/' || c == '_' || c == '-') + && t.replace('\\', "/") + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '/' | '-')) + { + return true; + } + false +} + +fn regex_is_mime(t: &str) -> bool { + // image/jpeg etc. + let bytes = t.as_bytes(); + if bytes.is_empty() || !bytes[0].is_ascii_alphabetic() { + return false; + } + let Some(slash) = t.find('/') else { + return false; + }; + if slash == 0 || slash + 1 >= t.len() { + return false; + } + t[..slash] + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '+' | '-')) + && t[slash + 1..] + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '+' | '-' | '/')) +} + +fn regex_is_uuid(b: &str) -> bool { + if b.len() != 36 { + return false; + } + let bytes = b.as_bytes(); + for (i, ch) in bytes.iter().enumerate() { + match i { + 8 | 13 | 18 | 23 => { + if *ch != b'-' { + return false; + } + } + _ => { + if !ch.is_ascii_hexdigit() { + return false; + } + } + } + } + true +} + +fn extract_attachment_names_from_field_value(value: &Value) -> Vec { + let mut names = Vec::new(); + + fn walk(v: &Value, depth: usize, names: &mut Vec) { + if depth > 16 { + return; + } + match v { + Value::Null => {} + Value::String(s) => { + if string_looks_like_attachment_ref(s) { + let b = basename_only(s); + if !b.is_empty() && !b.contains("..") { + names.push(b); + } + } + } + Value::Array(arr) => { + for el in arr { + walk(el, depth + 1, names); + } + } + Value::Object(obj) => { + let id = obj + .get("attachmentId") + .and_then(|x| x.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + .or_else(|| { + obj.get("attachment_id") + .and_then(|x| x.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + }); + if let Some(id) = id { + names.push(basename_only(id)); + } + if let Some(fn_) = obj.get("filename").and_then(|x| x.as_str()) + && !fn_.trim().is_empty() + { + let b = basename_only(fn_); + if !b.is_empty() && !b.contains("..") { + names.push(b); + } + } + for (k, val) in obj { + if k == "filename" || k == "attachmentId" || k == "attachment_id" { + continue; + } + walk(val, depth + 1, names); + } + } + _ => {} + } + } + + walk(value, 0, &mut names); + let mut uniq = HashSet::new(); + names + .into_iter() + .filter(|n| !n.is_empty() && uniq.insert(n.clone())) + .collect() +} + +fn values_at_schema_paths(data: &Value, paths: &[Vec]) -> Vec { + let mut values = Vec::new(); + + fn follow( + current: &Value, + segments: &[SchemaPathSegment], + idx: usize, + values: &mut Vec, + ) { + if idx >= segments.len() { + values.push(current.clone()); + return; + } + match &segments[idx] { + SchemaPathSegment::Each => { + if let Value::Array(arr) = current { + for el in arr { + follow(el, segments, idx + 1, values); + } + } + } + SchemaPathSegment::Key(key) => { + if let Value::Object(obj) = current + && let Some(next) = obj.get(key) + { + follow(next, segments, idx + 1, values); + } + } + } + } + + if !is_object(data) { + return values; + } + for p in paths { + follow(data, p, 0, &mut values); + } + values +} + +pub fn referenced_attachment_names_from_schema_and_data( + form_schema: &Value, + data: &Value, +) -> HashSet { + let paths = collect_attachment_paths_from_schema(form_schema); + let mut names = HashSet::new(); + for v in values_at_schema_paths(data, &paths) { + for n in extract_attachment_names_from_field_value(&v) { + names.insert(n); + } + } + names +} + +pub fn referenced_attachment_names_heuristic(data: &Value) -> HashSet { + let mut names = HashSet::new(); + + fn walk(v: &Value, depth: usize, names: &mut HashSet) { + if depth > 14 { + return; + } + match v { + Value::Array(arr) => { + for el in arr { + walk(el, depth + 1, names); + } + } + Value::Object(obj) => { + for (k, val) in obj { + let kl = k.to_lowercase(); + if (k == "attachmentId" || kl == "attachment_id") + && let Some(s) = val.as_str() + && !s.trim().is_empty() + { + names.insert(basename_only(s)); + } + if kl == "attachments" + && let Value::Array(arr) = val + { + for el in arr { + if let Value::Object(el_obj) = el { + let id = el_obj + .get("attachmentId") + .and_then(|x| x.as_str()) + .or_else(|| { + el_obj.get("attachment_id").and_then(|x| x.as_str()) + }) + .or_else(|| el_obj.get("id").and_then(|x| x.as_str())); + if let Some(id) = id + && !id.trim().is_empty() + { + names.insert(basename_only(id)); + } + if let Some(fn_) = el_obj.get("filename").and_then(|x| x.as_str()) + && !fn_.trim().is_empty() + { + names.insert(basename_only(fn_)); + } + } + } + } + walk(val, depth + 1, names); + } + } + _ => {} + } + } + + walk(data, 0, &mut names); + names +} + +/// Schema paths + heuristic (same union as TS `referencedNamesForObservation`). +pub fn referenced_attachment_names_for_observation( + form_schema: Option<&Value>, + data: &Value, +) -> HashSet { + let mut names = HashSet::new(); + if let Some(schema) = form_schema { + names.extend(referenced_attachment_names_from_schema_and_data( + schema, data, + )); + } + names.extend(referenced_attachment_names_heuristic(data)); + names +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn schema_photo_path_extracts_filename() { + let schema = json!({ + "type": "object", + "properties": { + "pic": { "type": "object", "format": "photo" } + } + }); + let data = json!({ "pic": { "filename": "a.jpg" } }); + let names = referenced_attachment_names_from_schema_and_data(&schema, &data); + assert!(names.contains("a.jpg")); + } + + #[test] + fn heuristic_finds_attachment_id() { + let data = json!({ "x": { "attachment_id": "uuid-here-12" } }); + let names = referenced_attachment_names_heuristic(&data); + assert!(names.contains("uuid-here-12")); + } +} diff --git a/desktop/src-tauri/src/import_validate/mod.rs b/desktop/src-tauri/src/import_validate/mod.rs new file mode 100644 index 000000000..d25fdc736 --- /dev/null +++ b/desktop/src-tauri/src/import_validate/mod.rs @@ -0,0 +1,532 @@ +//! Parallel import parse + JSON Schema validation + attachment reference checks. +//! +//! Mirrors Desktop TS `importValidation.ts` / `attachmentReferenceExtraction.ts` closely enough +//! for import preflight (AJV with `strict: false` and `validateFormats: false`). + +mod attachments; + +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::sync::Arc; + +use jsonschema::{Draft, Validator}; +use rayon::prelude::*; +use serde::Serialize; +use serde_json::Value; + +use crate::{ApiObservation, ParsedImportFileResult, parse_import_json_file}; + +pub use attachments::referenced_attachment_names_for_observation; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportIssue { + pub severity: &'static str, + pub code: String, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub observation_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub form_type: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportValidateBatchResult { + pub files: Vec, + pub issues: Vec, + pub observation_count: usize, + pub form_type_count: usize, + pub referenced_attachment_names: Vec, + pub missing_attachment_names: Vec, + pub orphan_attachment_names: Vec, +} + +enum SchemaCompileState { + Ok(Arc), + CompileError(String), + Missing, +} + +/// Build reusable validators once per form type (AJV-like: formats not enforced). +fn compile_validators( + form_types: &HashSet, + schemas_by_form_type: &HashMap, +) -> HashMap { + let mut out = HashMap::new(); + for ft in form_types { + match schemas_by_form_type.get(ft) { + None => { + out.insert(ft.clone(), SchemaCompileState::Missing); + } + Some(schema) => match build_validator(schema) { + Ok(v) => { + out.insert(ft.clone(), SchemaCompileState::Ok(Arc::new(v))); + } + Err(e) => { + out.insert(ft.clone(), SchemaCompileState::CompileError(e)); + } + }, + } + } + out +} + +fn build_validator(schema: &Value) -> Result { + jsonschema::options() + .with_draft(Draft::Draft7) + .should_validate_formats(false) + .build(schema) + .map_err(|e| e.to_string()) +} + +fn normalize_basename(s: &str) -> String { + s.trim().to_lowercase() +} + +fn schema_error_message(instance_path: &str, error: &str) -> String { + let path = if instance_path.is_empty() { + "(root)" + } else { + instance_path + }; + format!("{path}: {error}") +} + +struct FileValidateOutcome { + file: ParsedImportFileResult, + issues: Vec, + referenced: HashSet, + form_types: HashSet, + observation_count: usize, +} + +fn push_observation_issues( + file_name: &str, + obs: &ApiObservation, + validators: &HashMap, + schemas_by_form_type: &HashMap, + issues: &mut Vec, + referenced: &mut HashSet, + form_types: &mut HashSet, +) { + let ft = obs + .form_type + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + + let schema = ft + .as_ref() + .and_then(|t| schemas_by_form_type.get(t.as_str())); + + match &ft { + None => { + issues.push(ImportIssue { + severity: "warning", + code: "missing_form_type".to_string(), + message: format!( + "Observation {} has no formType; schema validation was skipped (attachment checks use heuristics only).", + obs.observation_id + ), + file_name: Some(file_name.to_string()), + observation_id: Some(obs.observation_id.clone()), + form_type: None, + }); + } + Some(form_type) => { + form_types.insert(form_type.clone()); + match validators.get(form_type) { + Some(SchemaCompileState::Missing) | None => {} + Some(SchemaCompileState::CompileError(_)) => {} + Some(SchemaCompileState::Ok(validator)) => { + for error in validator.iter_errors(&obs.data) { + let path = error.instance_path.to_string(); + issues.push(ImportIssue { + severity: "error", + code: "schema_validation".to_string(), + message: format!( + "{}: {}", + obs.observation_id, + schema_error_message(&path, &error.to_string()) + ), + file_name: Some(file_name.to_string()), + observation_id: Some(obs.observation_id.clone()), + form_type: Some(form_type.clone()), + }); + } + } + } + } + } + + for name in referenced_attachment_names_for_observation(schema, &obs.data) { + referenced.insert(name); + } +} + +/// Parse + validate import JSON paths in parallel against preloaded form schemas. +pub fn parse_and_validate_paths( + paths: Vec, + schemas_by_form_type: &HashMap, + staged_attachment_basenames: &[String], +) -> ImportValidateBatchResult { + if paths.is_empty() { + return ImportValidateBatchResult { + files: Vec::new(), + issues: Vec::new(), + observation_count: 0, + form_type_count: 0, + referenced_attachment_names: Vec::new(), + missing_attachment_names: Vec::new(), + orphan_attachment_names: Vec::new(), + }; + } + + // Pass 1: parallel parse (ordered). + let mut parsed_index: Vec<(usize, ParsedImportFileResult)> = paths + .par_iter() + .enumerate() + .map(|(i, raw)| { + let p = Path::new(raw.trim()); + (i, parse_import_json_file(p)) + }) + .collect(); + parsed_index.sort_by_key(|(i, _)| *i); + + let mut form_types = HashSet::new(); + for (_, file) in &parsed_index { + if file.error.is_some() { + continue; + } + for obs in &file.observations { + if let Some(ft) = obs + .form_type + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + { + form_types.insert(ft.to_string()); + } + } + } + + let validators = compile_validators(&form_types, schemas_by_form_type); + + // Pass 2: parallel schema + attachment checks on already-parsed files. + let mut outcomes: Vec<(usize, FileValidateOutcome)> = parsed_index + .into_par_iter() + .map(|(i, file)| { + let mut issues = Vec::new(); + let mut referenced = HashSet::new(); + let mut file_form_types = HashSet::new(); + let mut observation_count = 0usize; + + if let Some(err) = &file.error { + issues.push(ImportIssue { + severity: "error", + code: "parse_file".to_string(), + message: format!("{}: {err}", file.file_name), + file_name: Some(file.file_name.clone()), + observation_id: None, + form_type: None, + }); + return ( + i, + FileValidateOutcome { + file, + issues, + referenced, + form_types: file_form_types, + observation_count, + }, + ); + } + + for obs in &file.observations { + observation_count += 1; + push_observation_issues( + &file.file_name, + obs, + &validators, + schemas_by_form_type, + &mut issues, + &mut referenced, + &mut file_form_types, + ); + } + + ( + i, + FileValidateOutcome { + file, + issues, + referenced, + form_types: file_form_types, + observation_count, + }, + ) + }) + .collect(); + outcomes.sort_by_key(|(i, _)| *i); + + let mut files = Vec::with_capacity(outcomes.len()); + let mut issues = Vec::new(); + let mut all_referenced = HashSet::new(); + let mut all_form_types = HashSet::new(); + let mut observation_count = 0usize; + + for (_, outcome) in outcomes { + observation_count += outcome.observation_count; + all_form_types.extend(outcome.form_types); + all_referenced.extend(outcome.referenced); + issues.extend(outcome.issues); + files.push(outcome.file); + } + + let mut missing_schema_types: Vec = all_form_types + .iter() + .filter(|ft| { + matches!( + validators.get(*ft), + Some(SchemaCompileState::Missing) | None + ) + }) + .cloned() + .collect(); + missing_schema_types.sort(); + for ft in missing_schema_types { + issues.push(ImportIssue { + severity: "error", + code: "missing_form_schema".to_string(), + message: format!("No form schema in the active app bundle for form type \"{ft}\"."), + file_name: None, + observation_id: None, + form_type: Some(ft), + }); + } + + let mut compile_error_types: Vec<(String, String)> = Vec::new(); + for ft in &all_form_types { + if let Some(SchemaCompileState::CompileError(msg)) = validators.get(ft) { + compile_error_types.push((ft.clone(), msg.clone())); + } + } + compile_error_types.sort_by(|a, b| a.0.cmp(&b.0)); + for (ft, msg) in compile_error_types { + issues.push(ImportIssue { + severity: "error", + code: "invalid_form_schema".to_string(), + message: format!("Could not compile JSON Schema for form type \"{ft}\": {msg}"), + file_name: None, + observation_id: None, + form_type: Some(ft), + }); + } + + let staged_norm: HashMap = { + let mut m = HashMap::new(); + for b in staged_attachment_basenames { + let k = normalize_basename(b); + if !k.is_empty() { + m.entry(k).or_insert_with(|| b.clone()); + } + } + m + }; + + let mut referenced_list: Vec = all_referenced.into_iter().collect(); + referenced_list.sort(); + + let mut missing = Vec::new(); + for ref_name in &referenced_list { + let kn = normalize_basename(ref_name); + if !kn.is_empty() && !staged_norm.contains_key(&kn) { + missing.push(ref_name.clone()); + } + } + missing.sort(); + + let referenced_norm: HashSet = referenced_list + .iter() + .map(|r| normalize_basename(r)) + .filter(|s| !s.is_empty()) + .collect(); + + let mut orphan: Vec = staged_norm + .iter() + .filter(|(norm, _)| !referenced_norm.contains(*norm)) + .map(|(_, display)| display.clone()) + .collect(); + orphan.sort(); + + for m in &missing { + issues.push(ImportIssue { + severity: "error", + code: "missing_attachment".to_string(), + message: format!("Referenced attachment \"{m}\" is not in the staged attachment list."), + file_name: None, + observation_id: None, + form_type: None, + }); + } + for o in &orphan { + issues.push(ImportIssue { + severity: "warning", + code: "orphan_attachment".to_string(), + message: format!( + "Staged attachment \"{o}\" is not referenced by any staged observation payload." + ), + file_name: None, + observation_id: None, + form_type: None, + }); + } + + ImportValidateBatchResult { + files, + issues, + observation_count, + form_type_count: all_form_types.len(), + referenced_attachment_names: referenced_list, + missing_attachment_names: missing, + orphan_attachment_names: orphan, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::fs; + + fn write_obs(dir: &std::path::Path, name: &str, body: &str) -> String { + let path = dir.join(name); + fs::write(&path, body).unwrap(); + path.to_string_lossy().to_string() + } + + #[test] + fn schema_validation_flags_type_mismatch() { + let mut schemas = HashMap::new(); + schemas.insert( + "PhotoForm".to_string(), + json!({ + "type": "object", + "properties": { + "pic": { "type": "object", "format": "photo" } + } + }), + ); + + let base = + std::env::temp_dir().join(format!("ode_import_validate_schema_{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + let path = write_obs( + &base, + "a.json", + r#"{ + "observationId": "o1", + "formType": "PhotoForm", + "updatedAt": "2026-01-01T00:00:00Z", + "data": { "pic": "not-an-object" } + }"#, + ); + + let result = parse_and_validate_paths(vec![path], &schemas, &[]); + assert!( + result.issues.iter().any(|i| i.code == "schema_validation"), + "issues: {:?}", + result.issues + ); + let _ = fs::remove_dir_all(&base); + } + + #[test] + fn photo_format_does_not_fail_when_formats_disabled() { + let mut schemas = HashMap::new(); + schemas.insert( + "PhotoForm".to_string(), + json!({ + "type": "object", + "properties": { + "pic": { "type": "object", "format": "photo" } + } + }), + ); + + let base = + std::env::temp_dir().join(format!("ode_import_validate_format_{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + let path = write_obs( + &base, + "a.json", + r#"{ + "observationId": "o1", + "formType": "PhotoForm", + "updatedAt": "2026-01-01T00:00:00Z", + "data": { "pic": { "filename": "used.jpg" } } + }"#, + ); + + let result = parse_and_validate_paths(vec![path], &schemas, &["used.jpg".to_string()]); + assert!( + !result.issues.iter().any(|i| i.code == "schema_validation"), + "issues: {:?}", + result.issues + ); + assert!( + result + .referenced_attachment_names + .contains(&"used.jpg".to_string()) + ); + let _ = fs::remove_dir_all(&base); + } + + #[test] + fn missing_and_orphan_attachments() { + let mut schemas = HashMap::new(); + schemas.insert( + "PhotoForm".to_string(), + json!({ + "type": "object", + "properties": { + "pic": { "type": "object", "format": "photo" } + } + }), + ); + + let base = + std::env::temp_dir().join(format!("ode_import_validate_att_{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + let path = write_obs( + &base, + "a.json", + r#"{ + "observationId": "o1", + "formType": "PhotoForm", + "updatedAt": "2026-01-01T00:00:00Z", + "data": { "pic": { "filename": "missing.jpg" } } + }"#, + ); + + let result = parse_and_validate_paths(vec![path], &schemas, &["orphan.jpg".to_string()]); + assert!( + result + .missing_attachment_names + .contains(&"missing.jpg".to_string()) + ); + assert!( + result + .orphan_attachment_names + .contains(&"orphan.jpg".to_string()) + ); + let _ = fs::remove_dir_all(&base); + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 76f7d717f..2ad027fa4 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -29,6 +29,7 @@ use zip::write::SimpleFileOptions; use zip::{CompressionMethod, ZipWriter}; mod data_export; +mod import_validate; mod observation_index; mod observation_query; mod sync_engine; @@ -611,13 +612,13 @@ struct SaveObservationRequest { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -struct ApiObservation { - observation_id: String, - data: Value, - form_type: Option, - updated_at: Option, +pub(crate) struct ApiObservation { + pub(crate) observation_id: String, + pub(crate) data: Value, + pub(crate) form_type: Option, + pub(crate) updated_at: Option, #[serde(default)] - extras: Option, + pub(crate) extras: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -950,6 +951,24 @@ fn open_db(ctx: &AppCtxHandle) -> Result, CustodianError> { }) } +/// Open SQLite for long-running background work (full index rebuild) without +/// holding `workspace_sqlite_lock` for the entire job. Holding that mutex for a +/// multi-minute rebuild blocks every other DB command (import refresh, list, +/// health) and makes the UI look stuck on the last "Writing observations…" step. +/// +/// Concurrent short writers still take the mutex; this connection uses a busy +/// timeout so rebuild waits on them instead of starving the UI. +fn open_db_for_background_index_rebuild(ctx: &AppCtxHandle) -> Result { + let db_path = resolve_db_path(ctx)?; + if let Some(parent) = db_path.parent() { + fs::create_dir_all(parent)?; + } + let conn = Connection::open(&db_path)?; + conn.busy_timeout(std::time::Duration::from_secs(120))?; + init_db(&conn)?; + Ok(conn) +} + /// Caller must hold `workspace_sqlite_lock`. Opens the DB, checkpoints WAL, then closes. fn quiesce_sqlite_unlocked(ctx: &AppCtxHandle) -> Result<(), CustodianError> { let db_path = resolve_db_path(ctx)?; @@ -1913,7 +1932,7 @@ fn spawn_observation_index_rebuild(app: tauri::AppHandle, ctx: AppCtxHandle, job if defs.is_empty() { return; } - let conn = match open_db(&ctx) { + let conn = match open_db_for_background_index_rebuild(&ctx) { Ok(c) => c, Err(err) => { emit_bundle_index_rebuild_progress( @@ -3980,11 +3999,11 @@ fn host_path_is_directory(path: String) -> bool { #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -struct ParsedImportFileResult { - file_name: String, - observations: Vec, +pub(crate) struct ParsedImportFileResult { + pub(crate) file_name: String, + pub(crate) observations: Vec, #[serde(skip_serializing_if = "Option::is_none")] - error: Option, + pub(crate) error: Option, } fn observation_id_from_obj(obj: &serde_json::Map) -> Option { @@ -4110,7 +4129,7 @@ fn extract_observations_from_json_value( Ok(observations) } -fn parse_import_json_file(path: &Path) -> ParsedImportFileResult { +pub(crate) fn parse_import_json_file(path: &Path) -> ParsedImportFileResult { let file_name = path .file_name() .and_then(|n| n.to_str()) @@ -4144,6 +4163,68 @@ fn parse_import_json_file(path: &Path) -> ParsedImportFileResult { } } +/// Load every `schema.json` under active/dev form roots (first-wins by form type). +fn load_bundle_form_schemas_for_ctx(ctx: &AppCtxHandle) -> Result, String> { + let roots = bundle_form_roots_for_ctx(ctx)?; + let mut out = HashMap::new(); + for root in roots { + let rd = match fs::read_dir(&root) { + Ok(r) => r, + Err(_) => continue, + }; + for entry in rd { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry.file_name().to_string_lossy().to_string(); + if !entry.file_type().map_err(|e| e.to_string())?.is_dir() { + continue; + } + if reserved_form_dir_name(&name) { + continue; + } + if out.contains_key(&name) { + continue; + } + let schema_path = entry.path().join("schema.json"); + let ui_path = entry.path().join("ui.json"); + if !(schema_path.is_file() && ui_path.is_file()) { + continue; + } + let form_schema: Value = + serde_json::from_str(&fs::read_to_string(&schema_path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + out.insert(name, form_schema); + } + } + Ok(out) +} + +/// Parse + schema-validate import JSON on the host (Rayon), loading form schemas from the active bundle. +#[tauri::command] +fn parse_and_validate_import_json_paths( + paths: Vec, + staged_attachment_basenames: Vec, + ctx: tauri::State<'_, AppCtxHandle>, +) -> Result { + if paths.is_empty() { + return Ok(import_validate::parse_and_validate_paths( + paths, + &HashMap::new(), + &staged_attachment_basenames, + )); + } + if paths.len() > MAX_IMPORT_SCAN_ENTRIES { + return Err(format!( + "Too many JSON paths to validate (max {MAX_IMPORT_SCAN_ENTRIES})" + )); + } + let schemas = load_bundle_form_schemas_for_ctx(&ctx)?; + Ok(import_validate::parse_and_validate_paths( + paths, + &schemas, + &staged_attachment_basenames, + )) +} + /// Parse observation JSON files on the host (parallel) in import order. #[tauri::command] fn parse_import_observation_json_paths( @@ -5360,10 +5441,11 @@ fn import_observations_run( conflicts += 1; } } - if !index_defs.is_empty() && !mark_pending { - // Sync pull: update indexes incrementally per page (no full rebuild follows). - // Local file import: skip here — `import_observations` schedules one background - // full rebuild after the batch commit (incremental work would be discarded). + if !index_defs.is_empty() { + // Sync pull and local file import both update the active generation + // incrementally. A full rebuild is reserved for bundle apply / empty + // index / explicit rebuild — not for adding a few hundred import rows + // on top of an already-indexed sync. let payload = serde_json::to_string(&observation.data).map_err(|e| e.to_string())?; let form_type = observation.form_type.as_deref().unwrap_or(""); observation_index::incremental_reindex( @@ -5406,7 +5488,9 @@ fn import_observations( let ctx_inner = ctx.inner().clone(); let mark = mark_pending.unwrap_or(false); let mut result = import_observations_run(observations, mark, &ctx_inner)?; - let should_schedule = schedule_index_rebuild.unwrap_or(mark) && result.imported > 0; + // Full rebuild is optional and off by default — import/sync maintain indexes + // incrementally. Callers that need a snapshot rebuild (rare) pass true. + let should_schedule = schedule_index_rebuild.unwrap_or(false) && result.imported > 0; if should_schedule { result.index_rebuild_scheduled = schedule_observation_index_rebuild(&app, &ctx_inner).is_some(); @@ -5528,12 +5612,52 @@ fn get_app_health(ctx: tauri::State<'_, AppCtxHandle>) -> Result) -> Result { +fn reset_local_workspace_data( + pending_only: Option, + ctx: tauri::State<'_, AppCtxHandle>, +) -> Result { + let pending_only = pending_only.unwrap_or(false); with_workspace_fs_exclusive(&ctx, |ctx| { let db_path = resolve_db_path(ctx)?; let conn = Connection::open(&db_path)?; init_db(&conn)?; + + if pending_only { + conn.execute( + "DELETE FROM observation_history WHERE observation_id IN ( + SELECT id FROM observations + WHERE dirty = 1 OR sync_status IN ('dirty', 'conflict') + )", + [], + )?; + conn.execute( + "DELETE FROM observation_index WHERE observation_id IN ( + SELECT id FROM observations + WHERE dirty = 1 OR sync_status IN ('dirty', 'conflict') + )", + [], + )?; + conn.execute( + "DELETE FROM observations WHERE dirty = 1 OR sync_status IN ('dirty', 'conflict')", + [], + )?; + conn.execute_batch("PRAGMA wal_checkpoint(FULL);")?; + drop(conn); + + let ws = resolve_active_workspace_dir(ctx)?; + let pending_dir = attachments_root(&ws).join(ATTACH_SUBDIR_PENDING); + if pending_dir.exists() { + fs::remove_dir_all(&pending_dir)?; + } + ensure_workspace_layout(&ws)?; + return Ok(()); + } + conn.execute("DELETE FROM observation_history", [])?; conn.execute("DELETE FROM observations", [])?; conn.execute("DELETE FROM observation_index", [])?; @@ -5672,6 +5796,7 @@ pub fn run() { copy_workspace_attachment_from_path, expand_import_staging_paths, parse_import_observation_json_paths, + parse_and_validate_import_json_paths, scan_import_json_sync_appearance, copy_workspace_attachments_batch, read_host_text_file, @@ -6002,10 +6127,8 @@ mod tests { #[test] fn scan_import_json_sync_appearance_keeps_unsynced_paths() { - let base = std::env::temp_dir().join(format!( - "ode_import_sync_scan_{}", - std::process::id() - )); + let base = + std::env::temp_dir().join(format!("ode_import_sync_scan_{}", std::process::id())); let _ = fs::remove_dir_all(&base); fs::create_dir_all(&base).unwrap(); let synced_path = base.join("synced.json"); diff --git a/desktop/src-tauri/src/observation_index.rs b/desktop/src-tauri/src/observation_index.rs index 62c0edc18..af2e74707 100644 --- a/desktop/src-tauri/src/observation_index.rs +++ b/desktop/src-tauri/src/observation_index.rs @@ -1,11 +1,15 @@ //! Local observation_index EAV table (never synced) + snapshot generation rebuild. +use rayon::prelude::*; use rusqlite::{Connection, params}; use serde::Deserialize; use serde_json::Value; use std::collections::HashSet; use std::path::Path; +/// Observations loaded / parsed per parallel chunk during a full rebuild. +const REBUILD_MAP_CHUNK_SIZE: usize = 512; + #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ObservationIndexDef { @@ -85,48 +89,92 @@ fn form_type_matches(form_type: &str, patterns: Option<&Vec>) -> bool { false } -fn json_path_to_key(path: &str) -> String { - path.strip_prefix("$.").unwrap_or(path).to_string() +fn json_path_to_key(path: &str) -> &str { + path.strip_prefix("$.").unwrap_or(path) } -fn extract_scalar(payload: &str, path: &str) -> Option { - let v: Value = serde_json::from_str(payload).ok()?; - let key = json_path_to_key(path); - v.get(&key).cloned() +#[derive(Debug, Clone)] +struct IndexRow { + observation_id: String, + index_key: String, + value_text: Option, + value_num: Option, } -pub fn reindex_observation( - conn: &Connection, +/// Parse payload once and emit every matching EAV row (CPU-bound map step). +fn extract_index_rows( observation_id: &str, form_type: &str, payload: &str, defs: &[ObservationIndexDef], - generation: i64, -) -> rusqlite::Result<()> { - conn.execute( - "DELETE FROM observation_index WHERE observation_id = ?1 AND index_generation = ?2", - params![observation_id, generation], - )?; +) -> Vec { + let Ok(v) = serde_json::from_str::(payload) else { + return Vec::new(); + }; + let mut out = Vec::with_capacity(defs.len()); for def in defs { if !form_type_matches(form_type, def.form_types.as_ref()) { continue; } - let Some(val) = extract_scalar(payload, &def.path) else { + let Some(val) = v.get(json_path_to_key(&def.path)) else { continue; }; if val.is_null() { continue; } - let (value_text, value_num) = scalar_to_columns(&val, def.value_type.as_deref()); - conn.execute( - "INSERT OR REPLACE INTO observation_index (observation_id, index_key, index_generation, value_text, value_num) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![observation_id, def.key, generation, value_text, value_num], - )?; + let (value_text, value_num) = scalar_to_columns(val, def.value_type.as_deref()); + out.push(IndexRow { + observation_id: observation_id.to_string(), + index_key: def.key.clone(), + value_text, + value_num, + }); + } + out +} + +fn insert_index_rows( + conn: &Connection, + generation: i64, + rows: &[IndexRow], +) -> rusqlite::Result<()> { + if rows.is_empty() { + return Ok(()); + } + // Full rebuild clears the target generation first; INSERT (not OR REPLACE) is fine + // and cheaper. Incremental reindex deletes the observation's rows first as well. + let mut stmt = conn.prepare( + "INSERT INTO observation_index (observation_id, index_key, index_generation, value_text, value_num) + VALUES (?1, ?2, ?3, ?4, ?5)", + )?; + for row in rows { + stmt.execute(params![ + row.observation_id, + row.index_key, + generation, + row.value_text, + row.value_num, + ])?; } Ok(()) } +pub fn reindex_observation( + conn: &Connection, + observation_id: &str, + form_type: &str, + payload: &str, + defs: &[ObservationIndexDef], + generation: i64, +) -> rusqlite::Result<()> { + conn.execute( + "DELETE FROM observation_index WHERE observation_id = ?1 AND index_generation = ?2", + params![observation_id, generation], + )?; + let rows = extract_index_rows(observation_id, form_type, payload, defs); + insert_index_rows(conn, generation, &rows) +} + fn scalar_to_columns(val: &Value, value_type: Option<&str>) -> (Option, Option) { if (value_type == Some("number") || val.is_number()) && let Some(n) = val.as_f64() @@ -174,15 +222,6 @@ pub fn rebuild_all_indexes( cb(0, total, Some("Indexing observations…")); } - let mut stmt = conn.prepare("SELECT id, form_type, payload FROM observations")?; - let rows = stmt.query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, String>(2)?, - )) - })?; - let progress_interval = if total < 50 { 1 } else if total < 500 { @@ -191,23 +230,59 @@ pub fn rebuild_all_indexes( 50 }; + // Map-join rebuild: + // 1. Load observation triples (desktop-scale; typically tens of thousands). + // 2. Parallel map: parse each payload once and extract all EAV rows. + // 3. Join: batch INSERT in one transaction (avoids per-row autocommit). + let observations: Vec<(String, String, String)> = { + let mut stmt = conn.prepare("SELECT id, form_type, payload FROM observations")?; + let mapped = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?.unwrap_or_default(), + row.get::<_, String>(2)?, + )) + })?; + mapped.collect::>>()? + }; + + // Drop secondary indexes for the bulk load; recreate after commit. + conn.execute_batch( + r#" + DROP INDEX IF EXISTS idx_observation_index_lookup; + DROP INDEX IF EXISTS idx_observation_index_lookup_num; + "#, + )?; + + let tx = conn.unchecked_transaction()?; let mut done = 0i64; - for row in rows { - let (id, form_type, payload) = row?; - let ft = form_type.unwrap_or_default(); - reindex_observation(conn, &id, &ft, &payload, defs, new_gen)?; - done += 1; + for chunk in observations.chunks(REBUILD_MAP_CHUNK_SIZE) { + let mapped: Vec = chunk + .par_iter() + .flat_map(|(id, ft, payload)| extract_index_rows(id, ft, payload, defs)) + .collect(); + insert_index_rows(&tx, new_gen, &mapped)?; + done += chunk.len() as i64; if let Some(ref mut cb) = progress && (total == 0 || done == total || done % progress_interval == 0) { cb(done, total, Some("Indexing observations…")); } } + tx.commit()?; if let Some(ref mut cb) = progress { cb(done, total, Some("Creating SQLite indexes…")); } + conn.execute_batch( + r#" + CREATE INDEX IF NOT EXISTS idx_observation_index_lookup + ON observation_index(index_generation, index_key, value_text, observation_id); + CREATE INDEX IF NOT EXISTS idx_observation_index_lookup_num + ON observation_index(index_generation, index_key, value_num, observation_id); + "#, + )?; recreate_sqlite_indexes(conn, defs)?; conn.execute( @@ -470,6 +545,41 @@ mod tests { assert_eq!(rows, 1); } + #[test] + fn rebuild_map_join_indexes_many_observations() { + let conn = test_conn(); + let defs = sample_defs(); + for i in 0..1200 { + conn.execute( + "INSERT INTO observations (id, form_type, payload) VALUES (?1, 'person', ?2)", + params![ + format!("obs{i}"), + format!(r#"{{"p_id":"P{i}","age":{}}}"#, i % 80) + ], + ) + .unwrap(); + } + let generation = rebuild_all_indexes(&conn, &defs, None).unwrap(); + assert_eq!(generation, 2); + let rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM observation_index WHERE index_generation = 2", + [], + |r| r.get(0), + ) + .unwrap(); + // Each person has p_id + age + assert_eq!(rows, 2400); + let lookup_ok: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_observation_index_lookup'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(lookup_ok, 1); + } + #[test] fn missing_sqlite_indexes_detects_absent_names() { let conn = test_conn(); diff --git a/desktop/src/App.css b/desktop/src/App.css index 4cc9d9302..7b2831b5a 100644 --- a/desktop/src/App.css +++ b/desktop/src/App.css @@ -1278,6 +1278,16 @@ textarea { flex-wrap: wrap; } +.app-sync-banner-text { + flex: 1 1 12rem; + min-width: 0; +} + +.app-sync-banner-save-report { + flex-shrink: 0; + align-self: center; +} + .activity-progress { flex: 1 1 100%; height: 4px; diff --git a/desktop/src/App.test.tsx b/desktop/src/App.test.tsx index 5eeb1827e..50fe4a056 100644 --- a/desktop/src/App.test.tsx +++ b/desktop/src/App.test.tsx @@ -61,6 +61,15 @@ vi.mock('./lib/tauriClient', () => ({ copyWorkspaceAttachmentFromPath: vi.fn(), expandImportStagingPaths: vi.fn().mockResolvedValue([]), parseImportObservationJsonPaths: vi.fn().mockResolvedValue([]), + parseAndValidateImportJsonPaths: vi.fn().mockResolvedValue({ + files: [], + issues: [], + observationCount: 0, + formTypeCount: 0, + referencedAttachmentNames: [], + missingAttachmentNames: [], + orphanAttachmentNames: [], + }), scanImportJsonSyncAppearance: vi.fn().mockResolvedValue({ fileCount: 0, observationCount: 0, diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx index 3ecf059ec..ce1096cf1 100644 --- a/desktop/src/App.tsx +++ b/desktop/src/App.tsx @@ -40,6 +40,7 @@ import { ensureBundleApplyEventPipeline, installGlobalIndexRebuildListener, } from './lib/bundleTauriEvents'; +import { tauriClient } from './lib/tauriClient'; import './App.css'; const DATA_NAV = [ @@ -222,6 +223,7 @@ function Shell() { const isWorkbench = location.pathname.startsWith('/workbench'); const navItems = isWorkbench ? WORKBENCH_NAV : DATA_NAV; const syncMessage = useCustodianStore(s => s.syncMessage); + const syncDetailReport = useCustodianStore(s => s.syncDetailReport); const clearSyncMessage = useCustodianStore(s => s.clearSyncMessage); const pushToast = useToastStore(s => s.pushToast); const syncActivity = useCustodianStore(selectSyncActivity); @@ -276,13 +278,18 @@ function Shell() { if (!syncMessage) { return; } - const isLong = syncMessage.includes('\n') || syncMessage.length > 120; - if (isLong) { + // Keep the banner when a downloadable detail report is attached, or when + // the message is long (multi-line / verbose). + const keepBanner = + Boolean(syncDetailReport) || + syncMessage.includes('\n') || + syncMessage.length > 120; + if (keepBanner) { return; } pushToast({ message: syncMessage, variant: 'success' }); clearSyncMessage(); - }, [syncMessage, pushToast, clearSyncMessage]); + }, [syncMessage, syncDetailReport, pushToast, clearSyncMessage]); const showActivityBanner = Boolean(activityText) && activityPresent && !activityBannerDismissed; @@ -290,7 +297,33 @@ function Shell() { const showSyncMessageBanner = Boolean(syncMessage) && syncMessage !== null && - (syncMessage.includes('\n') || syncMessage.length > 120); + (Boolean(syncDetailReport) || + syncMessage.includes('\n') || + syncMessage.length > 120); + + async function saveSyncDetailReport() { + if (!syncDetailReport) { + return; + } + try { + const { save } = await import('@tauri-apps/plugin-dialog'); + const stamp = new Date().toISOString().slice(0, 19).replace(/[:T]/g, '-'); + const path = await save({ + defaultPath: `ode-push-attachment-report-${stamp}.txt`, + filters: [{ name: 'Text', extensions: ['txt'] }], + }); + if (path == null) { + return; + } + await tauriClient.writeTextFile(path, syncDetailReport); + pushToast({ message: 'Report saved.', variant: 'success' }); + } catch (e) { + pushToast({ + message: e instanceof Error ? e.message : String(e), + variant: 'error', + }); + } + } return ( <> @@ -372,7 +405,20 @@ function Shell() { ) : null} {showSyncMessageBanner ? (

-
{syncMessage}
+
+ {syncMessage} + {syncDetailReport ? ( + + ) : null} +
+ +
+ + + ); +} diff --git a/desktop/src/lib/importSummary.ts b/desktop/src/lib/importSummary.ts index 1ed6b9246..17c0efe86 100644 --- a/desktop/src/lib/importSummary.ts +++ b/desktop/src/lib/importSummary.ts @@ -1,6 +1,7 @@ import type { ApiObservation, HostTextReadResult, + ImportHostIssue, ObservationExtras, } from '../types/domain'; import { tauriClient } from './tauriClient'; @@ -368,6 +369,41 @@ export async function parseObservationJsonPathsViaRust( return chunkResults.flat(); } +/** + * Host-side parallel parse + schema/attachment validation for the full staged set. + * Prefer this over parse-then-AJV for large imports. + */ +export async function parseAndValidateImportJsonViaRust( + items: readonly { name: string; nativePath: string }[], + stagedAttachmentBasenames: readonly string[], +): Promise<{ + parsedFiles: ParsedObservationFile[]; + issues: ImportHostIssue[]; + observationCount: number; + formTypeCount: number; + referencedAttachmentNames: string[]; + missingAttachmentNames: string[]; + orphanAttachmentNames: string[]; +}> { + const paths = items.map(it => it.nativePath); + const result = await tauriClient.parseAndValidateImportJsonPaths(paths, [ + ...stagedAttachmentBasenames, + ]); + return { + parsedFiles: result.files.map(r => ({ + fileName: r.fileName, + observations: r.observations, + error: r.error, + })), + issues: result.issues, + observationCount: result.observationCount, + formTypeCount: result.formTypeCount, + referencedAttachmentNames: result.referencedAttachmentNames, + missingAttachmentNames: result.missingAttachmentNames, + orphanAttachmentNames: result.orphanAttachmentNames, + }; +} + export function flattenObservations( parsed: ParsedObservationFile[], ): ApiObservation[] { diff --git a/desktop/src/lib/pushAttachmentAudit.test.ts b/desktop/src/lib/pushAttachmentAudit.test.ts new file mode 100644 index 000000000..dfe2ce958 --- /dev/null +++ b/desktop/src/lib/pushAttachmentAudit.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; +import { + formatMissingAttachmentHighlight, + formatMissingAttachmentReport, + type MissingAttachmentIssue, +} from './pushAttachmentAudit'; + +const sample: MissingAttachmentIssue[] = [ + { + id: 'obs_1', + formType: 'p_consent', + missing: ['a.jpg', 'a'], + }, + { + id: 'obs_2', + formType: 'p_consent', + missing: ['b.jpg'], + }, +]; + +describe('formatMissingAttachmentHighlight', () => { + it('stays short for forced and skipped modes', () => { + const forced = formatMissingAttachmentHighlight(sample, 'forced'); + const skipped = formatMissingAttachmentHighlight(sample, 'skipped'); + expect(forced).toContain('Included 2 observations'); + expect(forced).toContain('(forced)'); + expect(forced).not.toContain('obs_1'); + expect(skipped).toContain('Skipped 2 observations'); + expect(skipped).not.toContain('a.jpg'); + expect(forced.length).toBeLessThan(120); + expect(skipped.length).toBeLessThan(120); + }); +}); + +describe('formatMissingAttachmentReport', () => { + it('includes full per-observation detail for download', () => { + const report = formatMissingAttachmentReport(sample, 'forced'); + expect(report).toContain('forced inclusion'); + expect(report).toContain('obs_1 (form: p_consent)'); + expect(report).toContain('- a.jpg'); + expect(report).toContain('obs_2 (form: p_consent)'); + }); +}); diff --git a/desktop/src/lib/pushAttachmentAudit.ts b/desktop/src/lib/pushAttachmentAudit.ts index eac5e645a..0dec4968e 100644 --- a/desktop/src/lib/pushAttachmentAudit.ts +++ b/desktop/src/lib/pushAttachmentAudit.ts @@ -105,3 +105,51 @@ export function formatMissingAttachmentSummary( ) .join('\n'); } + +/** One-line UI highlight (no per-observation dump). */ +export function formatMissingAttachmentHighlight( + issues: MissingAttachmentIssue[], + mode: 'skipped' | 'forced', +): string { + if (issues.length === 0) { + return ''; + } + const n = issues.length; + const noun = n === 1 ? 'observation' : 'observations'; + if (mode === 'forced') { + return ` Included ${n} ${noun} with missing attachment(s) (forced).`; + } + return ` Skipped ${n} ${noun} with missing attachment file(s).`; +} + +/** Full multi-line report suitable for saving to a text file. */ +export function formatMissingAttachmentReport( + issues: MissingAttachmentIssue[], + mode: 'skipped' | 'forced', + meta?: { headline?: string }, +): string { + const lines: string[] = []; + lines.push('ODE Desktop — push attachment report'); + lines.push(`Generated: ${new Date().toISOString()}`); + lines.push( + `Mode: ${mode === 'forced' ? 'forced inclusion' : 'skipped from push'}`, + ); + if (meta?.headline?.trim()) { + lines.push(''); + lines.push(meta.headline.trim()); + } + lines.push(''); + lines.push( + `Summary: ${issues.length} observation(s) with missing attachment file(s).`, + ); + lines.push(''); + lines.push('Details:'); + for (const issue of issues) { + lines.push(`${issue.id} (form: ${issue.formType})`); + for (const name of issue.missing) { + lines.push(` - ${name}`); + } + } + lines.push(''); + return lines.join('\n'); +} diff --git a/desktop/src/lib/tauriClient.ts b/desktop/src/lib/tauriClient.ts index 6a9d63ee2..2852db248 100644 --- a/desktop/src/lib/tauriClient.ts +++ b/desktop/src/lib/tauriClient.ts @@ -10,6 +10,7 @@ import type { ImportResult, ImportStagingScanEntry, ImportSyncAppearanceScanResult, + ImportValidateBatchResult, ParsedImportFileResult, AttachmentCopyBatchResult, HostTextReadResult, @@ -156,6 +157,18 @@ export const tauriClient = { 'parse_import_observation_json_paths', { paths }, ), + /** Parallel host parse + JSON Schema / attachment validation (loads bundle schemas once). */ + parseAndValidateImportJsonPaths: ( + paths: string[], + stagedAttachmentBasenames: string[], + ) => + invokeSafe( + 'parse_and_validate_import_json_paths', + { + paths, + stagedAttachmentBasenames, + }, + ), scanImportJsonSyncAppearance: (paths: string[]) => invokeSafe( 'scan_import_json_sync_appearance', @@ -289,8 +302,8 @@ export const tauriClient = { /** * @param markPending When true (file import), observations are stored as pending push. * When false/omitted, rows match server pull semantics (synced / conflict rules). - * @param scheduleIndexRebuild When false, skips the post-import full index rebuild (use on - * intermediate write batches; default true for file import, false for server pull). + * @param scheduleIndexRebuild When true, schedules a full background index rebuild + * after the write. Default false — import and sync update indexes incrementally. */ importObservations: ( observations: ApiObservation[], @@ -304,8 +317,10 @@ export const tauriClient = { markObservationsPushed: (ids: string[]) => invokeSafe('mark_observations_pushed', { ids }), getAppHealth: () => invokeSafe('get_app_health'), - resetLocalWorkspaceData: () => - invokeSafe('reset_local_workspace_data'), + resetLocalWorkspaceData: (options?: { pendingOnly?: boolean }) => + invokeSafe('reset_local_workspace_data', { + pendingOnly: options?.pendingOnly ?? false, + }), synkLogin: (req: SyncLoginRequest) => invokeSafe('synk_login', { req }), diff --git a/desktop/src/pages/ImportPage.tsx b/desktop/src/pages/ImportPage.tsx index 9d1cb53bd..714364a38 100644 --- a/desktop/src/pages/ImportPage.tsx +++ b/desktop/src/pages/ImportPage.tsx @@ -6,7 +6,6 @@ import { groupIssuesBySeverityAndCategory, normalizeBasename, referencedNamesForObservation, - runImportValidation, type ImportIssue, type ImportIssueCategory, type ImportValidationReport, @@ -15,7 +14,7 @@ import type { ApiObservation, BundleFormSpec } from '../types/domain'; import { flattenObservations, mapPool, - parseObservationJsonPathsViaRust, + parseAndValidateImportJsonViaRust, partitionImportObservationsBySyncAppearance, summarizeImportFiles, } from '../lib/importSummary'; @@ -145,6 +144,8 @@ const CATEGORY_LABELS: Record = { function ValidationAccordion({ issues }: { issues: ImportIssue[] }) { const grouped = groupIssuesBySeverityAndCategory(issues); const [openKeys, setOpenKeys] = useState>(new Set()); + const errorCount = issues.filter(i => i.severity === 'error').length; + const warningCount = issues.filter(i => i.severity === 'warning').length; function toggle(key: string) { setOpenKeys(prev => { @@ -169,38 +170,54 @@ function ValidationAccordion({ issues }: { issues: ImportIssue[] }) { } } - if (sections.length === 0) { - return

No issues reported.

; - } - return ( -
- {sections.map(sec => ( -
- - {openKeys.has(sec.key) ? ( -
-
    - {sec.items.slice(0, 30).map((issue, i) => ( -
  • {issue.message}
  • - ))} -
- {sec.items.length > 30 ? ( -

… and {sec.items.length - 30} more

+
+ {errorCount === 0 ? ( +

No validation errors were found.

+ ) : ( +

+ {errorCount} validation error{errorCount === 1 ? '' : 's'} + {warningCount > 0 + ? ` · ${warningCount} warning${warningCount === 1 ? '' : 's'}` + : ''} + . Review below — you can still import. +

+ )} + {errorCount === 0 && warningCount > 0 ? ( +

+ {warningCount} warning{warningCount === 1 ? '' : 's'} reported. +

+ ) : null} + {sections.length > 0 ? ( +
+ {sections.map(sec => ( +
+ + {openKeys.has(sec.key) ? ( +
+
    + {sec.items.slice(0, 30).map((issue, i) => ( +
  • {issue.message}
  • + ))} +
+ {sec.items.length > 30 ? ( +

… and {sec.items.length - 30} more

+ ) : null} +
) : null}
- ) : null} + ))}
- ))} + ) : null}
); } @@ -300,6 +317,7 @@ export function ImportPage() { if (!expanded.length) { return; } + setPreviewReport(null); addScanEntries(expanded); if (expanded.some(e => e.isJson)) { await offerSkipAlreadySynced(); @@ -426,6 +444,7 @@ export function ImportPage() { MAX_INDIVIDUAL_FILES, ); if (expanded.length) { + setPreviewReport(null); addScanEntries(expanded); } } catch (e) { @@ -435,85 +454,64 @@ export function ImportPage() { } }, [addScanEntries, setError, setImportActivity]); - const runFullImport = useCallback(async () => { + const runValidate = useCallback(async () => { if (stagedJson.length === 0) { return; } const statusCtl = createThrottledImportStatus(setImportActivity); - setPreviewReport(null); setMessage(null); setError(null); - statusCtl.push('Reading observation JSON…'); try { await ensureBundleApplyEventPipeline(); - const parsed = await parseObservationJsonPathsViaRust( + statusCtl.push( + `Reading and validating JSON (${stagedJson.length} files)…`, + ); + const hostReport = await parseAndValidateImportJsonViaRust( stagedJson.map(s => ({ name: s.name, nativePath: s.nativePath })), - (done, tot) => statusCtl.push(`Reading JSON (${done}/${tot})…`), + stagedAttachments.map(s => s.name), ); - const formTypes = new Set(); - for (const p of parsed) { - if (p.error) { - continue; - } - for (const obs of p.observations) { - if (obs.formType?.trim()) { - formTypes.add(obs.formType.trim()); - } - } - } - - const formSpecsByType = new Map(); - const ftArr = [...formTypes].sort(); - if (ftArr.length > 0) { - let schemaDone = 0; - await mapPool(ftArr, 8, async ft => { - try { - const spec = await tauriClient.readBundleFormSpec(ft); - formSpecsByType.set(ft, spec); - } catch { - /* missing schema reported inside runImportValidation */ - } finally { - schemaDone += 1; - statusCtl.push( - `Loading form schemas (${schemaDone}/${ftArr.length})…`, - ); - } - }); - } - - statusCtl.push('Validating…'); - const basenames = stagedAttachments.map(s => s.name); - const report = runImportValidation({ - parsedFiles: parsed, - formSpecsByType, - stagedAttachmentBasenames: basenames, - onFileValidated: (fi, tot, name) => { - if (tot <= 40 || fi === tot - 1 || (fi + 1) % 50 === 0) { - statusCtl.push(`Validating (${fi + 1}/${tot}) ${name}…`); - } - }, + const issues: ImportIssue[] = hostReport.issues.map(i => ({ + severity: i.severity === 'warning' ? 'warning' : 'error', + code: i.code, + message: i.message, + fileName: i.fileName, + observationId: i.observationId, + formType: i.formType ?? null, + })); + + setPreviewReport({ + issues, + parsedFiles: hostReport.parsedFiles, + observationCount: hostReport.observationCount, + formTypeCount: hostReport.formTypeCount, + stagedAttachmentBasenames: stagedAttachments.map(s => s.name), + referencedAttachmentNames: hostReport.referencedAttachmentNames, + missingAttachmentNames: hostReport.missingAttachmentNames, + orphanAttachmentNames: hostReport.orphanAttachmentNames, }); + } catch (e) { + setPreviewReport(null); + setError(messageFromUnknown(e, 'Validation failed')); + } finally { + statusCtl.dispose(); + setImportActivity(null); + } + }, [stagedJson, stagedAttachments, setMessage, setError, setImportActivity]); - if (report.issues.length > 0) { - const errCount = report.issues.filter( - i => i.severity === 'error', - ).length; - const warnCount = report.issues.filter( - i => i.severity === 'warning', - ).length; - const ok = await confirm( - `${errCount} error(s), ${warnCount} warning(s). Import anyway?`, - { title: 'Validation issues', kind: 'warning' }, - ); - if (!ok) { - setPreviewReport(report); - return; - } - } + const runImportFromReport = useCallback(async () => { + if (!previewReport) { + return; + } + const statusCtl = createThrottledImportStatus(setImportActivity); + setMessage(null); + setError(null); - const allObservations = flattenObservations(report.parsedFiles); + try { + await ensureBundleApplyEventPipeline(); + + const allObservations = flattenObservations(previewReport.parsedFiles); const syncPartition = partitionImportObservationsBySyncAppearance(allObservations); // Staging already offered skip/keep; when skip was chosen, drop any @@ -537,10 +535,35 @@ export function ImportPage() { return; } + // Form schemas only needed for attachment refs on the rows we actually write. + const formTypes = new Set(); + for (const obs of observations) { + if (obs.formType?.trim()) { + formTypes.add(obs.formType.trim()); + } + } + const formSpecsByType = new Map(); + const ftArr = [...formTypes].sort(); + if (ftArr.length > 0 && stagedAttachments.length > 0) { + let schemaDone = 0; + await mapPool(ftArr, 8, async ft => { + try { + const spec = await tauriClient.readBundleFormSpec(ft); + formSpecsByType.set(ft, spec); + } catch { + /* attachment refs fall back to heuristics */ + } finally { + schemaDone += 1; + statusCtl.push( + `Loading form schemas (${schemaDone}/${ftArr.length})…`, + ); + } + }); + } + const writeTotal = observations.length; let imported = 0; let conflicts = 0; - let indexRebuildScheduled = false; for ( let offset = 0; @@ -552,103 +575,107 @@ export function ImportPage() { offset + IMPORT_WRITE_CHUNK_SIZE, ); const written = Math.min(offset + chunk.length, writeTotal); - const isLast = written >= writeTotal; statusCtl.push(`Writing observations (${written}/${writeTotal})…`); + // Indexes are updated incrementally inside import (same as sync pull). const chunkResult = await tauriClient.importObservations(chunk, { markPending: true, - scheduleIndexRebuild: isLast, + scheduleIndexRebuild: false, }); imported += chunkResult.imported; conflicts += chunkResult.conflicts; - indexRebuildScheduled = - indexRebuildScheduled || !!chunkResult.indexRebuildScheduled; } - const result = { imported, conflicts, indexRebuildScheduled }; + const result = { imported, conflicts }; - const stagedNorm = new Map(); - for (const s of stagedAttachments) { - const k = normalizeBasename(s.name); - if (k && !stagedNorm.has(k)) { - stagedNorm.set(k, s.name); + const copyItems: { sourcePath: string; attachmentId: string }[] = []; + const copyErrors: string[] = []; + let attachmentsCopied = 0; + if (stagedAttachments.length > 0) { + statusCtl.push('Resolving attachment references…'); + const stagedNorm = new Map(); + for (const s of stagedAttachments) { + const k = normalizeBasename(s.name); + if (k && !stagedNorm.has(k)) { + stagedNorm.set(k, s.name); + } } - } - const refNorm = new Set( - referencedAttachmentNamesForObservations(observations, formSpecsByType) - .map(n => normalizeBasename(n)) - .filter(Boolean), - ); + const refNorm = new Set( + referencedAttachmentNamesForObservations( + observations, + formSpecsByType, + ) + .map(n => normalizeBasename(n)) + .filter(Boolean), + ); - const copyItems: { sourcePath: string; attachmentId: string }[] = []; - for (const s of stagedAttachments) { - const kn = normalizeBasename(s.name); - if (!kn || !refNorm.has(kn)) { - continue; + for (const s of stagedAttachments) { + const kn = normalizeBasename(s.name); + if (!kn || !refNorm.has(kn)) { + continue; + } + const attachmentId = stagedNorm.get(kn) ?? s.name; + copyItems.push({ + sourcePath: s.nativePath, + attachmentId, + }); } - const attachmentId = stagedNorm.get(kn) ?? s.name; - copyItems.push({ - sourcePath: s.nativePath, - attachmentId, - }); - } - const copyErrors: string[] = []; - let attachmentsCopied = 0; - if (copyItems.length > 0) { - const copyTotal = copyItems.length; - const copyStatusCtl = createThrottledImportStatus( - setImportActivity, - 250, - ); - copyStatusCtl.push(formatAttachmentCopyProgress(0, copyTotal)); - const { listen } = await import('@tauri-apps/api/event'); - let unlisten: (() => void) | undefined; - try { - for ( - let offset = 0; - offset < copyItems.length; - offset += ATTACHMENT_COPY_CHUNK_SIZE - ) { - const chunk = copyItems.slice( - offset, - offset + ATTACHMENT_COPY_CHUNK_SIZE, - ); - const chunkIndex = - Math.floor(offset / ATTACHMENT_COPY_CHUNK_SIZE) + 1; - const chunkCount = Math.ceil( - copyItems.length / ATTACHMENT_COPY_CHUNK_SIZE, - ); - if (chunkCount > 1) { + if (copyItems.length > 0) { + const copyTotal = copyItems.length; + const copyStatusCtl = createThrottledImportStatus( + setImportActivity, + 250, + ); + copyStatusCtl.push(formatAttachmentCopyProgress(0, copyTotal)); + const { listen } = await import('@tauri-apps/api/event'); + let unlisten: (() => void) | undefined; + try { + for ( + let offset = 0; + offset < copyItems.length; + offset += ATTACHMENT_COPY_CHUNK_SIZE + ) { + const chunk = copyItems.slice( + offset, + offset + ATTACHMENT_COPY_CHUNK_SIZE, + ); + const chunkIndex = + Math.floor(offset / ATTACHMENT_COPY_CHUNK_SIZE) + 1; + const chunkCount = Math.ceil( + copyItems.length / ATTACHMENT_COPY_CHUNK_SIZE, + ); + if (chunkCount > 1) { + copyStatusCtl.push( + `Copying attachments batch ${chunkIndex}/${chunkCount}…`, + ); + } + unlisten?.(); + unlisten = await listen<{ + done: number; + total: number; + attachmentId: string; + }>('import/attachment-copy-progress', e => { + const globalDone = offset + e.payload.done; + copyStatusCtl.push( + formatAttachmentCopyProgress(globalDone, copyTotal), + ); + }); + const batchResult = + await tauriClient.copyWorkspaceAttachmentsBatch(chunk); + copyErrors.push(...batchResult.errors); + attachmentsCopied += batchResult.copied; copyStatusCtl.push( - `Copying attachments batch ${chunkIndex}/${chunkCount}…`, + formatAttachmentCopyProgress( + Math.min(offset + chunk.length, copyTotal), + copyTotal, + ), ); } + } finally { unlisten?.(); - unlisten = await listen<{ - done: number; - total: number; - attachmentId: string; - }>('import/attachment-copy-progress', e => { - const globalDone = offset + e.payload.done; - copyStatusCtl.push( - formatAttachmentCopyProgress(globalDone, copyTotal), - ); - }); - const batchResult = - await tauriClient.copyWorkspaceAttachmentsBatch(chunk); - copyErrors.push(...batchResult.errors); - attachmentsCopied += batchResult.copied; - copyStatusCtl.push( - formatAttachmentCopyProgress( - Math.min(offset + chunk.length, copyTotal), - copyTotal, - ), - ); + copyStatusCtl.dispose(); } - } finally { - unlisten?.(); - copyStatusCtl.dispose(); } } @@ -661,14 +688,11 @@ export function ImportPage() { ? ` Skipped ${skippedSyncedCount} already-synced observation(s).` : ''; const baseMsg = `Imported ${result.imported} observations (${result.conflicts} conflicts).${skipMsg}`; - const indexMsg = result.indexRebuildScheduled - ? ' Rebuilding observation indexes in the background (see activity banner).' - : ''; const attMsg = copyItems.length > 0 ? ` Copied ${attachmentsCopied}/${copyItems.length} referenced attachment(s) to queue.${copyErrors.length ? ` Errors: ${copyErrors.slice(0, 5).join('; ')}${copyErrors.length > 5 ? '…' : ''}` : ''}` : ''; - setMessage(`${baseMsg}${indexMsg}${attMsg}`); + setMessage(`${baseMsg}${attMsg}`); clearStagedFiles(); setPreferSkipSynced(false); setSkippedSyncedAtStaging(0); @@ -680,7 +704,7 @@ export function ImportPage() { setImportActivity(null); } }, [ - stagedJson, + previewReport, stagedAttachments, preferSkipSynced, skippedSyncedAtStaging, @@ -760,6 +784,7 @@ export function ImportPage() { clearStagingLists(); setPreferSkipSynced(false); setSkippedSyncedAtStaging(0); + setPreviewReport(null); }}> Clear staging @@ -778,7 +803,10 @@ export function ImportPage() { className="import-staging-row-remove" aria-label={`Remove ${s.name}`} disabled={busy} - onClick={() => removeStagedJson(s.nativePath)}> + onClick={() => { + removeStagedJson(s.nativePath); + setPreviewReport(null); + }}> × @@ -807,7 +835,10 @@ export function ImportPage() { className="import-staging-row-remove" aria-label={`Remove ${s.name}`} disabled={busy} - onClick={() => removeStagedAttachment(s.nativePath)}> + onClick={() => { + removeStagedAttachment(s.nativePath); + setPreviewReport(null); + }}> × @@ -833,23 +864,35 @@ export function ImportPage() { type="button" className="btn-icon" disabled={busy || stagedJson.length === 0} - onClick={() => void runFullImport()}> + onClick={() => void runValidate()}> - download + fact_check - {busy ? 'Working…' : 'Import into local store'} + {busy ? 'Working…' : 'Validate'}
{previewReport && preflightSummary ? (
-

Validation summary

+

Validation results

{preflightSummary.observationCount} observations ·{' '} {preflightSummary.formTypeCount} form types

+
+ +
) : null} diff --git a/desktop/src/pages/SyncPage.tsx b/desktop/src/pages/SyncPage.tsx index 9bd45b42b..fa1a0dbb4 100644 --- a/desktop/src/pages/SyncPage.tsx +++ b/desktop/src/pages/SyncPage.tsx @@ -1,6 +1,7 @@ import { useCallback, useEffect, useState } from 'react'; import { Link } from 'react-router-dom'; import { ForcePushMissingAttachmentsDialog } from '../components/ForcePushMissingAttachmentsDialog'; +import { ResetLocalDataDialog } from '../components/ResetLocalDataDialog'; import { useProfileAutoSynkAuth } from '../hooks/useProfileAutoSynkAuth'; import { tauriClient } from '../lib/tauriClient'; import { confirmDestructiveAction } from '../lib/destructivePolicy'; @@ -56,6 +57,7 @@ export function SyncPage() { const [missingAttachmentIssues, setMissingAttachmentIssues] = useState< MissingAttachmentIssue[] | null >(null); + const [resetLocalOpen, setResetLocalOpen] = useState(false); const [indexStatus, setIndexStatus] = useState<{ activeGeneration: number; lastRebuildAt?: string | null; @@ -193,15 +195,8 @@ export function SyncPage() { } async function resetLocalData() { - if ( - !(await confirmDestructiveAction( - 'local_reset', - 'Remove all observations and attachment files from this device and reset sync offsets.', - )) - ) { - return; - } - await resetLocalWorkspaceData(); + await loadHealth(); + setResetLocalOpen(true); } return ( @@ -365,6 +360,19 @@ export function SyncPage() { void runPush(force); }} /> + + { + setResetLocalOpen(false); + if (!choice) { + return; + } + void resetLocalWorkspaceData({ pendingOnly: choice.pendingOnly }); + }} + /> ); } diff --git a/desktop/src/store/useCustodianStore.ts b/desktop/src/store/useCustodianStore.ts index 745544287..a013f7bb4 100644 --- a/desktop/src/store/useCustodianStore.ts +++ b/desktop/src/store/useCustodianStore.ts @@ -7,7 +7,11 @@ import { setCustodianSyncProgressHandler, SyncPausedError, } from '../lib/syncTauriEvents'; -import { partitionPendingPushObservations } from '../lib/pushAttachmentAudit'; +import { + formatMissingAttachmentHighlight, + formatMissingAttachmentReport, + partitionPendingPushObservations, +} from '../lib/pushAttachmentAudit'; import { getOrCreateClientId, syncGateway } from '../services/synk'; import { isSyncHttpUnauthorized } from '../services/synk/syncErrors'; import type { @@ -229,6 +233,8 @@ interface CustodianState { loading: boolean; error: string | null; syncMessage: string | null; + /** Full text for optional “Save report” (e.g. missing-attachment push details). */ + syncDetailReport: string | null; syncActivity: { op: 'pull' | 'push' | 'reset'; statusText: string; @@ -295,7 +301,9 @@ interface CustodianState { syncPauseInFlight: () => Promise; syncContinueInFlight: () => Promise; syncCancelJob: (jobId?: string | null) => Promise; - resetLocalWorkspaceData: () => Promise; + resetLocalWorkspaceData: (options?: { + pendingOnly?: boolean; + }) => Promise; } /** Tauri invoke often rejects with a string; preserve the real message for the UI. */ @@ -392,6 +400,7 @@ export const useCustodianStore = create((set, get) => ({ loading: false, error: null, syncMessage: null, + syncDetailReport: null, syncActivity: null, bundleActivity: null, exportActivity: null, @@ -540,6 +549,7 @@ export const useCustodianStore = create((set, get) => ({ activeProfileId: s.activeProfileId, profiles: s.profiles, syncMessage: null, + syncDetailReport: null, }); await reloadProfileScopedData(set, get); await get().refreshPausedSyncJob(); @@ -576,7 +586,7 @@ export const useCustodianStore = create((set, get) => ({ setSelectedObservationId: id => set({ selectedObservationId: id }), clearError: () => set({ error: null }), - clearSyncMessage: () => set({ syncMessage: null }), + clearSyncMessage: () => set({ syncMessage: null, syncDetailReport: null }), loadWorkspace: async () => withErrorHandling(set, async () => { @@ -649,7 +659,10 @@ export const useCustodianStore = create((set, get) => ({ await tauriClient.saveObservation(request); await get().loadObservations(); await get().loadHealth(); - set({ syncMessage: 'Saved locally. Observation is now pending push.' }); + set({ + syncMessage: 'Saved locally. Observation is now pending push.', + syncDetailReport: null, + }); }), synkLogin: async request => @@ -661,6 +674,7 @@ export const useCustodianStore = create((set, get) => ({ set({ authSessionsByProfileId: next, syncMessage: 'Authenticated with Synkronus.', + syncDetailReport: null, }); }), @@ -673,6 +687,7 @@ export const useCustodianStore = create((set, get) => ({ ); set({ syncMessage: null, + syncDetailReport: null, syncActivity: { op: 'pull', statusText: 'Pulling…' }, }); try { @@ -703,6 +718,7 @@ export const useCustodianStore = create((set, get) => ({ await get().loadHealth(); set({ syncMessage: capture.current.trim() || 'Pull finished.', + syncDetailReport: null, }); } finally { setCustodianSyncProgressHandler(null); @@ -720,6 +736,7 @@ export const useCustodianStore = create((set, get) => ({ ); set({ syncMessage: null, + syncDetailReport: null, syncActivity: { op: 'push', statusText: 'Preparing push…' }, }); try { @@ -733,7 +750,10 @@ export const useCustodianStore = create((set, get) => ({ const pendingPushObservations = await tauriClient.listDirtyObservations(); if (pendingPushObservations.length === 0) { - set({ syncMessage: 'No pending observations to push.' }); + set({ + syncMessage: 'No pending observations to push.', + syncDetailReport: null, + }); return 0; } @@ -744,25 +764,21 @@ export const useCustodianStore = create((set, get) => ({ forceMissing, ); - const skipSummary = - missingAttachmentIssues.length > 0 && !forceMissing - ? ` Skipped ${missingAttachmentIssues.length} observation(s) with missing attachment file(s): ${missingAttachmentIssues - .map( - s => - `${s.id} (form: ${s.formType}; missing: ${s.missing.map(n => `"${n}"`).join(', ')})`, - ) - .join('; ')}.` - : ''; - - const forceMissingSummary = - missingAttachmentIssues.length > 0 && forceMissing - ? ` Included ${missingAttachmentIssues.length} observation(s) with missing attachment(s) (forced): ${missingAttachmentIssues - .map( - s => - `${s.id} (form: ${s.formType}; missing: ${s.missing.map(n => `"${n}"`).join(', ')})`, - ) - .join('; ')}.` + const attachmentMode = forceMissing ? 'forced' : 'skipped'; + const attachmentHighlight = + missingAttachmentIssues.length > 0 + ? formatMissingAttachmentHighlight( + missingAttachmentIssues, + attachmentMode, + ) : ''; + const attachmentReport = + missingAttachmentIssues.length > 0 + ? formatMissingAttachmentReport( + missingAttachmentIssues, + attachmentMode, + ) + : null; if ( missingAttachmentIssues.length > 0 && @@ -779,7 +795,8 @@ export const useCustodianStore = create((set, get) => ({ if (readyToPush.length === 0) { set({ - syncMessage: `Nothing pushed.${skipSummary}`.trim(), + syncMessage: `Nothing pushed.${attachmentHighlight}`.trim(), + syncDetailReport: attachmentReport, }); return 0; } @@ -793,7 +810,9 @@ export const useCustodianStore = create((set, get) => ({ xOdeVersion: SYNKRONUS_CLIENT_VERSION, pushPrepare: { readyObservationIds: readyToPush.map(o => o.id), - skipSummary: skipSummary.trim() ? skipSummary : undefined, + skipSummary: attachmentHighlight.trim() + ? attachmentHighlight.trim() + : undefined, }, }); const resumePayloadBound = (): SyncResumeJobRequest => ({ @@ -812,8 +831,8 @@ export const useCustodianStore = create((set, get) => ({ ? Number(acceptedMatch[1]) : readyToPush.length; set({ - syncMessage: - `${capture.current.trim()}${skipSummary}${forceMissingSummary}`.trim(), + syncMessage: `${capture.current.trim()}${attachmentHighlight}`.trim(), + syncDetailReport: attachmentReport, }); return accepted; } finally { @@ -832,6 +851,7 @@ export const useCustodianStore = create((set, get) => ({ ); set({ syncMessage: null, + syncDetailReport: null, syncActivity: { op: 'reset', statusText: 'Resetting server repository…', @@ -870,6 +890,7 @@ export const useCustodianStore = create((set, get) => ({ syncMessage: capture.current.trim() || 'Server repository reset and pull finished.', + syncDetailReport: null, }); } finally { setCustodianSyncProgressHandler(null); @@ -935,7 +956,10 @@ export const useCustodianStore = create((set, get) => ({ await get().loadObservations(); await get().loadHealth(); if (capture.current.trim()) { - set({ syncMessage: capture.current.trim() }); + set({ + syncMessage: capture.current.trim(), + syncDetailReport: null, + }); } } finally { setCustodianSyncProgressHandler(null); @@ -959,14 +983,17 @@ export const useCustodianStore = create((set, get) => ({ await get().loadHealth(); }, - resetLocalWorkspaceData: async () => + resetLocalWorkspaceData: async options => withErrorHandling(set, async () => { - await tauriClient.resetLocalWorkspaceData(); + const pendingOnly = options?.pendingOnly === true; + await tauriClient.resetLocalWorkspaceData({ pendingOnly }); await reloadProfileScopedData(set, get); set({ selectedObservationId: null, - syncMessage: - 'Local data reset: observations cleared, attachments removed, sync offsets reset.', + syncMessage: pendingOnly + ? 'Pending observations cleared. Synced data and sync offsets kept.' + : 'Local data reset: observations cleared, attachments removed, sync offsets reset.', + syncDetailReport: null, }); }), })); diff --git a/desktop/src/types/domain.ts b/desktop/src/types/domain.ts index 1f008823a..ac3fca803 100644 --- a/desktop/src/types/domain.ts +++ b/desktop/src/types/domain.ts @@ -95,6 +95,27 @@ export interface ImportSyncAppearanceScanResult { unsyncedPaths: string[]; } +/** One issue from host-side import validation ({@link parseAndValidateImportJsonPaths}). */ +export interface ImportHostIssue { + severity: 'error' | 'warning' | string; + code: string; + message: string; + fileName?: string; + observationId?: string; + formType?: string | null; +} + +/** Result of parallel Rust parse + schema/attachment validation. */ +export interface ImportValidateBatchResult { + files: ParsedImportFileResult[]; + issues: ImportHostIssue[]; + observationCount: number; + formTypeCount: number; + referencedAttachmentNames: string[]; + missingAttachmentNames: string[]; + orphanAttachmentNames: string[]; +} + export interface AttachmentCopyBatchResult { copied: number; failed: number; From db544e6f9f98334191ee7395de3abae34cba3b33 Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Sun, 16 Aug 2026 15:02:11 +0200 Subject: [PATCH 30/32] feat(formulus): add log viewer to Help&Support, optmizie sync implementation --- formulus/App.tsx | 2 + .../src/main/assets/webview/formulus-load.js | 97 +++- .../formulus/DiagnosticsModule.kt | 43 ++ .../formulus/DiagnosticsStore.kt | 146 ++++++ .../formulus/MainApplication.kt | 1 + .../formulus/UserAppPackage.kt | 2 +- .../main/res/drawable/ic_stat_formulus.xml | 28 ++ formulus/index.js | 2 + formulus/metro.config.js | 14 +- formulus/src/api/synkronus/Auth.ts | 7 +- .../synkronus/__tests__/downloadPool.test.ts | 93 ++++ formulus/src/api/synkronus/download.ts | 1 + formulus/src/api/synkronus/downloadPool.ts | 67 +++ formulus/src/api/synkronus/index.ts | 435 ++++++++---------- formulus/src/components/CustomAppWebView.tsx | 23 +- formulus/src/components/FormplayerModal.tsx | 7 - .../src/components/common/FormListTable.tsx | 155 +++++++ .../common/ObservationListTable.tsx | 196 ++++++++ .../components/common/ObservationPager.tsx | 117 +++++ formulus/src/components/common/index.ts | 3 + .../__tests__/observationIndexGuards.test.ts | 121 ++++- .../__tests__/observationListQuery.test.ts | 68 +++ formulus/src/database/observationListQuery.ts | 159 +++++++ .../repositories/LocalRepoInterface.ts | 17 + .../database/repositories/WatermelonDBRepo.ts | 288 ++++++++---- .../repositories/__tests__/LocalRepo.test.ts | 28 +- .../__tests__/WatermelonDBRepo.test.ts | 41 ++ formulus/src/database/schema.ts | 1 + formulus/src/diagnostics/DiagnosticLog.ts | 210 +++++++++ formulus/src/diagnostics/DirtyExitGate.tsx | 65 +++ .../__tests__/DiagnosticLog.test.ts | 73 +++ .../__tests__/classifyExit.test.ts | 79 ++++ .../__tests__/consumeDirtyExit.test.ts | 57 +++ .../__tests__/exportDiagnostics.test.ts | 41 ++ .../src/diagnostics/__tests__/logger.test.ts | 62 +++ .../src/diagnostics/__tests__/redact.test.ts | 65 +++ formulus/src/diagnostics/classifyExit.ts | 60 +++ formulus/src/diagnostics/consumeDirtyExit.ts | 67 +++ formulus/src/diagnostics/exportDiagnostics.ts | 75 +++ .../src/diagnostics/exportDiagnosticsText.ts | 39 ++ formulus/src/diagnostics/index.ts | 14 + .../src/diagnostics/installErrorHandlers.ts | 58 +++ formulus/src/diagnostics/logger.ts | 162 +++++++ formulus/src/diagnostics/memoryFs.ts | 41 ++ formulus/src/diagnostics/nativeExits.ts | 23 + formulus/src/diagnostics/paths.ts | 25 + formulus/src/diagnostics/redact.ts | 105 +++++ formulus/src/diagnostics/sessionHeartbeat.ts | 29 ++ formulus/src/diagnostics/types.ts | 67 +++ formulus/src/hooks/useForms.ts | 53 ++- formulus/src/hooks/useObservations.ts | 190 ++++---- formulus/src/locales/en.json | 27 +- formulus/src/locales/fr.json | 27 +- formulus/src/locales/pt.json | 27 +- formulus/src/screens/FormsScreen.tsx | 86 ++-- formulus/src/screens/HelpScreen.tsx | 158 ++++++- .../src/screens/ObservationDetailScreen.tsx | 18 +- formulus/src/screens/ObservationsScreen.tsx | 247 +++------- formulus/src/screens/SyncScreen.tsx | 18 +- formulus/src/services/AppConfigService.ts | 6 - formulus/src/services/AppVersionService.ts | 2 - formulus/src/services/ClientIdService.ts | 1 - formulus/src/services/ExtensionService.ts | 55 --- formulus/src/services/FormService.ts | 76 ++- formulus/src/services/GeolocationService.ts | 1 - formulus/src/services/NotificationService.ts | 23 +- .../src/services/ObservationIndexService.ts | 317 ++++++++++--- formulus/src/services/QRSettingsService.ts | 2 - formulus/src/services/SyncService.ts | 109 +++-- .../services/__tests__/FormService.test.ts | 2 + .../src/sync/__tests__/syncProgressUi.test.ts | 2 +- formulus/src/sync/syncConstants.ts | 22 + .../src/utils/__tests__/dateUtils.test.ts | 13 + formulus/src/utils/dateUtils.ts | 21 + .../src/webview/FormulusMessageHandlers.ts | 97 +--- .../src/webview/FormulusWebViewHandler.ts | 11 + 76 files changed, 4122 insertions(+), 1068 deletions(-) create mode 100644 formulus/android/app/src/main/java/org/opendataensemble/formulus/DiagnosticsModule.kt create mode 100644 formulus/android/app/src/main/java/org/opendataensemble/formulus/DiagnosticsStore.kt create mode 100644 formulus/android/app/src/main/res/drawable/ic_stat_formulus.xml create mode 100644 formulus/src/api/synkronus/__tests__/downloadPool.test.ts create mode 100644 formulus/src/api/synkronus/downloadPool.ts create mode 100644 formulus/src/components/common/FormListTable.tsx create mode 100644 formulus/src/components/common/ObservationListTable.tsx create mode 100644 formulus/src/components/common/ObservationPager.tsx create mode 100644 formulus/src/database/__tests__/observationListQuery.test.ts create mode 100644 formulus/src/database/observationListQuery.ts create mode 100644 formulus/src/diagnostics/DiagnosticLog.ts create mode 100644 formulus/src/diagnostics/DirtyExitGate.tsx create mode 100644 formulus/src/diagnostics/__tests__/DiagnosticLog.test.ts create mode 100644 formulus/src/diagnostics/__tests__/classifyExit.test.ts create mode 100644 formulus/src/diagnostics/__tests__/consumeDirtyExit.test.ts create mode 100644 formulus/src/diagnostics/__tests__/exportDiagnostics.test.ts create mode 100644 formulus/src/diagnostics/__tests__/logger.test.ts create mode 100644 formulus/src/diagnostics/__tests__/redact.test.ts create mode 100644 formulus/src/diagnostics/classifyExit.ts create mode 100644 formulus/src/diagnostics/consumeDirtyExit.ts create mode 100644 formulus/src/diagnostics/exportDiagnostics.ts create mode 100644 formulus/src/diagnostics/exportDiagnosticsText.ts create mode 100644 formulus/src/diagnostics/index.ts create mode 100644 formulus/src/diagnostics/installErrorHandlers.ts create mode 100644 formulus/src/diagnostics/logger.ts create mode 100644 formulus/src/diagnostics/memoryFs.ts create mode 100644 formulus/src/diagnostics/nativeExits.ts create mode 100644 formulus/src/diagnostics/paths.ts create mode 100644 formulus/src/diagnostics/redact.ts create mode 100644 formulus/src/diagnostics/sessionHeartbeat.ts create mode 100644 formulus/src/diagnostics/types.ts create mode 100644 formulus/src/utils/__tests__/dateUtils.test.ts diff --git a/formulus/App.tsx b/formulus/App.tsx index 6d45479d3..d2d6cb2e0 100644 --- a/formulus/App.tsx +++ b/formulus/App.tsx @@ -30,6 +30,7 @@ import MainAppNavigator from './src/navigation/MainAppNavigator'; import { FormInitData } from './src/webview/FormulusInterfaceDefinition.ts'; import { FormSpec } from './src/services'; import { initFormulusI18n, i18n } from './src/i18n'; +import { DirtyExitGate } from './src/diagnostics/DirtyExitGate'; /** * Inner component that consumes the AppTheme context to build a dynamic @@ -268,6 +269,7 @@ function AppInner(): React.JSX.Element { barStyle={isDark ? 'light-content' : 'dark-content'} backgroundColor={themeColors.surface} /> + {formplayerStack.map((entry, index) => ( diff --git a/formulus/android/app/src/main/assets/webview/formulus-load.js b/formulus/android/app/src/main/assets/webview/formulus-load.js index 7c8e7b2b8..497123b85 100644 --- a/formulus/android/app/src/main/assets/webview/formulus-load.js +++ b/formulus/android/app/src/main/assets/webview/formulus-load.js @@ -1,9 +1,9 @@ /** * Formulus Load Script - * + * * This is a standalone script that client code must include to access the Formulus API. * It handles complete injection failure and recovery. - * + * * Usage: * * */ -(function() { +(function () { 'use strict'; // Prevent multiple inclusions @@ -25,7 +25,7 @@ * The ONLY function client code should use to access Formulus API * This function is completely self-contained and can recover from any injection failure */ - window.getFormulus = function() { + window.getFormulus = function () { return new Promise((resolve, reject) => { console.log('getFormulus: Starting API load...'); @@ -42,51 +42,104 @@ function checkExistingAPI() { const api = window.formulus || window.globalThis?.formulus; - return api && typeof api === 'object' && typeof api.getVersion === 'function'; + return ( + api && typeof api === 'object' && typeof api.getVersion === 'function' + ); } function getExistingAPI() { - return window.formulus || window.globalThis?.formulus; + const api = window.formulus || window.globalThis?.formulus; + ensureGetObservationsByQuery(api); + return api; + } + + function ensureGetObservationsByQuery(api) { + if (!api) return; + // Always replace: injected API may have buggy getObservationsByQuery that delegates to getObservations (drops whereClause) + api.getObservationsByQuery = function (options) { + return new Promise(function (resolve, reject) { + const messageId = + 'msg_' + Date.now() + '_' + Math.floor(Math.random() * 1000); + const callback = function (event) { + try { + var data = + typeof event.data === 'string' + ? JSON.parse(event.data) + : event.data; + if ( + data.type === 'getObservationsByQuery_response' && + data.messageId === messageId + ) { + window.removeEventListener('message', callback); + if (data.error) reject(new Error(data.error)); + else resolve(data.result); + } + } catch (e) { + window.removeEventListener('message', callback); + reject(e); + } + }; + window.addEventListener('message', callback); + window.ReactNativeWebView.postMessage( + JSON.stringify({ + type: 'getObservationsByQuery', + messageId: messageId, + formType: options.formType, + isDraft: options.isDraft, + includeDeleted: options.includeDeleted, + filter: options.filter, + whereClause: options.whereClause, + }), + ); + }); + }; } function initiateRecovery() { // Request re-injection from React Native host if (window.ReactNativeWebView) { console.log('getFormulus: Requesting API re-injection from host...'); - window.ReactNativeWebView.postMessage(JSON.stringify({ - type: 'requestApiReinjection', - timestamp: Date.now(), - reason: 'api_load_recovery' - })); + window.ReactNativeWebView.postMessage( + JSON.stringify({ + type: 'requestApiReinjection', + timestamp: Date.now(), + reason: 'api_load_recovery', + }), + ); } else { - console.warn('getFormulus: ReactNativeWebView not available, cannot request re-injection'); + console.warn( + 'getFormulus: ReactNativeWebView not available, cannot request re-injection', + ); } // Wait for re-injection to complete let attempts = 0; const maxAttempts = 50; // 5 seconds with 100ms intervals - + const checkForRecovery = () => { attempts++; - - console.log(`getFormulus: Recovery attempt ${attempts}/${maxAttempts}`); - + + console.log( + `getFormulus: Recovery attempt ${attempts}/${maxAttempts}`, + ); + // Check if we now have a working API if (checkExistingAPI()) { console.log('getFormulus: API recovery successful'); resolve(getExistingAPI()); return; } - + if (attempts >= maxAttempts) { - const errorMsg = 'Formulus API load failed: No API available after maximum recovery attempts'; + const errorMsg = + 'Formulus API load failed: No API available after maximum recovery attempts'; console.error('getFormulus:', errorMsg); reject(new Error(errorMsg)); } else { setTimeout(checkForRecovery, 100); } }; - + // Start checking immediately checkForRecovery(); } @@ -94,9 +147,11 @@ }; // Also expose a synchronous check function for quick availability testing - window.formulusAvailable = function() { + window.formulusAvailable = function () { const api = window.formulus || window.globalThis?.formulus; - return api && typeof api === 'object' && typeof api.getVersion === 'function'; + return ( + api && typeof api === 'object' && typeof api.getVersion === 'function' + ); }; console.log('getFormulus: Load script ready'); diff --git a/formulus/android/app/src/main/java/org/opendataensemble/formulus/DiagnosticsModule.kt b/formulus/android/app/src/main/java/org/opendataensemble/formulus/DiagnosticsModule.kt new file mode 100644 index 000000000..2afb62072 --- /dev/null +++ b/formulus/android/app/src/main/java/org/opendataensemble/formulus/DiagnosticsModule.kt @@ -0,0 +1,43 @@ +package org.opendataensemble.formulus + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReactContextBaseJavaModule +import com.facebook.react.bridge.ReactMethod +import org.json.JSONObject + +class DiagnosticsModule(reactContext: ReactApplicationContext) : + ReactContextBaseJavaModule(reactContext) { + + override fun getName(): String { + return "DiagnosticsModule" + } + + @ReactMethod + fun getRecentExits(max: Int, promise: Promise) { + try { + val lines = DiagnosticsStore.readRecentExitLines(reactApplicationContext, max) + val array = Arguments.createArray() + for (line in lines) { + try { + val obj = JSONObject(line) + val map = Arguments.createMap() + map.putDouble("timestamp", obj.optLong("timestamp").toDouble()) + map.putString("reason", obj.optString("reason")) + map.putInt("status", obj.optInt("status")) + map.putInt("importance", obj.optInt("importance")) + map.putInt("pssKb", obj.optInt("pssKb")) + map.putInt("rssKb", obj.optInt("rssKb")) + map.putString("description", obj.optString("description")) + array.pushMap(map) + } catch (_: Throwable) { + // skip malformed + } + } + promise.resolve(array) + } catch (e: Throwable) { + promise.reject("diagnostics_exits", e) + } + } +} diff --git a/formulus/android/app/src/main/java/org/opendataensemble/formulus/DiagnosticsStore.kt b/formulus/android/app/src/main/java/org/opendataensemble/formulus/DiagnosticsStore.kt new file mode 100644 index 000000000..231d88770 --- /dev/null +++ b/formulus/android/app/src/main/java/org/opendataensemble/formulus/DiagnosticsStore.kt @@ -0,0 +1,146 @@ +package org.opendataensemble.formulus + +import android.app.ActivityManager +import android.app.ApplicationExitInfo +import android.content.Context +import android.os.Build +import org.json.JSONObject +import java.io.File + +/** + * Writes Android [ApplicationExitInfo] records to filesDir/diagnostics/exits.ndjson + * so JS can read the same path as RNFS.DocumentDirectoryPath/diagnostics/. + */ +object DiagnosticsStore { + const val DIR_NAME = "diagnostics" + const val EXITS_FILE = "exits.ndjson" + private const val MAX_BYTES = 256 * 1024 + private const val DESCRIPTION_MAX = 500 + + fun recordHistoricalExits(context: Context) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + return + } + try { + val am = + context.getSystemService(Context.ACTIVITY_SERVICE) as? ActivityManager + ?: return + val exits = am.getHistoricalProcessExitReasons(context.packageName, 0, 5) + if (exits.isNullOrEmpty()) { + return + } + val file = exitsFile(context) + val known = readTimestamps(file) + val batch = StringBuilder() + for (info in exits) { + val ts = info.timestamp + if (!known.add(ts)) { + continue + } + batch.append(toJson(info)).append('\n') + } + if (batch.isNotEmpty()) { + appendRotated(file, batch.toString()) + } + } catch (_: Throwable) { + // Never block app start. + } + } + + fun readRecentExitLines(context: Context, max: Int): List { + val file = exitsFile(context) + if (!file.exists()) { + return emptyList() + } + return try { + file.readLines().filter { it.isNotBlank() }.takeLast(max.coerceAtLeast(0)) + } catch (_: Throwable) { + emptyList() + } + } + + private fun exitsFile(context: Context): File { + val dir = File(context.filesDir, DIR_NAME) + if (!dir.exists()) { + dir.mkdirs() + } + return File(dir, EXITS_FILE) + } + + private fun readTimestamps(file: File): MutableSet { + val out = mutableSetOf() + if (!file.exists()) { + return out + } + try { + file.forEachLine { line -> + if (line.isBlank()) return@forEachLine + try { + val ts = JSONObject(line).optLong("timestamp", -1L) + if (ts >= 0L) { + out.add(ts) + } + } catch (_: Throwable) { + // skip malformed + } + } + } catch (_: Throwable) { + // ignore + } + return out + } + + private fun toJson(info: ApplicationExitInfo): String { + val obj = JSONObject() + obj.put("timestamp", info.timestamp) + obj.put("reason", reasonName(info.reason)) + obj.put("status", info.status) + obj.put("importance", info.importance) + obj.put("pssKb", info.pss) + obj.put("rssKb", info.rss) + val description = info.description?.take(DESCRIPTION_MAX) ?: "" + obj.put("description", description) + return obj.toString() + } + + private fun reasonName(reason: Int): String { + return when (reason) { + ApplicationExitInfo.REASON_EXIT_SELF -> "REASON_EXIT_SELF" + ApplicationExitInfo.REASON_SIGNALED -> "REASON_SIGNALED" + ApplicationExitInfo.REASON_LOW_MEMORY -> "REASON_LOW_MEMORY" + ApplicationExitInfo.REASON_CRASH -> "REASON_CRASH" + ApplicationExitInfo.REASON_CRASH_NATIVE -> "REASON_CRASH_NATIVE" + ApplicationExitInfo.REASON_ANR -> "REASON_ANR" + ApplicationExitInfo.REASON_INITIALIZATION_FAILURE -> + "REASON_INITIALIZATION_FAILURE" + ApplicationExitInfo.REASON_PERMISSION_CHANGE -> "REASON_PERMISSION_CHANGE" + ApplicationExitInfo.REASON_EXCESSIVE_RESOURCE_USAGE -> + "REASON_EXCESSIVE_RESOURCE_USAGE" + ApplicationExitInfo.REASON_USER_REQUESTED -> "REASON_USER_REQUESTED" + ApplicationExitInfo.REASON_USER_STOPPED -> "REASON_USER_STOPPED" + ApplicationExitInfo.REASON_DEPENDENCY_DIED -> "REASON_DEPENDENCY_DIED" + ApplicationExitInfo.REASON_OTHER -> "REASON_OTHER" + else -> { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU && + reason == ApplicationExitInfo.REASON_FREEZER + ) { + "REASON_FREEZER" + } else { + "REASON_$reason" + } + } + } + } + + private fun appendRotated(file: File, chunk: String) { + val nextSize = (if (file.exists()) file.length() else 0L) + chunk.length + if (nextSize > MAX_BYTES && file.exists()) { + val backup = File(file.parentFile, "${file.name}.1") + if (backup.exists()) { + backup.delete() + } + file.renameTo(backup) + } + file.appendText(chunk) + } +} diff --git a/formulus/android/app/src/main/java/org/opendataensemble/formulus/MainApplication.kt b/formulus/android/app/src/main/java/org/opendataensemble/formulus/MainApplication.kt index ec093d426..163ecce95 100644 --- a/formulus/android/app/src/main/java/org/opendataensemble/formulus/MainApplication.kt +++ b/formulus/android/app/src/main/java/org/opendataensemble/formulus/MainApplication.kt @@ -23,6 +23,7 @@ class MainApplication : Application(), ReactApplication { override fun onCreate() { super.onCreate() + DiagnosticsStore.recordHistoricalExits(this) WebView.setWebContentsDebuggingEnabled(true) loadReactNative(this) } diff --git a/formulus/android/app/src/main/java/org/opendataensemble/formulus/UserAppPackage.kt b/formulus/android/app/src/main/java/org/opendataensemble/formulus/UserAppPackage.kt index 00307cf32..eaf07fe48 100644 --- a/formulus/android/app/src/main/java/org/opendataensemble/formulus/UserAppPackage.kt +++ b/formulus/android/app/src/main/java/org/opendataensemble/formulus/UserAppPackage.kt @@ -7,7 +7,7 @@ import com.facebook.react.uimanager.ViewManager class UserAppPackage : ReactPackage { override fun createNativeModules(reactContext: ReactApplicationContext): List { - return listOf(UserAppModule(reactContext)) + return listOf(UserAppModule(reactContext), DiagnosticsModule(reactContext)) } override fun createViewManagers(reactContext: ReactApplicationContext): List> { diff --git a/formulus/android/app/src/main/res/drawable/ic_stat_formulus.xml b/formulus/android/app/src/main/res/drawable/ic_stat_formulus.xml new file mode 100644 index 000000000..fd78f1c76 --- /dev/null +++ b/formulus/android/app/src/main/res/drawable/ic_stat_formulus.xml @@ -0,0 +1,28 @@ + + + diff --git a/formulus/index.js b/formulus/index.js index d276d516d..a693cb52a 100644 --- a/formulus/index.js +++ b/formulus/index.js @@ -7,11 +7,13 @@ import notifee from '@notifee/react-native'; // Initialize axios interceptors BEFORE any other imports that might make API calls // This ensures version mismatch errors are handled from the very first request import { setupSynkronusClientInterceptors } from './src/api/synkronus/client'; +import { installErrorHandlers } from './src/diagnostics'; import App from './App'; import { name as appName } from './app.json'; // Set up interceptors immediately - before any React components or contexts load setupSynkronusClientInterceptors(); +installErrorHandlers(); if (Platform.OS === 'android') { notifee.registerForegroundService(() => { diff --git a/formulus/metro.config.js b/formulus/metro.config.js index b09423d6b..7b269fc6c 100644 --- a/formulus/metro.config.js +++ b/formulus/metro.config.js @@ -5,9 +5,16 @@ import { fileURLToPath } from 'node:url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); -/** Monorepo root (parent of formulus) so Metro can resolve @ode/tokens and @ode/components */ +/** Monorepo root — used only to resolve @ode/* paths, not as a watch root. */ const monorepoRoot = path.resolve(__dirname, '..'); +/** Workspace packages Formulus imports. Do not watch the rest of the repo. */ +const workspaceWatchFolders = [ + path.resolve(monorepoRoot, 'packages/tokens'), + path.resolve(monorepoRoot, 'packages/components'), + path.resolve(monorepoRoot, 'packages/observation-query'), +]; + /** * Force a single React/react-native instance so hooks work (avoids "Invalid hook call" / "useState of null"). * Without this, @ode/components would use its own node_modules/react and we'd have two React copies. @@ -56,7 +63,10 @@ const extraModules = { * @type {import('@react-native/metro-config').MetroConfig} */ const config = { - watchFolders: [monorepoRoot], + // Default config already watches formulus/. Extra roots are only the + // @ode packages we import — watching the monorepo root also walks + // desktop/, synkronus/, and their node_modules (ENOSPC on typical Linux). + watchFolders: workspaceWatchFolders, resolver: { unstable_enableSymlinks: true, unstable_enablePackageExports: true, diff --git a/formulus/src/api/synkronus/Auth.ts b/formulus/src/api/synkronus/Auth.ts index a73d1db40..9b23a7157 100644 --- a/formulus/src/api/synkronus/Auth.ts +++ b/formulus/src/api/synkronus/Auth.ts @@ -2,6 +2,7 @@ import { synkronusApi } from './index'; import AsyncStorage from '@react-native-async-storage/async-storage'; import * as Keychain from 'react-native-keychain'; import { ODE_VERSION } from '../../version'; +import { logger } from '../../diagnostics/logger'; export type UserRole = 'read-only' | 'read-write' | 'admin'; @@ -100,7 +101,7 @@ export const login = async ( username: string, password: string, ): Promise => { - console.log('Logging in with', username); + logger.info('auth', 'login ok'); const api = await synkronusApi.getApi(); synkronusApi.clearTokenCache(); @@ -155,7 +156,6 @@ export const getApiAuthToken = async (): Promise => { try { const token = await AsyncStorage.getItem('@token'); if (token) { - console.debug('Token retrieved from AsyncStorage.'); return token; } console.warn('No token found in AsyncStorage.'); @@ -199,9 +199,8 @@ export const autoLogin = async (): Promise => { return null; } - console.log('🔄 Attempting auto-login with stored credentials'); const userInfo = await login(credentials.username, credentials.password); - console.log('✅ Auto-login successful - token refreshed'); + logger.info('auth', 'auto-login ok'); return userInfo; } catch (error: unknown) { const httpError = error as HttpError; diff --git a/formulus/src/api/synkronus/__tests__/downloadPool.test.ts b/formulus/src/api/synkronus/__tests__/downloadPool.test.ts new file mode 100644 index 000000000..f7bfdea16 --- /dev/null +++ b/formulus/src/api/synkronus/__tests__/downloadPool.test.ts @@ -0,0 +1,93 @@ +import { + failedDownloadCount, + runWithConcurrency, + SYNC_CANCELLED_MESSAGE, +} from '../downloadPool'; + +function delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe('runWithConcurrency', () => { + it('returns results aligned to input indexes when completion order differs', async () => { + const items = [30, 5, 15, 1]; + const results = await runWithConcurrency(items, 3, async (ms, index) => { + await delay(ms); + return index; + }); + expect(results).toEqual([0, 1, 2, 3]); + }); + + it('never runs more workers than the concurrency cap', async () => { + let inFlight = 0; + let maxInFlight = 0; + await runWithConcurrency([1, 2, 3, 4, 5, 6], 3, async n => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await delay(15); + inFlight -= 1; + return n; + }); + expect(maxInFlight).toBe(3); + }); + + it('treats a non-positive concurrency as 1', async () => { + let inFlight = 0; + let maxInFlight = 0; + await runWithConcurrency([1, 2, 3], 0, async n => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await delay(10); + inFlight -= 1; + return n; + }); + expect(maxInFlight).toBe(1); + }); + + it('returns an empty list without starting workers', async () => { + const worker = jest.fn(); + await expect(runWithConcurrency([], 4, worker)).resolves.toEqual([]); + expect(worker).not.toHaveBeenCalled(); + }); + + it('stops handing out work on cancel and notifies in-flight jobs once', async () => { + let cancelled = false; + const onCancelInFlight = jest.fn(); + const started: number[] = []; + + const run = runWithConcurrency( + [1, 2, 3, 4, 5, 6], + 2, + async n => { + started.push(n); + await delay(30); + if (n === 1) { + cancelled = true; + } + return n; + }, + { + isCancelled: () => cancelled, + onCancelInFlight, + }, + ); + + await expect(run).rejects.toThrow(SYNC_CANCELLED_MESSAGE); + expect(onCancelInFlight).toHaveBeenCalledTimes(1); + expect(started.length).toBeLessThan(6); + expect(started.length).toBeGreaterThanOrEqual(2); + }); +}); + +describe('failedDownloadCount', () => { + it('counts unsuccessful results', () => { + expect( + failedDownloadCount([ + { success: true }, + { success: false }, + { success: true }, + { success: false }, + ]), + ).toBe(2); + }); +}); diff --git a/formulus/src/api/synkronus/download.ts b/formulus/src/api/synkronus/download.ts index beb67f8f2..3b0cf48ed 100644 --- a/formulus/src/api/synkronus/download.ts +++ b/formulus/src/api/synkronus/download.ts @@ -24,6 +24,7 @@ export interface SynkronusDownloadOptions { * Automatically includes x-ode-version header. */ export function synkronusDownload(options: SynkronusDownloadOptions): { + jobId: number; promise: Promise; } { return RNFS.downloadFile({ diff --git a/formulus/src/api/synkronus/downloadPool.ts b/formulus/src/api/synkronus/downloadPool.ts new file mode 100644 index 000000000..e7e9c818d --- /dev/null +++ b/formulus/src/api/synkronus/downloadPool.ts @@ -0,0 +1,67 @@ +/** + * Bounded async pool for attachment (and similar) downloads. + * + * Workers share a single-threaded index, so in-flight work never exceeds + * `concurrency`. Cancel stops handing out new items and lets the caller abort + * native jobs already started. The function still waits for those workers to + * settle so we do not leak downloads, then throws `Sync cancelled`. + */ + +export const SYNC_CANCELLED_MESSAGE = 'Sync cancelled'; + +export async function runWithConcurrency( + items: readonly T[], + concurrency: number, + worker: (item: T, index: number) => Promise, + options?: { + isCancelled?: () => boolean; + onCancelInFlight?: () => void; + }, +): Promise { + if (items.length === 0) { + return []; + } + + const limit = Math.max( + 1, + Math.min(Math.floor(concurrency) || 1, items.length), + ); + const results: R[] = new Array(items.length); + let nextIndex = 0; + let cancelNotified = false; + + const cancelled = (): boolean => { + if (!options?.isCancelled?.()) { + return false; + } + if (!cancelNotified) { + cancelNotified = true; + options.onCancelInFlight?.(); + } + return true; + }; + + const runWorker = async (): Promise => { + while (!cancelled()) { + const i = nextIndex; + if (i >= items.length) { + return; + } + nextIndex += 1; + results[i] = await worker(items[i], i); + } + }; + + await Promise.all(Array.from({ length: limit }, () => runWorker())); + + if (cancelled()) { + throw new Error(SYNC_CANCELLED_MESSAGE); + } + return results; +} + +export function failedDownloadCount( + results: ReadonlyArray<{ success: boolean }>, +): number { + return results.reduce((n, r) => n + (r.success ? 0 : 1), 0); +} diff --git a/formulus/src/api/synkronus/index.ts b/formulus/src/api/synkronus/index.ts index 626f4bac6..a9d80b025 100644 --- a/formulus/src/api/synkronus/index.ts +++ b/formulus/src/api/synkronus/index.ts @@ -24,6 +24,7 @@ import { unzip } from 'react-native-zip-archive'; import { synkronusDownload } from './download'; import { ODE_VERSION } from '../../version'; import { pendingRoot, syncedRoot } from '../../services/attachmentStorage'; +import { logger } from '../../diagnostics/logger'; import { isRepositoryResetRequiredError, parseRepositoryResetFromAxios, @@ -32,6 +33,11 @@ import { import type { AxiosError, AxiosResponse } from 'axios'; import { effectiveRepositoryGenerationForRequest } from './repositoryGenerationRequest'; import { pullPageOutcome } from './pullCursor'; +import { + ATTACHMENT_DOWNLOAD_CONCURRENCY, + PULL_PAGE_SIZE, +} from '../../sync/syncConstants'; +import { failedDownloadCount, runWithConcurrency } from './downloadPool'; import { formatCountProgress, type SynkronusSyncOptions, @@ -63,11 +69,11 @@ function logRepositoryGenerationSync( message: string, payload?: Record, ): void { - if (payload != null) { - console.log(`[RepositoryGeneration] ${message}`, payload); - } else { - console.log(`[RepositoryGeneration] ${message}`); - } + const extras = + payload && typeof payload.observationDataVersion === 'number' + ? { counts: payload.observationDataVersion as number } + : undefined; + logger.info('sync', message, extras); } function logAxiosErrorForRepoGen(operation: string, error: unknown): void { @@ -101,6 +107,7 @@ interface DownloadResult { function throwIfSyncCancelled(isCancelled?: () => boolean): void { if (isCancelled?.()) { + logger.info('sync', 'cancel observed, aborting'); throw new Error('Sync cancelled'); } } @@ -242,7 +249,6 @@ class SynkronusApi { const removeIfExists = async (path: string) => { try { if (await RNFS.exists(path)) { - console.debug(`Removing files from ${path}`); await RNFS.unlink(path); } await RNFS.mkdir(path); @@ -293,10 +299,6 @@ class SynkronusApi { prefix: string, progressCallback?: (progressPercent: number) => void, ): Promise { - console.debug( - `Downloading files with prefix "${prefix}" to: ${outputRootDirectory}`, - ); - const config = await this.getConfig(); const filesToDownload = manifest.files.filter(file => file.path.startsWith(prefix), @@ -439,10 +441,6 @@ class SynkronusApi { details: i18n.t('sync.progress.checkingAttachments'), }); - console.debug( - `Getting attachment manifest since version ${lastAttachmentVersion}`, - ); - const api = await this.getApi(); const manifestClientGen = await this.getRepositoryGenerationForRequestOrNull(); @@ -474,12 +472,8 @@ class SynkronusApi { // Handle null operations array (server returns null when no operations) const operations = manifest.operations || []; - console.debug( - `Received attachment manifest: ${operations.length} operations at version ${manifest.current_version}`, - ); if (operations.length === 0) { - console.debug('No attachment operations to perform'); await AsyncStorage.setItem( '@last_attachment_version', manifest.current_version.toString(), @@ -498,10 +492,6 @@ class SynkronusApi { const deleteOps = operations.filter(op => op.operation === 'delete'); const totalSteps = deleteOps.length + downloadOps.length; - console.debug( - `Processing ${downloadOps.length} downloads, ${deleteOps.length} deletions`, - ); - let doneSteps = 0; const bumpProgress = (currentItem?: string) => { reportSyncProgress(report, { @@ -541,9 +531,6 @@ class SynkronusApi { '@last_attachment_version', manifest.current_version.toString(), ); - console.debug( - `Attachment sync completed at version ${manifest.current_version}`, - ); } catch (error: unknown) { this.rethrowIfRepositoryResetConflict(error); if (isRepositoryResetRequiredError(error)) { @@ -570,19 +557,11 @@ class SynkronusApi { try { const syncedPath = `${syncedDirectory}/${op.attachment_id}`; const pendingPath = `${pendingDirectory}/${op.attachment_id}`; - let deleted = false; if (await RNFS.exists(syncedPath)) { await RNFS.unlink(syncedPath); - deleted = true; } if (await RNFS.exists(pendingPath)) { await RNFS.unlink(pendingPath); - deleted = true; - } - if (deleted) { - console.debug(`Deleted attachment: ${op.attachment_id}`); - } else { - console.debug(`Attachment already deleted: ${op.attachment_id}`); } } catch (error) { console.error( @@ -605,6 +584,9 @@ class SynkronusApi { }, ): Promise { const isCancelled = options?.isCancelled; + if (downloadOps.length === 0) { + return; + } const syncedDirectory = syncedRoot(); await RNFS.mkdir(syncedDirectory); @@ -624,39 +606,39 @@ class SynkronusApi { // with `version > cursor`, so being told to download at all means either // (a) we don't have the file yet or (b) our local copy is stale or // corrupt — re-fetching is always the correct action. + const started = Date.now(); const results = await this.downloadRawFiles(urls, localPaths, undefined, { overwrite: true, isCancelled, - onFileStart: (index: number) => { - const op = downloadOps[index]; - reportSyncProgress(options?.onProgress, { - phase: 'pull_attachments', - current: (options?.startDone ?? 0) + index, - total: options?.totalSteps ?? downloadOps.length, - details: formatCountProgress( - (options?.startDone ?? 0) + index, - options?.totalSteps ?? downloadOps.length, - ), - currentItem: op.attachment_id, - }); - }, + concurrency: ATTACHMENT_DOWNLOAD_CONCURRENCY, onFileComplete: (index: number) => { options?.onStepComplete?.(downloadOps[index].attachment_id); }, }); + const failed = failedDownloadCount(results); + logger.info( + 'sync', + `attachments download=${Date.now() - started}ms files=${downloadOps.length} concurrency=${ATTACHMENT_DOWNLOAD_CONCURRENCY} failed=${failed}`, + { phase: 'pull_attachments', counts: downloadOps.length }, + ); + results.forEach((result, index) => { const op = downloadOps[index]; - if (result.success) { - console.debug( - `Downloaded attachment: ${op.attachment_id} (${result.bytesWritten} bytes)`, - ); - } else { + if (!result.success) { console.error( `Failed to download attachment ${op.attachment_id}: ${result.message}`, ); } }); + + // Leave `@last_attachment_version` unadvanced so the next sync retries + // the missing files instead of skipping them forever. + if (failed > 0) { + throw new Error( + `Failed to download ${failed} attachment${failed === 1 ? '' : 's'}`, + ); + } } private async getAttachmentsUploadManifest(): Promise { @@ -752,17 +734,28 @@ class SynkronusApi { isCancelled?: () => boolean; onFileStart?: (index: number) => void; onFileComplete?: (index: number) => void; + concurrency?: number; }, ): Promise { - const results: DownloadResult[] = []; if (urls.length !== localFilePaths.length) { throw new Error( 'URLs and local file paths arrays must have the same length', ); } const totalFiles = urls.length; - console.debug('URLS:', urls); - console.debug('Local file paths:', localFilePaths); + const concurrency = options?.concurrency ?? 1; + const activeJobIds = new Set(); + let completed = 0; + const stopActiveDownloads = () => { + for (const jobId of activeJobIds) { + try { + RNFS.stopDownload(jobId); + } catch { + // Native cancel is best-effort. + } + } + activeJobIds.clear(); + }; const singleFileCallback = ( currentIndex: number, progress: RNFS.DownloadProgressCallbackResult, @@ -771,47 +764,58 @@ class SynkronusApi { const overallProgress = ((currentIndex + fileProgress) / totalFiles) * 100; - console.debug( - `Downloading file: ${urls[currentIndex]} ${Math.round( - fileProgress * 100, - )}%`, - ); progressCallback?.(Math.round(overallProgress)); }; - for (let i = 0; i < totalFiles; i++) { - throwIfSyncCancelled(options?.isCancelled); - const url = urls[i]; - const localFilePath = localFilePaths[i]; - options?.onFileStart?.(i); - try { - console.debug(`Downloading file: ${url}`); - const result = await this.downloadRawFile( - url, - localFilePath, - (progress: RNFS.DownloadProgressCallbackResult) => - singleFileCallback(i, progress), - options, - ); - console.debug( - `Downloaded file: ${localFilePath} (size: ${result.bytesWritten})`, - ); - results.push(result); - } catch (error) { - console.error(`Failed to download file ${localFilePath}: ${error}`); - results.push({ - success: false, - message: `Failed to download file ${localFilePath}: ${error}`, - filePath: localFilePath, - bytesWritten: 0, - }); - } - options?.onFileComplete?.(i); - const progressPercent = Math.round((i / totalFiles) * 100); - progressCallback?.(progressPercent); - } - console.debug('Files downloaded'); - return results; + return runWithConcurrency( + urls, + concurrency, + async (url, i) => { + const localFilePath = localFilePaths[i]; + options?.onFileStart?.(i); + try { + const result = await this.downloadRawFile( + url, + localFilePath, + concurrency === 1 + ? (progress: RNFS.DownloadProgressCallbackResult) => + singleFileCallback(i, progress) + : undefined, + { + overwrite: options?.overwrite, + isCancelled: options?.isCancelled, + onJobStart: jobId => { + activeJobIds.add(jobId); + }, + onJobEnd: jobId => { + activeJobIds.delete(jobId); + }, + }, + ); + return result; + } catch (error) { + console.error(`Failed to download file ${localFilePath}: ${error}`); + return { + success: false, + message: `Failed to download file ${localFilePath}: ${error}`, + filePath: localFilePath, + bytesWritten: 0, + }; + } finally { + completed += 1; + options?.onFileComplete?.(i); + if (concurrency > 1) { + progressCallback?.(Math.round((completed / totalFiles) * 100)); + } else { + progressCallback?.(Math.round((i / totalFiles) * 100)); + } + } + }, + { + isCancelled: options?.isCancelled, + onCancelInFlight: stopActiveDownloads, + }, + ); } private async downloadRawFile( url: string, @@ -819,8 +823,15 @@ class SynkronusApi { progressCallback?: ( progressPercent: RNFS.DownloadProgressCallbackResult, ) => void, - options?: { overwrite?: boolean }, + options?: { + overwrite?: boolean; + isCancelled?: () => boolean; + onJobStart?: (jobId: number) => void; + onJobEnd?: (jobId: number) => void; + }, ): Promise { + throwIfSyncCancelled(options?.isCancelled); + if (await RNFS.exists(localFilePath)) { if (options?.overwrite) { // Caller is re-fetching from authoritative source (e.g. attachment @@ -854,23 +865,40 @@ class SynkronusApi { } } + throwIfSyncCancelled(options?.isCancelled); + const authToken = this.fastGetToken_cachedToken ?? (await this.fastGetToken()); - console.debug(`Downloading from: ${url}`); - const result = await synkronusDownload({ + const watchCancel = Boolean(options?.isCancelled); + const download = synkronusDownload({ fromUrl: url, toFile: localFilePath, authToken, background: true, - progressInterval: 500, // fire at most every 500ms if progressCallback is provided - progressDivider: progressCallback ? 1 : 100, // fire at most on every percentage change if progressCallback is provided + progressInterval: 500, + // Check cancel every 500ms even when the caller does not want byte progress. + progressDivider: progressCallback || watchCancel ? 1 : 100, progress: progress => { + if (options?.isCancelled?.()) { + try { + RNFS.stopDownload(progress.jobId); + } catch { + // Native cancel is best-effort. + } + } if (progressCallback) { progressCallback(progress); } }, - }).promise; + }); + options?.onJobStart?.(download.jobId); + let result: RNFS.DownloadResult; + try { + result = await download.promise; + } finally { + options?.onJobEnd?.(download.jobId); + } if (result.statusCode !== 200) { console.error( @@ -884,9 +912,6 @@ class SynkronusApi { }; } - console.debug( - `Successfully downloaded and saved (binary): ${localFilePath} (${result.bytesWritten} bytes)`, - ); return { success: true, message: `Successfully downloaded and saved (binary): ${localFilePath} (${result.bytesWritten} bytes)`, @@ -903,11 +928,10 @@ class SynkronusApi { const isCancelled = options?.isCancelled; const total = attachments.length; if (attachments.length === 0) { - console.debug('No attachments to upload'); return []; } + logger.info('sync', `uploading ${total} attachments`, { counts: total }); - console.debug('Starting attachments upload...', attachments); const pendingDirectory = pendingRoot(); const syncedDirectory = syncedRoot(); const api = await this.getApi(); @@ -964,11 +988,7 @@ class SynkronusApi { this.rethrowIfRepositoryResetConflict(err); } - if (alreadyOnServer) { - console.debug( - `Attachment ${attachmentId} already on server; skipping PUT`, - ); - } else { + if (!alreadyOnServer) { const mimeType = this.getMimeTypeFromFilename(attachmentId); const file = { uri: `file://${pendingFilePath}`, @@ -976,9 +996,6 @@ class SynkronusApi { name: attachmentId, } as unknown as File; - console.debug( - `Uploading attachment: ${attachmentId} (${fileStats.size} bytes)`, - ); await api.uploadAttachment({ attachmentId, file, @@ -1001,12 +1018,6 @@ class SynkronusApi { filePath: syncedFilePath, bytesWritten: fileStats.size, }); - - console.debug( - alreadyOnServer - ? `Confirmed existing attachment: ${attachmentId}` - : `Successfully uploaded attachment: ${attachmentId}`, - ); } catch (error: unknown) { this.rethrowIfRepositoryResetConflict(error); console.error(`Failed to upload attachment ${attachmentId}:`, error); @@ -1027,7 +1038,6 @@ class SynkronusApi { }); } - console.debug('Attachments upload completed', results); return results; } @@ -1094,68 +1104,71 @@ class SynkronusApi { const repo = databaseService.getLocalRepo(); const api = await this.getApi(); const schemaTypes = undefined; // TODO: Feature: Maybe allow partial sync - let res; + let res: AxiosResponse | undefined; let currentSince = since; let totalServerRecordsThisPull = 0; let pullPage = 0; let hasMorePages = true; let finalVersion = since; - reportSyncProgress(report, { - phase: 'pull_observations', - current: 0, - total: 0, - indeterminate: true, - details: i18n.t('sync.progress.connecting'), - }); - - do { + const fetchPullPage = async (sinceVersion: number) => { throwIfSyncCancelled(isCancelled); - pullPage += 1; const clientGen = await this.getRepositoryGenerationForRequestOrNull(); - logRepositoryGenerationSync('syncPull request', { - clientXRepositoryGeneration: clientGen ?? '(omitted)', - sinceVersion: currentSince, - }); - + const fetchStarted = Date.now(); try { - res = await api.syncPull({ + const response = await api.syncPull({ xOdeVersion: ODE_VERSION, + limit: PULL_PAGE_SIZE, syncPullRequest: { client_id: clientId, ...(clientGen != null ? { repository_generation: clientGen } : {}), since: { - version: currentSince, + version: sinceVersion, }, schema_types: schemaTypes, }, xRepositoryGeneration: clientGen ?? undefined, }); + logger.info( + 'sync', + `pull fetch=${Date.now() - fetchStarted}ms records=${ + response.data.records?.length ?? 0 + }`, + { + phase: 'fetch', + counts: response.data.records?.length ?? 0, + }, + ); + return { response, clientGen }; } catch (err: unknown) { logAxiosErrorForRepoGen('syncPull', err); this.rethrowIfRepositoryResetConflict(err); throw err; } + }; - const pullRes = res as AxiosResponse; - logRepositoryGenerationSync('syncPull response OK', { - clientSent: clientGen ?? '(omitted)', - sinceVersion: currentSince, - bodyRepositoryGeneration: res.data.repository_generation, - bodyCurrentVersion: res.data.current_version, - bodyHasMore: res.data.has_more, - recordsInThisPage: res.data.records?.length ?? 0, - headerXRepositoryGeneration: headerRepositoryGeneration( - pullRes.headers, - ), - note: 'repository_generation = server epoch (resets); current_version = observation stream cursor — a 4 and a 1 here are not a mismatch.', - }); + // Next HTTP page, started after we know has_more so it overlaps apply+index. + // Discarded if apply fails (cursor is not advanced). Never apply two pages + // at once — SQLite is a single writer. + let pendingPage: ReturnType | null = null; - console.debug( - `Pull response: page ${pullPage}, ${ - res.data.records?.length ?? 0 - } record(s), has_more=${String(res.data.has_more)}`, - ); + reportSyncProgress(report, { + phase: 'pull_observations', + current: 0, + total: 0, + indeterminate: true, + details: i18n.t('sync.progress.connecting'), + }); + + do { + throwIfSyncCancelled(isCancelled); + pullPage += 1; + const waitStarted = Date.now(); + const fetched = await (pendingPage ?? fetchPullPage(currentSince)); + const waitMs = Date.now() - waitStarted; + pendingPage = null; + const clientGen = fetched.clientGen; + res = fetched.response; this.ensureRepoGenResponseMatchesSent( 'syncPull', @@ -1166,42 +1179,16 @@ class SynkronusApi { res.data.repository_generation, ); - // 1. Pull and map changes from the API + const mapStarted = Date.now(); const domainObservations = res.data.records ? res.data.records.map(ObservationMapper.fromApi) : []; + const mapMs = Date.now() - mapStarted; totalServerRecordsThisPull += domainObservations.length; - // 2. Apply to local db (local dirty records will not be applied = last update wins). - // Skipped rows get a `last_write_won` tag (see syncConstants / WatermelonDBRepo). - // Report before and during this step: on a first-time pull the page - // can be thousands of rows, and indexing them used to leave the - // progress card sitting on "Connecting…" until the flush returned. - if (domainObservations.length > 0) { - reportSyncProgress(report, { - phase: 'pull_observations', - current: pullPage, - total: 0, - indeterminate: true, - details: i18n.t('sync.progress.savingRecords', { - count: domainObservations.length, - }), - }); - } - const pulledChanges = await repo.applyServerChanges(domainObservations, { - onIndexProgress: ({ current, total }) => { - if (total <= 0) return; - reportSyncProgress(report, { - phase: 'index_rebuild', - current, - total, - details: formatCountProgress(current, total), - }); - }, - }); - console.debug(`Applied ${pulledChanges} changes to local database`); - + // One line for the whole page: count when HTTP arrives, then leave it + // up through apply/index. Toggling "Saving…" made the count unreadable. reportSyncProgress(report, { phase: 'pull_observations', current: pullPage, @@ -1217,15 +1204,31 @@ class SynkronusApi { : i18n.t('sync.progress.downloading'), }); - // 3. Advance the cursor and persist it before fetching the next page, so - // an interrupted pull resumes here instead of restarting from zero. - // See pullCursor.ts for why this cannot skip records. + // Cursor math depends only on the response. Start the next fetch before + // apply so the RTT is hidden behind SQLite work. Persist the cursor + // only after apply succeeds (see pullCursor.ts). const pageOutcome = pullPageOutcome(res.data, currentSince); if (pageOutcome.kind === 'unusable') { throw new Error( `Sync pull stopped after page ${pullPage}: ${pageOutcome.reason}`, ); } + if (pageOutcome.kind === 'continue') { + pendingPage = fetchPullPage(pageOutcome.nextSince); + } + + // Apply + incremental index. Stay on pull_observations — flipping to + // index_rebuild made the card blink "Preparing data for search" on + // every page. That title is for the full rebuild after a bundle change. + const applyStarted = Date.now(); + await repo.applyServerChanges(domainObservations, { + isCancelled, + }); + logger.info( + 'sync', + `pull page=${pullPage} records=${domainObservations.length} wait=${waitMs}ms map=${mapMs}ms apply=${Date.now() - applyStarted}ms`, + { phase: 'page', counts: domainObservations.length }, + ); hasMorePages = pageOutcome.kind === 'continue'; const cursor = @@ -1239,17 +1242,12 @@ class SynkronusApi { } await AsyncStorage.setItem('@last_seen_version', String(cursor)); - console.debug( - `Pull cursor persisted at version ${cursor}${ - hasMorePages ? ' (more pages follow)' : '' - }`, - ); } while (hasMorePages); logRepositoryGenerationSync('syncPull all pages done', { totalServerRecordsReceived: totalServerRecordsThisPull, - finalBodyCurrentVersion: res.data.current_version, - finalBodyRepositoryGeneration: res.data.repository_generation, + finalBodyCurrentVersion: res?.data.current_version, + finalBodyRepositoryGeneration: res?.data.repository_generation, persistedObservationCursor: finalVersion, }); @@ -1308,16 +1306,11 @@ class SynkronusApi { // 1. Get pending changes from watermelondb const repo = databaseService.getLocalRepo(); const localChanges = await repo.getPendingChanges(); - console.debug(`Found ${localChanges.length} local changes to push`); // 2. Upload attachments first (if requested and available) let attachmentUploadResults: DownloadResult[] = []; if (includeAttachments) { const attachments = await this.getAttachmentsUploadManifest(); - console.debug( - `Found ${attachments.length} pending attachments to upload:`, - attachments, - ); if (attachments.length > 0) { attachmentUploadResults = await this.uploadAttachments( @@ -1337,13 +1330,6 @@ class SynkronusApi { // Continue with observation sync even if some attachments failed // The server should handle missing attachments gracefully } - - const successfulUploads = attachmentUploadResults.filter( - result => result.success, - ); - console.debug( - `Successfully uploaded ${successfulUploads.length}/${attachments.length} attachments`, - ); } } @@ -1351,7 +1337,6 @@ class SynkronusApi { // 3. Check if we have observations to push if (localChanges.length === 0) { - console.debug('No local changes to push'); reportSyncProgress(report, { phase: 'push_observations', current: 1, @@ -1369,16 +1354,6 @@ class SynkronusApi { }, ); - // If we uploaded attachments, report that - if (includeAttachments && attachmentUploadResults.length > 0) { - const successfulUploads = attachmentUploadResults.filter( - result => result.success, - ); - console.debug( - `Push completed: 0 observations, ${successfulUploads.length}/${attachmentUploadResults.length} attachments uploaded`, - ); - } - return Number(await AsyncStorage.getItem('@last_seen_version')) || 0; } @@ -1415,9 +1390,6 @@ class SynkronusApi { }), }); - console.debug( - `Pushing ${localChanges.length} observations with transmission ID: ${transmissionId}`, - ); const res = await api.syncPush(request); logRepositoryGenerationSync('syncPush response OK', { @@ -1437,15 +1409,11 @@ class SynkronusApi { await this.persistRepositoryGenerationFromResponse( res.data.repository_generation, ); - console.debug( - `Successfully pushed ${localChanges.length} observations. Server version: ${res.data.current_version}`, - ); // 4. Update local database sync status await repo.markObservationsAsSynced( localChanges.map(record => record.observationId), ); - console.debug(`Marked ${localChanges.length} observations as synced`); // 5. Update last seen version await AsyncStorage.setItem( @@ -1453,19 +1421,13 @@ class SynkronusApi { res.data.current_version.toString(), ); - // 6. Log summary if (includeAttachments && attachmentUploadResults.length > 0) { const successfulUploads = attachmentUploadResults.filter( result => result.success, ).length; - const totalUploads = attachmentUploadResults.length; - console.debug( - `Push completed: ${localChanges.length} observations, ${successfulUploads}/${totalUploads} attachments uploaded`, - ); - } else { - console.debug( - `Push completed: ${localChanges.length} observations (attachments not included)`, - ); + logger.info('sync', `uploaded ${successfulUploads} attachments`, { + counts: successfulUploads, + }); } reportSyncProgress(report, { @@ -1497,11 +1459,6 @@ class SynkronusApi { includeAttachments: boolean = false, options?: SynkronusSyncOptions, ) { - console.debug( - includeAttachments - ? 'Syncing observations with attachments' - : 'Syncing observations', - ); const rawStored = await AsyncStorage.getItem( REPOSITORY_GENERATION_STORAGE_KEY, ); @@ -1512,10 +1469,8 @@ class SynkronusApi { includeAttachments, }); const version = await this.pullObservations(includeAttachments, options); - console.debug('Pull completed @ data version ' + version); throwIfSyncCancelled(options?.isCancelled); await this.pushObservations(includeAttachments, options); - console.debug('Push completed'); const storageAfter = await AsyncStorage.getItem( REPOSITORY_GENERATION_STORAGE_KEY, ); diff --git a/formulus/src/components/CustomAppWebView.tsx b/formulus/src/components/CustomAppWebView.tsx index 02cc3e4fc..f6fba5098 100644 --- a/formulus/src/components/CustomAppWebView.tsx +++ b/formulus/src/components/CustomAppWebView.tsx @@ -18,6 +18,7 @@ import { appEvents, Listener } from '../webview/FormulusMessageHandlers'; import { FormInitData } from '../webview/FormulusInterfaceDefinition'; import { colors } from '../theme/colors'; import { loadSettingsHydrationFromStorage } from '../services/SettingsHydrationCache'; +import { logger } from '../diagnostics/logger'; export interface CustomAppWebViewHandle { reload: () => void; @@ -274,10 +275,9 @@ const CustomAppWebView = forwardRef< // Handle API re-injection requests from WebView if (eventData.type === 'requestApiReinjection') { - console.log( - `[CustomAppWebView - ${ - appName || 'Default' - }] WebView requested API re-injection`, + logger.debug( + 'webview', + `API re-injection requested (${appName || 'Default'})`, ); // Perform immediate re-injection @@ -427,9 +427,7 @@ const CustomAppWebView = forwardRef< useEffect(() => { const handleAppStateChange = (nextAppState: string) => { if (nextAppState === 'active') { - console.log( - '[CustomAppWebView] App became active, triggering handleReceiveFocus', - ); + logger.debug('webview', 'app became active, handleReceiveFocus'); // Call handleReceiveFocus on the messageManager when app becomes active if ( messageManager && @@ -477,17 +475,10 @@ const CustomAppWebView = forwardRef< onMessage={messageManager.handleWebViewMessage} onError={handleError} onLoadStart={() => - console.debug( - `[CustomAppWebView - ${appName || 'Default'}] Starting to load URL:`, - appUrl, - ) + logger.debug('webview', `starting load (${appName || 'Default'})`) } onLoadEnd={() => { - console.debug( - `[CustomAppWebView - ${ - appName || 'Default' - }] Finished loading URL: ${appUrl}`, - ); + logger.debug('webview', `finished load (${appName || 'Default'})`); if (webViewRef.current) { // Ensure API is available after load const ensureApiScript = ` diff --git a/formulus/src/components/FormplayerModal.tsx b/formulus/src/components/FormplayerModal.tsx index 417711c0d..837cac685 100644 --- a/formulus/src/components/FormplayerModal.tsx +++ b/formulus/src/components/FormplayerModal.tsx @@ -233,7 +233,6 @@ const FormplayerModal = forwardRef( // Handle WebView load complete const handleWebViewLoad = () => { - console.log('[FormplayerModal] WebView finished loading'); setWebViewReady(true); // WebView is now ready to receive form initialization }; @@ -441,9 +440,6 @@ const FormplayerModal = forwardRef( // Read the source code so the WebView can evaluate it directly const source = await RNFS.readFile(jsPath, 'utf8'); custom_types[folder.name] = { source }; - console.log( - `[FormplayerModal] Custom question type: "${folder.name}" (${source.length} bytes from ${jsPath})`, - ); } else { console.warn( `[FormplayerModal] Skipping "${folder.name}": no renderer.js or index.js found`, @@ -472,9 +468,6 @@ const FormplayerModal = forwardRef( // Read the source code so the WebView can evaluate it directly const source = await RNFS.readFile(indexPath, 'utf8'); validators[folder.name] = { source }; - console.log( - `[FormplayerModal] Custom validator: "${folder.name}" (${source.length} bytes from ${indexPath})`, - ); } else { console.warn( `[FormplayerModal] Skipping validator "${folder.name}": no index.js found`, diff --git a/formulus/src/components/common/FormListTable.tsx b/formulus/src/components/common/FormListTable.tsx new file mode 100644 index 000000000..99d76bb21 --- /dev/null +++ b/formulus/src/components/common/FormListTable.tsx @@ -0,0 +1,155 @@ +import React, { memo, useCallback } from 'react'; +import { View, Text, Pressable, StyleSheet, ScrollView } from 'react-native'; +import Icon from '@react-native-vector-icons/material-design-icons'; +import { useTranslation } from 'react-i18next'; +import { useAppTheme } from '../../contexts/AppThemeContext'; +import { + odeSpacing, + odeTypography, + odeBorderWidth, +} from '../../theme/odeDesign'; +import type { FormSpec } from '../../services/FormService'; + +type FormListTableProps = { + forms: FormSpec[]; + observationCounts: Record; + onCreate: (formId: string) => void; +}; + +type FormTableRowProps = { + form: FormSpec; + count: number | undefined; + onCreate: (formId: string) => void; + cellColor: string; + divider: string; + primary: string; + newLabel: string; +}; + +const FormTableRow = memo( + ({ form, count, onCreate, cellColor, divider, primary, newLabel }) => { + const onPress = useCallback(() => onCreate(form.id), [onCreate, form.id]); + return ( + + + {form.name || form.id} + + + {count == null ? '—' : String(count)} + + + + + + ); + }, +); + +const FormListTable: React.FC = ({ + forms, + observationCounts, + onCreate, +}) => { + const { t } = useTranslation(); + const { themeColors } = useAppTheme(); + const headerColor = themeColors.onSurface as string; + const cellColor = themeColors.onSurface as string; + const divider = themeColors.divider as string; + const primary = themeColors.primary as string; + const newLabel = t('forms.colNewObservation'); + + return ( + + + + {t('forms.colFormType')} + + + {t('forms.colObservationCount')} + + + {t('forms.colNewObservation')} + + + {forms.map(form => ( + + ))} + + ); +}; + +const styles = StyleSheet.create({ + vScroll: { + flex: 1, + }, + vContent: { + flexGrow: 1, + paddingBottom: odeSpacing.md, + }, + row: { + flexDirection: 'row', + alignItems: 'center', + borderBottomWidth: odeBorderWidth.hairline, + paddingVertical: odeSpacing.sm, + paddingHorizontal: odeSpacing.sm, + }, + headerRow: { + paddingTop: odeSpacing.xs, + }, + header: { + fontWeight: '700', + fontSize: odeTypography.caption, + }, + cellForm: { + flex: 1.4, + minWidth: 0, + fontSize: odeTypography.bodySm, + paddingRight: odeSpacing.xs, + }, + cellCount: { + width: 216, + fontSize: odeTypography.bodySm, + paddingRight: odeSpacing.sm, + textAlign: 'right', + }, + cellNew: { + width: 160, + alignItems: 'flex-end', + justifyContent: 'center', + }, + newHeader: { + textAlign: 'right', + }, +}); + +export default FormListTable; diff --git a/formulus/src/components/common/ObservationListTable.tsx b/formulus/src/components/common/ObservationListTable.tsx new file mode 100644 index 000000000..37de90804 --- /dev/null +++ b/formulus/src/components/common/ObservationListTable.tsx @@ -0,0 +1,196 @@ +import React, { memo, useCallback } from 'react'; +import { View, Text, Pressable, StyleSheet, ScrollView } from 'react-native'; +import { useTranslation } from 'react-i18next'; +import { useAppTheme } from '../../contexts/AppThemeContext'; +import colors from '../../theme/colors'; +import { + odeSpacing, + odeTypography, + odeBorderWidth, +} from '../../theme/odeDesign'; +import { + formatObservationIdShort, + type ObservationListRow, +} from '../../database/observationListQuery'; +import { isObservationFullySynced } from '../../utils/observationSyncStatus'; +import { formatDateTimeShort } from '../../utils/dateUtils'; + +type ObservationListTableProps = { + rows: ObservationListRow[]; + formNames: Record; + onPressRow: (row: ObservationListRow) => void; +}; + +type ObservationTableRowProps = { + row: ObservationListRow; + formLabel: string; + onPressRow: (row: ObservationListRow) => void; + cellColor: string; + divider: string; + pendingColor: string; + syncedColor: string; + syncedLabel: string; + pendingLabel: string; +}; + +const ObservationTableRow = memo( + ({ + row, + formLabel, + onPressRow, + cellColor, + divider, + pendingColor, + syncedColor, + syncedLabel, + pendingLabel, + }) => { + const synced = isObservationFullySynced(row); + const onPress = useCallback(() => onPressRow(row), [onPressRow, row]); + return ( + + + {formLabel} + + + {formatDateTimeShort(row.createdAt)} + + + {synced ? syncedLabel : pendingLabel} + + + {row.author || '—'} + + + {formatObservationIdShort(row.observationId)} + + + ); + }, +); + +const ObservationListTable: React.FC = ({ + rows, + formNames, + onPressRow, +}) => { + const { t } = useTranslation(); + const { themeColors } = useAppTheme(); + const headerColor = themeColors.onSurface as string; + const cellColor = themeColors.onSurface as string; + const divider = themeColors.divider as string; + const pendingColor = colors.semantic.warning[500] as string; + const syncedColor = colors.brand.primary['500'] as string; + const syncedLabel = t('filters.synced'); + const pendingLabel = t('filters.pending'); + + return ( + + + + {t('observations.colForm')} + + + {t('observations.colCreated')} + + + {t('observations.colSync')} + + + {t('observations.colAuthor')} + + + {t('observations.colId')} + + + {rows.map(row => ( + + ))} + + ); +}; + +const styles = StyleSheet.create({ + vScroll: { + flex: 1, + }, + vContent: { + flexGrow: 1, + paddingBottom: odeSpacing.md, + }, + row: { + flexDirection: 'row', + alignItems: 'center', + borderBottomWidth: odeBorderWidth.hairline, + paddingVertical: odeSpacing.sm, + paddingHorizontal: odeSpacing.sm, + }, + headerRow: { + paddingTop: odeSpacing.xs, + }, + header: { + fontWeight: '700', + fontSize: odeTypography.caption, + }, + cellForm: { + flex: 1.3, + minWidth: 0, + fontSize: odeTypography.bodySm, + paddingRight: odeSpacing.xs, + }, + cellWhen: { + flex: 1.5, + minWidth: 0, + fontSize: odeTypography.bodySm, + paddingRight: odeSpacing.xs, + }, + cellSync: { + flex: 0.7, + minWidth: 0, + fontSize: odeTypography.bodySm, + paddingRight: odeSpacing.xs, + }, + cellAuthor: { + flex: 1, + minWidth: 0, + fontSize: odeTypography.bodySm, + paddingRight: odeSpacing.xs, + }, + cellId: { + flex: 0.8, + minWidth: 0, + fontSize: odeTypography.bodySm, + }, + mono: { + fontFamily: 'monospace', + }, +}); + +export default ObservationListTable; diff --git a/formulus/src/components/common/ObservationPager.tsx b/formulus/src/components/common/ObservationPager.tsx new file mode 100644 index 000000000..3de2fd14f --- /dev/null +++ b/formulus/src/components/common/ObservationPager.tsx @@ -0,0 +1,117 @@ +import React from 'react'; +import { View, Text, Pressable, StyleSheet } from 'react-native'; +import { useTranslation } from 'react-i18next'; +import { useAppTheme } from '../../contexts/AppThemeContext'; +import { odeSpacing, odeTypography, odeRadius } from '../../theme/odeDesign'; +import { buildObservationPagerItems } from '../../database/observationListQuery'; + +type ObservationPagerProps = { + page: number; + totalPages: number; + onPageChange: (page: number) => void; +}; + +const ObservationPager: React.FC = ({ + page, + totalPages, + onPageChange, +}) => { + const { t } = useTranslation(); + const { themeColors } = useAppTheme(); + const items = buildObservationPagerItems(page, totalPages); + const onSurface = themeColors.onSurface as string; + const primary = themeColors.primary as string; + + return ( + + {items.map((item, index) => { + if (item.kind === 'ellipsis') { + return ( + + … + + ); + } + if (item.kind === 'prev' || item.kind === 'next') { + const label = item.kind === 'prev' ? '<' : '>'; + return ( + onPageChange(item.page)} + disabled={item.disabled} + accessibilityRole="button" + accessibilityLabel={ + item.kind === 'prev' + ? t('observations.pagerPrev') + : t('observations.pagerNext') + } + style={styles.hit}> + + {label} + + + ); + } + if (item.current) { + return ( + + {item.page} + + ); + } + return ( + onPageChange(item.page)} + accessibilityRole="button" + accessibilityLabel={t('observations.pagerPage', { + page: item.page, + })} + style={styles.hit}> + {item.page} + + ); + })} + + ); +}; + +const styles = StyleSheet.create({ + row: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'center', + flexWrap: 'wrap', + gap: odeSpacing.sm, + paddingVertical: odeSpacing.sm, + }, + hit: { + minWidth: 32, + minHeight: 32, + alignItems: 'center', + justifyContent: 'center', + borderRadius: odeRadius.inner, + }, + item: { + fontSize: odeTypography.body, + fontWeight: '600', + paddingHorizontal: odeSpacing.xs, + }, + current: { + textDecorationLine: 'none', + }, +}); + +export default ObservationPager; diff --git a/formulus/src/components/common/index.ts b/formulus/src/components/common/index.ts index d0b81b0e5..131f6dfde 100644 --- a/formulus/src/components/common/index.ts +++ b/formulus/src/components/common/index.ts @@ -3,7 +3,10 @@ export { default as ConfirmModal } from './ConfirmModal'; export type { ConfirmButton } from './ConfirmModal'; export { default as Input } from './Input'; export { default as FormCard } from './FormCard'; +export { default as FormListTable } from './FormListTable'; export { default as ObservationCard } from './ObservationCard'; +export { default as ObservationListTable } from './ObservationListTable'; +export { default as ObservationPager } from './ObservationPager'; export { default as EmptyState } from './EmptyState'; export { default as FilterBar } from './FilterBar'; export { default as StatusTabs } from './StatusTabs'; diff --git a/formulus/src/database/__tests__/observationIndexGuards.test.ts b/formulus/src/database/__tests__/observationIndexGuards.test.ts index 99a19e91d..c3e5322f0 100644 --- a/formulus/src/database/__tests__/observationIndexGuards.test.ts +++ b/formulus/src/database/__tests__/observationIndexGuards.test.ts @@ -38,7 +38,10 @@ jest.mock('../../services/AppConfigService', () => ({ import ObservationIndexService, { computeDefsSignature, + deleteIndexSqls, + extractIndexRows, INDEX_WRITE_BATCH_SIZE, + insertIndexSqls, } from '../../services/ObservationIndexService'; const EMPTY_SIGNATURE = computeDefsSignature([]); @@ -178,6 +181,110 @@ describe('ObservationIndexService guards', () => { }); }); + describe('extractIndexRows', () => { + const defs = [ + { key: 'hh_id', path: '$.hh_id' }, + { key: 'af', path: '$.af' }, + ]; + + it('parses JSON once and emits a row per matching key', () => { + const { rows, nonScalarKeys } = extractIndexRows( + 'obs-1', + 'household', + JSON.stringify({ hh_id: 'HH-1', af: 12 }), + defs, + 1, + ); + expect(nonScalarKeys).toEqual([]); + expect(rows).toEqual([ + { + id: 'obs-1:hh_id:1', + observationId: 'obs-1', + indexKey: 'hh_id', + generation: 1, + valueText: 'HH-1', + valueNum: null, + }, + { + id: 'obs-1:af:1', + observationId: 'obs-1', + indexKey: 'af', + generation: 1, + valueText: null, + valueNum: 12, + }, + ]); + }); + + it('accepts an already-parsed object so pull can skip JSON.parse', () => { + const { rows } = extractIndexRows( + 'obs-1', + 'household', + { hh_id: 'HH-1' }, + [defs[0]], + 1, + ); + expect(rows).toHaveLength(1); + expect(rows[0].valueText).toBe('HH-1'); + }); + + it('returns no rows for invalid JSON', () => { + expect( + extractIndexRows('obs-1', 'household', '{not-json', defs, 1), + ).toEqual({ rows: [], nonScalarKeys: [] }); + }); + }); + + describe('index SQL helpers', () => { + it('deletes a batch with one IN list', () => { + expect(deleteIndexSqls(['obs-1', 'obs-2'], 1)).toEqual([ + [ + 'DELETE FROM observation_index WHERE observation_id IN (?,?) AND index_generation = ?', + ['obs-1', 'obs-2', 1], + ], + ]); + }); + + it('inserts many EAV rows in one VALUES list', () => { + const sqls = insertIndexSqls([ + { + id: 'obs-1:hh_id:1', + observationId: 'obs-1', + indexKey: 'hh_id', + generation: 1, + valueText: 'HH-1', + valueNum: null, + }, + { + id: 'obs-2:hh_id:1', + observationId: 'obs-2', + indexKey: 'hh_id', + generation: 1, + valueText: 'HH-2', + valueNum: null, + }, + ]); + expect(sqls).toHaveLength(1); + expect(sqls[0][0]).toBe( + 'INSERT INTO observation_index (id, observation_id, index_key, index_generation, value_text, value_num) VALUES (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?)', + ); + expect(sqls[0][1]).toEqual([ + 'obs-1:hh_id:1', + 'obs-1', + 'hh_id', + 1, + 'HH-1', + null, + 'obs-2:hh_id:1', + 'obs-2', + 'hh_id', + 1, + 'HH-2', + null, + ]); + }); + }); + describe('incrementalReindexMany', () => { it('flushes in bounded writes instead of one statement list for the whole page', async () => { configIndexes.push({ key: 'hh_id', path: '$.hh_id' }); @@ -189,8 +296,9 @@ describe('ObservationIndexService guards', () => { dataJson: JSON.stringify({ hh_id: `HH-${i}` }), }), ); - rawResults.push([{ active_generation: 1 }], [{ active_generation: 1 }]); + rawResults.push([{ active_generation: 1 }]); mockDb.write.mockClear(); + mockDb.adapter.unsafeExecute.mockClear(); const onProgress = jest.fn(); await service.incrementalReindexMany(rows, onProgress); @@ -208,6 +316,17 @@ describe('ObservationIndexService guards', () => { current: INDEX_WRITE_BATCH_SIZE + 50, total: INDEX_WRITE_BATCH_SIZE + 50, }); + + const firstFlush = mockDb.adapter.unsafeExecute.mock.calls[0][0] + .sqls as Array<[string, unknown[]]>; + expect(firstFlush[0][0]).toMatch( + /^DELETE FROM observation_index WHERE observation_id IN \(/, + ); + expect(firstFlush[0][1]).toHaveLength(INDEX_WRITE_BATCH_SIZE + 1); + expect(firstFlush[1][0]).toMatch( + /^INSERT INTO observation_index \(id, observation_id, index_key, index_generation, value_text, value_num\) VALUES /, + ); + expect(firstFlush[1][0]).toContain('(?, ?, ?, ?, ?, ?),'); }); }); }); diff --git a/formulus/src/database/__tests__/observationListQuery.test.ts b/formulus/src/database/__tests__/observationListQuery.test.ts new file mode 100644 index 000000000..3214f0e82 --- /dev/null +++ b/formulus/src/database/__tests__/observationListQuery.test.ts @@ -0,0 +1,68 @@ +import { + buildObservationListSql, + buildObservationPagerItems, + formatObservationIdShort, + OBSERVATION_LIST_PAGE_SIZE, +} from '../observationListQuery'; +import { MIN_VALID_SYNCED_AT_MS } from '../../utils/observationSyncStatus'; + +describe('buildObservationListSql', () => { + it('selects envelope columns only and pages with LIMIT/OFFSET', () => { + const built = buildObservationListSql({ page: 3 }); + expect(built.listSql).toContain( + 'SELECT observation_id, form_type, created_at, updated_at, synced_at, author', + ); + expect(built.listSql).not.toContain(' data'); + expect(built.listSql).toContain('LIMIT ? OFFSET ?'); + expect(built.listParams.slice(-2)).toEqual([ + OBSERVATION_LIST_PAGE_SIZE, + 2 * OBSERVATION_LIST_PAGE_SIZE, + ]); + expect(built.countSql).toBe( + 'SELECT COUNT(*) AS cnt FROM observations WHERE deleted = 0', + ); + }); + + it('filters form type, pending sync, and escaped search', () => { + const built = buildObservationListSql({ + page: 1, + formType: 'censo_milda', + syncStatus: 'pending', + search: 'ab%_c', + }); + expect(built.listSql).toContain('form_type = ?'); + expect(built.listSql).toContain('updated_at > synced_at'); + expect(built.listParams).toContain('censo_milda'); + expect(built.listParams).toContain(MIN_VALID_SYNCED_AT_MS); + expect(built.listParams).toContain('%ab\\%\\_c%'); + }); +}); + +describe('buildObservationPagerItems', () => { + it('shows 1, 2, …, last on page one', () => { + const labels = buildObservationPagerItems(1, 10).map(item => + item.kind === 'page' + ? `${item.page}${item.current ? '*' : ''}` + : item.kind === 'ellipsis' + ? '…' + : item.kind, + ); + expect(labels).toEqual(['prev', '1*', '2', '…', '10', 'next']); + }); + + it('shows n-1, n, n+1 when not on page one', () => { + const labels = buildObservationPagerItems(4, 10).map(item => + item.kind === 'page' + ? `${item.page}${item.current ? '*' : ''}` + : item.kind, + ); + expect(labels).toEqual(['prev', '3', '4*', '5', 'next']); + }); +}); + +describe('formatObservationIdShort', () => { + it('keeps short ids and clips long ones', () => { + expect(formatObservationIdShort('abcd1234')).toBe('abcd1234'); + expect(formatObservationIdShort('abcdefghijklmnop')).toBe('abcd…mnop'); + }); +}); diff --git a/formulus/src/database/observationListQuery.ts b/formulus/src/database/observationListQuery.ts new file mode 100644 index 000000000..cf2469d28 --- /dev/null +++ b/formulus/src/database/observationListQuery.ts @@ -0,0 +1,159 @@ +import { MIN_VALID_SYNCED_AT_MS } from '../utils/observationSyncStatus'; +import type { SyncStatus } from '../components/common/SyncStatusButtons'; + +export const OBSERVATION_LIST_PAGE_SIZE = 15; + +export type ObservationListRow = { + observationId: string; + formType: string; + createdAt: Date; + updatedAt: Date; + syncedAt: Date | null; + author: string | null; +}; + +export type ObservationListQuery = { + formType?: string | null; + syncStatus?: SyncStatus; + search?: string; + page: number; + pageSize?: number; +}; + +export type ObservationListPage = { + rows: ObservationListRow[]; + total: number; + page: number; + pageSize: number; + totalPages: number; +}; + +const LIST_COLUMNS = `observation_id, form_type, created_at, updated_at, synced_at, author`; + +function likeContains(term: string): string { + return `%${term + .replace(/\\/g, '\\\\') + .replace(/%/g, '\\%') + .replace(/_/g, '\\_')}%`; +} + +function appendFilters( + where: string[], + params: Array, + query: ObservationListQuery, +): void { + where.push('deleted = 0'); + if (query.formType?.trim()) { + where.push('form_type = ?'); + params.push(query.formType.trim()); + } + if (query.syncStatus === 'synced') { + where.push( + 'synced_at IS NOT NULL AND synced_at > ? AND updated_at <= synced_at', + ); + params.push(MIN_VALID_SYNCED_AT_MS); + } else if (query.syncStatus === 'pending') { + where.push( + '(synced_at IS NULL OR synced_at <= ? OR updated_at > synced_at)', + ); + params.push(MIN_VALID_SYNCED_AT_MS); + } + const search = query.search?.trim(); + if (search) { + const like = likeContains(search); + where.push( + "(observation_id LIKE ? ESCAPE '\\' OR form_type LIKE ? ESCAPE '\\' OR IFNULL(author, '') LIKE ? ESCAPE '\\')", + ); + params.push(like, like, like); + } +} + +export function buildObservationListSql(query: ObservationListQuery): { + listSql: string; + listParams: Array; + countSql: string; + countParams: Array; + page: number; + pageSize: number; + offset: number; +} { + const pageSize = Math.max(1, query.pageSize ?? OBSERVATION_LIST_PAGE_SIZE); + const page = Math.max(1, query.page); + const offset = (page - 1) * pageSize; + const where: string[] = []; + const params: Array = []; + appendFilters(where, params, query); + const whereSql = `WHERE ${where.join(' AND ')}`; + return { + listSql: `SELECT ${LIST_COLUMNS} FROM observations ${whereSql} ORDER BY created_at DESC LIMIT ? OFFSET ?`, + listParams: [...params, pageSize, offset], + countSql: `SELECT COUNT(*) AS cnt FROM observations ${whereSql}`, + countParams: [...params], + page, + pageSize, + offset, + }; +} + +export function mapObservationListRow( + row: Record, +): ObservationListRow { + const syncedRaw = row.synced_at; + const syncedAt = + syncedRaw == null || syncedRaw === '' ? null : new Date(Number(syncedRaw)); + return { + observationId: String(row.observation_id ?? row.id ?? ''), + formType: String(row.form_type ?? ''), + createdAt: new Date(Number(row.created_at ?? 0)), + updatedAt: new Date(Number(row.updated_at ?? 0)), + syncedAt, + author: row.author ? String(row.author) : null, + }; +} + +export function formatObservationIdShort(id: string): string { + if (id.length <= 8) { + return id; + } + return `${id.slice(0, 4)}…${id.slice(-4)}`; +} + +export type PagerItem = + | { kind: 'prev' | 'next'; page: number; disabled: boolean } + | { kind: 'page'; page: number; current: boolean } + | { kind: 'ellipsis' }; + +/** + * Page 1: < 1 2 … last > + * Other: < n-1 n n+1 > + */ +export function buildObservationPagerItems( + page: number, + totalPages: number, +): PagerItem[] { + const last = Math.max(1, totalPages); + const current = Math.min(Math.max(1, page), last); + const items: PagerItem[] = [ + { kind: 'prev', page: current - 1, disabled: current <= 1 }, + ]; + if (current === 1) { + items.push({ kind: 'page', page: 1, current: true }); + if (last >= 2) { + items.push({ kind: 'page', page: 2, current: false }); + } + if (last > 3) { + items.push({ kind: 'ellipsis' }); + } + if (last > 2) { + items.push({ kind: 'page', page: last, current: false }); + } + } else { + items.push({ kind: 'page', page: current - 1, current: false }); + items.push({ kind: 'page', page: current, current: true }); + if (current < last) { + items.push({ kind: 'page', page: current + 1, current: false }); + } + } + items.push({ kind: 'next', page: current + 1, disabled: current >= last }); + return items; +} diff --git a/formulus/src/database/repositories/LocalRepoInterface.ts b/formulus/src/database/repositories/LocalRepoInterface.ts index 1ebc12196..b3e840d78 100644 --- a/formulus/src/database/repositories/LocalRepoInterface.ts +++ b/formulus/src/database/repositories/LocalRepoInterface.ts @@ -4,6 +4,10 @@ import { NewObservationInput, UpdateObservationInput, } from '../models/Observation'; +import type { + ObservationListPage, + ObservationListQuery, +} from '../observationListQuery'; /** * Interface for local data repository operations * This allows us to abstract the storage implementation for testability @@ -30,6 +34,18 @@ export interface LocalRepoInterface { */ getObservationsByFormType(formType: string): Promise; + /** + * All non-deleted observations (every form type). Used by the Observations list. + */ + getActiveObservations(): Promise; + + /** + * Envelope-only page for the Observations table. Never reads `data`. + */ + listObservationsPage( + query: ObservationListQuery, + ): Promise; + /** * Query observations with structured filter AST (SQLite + local indexes). */ @@ -74,6 +90,7 @@ export interface LocalRepoInterface { changes: Observation[], options?: { onIndexProgress?: (progress: { current: number; total: number }) => void; + isCancelled?: () => boolean; }, ): Promise; diff --git a/formulus/src/database/repositories/WatermelonDBRepo.ts b/formulus/src/database/repositories/WatermelonDBRepo.ts index a82aa2a83..21c2acac2 100644 --- a/formulus/src/database/repositories/WatermelonDBRepo.ts +++ b/formulus/src/database/repositories/WatermelonDBRepo.ts @@ -18,6 +18,13 @@ import { ToastService } from '../../services/ToastService'; import { clientIdService } from '../../services/ClientIdService'; import { getUserInfo } from '../../api/synkronus/Auth'; import { LAST_WRITE_WON_TAG } from '../../sync/syncConstants'; +import { logger } from '../../diagnostics/logger'; +import { + buildObservationListSql, + mapObservationListRow, + type ObservationListPage, + type ObservationListQuery, +} from '../observationListQuery'; function parseTagsColumn(raw: string | undefined): string[] { if (!raw?.trim()) { @@ -38,6 +45,36 @@ function serializeTagsColumn(tags: string[]): string { return tags.length > 0 ? JSON.stringify(tags) : ''; } +function toTimestampMs( + value: Date | number | string | null | undefined, +): number | null { + if (value == null || value === '') { + return null; + } + const ms = + value instanceof Date ? value.getTime() : new Date(value).getTime(); + return Number.isFinite(ms) ? ms : null; +} + +/** + * Watermelon owns created_at / updated_at via @readonly @date and stamps "now" + * on create/update. Pull must write the Synkronus envelope times through + * _setRaw or the list shows the local insert (sync) time instead. + */ +function applyEnvelopeTimestamps( + record: ObservationModel, + change: Observation, +): void { + const createdMs = toTimestampMs(change.createdAt); + if (createdMs != null) { + record._setRaw('created_at', createdMs); + } + const updatedMs = toTimestampMs(change.updatedAt); + if (updatedMs != null) { + record._setRaw('updated_at', updatedMs); + } +} + /** * WatermelonDB implementation of the LocalRepoInterface * This implementation is designed to work well with the Synkronus API's pull/push synchronization @@ -45,6 +82,7 @@ function serializeTagsColumn(tags: string[]): string { export class WatermelonDBRepo implements LocalRepoInterface { private database: Database; private observationsCollection: Collection; + private columnIndexesPromise: Promise | null = null; constructor(database: Database) { this.database = database; @@ -53,6 +91,53 @@ export class WatermelonDBRepo implements LocalRepoInterface { // Touch the index service early so the `bundleUpdated` listener and the // initial-rebuild bootstrap kick off before any sync activity. ObservationIndexService.getInstance(this.database); + void this.ensureColumnIndexes(); + } + + /** + * Column indexes on `observations` (not the custom-app EAV table). + * `form_type` is always required for list/query paths. CREATE INDEX IF NOT + * EXISTS is a no-op when Watermelon already created them from the schema. + */ + private ensureColumnIndexes(): Promise { + if (!this.columnIndexesPromise) { + this.columnIndexesPromise = (async () => { + try { + await this.database.write(async () => { + await this.database.adapter.unsafeExecute({ + sqls: [ + [ + 'CREATE INDEX IF NOT EXISTS observations_form_type ON observations(form_type)', + [], + ], + [ + 'CREATE INDEX IF NOT EXISTS observations_deleted ON observations(deleted)', + [], + ], + [ + 'CREATE INDEX IF NOT EXISTS observations_form_type_deleted ON observations(form_type, deleted)', + [], + ], + ], + }); + }); + logger.info('db', 'observation column indexes ready', { + phase: 'index', + success: true, + }); + } catch (err) { + this.columnIndexesPromise = null; + logger.warn( + 'db', + err instanceof Error + ? err.message + : 'failed to ensure observation column indexes', + { phase: 'index', success: false }, + ); + } + })(); + } + return this.columnIndexesPromise; } /** @@ -62,7 +147,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { */ async saveObservation(input: NewObservationInput): Promise { try { - console.log('Saving observation:', input); + logger.info('db', 'saving observation', { formType: input.formType }); // Use pre-cached GPS when available, fall back to fresh capture let geolocation = null; @@ -141,8 +226,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { throw new Error('Failed to create observation record'); } - console.log('Successfully created observation with ID:', observationId); - await ObservationIndexService.getInstance( this.database, ).incrementalReindex(observationId, input.formType, stringifiedData); @@ -179,20 +262,12 @@ export class WatermelonDBRepo implements LocalRepoInterface { */ async getObservation(id: string): Promise { try { - console.log(`Looking up observation with ID: ${id}`); - // First try direct lookup by ID (WatermelonDB's internal ID) try { const observation = await this.observationsCollection.find(id); - console.log(`Found observation directly by ID: ${observation.id}`); return this.mapObservationModelToInterface(observation); - } catch (error) { + } catch { // ID not found, continue to next approach - console.log( - `Direct lookup by ID failed, trying by observationId: ${ - (error as Error).message - }`, - ); } // If not found by ID, try to find by observationId field @@ -203,36 +278,11 @@ export class WatermelonDBRepo implements LocalRepoInterface { .query(Q.where('observation_id', id)) .fetch(); - console.log( - `Query for observation_id=${id} returned ${observations.length} results`, - ); - if (observations.length > 0) { const observation = observations[0]; - console.log( - `Found observation via observationId query: ${observation.id}`, - ); return this.mapObservationModelToInterface(observation); } - // Not found by either method - // As a last resort, try to fetch all observations to see what's in the database - const allObservations = await this.observationsCollection.query().fetch(); - console.log( - `No observation found with ID: ${id}. Total observations in database: ${allObservations.length}`, - ); - - if (allObservations.length > 0) { - console.log( - 'Available observations:', - allObservations.map(o => ({ - id: o.id, - observationId: o.observationId, - formType: o.formType, - })), - ); - } - return null; } catch (error) { console.error( @@ -254,8 +304,15 @@ export class WatermelonDBRepo implements LocalRepoInterface { filter?: ObservationFilter; }): Promise { try { + await this.ensureColumnIndexes(); const indexService = ObservationIndexService.getInstance(this.database); + const ensureStarted = Date.now(); await indexService.ensureInitialRebuild(); + logger.info( + 'observations', + `queryObservations ensureInitialRebuild ${Date.now() - ensureStarted}ms`, + { formType: options.formType, phase: 'ensure' }, + ); let indexKeys = indexKeysFromConfig(indexService.getIndexDefs()); if (indexKeys.size > 0 && !(await indexService.isIndexUsable())) { @@ -308,14 +365,27 @@ export class WatermelonDBRepo implements LocalRepoInterface { return []; } - // Query for observations with form_type matching and exclude soft-deleted - const observations = await this.observationsCollection + await this.ensureColumnIndexes(); + logger.info('observations', 'getByFormType query start', { + formType: formId, + phase: 'query', + }); + const queryStarted = Date.now(); + const rows = await this.observationsCollection .query(Q.where('form_type', formId), Q.where('deleted', false)) - .fetch(); - - return observations.map(observation => - this.mapObservationModelToInterface(observation), + .unsafeFetchRaw(); + const fetchedMs = Date.now() - queryStarted; + const mapped = rows.map(raw => this.mapRawObservationRow(raw)); + logger.info( + 'observations', + `getByFormType fetch=${fetchedMs}ms map=${Date.now() - queryStarted - fetchedMs}ms`, + { + formType: formId, + phase: 'query', + counts: mapped.length, + }, ); + return mapped; } catch (error) { console.error( 'Error getting observations by form type ID:', @@ -325,6 +395,56 @@ export class WatermelonDBRepo implements LocalRepoInterface { } } + async getActiveObservations(): Promise { + await this.ensureColumnIndexes(); + logger.info('observations', 'getActive query start', { phase: 'query' }); + const queryStarted = Date.now(); + const rows = await this.observationsCollection + .query(Q.where('deleted', false)) + .unsafeFetchRaw(); + const fetchedMs = Date.now() - queryStarted; + const mapped = rows.map(raw => this.mapRawObservationRow(raw)); + logger.info( + 'observations', + `getActive fetch=${fetchedMs}ms map=${Date.now() - queryStarted - fetchedMs}ms`, + { phase: 'query', counts: mapped.length }, + ); + return mapped; + } + + async listObservationsPage( + query: ObservationListQuery, + ): Promise { + await this.ensureColumnIndexes(); + const built = buildObservationListSql(query); + const started = Date.now(); + const [rawRows, countRows] = await Promise.all([ + this.observationsCollection + .query(Q.unsafeSqlQuery(built.listSql, built.listParams)) + .unsafeFetchRaw(), + this.observationsCollection + .query(Q.unsafeSqlQuery(built.countSql, built.countParams)) + .unsafeFetchRaw(), + ]); + const total = Number( + (countRows[0] as { cnt?: number } | undefined)?.cnt ?? 0, + ); + const rows = rawRows.map(row => mapObservationListRow(row)); + const totalPages = Math.max(1, Math.ceil(total / built.pageSize)); + logger.info('observations', `listPage fetch=${Date.now() - started}ms`, { + phase: 'query', + counts: rows.length, + formType: query.formType ?? undefined, + }); + return { + rows, + total, + page: built.page, + pageSize: built.pageSize, + totalPages, + }; + } + /** * All local observation rows (including soft-deleted), for backup/export. */ @@ -348,11 +468,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { */ async updateObservation(input: UpdateObservationInput): Promise { try { - console.log( - 'Updating observation with ObservationId:', - input.observationId, - ); - const record = await this.findObservationRecord(input.observationId); if (!record) { @@ -387,8 +502,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { this.database, ).incrementalReindex(record.id, record.formType, stringifiedData); await this.database.get('observations').query().fetch(); - const updatedRecord = await this.observationsCollection.find(record.id); - console.log('Successfully updated observation:', updatedRecord.id); } return success; @@ -408,8 +521,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { */ async deleteObservation(id: string): Promise { try { - console.log('Deleting observation with ObservationId:', id); - const record = await this.findObservationRecord(id); if (!record) { @@ -430,13 +541,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { if (success) { // Force a database sync await this.database.get('observations').query().fetch(); - - // Verify the record was updated by querying for it again - const updatedRecord = await this.observationsCollection.find(record.id); - console.log( - 'Successfully marked observation as deleted:', - updatedRecord.id, - ); } return success; @@ -456,20 +560,14 @@ export class WatermelonDBRepo implements LocalRepoInterface { */ async markObservationAsSynced(id: string): Promise { try { - console.log(`Marking observation as synced: ${id}`); - // Find the observation using our improved lookup approach let record: ObservationModel | null = null; // Try to find by direct ID first try { record = await this.observationsCollection.find(id); - } catch (error) { - console.log( - `Direct lookup by ID failed, trying by observationId: ${ - (error as Error).message - }`, - ); + } catch { + // not found by primary key } // If not found by ID, try to find by observationId field @@ -480,9 +578,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { if (observations.length > 0) { record = observations[0]; - console.log( - `Found observation via observationId query: ${record.id}`, - ); } } @@ -504,13 +599,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { if (success) { // Force a database sync await this.database.get('observations').query().fetch(); - - // Verify the record was updated by querying for it again - const updatedRecord = await this.observationsCollection.find(record.id); - console.log( - 'Successfully marked observation as synced:', - updatedRecord.id, - ); } return success; @@ -531,6 +619,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { changes: Observation[], options?: { onIndexProgress?: (progress: { current: number; total: number }) => void; + isCancelled?: () => boolean; }, ): Promise { if (!changes.length) { @@ -542,6 +631,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { // so indexing the server payload for it would leave the index describing // values the stored record does not have — invisible until a full rebuild. const applied: Observation[] = []; + const writeStarted = Date.now(); const count = await this.database.write(async () => { const existingRecords = await this.observationsCollection @@ -556,11 +646,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { .map(change => { const existing = existingMap.get(change.observationId); if (existing) { - console.debug(`Preparing update for observation: ${existing.id}`); if (existing.updatedAt > existing.syncedAt) { - console.debug( - `Skipping server change for ${existing.id} because it's locally dirty`, - ); const currentTags = parseTagsColumn(existing.tags); if (currentTags.includes(LAST_WRITE_WON_TAG)) { return null; @@ -593,11 +679,9 @@ export class WatermelonDBRepo implements LocalRepoInterface { : ''; } record.syncedAt = new Date(); + applyEnvelopeTimestamps(record, change); }); } - console.debug( - `Preparing create for new observation: ${change.observationId}`, - ); applied.push(change); return this.observationsCollection.prepareCreate(record => { record._raw.id = change.observationId; @@ -616,6 +700,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { : ''; record.deleted = change.deleted ?? false; record.syncedAt = new Date(); + applyEnvelopeTimestamps(record, change); }); }) .filter((op): op is NonNullable => op != null); @@ -624,19 +709,33 @@ export class WatermelonDBRepo implements LocalRepoInterface { } return batchOps.length; }); + const writeMs = Date.now() - writeStarted; + + if (options?.isCancelled?.()) { + logger.info( + 'sync', + `apply write=${writeMs}ms rows=${count} skipped index (cancelled)`, + { phase: 'apply', counts: count }, + ); + throw new Error('Sync cancelled'); + } const indexService = ObservationIndexService.getInstance(this.database); const indexRows = applied.map(change => ({ observationId: change.observationId, formType: change.formType, - dataJson: - typeof change.data === 'string' - ? change.data - : JSON.stringify(change.data), + dataJson: change.data, })); + const indexStarted = Date.now(); await indexService.incrementalReindexMany( indexRows, options?.onIndexProgress, + options?.isCancelled, + ); + logger.info( + 'sync', + `apply write=${writeMs}ms index=${Date.now() - indexStarted}ms rows=${count}`, + { phase: 'apply', counts: count }, ); return count; @@ -707,15 +806,11 @@ export class WatermelonDBRepo implements LocalRepoInterface { pushChanges: (observations: Observation[]) => Promise, ): Promise { try { - console.log('Starting synchronization process'); - // Step 1: Pull changes from the server const serverChanges = await pullChanges(); - console.log(`Received ${serverChanges.length} changes from server`); // Step 2: Apply server changes to local database - const pulledChanges = await this.applyServerChanges(serverChanges); - console.log(`Applied ${pulledChanges} changes to local database`); + await this.applyServerChanges(serverChanges); // Step 3: Get local changes to push to server // Get all observations that haven't been synced or were updated after last sync @@ -728,8 +823,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { ) .fetch(); - console.log(`Found ${localChanges.length} local changes to push`); - // Step 4: Push local changes to server if (localChanges.length > 0) { // Convert WatermelonDB records to plain objects for the API @@ -739,7 +832,6 @@ export class WatermelonDBRepo implements LocalRepoInterface { // Push changes to server await pushChanges(localObservations); - console.log(`Pushed ${localObservations.length} changes to server`); // Mark all pushed observations as synced await this.database.write(async () => { @@ -749,11 +841,7 @@ export class WatermelonDBRepo implements LocalRepoInterface { }); } }); - - console.log('All pushed observations marked as synced'); } - - console.log('Synchronization completed successfully'); } catch (error) { console.error( 'Error during synchronization:', diff --git a/formulus/src/database/repositories/__tests__/LocalRepo.test.ts b/formulus/src/database/repositories/__tests__/LocalRepo.test.ts index fc2fa88cd..2147eb428 100644 --- a/formulus/src/database/repositories/__tests__/LocalRepo.test.ts +++ b/formulus/src/database/repositories/__tests__/LocalRepo.test.ts @@ -32,12 +32,38 @@ class MockLocalRepo implements LocalRepoInterface { return this.observations.get(id) || null; } - async getObservationsByFormId(formId: string): Promise { + async getObservationsByFormType(formId: string): Promise { return Array.from(this.observations.values()).filter( obs => obs.formType === formId && !obs.deleted, ); } + async getObservationsByFormId(formId: string): Promise { + return this.getObservationsByFormType(formId); + } + + async getActiveObservations(): Promise { + return Array.from(this.observations.values()).filter(obs => !obs.deleted); + } + + async listObservationsPage() { + const rows = await this.getActiveObservations(); + return { + rows: rows.map(obs => ({ + observationId: obs.observationId, + formType: obs.formType, + createdAt: obs.createdAt, + updatedAt: obs.updatedAt, + syncedAt: obs.syncedAt, + author: obs.author ?? null, + })), + total: rows.length, + page: 1, + pageSize: rows.length || 1, + totalPages: 1, + }; + } + async updateObservation( id: string, observation: Partial, diff --git a/formulus/src/database/repositories/__tests__/WatermelonDBRepo.test.ts b/formulus/src/database/repositories/__tests__/WatermelonDBRepo.test.ts index 2c5a917a8..e9106a748 100644 --- a/formulus/src/database/repositories/__tests__/WatermelonDBRepo.test.ts +++ b/formulus/src/database/repositories/__tests__/WatermelonDBRepo.test.ts @@ -623,6 +623,47 @@ describe('WatermelonDBRepo', () => { expect(viaLookup!.deviceId).toBe('device-a'); expect(apiPayload.author).toBe('alice'); expect(apiPayload.device_id).toBe('device-a'); + expect(model.createdAt.toISOString()).toBe('2025-01-02T10:00:00.000Z'); + expect(model.updatedAt.toISOString()).toBe('2025-01-02T11:00:00.000Z'); + expect(domain.createdAt.toISOString()).toBe('2025-01-02T10:00:00.000Z'); + expect(domain.updatedAt.toISOString()).toBe('2025-01-02T11:00:00.000Z'); + }); + + test('applyServerChanges writes envelope createdAt/updatedAt on update', async () => { + const serverObservationId = 'obs_pulled_timestamp_repair'; + await repo.applyServerChanges([ + { + observationId: serverObservationId, + formType: 'register_coffee', + formVersion: '1.0', + createdAt: new Date('2020-01-01T00:00:00.000Z'), + updatedAt: new Date('2020-01-01T00:00:00.000Z'), + syncedAt: null, + deleted: false, + data: { name: 'old' }, + geolocation: null, + }, + ]); + + await repo.applyServerChanges([ + { + observationId: serverObservationId, + formType: 'register_coffee', + formVersion: '1.0', + createdAt: new Date('2019-06-15T08:30:00.000Z'), + updatedAt: new Date('2024-03-01T12:00:00.000Z'), + syncedAt: null, + deleted: false, + data: { name: 'repaired' }, + geolocation: null, + }, + ]); + + const repaired = await repo.getObservation(serverObservationId); + expect(repaired).not.toBeNull(); + expect(repaired!.createdAt.toISOString()).toBe('2019-06-15T08:30:00.000Z'); + expect(repaired!.updatedAt.toISOString()).toBe('2024-03-01T12:00:00.000Z'); + expect(repaired!.data).toEqual({ name: 'repaired' }); }); /** diff --git a/formulus/src/database/schema.ts b/formulus/src/database/schema.ts index 31c0f6af0..558c5b0f7 100644 --- a/formulus/src/database/schema.ts +++ b/formulus/src/database/schema.ts @@ -8,6 +8,7 @@ export const schemas = appSchema({ name: 'observations', columns: [ { name: 'observation_id', type: 'string', isIndexed: true }, + // Platform-owned. Always indexed — not part of custom-app observationIndexes. { name: 'form_type', type: 'string', isIndexed: true }, { name: 'form_version', type: 'string' }, { name: 'deleted', type: 'boolean', isIndexed: true }, diff --git a/formulus/src/diagnostics/DiagnosticLog.ts b/formulus/src/diagnostics/DiagnosticLog.ts new file mode 100644 index 000000000..a79c14058 --- /dev/null +++ b/formulus/src/diagnostics/DiagnosticLog.ts @@ -0,0 +1,210 @@ +import type { DiagnosticEvent, DiagnosticFs, ProcessExitRecord } from './types'; +import { + backupPath, + eventsPath, + exitsPath, + MAX_LOG_BYTES, + sessionPath, +} from './paths'; + +const DEFAULT_DOC_DIR = '__unset__'; + +let fsImpl: DiagnosticFs | null = null; +let documentDirectoryPath = DEFAULT_DOC_DIR; +let loadDefaultFsPromise: Promise | null = null; + +export function configureDiagnosticLog(options: { + fs: DiagnosticFs; + documentDirectoryPath: string; +}): void { + fsImpl = options.fs; + documentDirectoryPath = options.documentDirectoryPath; +} + +export function resetDiagnosticLogForTests(): void { + fsImpl = null; + documentDirectoryPath = DEFAULT_DOC_DIR; + loadDefaultFsPromise = null; +} + +async function getFs(): Promise { + if (fsImpl) { + return fsImpl; + } + if (!loadDefaultFsPromise) { + loadDefaultFsPromise = (async () => { + // RNFS ships Flow syntax; keep this lazy so Jest never loads it. + /* eslint-disable @typescript-eslint/no-require-imports */ + const RNFS = + require('react-native-fs') as typeof import('react-native-fs'); + /* eslint-enable @typescript-eslint/no-require-imports */ + documentDirectoryPath = RNFS.DocumentDirectoryPath; + const adapter: DiagnosticFs = { + exists: path => RNFS.exists(path), + readFile: path => RNFS.readFile(path, 'utf8'), + writeFile: (path, contents) => RNFS.writeFile(path, contents, 'utf8'), + appendFile: (path, contents) => RNFS.appendFile(path, contents, 'utf8'), + unlink: path => RNFS.unlink(path), + mkdir: path => RNFS.mkdir(path), + stat: async path => { + const info = await RNFS.stat(path); + return { size: Number(info.size) || 0 }; + }, + }; + fsImpl = adapter; + return adapter; + })(); + } + return loadDefaultFsPromise; +} + +function docDir(): string { + return documentDirectoryPath; +} + +export function getEventsFilePath(): string { + return eventsPath(docDir()); +} + +export function getExitsFilePath(): string { + return exitsPath(docDir()); +} + +export function getSessionFilePath(): string { + return sessionPath(docDir()); +} + +async function ensureDir(): Promise { + const fs = await getFs(); + const dir = eventsPath(docDir()).replace(/\/[^/]+$/, ''); + if (!(await fs.exists(dir))) { + await fs.mkdir(dir); + } + return fs; +} + +async function rotateIfNeeded( + fs: DiagnosticFs, + filePath: string, + incomingBytes: number, +): Promise { + if (!(await fs.exists(filePath))) { + return; + } + const { size } = await fs.stat(filePath); + if (size + incomingBytes <= MAX_LOG_BYTES) { + return; + } + const backup = backupPath(filePath); + if (await fs.exists(backup)) { + await fs.unlink(backup); + } + const current = await fs.readFile(filePath); + await fs.writeFile(backup, current); + await fs.writeFile(filePath, ''); +} + +export async function appendEvent(event: DiagnosticEvent): Promise { + const line = `${JSON.stringify(event)}\n`; + const fs = await ensureDir(); + const path = getEventsFilePath(); + await rotateIfNeeded(fs, path, line.length); + await fs.appendFile(path, line); +} + +export async function readRecentEvents( + max: number = 30, +): Promise { + const fs = await getFs(); + const path = getEventsFilePath(); + if (!(await fs.exists(path))) { + return []; + } + const raw = await fs.readFile(path); + const events: DiagnosticEvent[] = []; + for (const line of raw.split('\n')) { + if (!line.trim()) { + continue; + } + try { + events.push(JSON.parse(line) as DiagnosticEvent); + } catch { + // skip malformed + } + } + return events.slice(-Math.max(0, max)).reverse(); +} + +export async function readExitRecords(): Promise { + const fs = await getFs(); + const path = getExitsFilePath(); + if (!(await fs.exists(path))) { + return []; + } + const raw = await fs.readFile(path); + const records: ProcessExitRecord[] = []; + for (const line of raw.split('\n')) { + if (!line.trim()) { + continue; + } + try { + records.push(JSON.parse(line) as ProcessExitRecord); + } catch { + // skip malformed + } + } + return records; +} + +export async function readLastExit(): Promise { + const records = await readExitRecords(); + return records.length > 0 ? records[records.length - 1] : null; +} + +export async function writeSession(session: { + startedAt: string; + appState: string; + cleanExit: boolean; +}): Promise { + const fs = await ensureDir(); + await fs.writeFile(getSessionFilePath(), JSON.stringify(session)); +} + +export async function readSession(): Promise<{ + startedAt: string; + appState: string; + cleanExit: boolean; +} | null> { + const fs = await getFs(); + const path = getSessionFilePath(); + if (!(await fs.exists(path))) { + return null; + } + try { + return JSON.parse(await fs.readFile(path)); + } catch { + return null; + } +} + +export async function clearDiagnosticFiles(): Promise { + const fs = await getFs(); + for (const path of [ + getEventsFilePath(), + getExitsFilePath(), + backupPath(getEventsFilePath()), + backupPath(getExitsFilePath()), + ]) { + if (await fs.exists(path)) { + await fs.unlink(path); + } + } +} + +export async function readFileIfExists(path: string): Promise { + const fs = await getFs(); + if (!(await fs.exists(path))) { + return ''; + } + return fs.readFile(path); +} diff --git a/formulus/src/diagnostics/DirtyExitGate.tsx b/formulus/src/diagnostics/DirtyExitGate.tsx new file mode 100644 index 000000000..2bf589ba0 --- /dev/null +++ b/formulus/src/diagnostics/DirtyExitGate.tsx @@ -0,0 +1,65 @@ +import { useEffect } from 'react'; +import { AppState } from 'react-native'; +import { useTranslation } from 'react-i18next'; +import { useConfirmModal } from '../contexts/ConfirmModalContext'; +import { consumePendingDirtyExit } from './consumeDirtyExit'; +import { exportDiagnosticsZip } from './exportDiagnostics'; +import { logger } from './logger'; +import { updateAppState } from './sessionHeartbeat'; + +/** + * After i18n + ConfirmModal are up: start a heartbeat session and show a + * one-shot dialog for a non-garden-variety previous exit. + */ +export function DirtyExitGate(): null { + const { showConfirm } = useConfirmModal(); + const { t } = useTranslation(); + + useEffect(() => { + let cancelled = false; + void (async () => { + try { + const dirty = await consumePendingDirtyExit(); + if (cancelled || !dirty) { + return; + } + showConfirm({ + title: t('help.diagnostics.unexpectedTitle'), + message: t('help.diagnostics.unexpectedMessage', { + reason: dirty.reason, + }), + buttons: [ + { text: t('help.diagnostics.ok'), onPress: () => undefined }, + { + text: t('help.diagnostics.saveLog'), + variant: 'primary', + onPress: () => { + void exportDiagnosticsZip().catch(error => { + logger.warn( + 'diagnostics', + error instanceof Error ? error.message : 'export failed', + ); + }); + }, + }, + ], + }); + } catch (error) { + logger.warn( + 'diagnostics', + error instanceof Error ? error.message : 'dirty-exit check failed', + ); + } + })(); + + const sub = AppState.addEventListener('change', state => { + void updateAppState(state); + }); + return () => { + cancelled = true; + sub.remove(); + }; + }, [showConfirm, t]); + + return null; +} diff --git a/formulus/src/diagnostics/__tests__/DiagnosticLog.test.ts b/formulus/src/diagnostics/__tests__/DiagnosticLog.test.ts new file mode 100644 index 000000000..7035bb1bf --- /dev/null +++ b/formulus/src/diagnostics/__tests__/DiagnosticLog.test.ts @@ -0,0 +1,73 @@ +import { + appendEvent, + clearDiagnosticFiles, + configureDiagnosticLog, + readRecentEvents, + resetDiagnosticLogForTests, +} from '../DiagnosticLog'; +import { createMemoryFs } from '../memoryFs'; +import { MAX_LOG_BYTES } from '../paths'; + +describe('DiagnosticLog', () => { + beforeEach(() => { + resetDiagnosticLogForTests(); + configureDiagnosticLog({ + fs: createMemoryFs(), + documentDirectoryPath: '/docs', + }); + }); + + it('appends events and returns newest first', async () => { + await appendEvent({ + ts: '2026-08-16T10:00:00.000Z', + kind: 'log', + level: 'info', + tag: 'sync', + message: 'start', + }); + await appendEvent({ + ts: '2026-08-16T10:00:01.000Z', + kind: 'log', + level: 'info', + tag: 'sync', + message: 'done', + }); + const recent = await readRecentEvents(10); + expect(recent.map(e => e.message)).toEqual(['done', 'start']); + }); + + it('rotates when the file would exceed the size cap', async () => { + const fs = createMemoryFs(); + configureDiagnosticLog({ fs, documentDirectoryPath: '/docs' }); + const huge = 'x'.repeat(MAX_LOG_BYTES - 20); + await appendEvent({ + ts: '2026-08-16T10:00:00.000Z', + kind: 'log', + level: 'info', + tag: 'sync', + message: huge, + }); + await appendEvent({ + ts: '2026-08-16T10:00:01.000Z', + kind: 'log', + level: 'info', + tag: 'sync', + message: 'overflow', + }); + expect(await fs.exists('/docs/diagnostics/events.ndjson.1')).toBe(true); + const recent = await readRecentEvents(5); + expect(recent[0].message).toBe('overflow'); + }); + + it('clears event and exit files', async () => { + await appendEvent({ + ts: '2026-08-16T10:00:00.000Z', + kind: 'log', + level: 'info', + tag: 'sync', + message: 'start', + }); + await clearDiagnosticFiles(); + expect(await readRecentEvents(10)).toEqual([]); + }); +}); diff --git a/formulus/src/diagnostics/__tests__/classifyExit.test.ts b/formulus/src/diagnostics/__tests__/classifyExit.test.ts new file mode 100644 index 000000000..9b1de3c05 --- /dev/null +++ b/formulus/src/diagnostics/__tests__/classifyExit.test.ts @@ -0,0 +1,79 @@ +import { + dirtyExitFromAei, + formatExitReason, + isDirtyExit, + isDirtyHeartbeat, +} from '../classifyExit'; + +describe('classifyExit', () => { + it('treats crash, ANR, OOM, and crash signals as dirty', () => { + expect(isDirtyExit({ timestamp: 1, reason: 'REASON_CRASH' })).toBe(true); + expect(isDirtyExit({ timestamp: 1, reason: 'REASON_ANR' })).toBe(true); + expect(isDirtyExit({ timestamp: 1, reason: 'REASON_LOW_MEMORY' })).toBe( + true, + ); + expect( + isDirtyExit({ timestamp: 1, reason: 'REASON_EXCESSIVE_RESOURCE_USAGE' }), + ).toBe(true); + expect( + isDirtyExit({ timestamp: 1, reason: 'REASON_SIGNALED', status: 11 }), + ).toBe(true); + }); + + it('treats user swipe-away and self-exit as garden-variety', () => { + expect(isDirtyExit({ timestamp: 1, reason: 'REASON_USER_REQUESTED' })).toBe( + false, + ); + expect(isDirtyExit({ timestamp: 1, reason: 'REASON_USER_STOPPED' })).toBe( + false, + ); + expect(isDirtyExit({ timestamp: 1, reason: 'REASON_EXIT_SELF' })).toBe( + false, + ); + expect(isDirtyExit({ timestamp: 1, reason: 'REASON_FREEZER' })).toBe(false); + expect( + isDirtyExit({ timestamp: 1, reason: 'REASON_SIGNALED', status: 9 }), + ).toBe(false); + }); + + it('prefers the AEI description for the popup reason', () => { + expect( + formatExitReason({ + timestamp: 1, + reason: 'REASON_CRASH', + description: 'Context.startForeground did not start in time', + }), + ).toBe('Context.startForeground did not start in time'); + expect( + dirtyExitFromAei({ + timestamp: 1_700_000_000_000, + reason: 'REASON_CRASH', + description: 'ForegroundServiceDidNotStartInTimeException', + }).reason, + ).toBe('ForegroundServiceDidNotStartInTimeException'); + }); + + it('marks a foreground session without cleanExit as dirty', () => { + expect( + isDirtyHeartbeat({ + startedAt: '2026-08-16T10:00:00.000Z', + appState: 'active', + cleanExit: false, + }), + ).toBe(true); + expect( + isDirtyHeartbeat({ + startedAt: '2026-08-16T10:00:00.000Z', + appState: 'background', + cleanExit: false, + }), + ).toBe(false); + expect( + isDirtyHeartbeat({ + startedAt: '2026-08-16T10:00:00.000Z', + appState: 'active', + cleanExit: true, + }), + ).toBe(false); + }); +}); diff --git a/formulus/src/diagnostics/__tests__/consumeDirtyExit.test.ts b/formulus/src/diagnostics/__tests__/consumeDirtyExit.test.ts new file mode 100644 index 000000000..cc6a94c75 --- /dev/null +++ b/formulus/src/diagnostics/__tests__/consumeDirtyExit.test.ts @@ -0,0 +1,57 @@ +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { consumePendingDirtyExit, SHOWN_EXIT_KEY } from '../consumeDirtyExit'; +import { + configureDiagnosticLog, + resetDiagnosticLogForTests, + writeSession, +} from '../DiagnosticLog'; +import { createMemoryFs } from '../memoryFs'; + +jest.mock('react-native', () => ({ + NativeModules: {}, + Platform: { OS: 'ios' }, +})); + +jest.mock('@react-native-async-storage/async-storage', () => ({ + getItem: jest.fn(), + setItem: jest.fn(), +})); + +describe('consumePendingDirtyExit', () => { + beforeEach(() => { + resetDiagnosticLogForTests(); + configureDiagnosticLog({ + fs: createMemoryFs(), + documentDirectoryPath: '/docs', + }); + (AsyncStorage.getItem as jest.Mock).mockResolvedValue(null); + (AsyncStorage.setItem as jest.Mock).mockResolvedValue(undefined); + }); + + it('returns a heartbeat dirty exit when the last session died in the foreground', async () => { + await writeSession({ + startedAt: '2026-08-16T09:00:00.000Z', + appState: 'active', + cleanExit: false, + }); + const dirty = await consumePendingDirtyExit(); + expect(dirty).toEqual({ + source: 'heartbeat', + timestamp: '2026-08-16T09:00:00.000Z', + reason: 'the app closed unexpectedly', + }); + expect(AsyncStorage.setItem).toHaveBeenCalledWith( + SHOWN_EXIT_KEY, + '2026-08-16T09:00:00.000Z', + ); + }); + + it('does not popup after a background kill', async () => { + await writeSession({ + startedAt: '2026-08-16T09:00:00.000Z', + appState: 'background', + cleanExit: true, + }); + await expect(consumePendingDirtyExit()).resolves.toBeNull(); + }); +}); diff --git a/formulus/src/diagnostics/__tests__/exportDiagnostics.test.ts b/formulus/src/diagnostics/__tests__/exportDiagnostics.test.ts new file mode 100644 index 000000000..f97d16145 --- /dev/null +++ b/formulus/src/diagnostics/__tests__/exportDiagnostics.test.ts @@ -0,0 +1,41 @@ +import { + buildSummaryText, + DIAGNOSTICS_ZIP_FILES, + serverHostnameOnly, +} from '../exportDiagnosticsText'; + +describe('exportDiagnostics', () => { + it('includes only the diagnostic zip members', () => { + expect([...DIAGNOSTICS_ZIP_FILES]).toEqual([ + 'events.ndjson', + 'exits.ndjson', + 'summary.txt', + ]); + expect(DIAGNOSTICS_ZIP_FILES).not.toEqual( + expect.arrayContaining(['observations.json', 'attachments']), + ); + }); + + it('keeps only the server hostname', () => { + expect(serverHostnameOnly('https://sync.example.org/api/v1?token=x')).toBe( + 'sync.example.org', + ); + expect(serverHostnameOnly(null)).toBe('(none)'); + }); + + it('builds a summary without observation payloads', () => { + const text = buildSummaryText({ + deviceModel: 'Blackview', + systemName: 'Android', + systemVersion: '14', + appVersion: '1.2.3 (45)', + serverHost: 'sync.example.org', + lastExitReason: 'Context.startForeground did not start in time', + breadcrumbs: ['2026-08-16T10:00:00.000Z fgs.start'], + }); + expect(text).toContain('Blackview'); + expect(text).toContain('fgs.start'); + expect(text).not.toContain('observation'); + expect(text).not.toContain('hh_id'); + }); +}); diff --git a/formulus/src/diagnostics/__tests__/logger.test.ts b/formulus/src/diagnostics/__tests__/logger.test.ts new file mode 100644 index 000000000..9591c22e7 --- /dev/null +++ b/formulus/src/diagnostics/__tests__/logger.test.ts @@ -0,0 +1,62 @@ +import { configureDiagnosticLog, readRecentEvents } from '../DiagnosticLog'; +import { + configureLogger, + logger, + persistWebViewConsole, + resetLoggerForTests, +} from '../logger'; +import { createMemoryFs } from '../memoryFs'; + +describe('logger', () => { + const consoleMock = { + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }; + + beforeEach(() => { + resetLoggerForTests(); + configureLogger({ console: consoleMock, persist: true }); + configureDiagnosticLog({ + fs: createMemoryFs(), + documentDirectoryPath: '/docs', + }); + consoleMock.debug.mockClear(); + consoleMock.info.mockClear(); + consoleMock.warn.mockClear(); + consoleMock.error.mockClear(); + }); + + it('does not persist debug lines', async () => { + logger.debug('sync', 'looking up observation'); + await logger.breadcrumb('sync', 'start', { counts: 3 }); + const events = await readRecentEvents(10); + expect(events.some(e => e.level === 'debug')).toBe(false); + expect(events.some(e => e.kind === 'breadcrumb')).toBe(true); + }); + + it('persists info after redaction and drops unknown extras', async () => { + logger.info('sync', 'Bearer super-secret-token pull done', { + counts: 4, + // @ts-expect-error intentional leak attempt + observationId: 'obs-1', + }); + // allow async persist + await new Promise(resolve => setTimeout(resolve, 0)); + const events = await readRecentEvents(5); + expect(events[0].message).toContain('[redacted]'); + expect(events[0].message).not.toContain('super-secret-token'); + expect(events[0].extras).toEqual({ counts: 4 }); + }); + + it('persists webview warn/error but not debug', async () => { + persistWebViewConsole('webview', 'debug', ['noisy']); + persistWebViewConsole('webview', 'error', ['boom', { data: { n: 1 } }]); + await new Promise(resolve => setTimeout(resolve, 0)); + const events = await readRecentEvents(10); + expect(events.some(e => e.message.includes('noisy'))).toBe(false); + expect(events.some(e => e.level === 'error')).toBe(true); + expect(events[0].message).not.toContain('data'); + }); +}); diff --git a/formulus/src/diagnostics/__tests__/redact.test.ts b/formulus/src/diagnostics/__tests__/redact.test.ts new file mode 100644 index 000000000..13de2fc22 --- /dev/null +++ b/formulus/src/diagnostics/__tests__/redact.test.ts @@ -0,0 +1,65 @@ +import { + joinLogArgs, + pickAllowedExtras, + redactText, + WEBVIEW_INFO_MESSAGE_MAX, +} from '../redact'; + +describe('redact', () => { + it('strips bearer tokens, emails, file URIs, and cookies', () => { + const raw = + 'Authorization Bearer abc.def.ghi cookie=secret file:///data/user/0/x/photo.jpg user@example.org'; + const out = redactText(raw); + expect(out).not.toContain('abc.def.ghi'); + expect(out).not.toContain('user@example.org'); + expect(out).not.toContain('file:///data'); + expect(out).not.toContain('secret'); + expect(out).toContain('[redacted]'); + expect(out).toContain('[email]'); + }); + + it('replaces lat/lon pairs and JSON blobs', () => { + const out = redactText( + 'at 9.012345, 38.765432 payload {"name":"Amina","hh_id":"HH-1","village":"x"}', + ); + expect(out).toContain('[latlon]'); + expect(out).toContain('[json]'); + expect(out).not.toContain('Amina'); + }); + + it('truncates long messages', () => { + const out = redactText('x'.repeat(800), 500); + expect(out.endsWith('…')).toBe(true); + expect(out.length).toBe(501); + }); + + it('allowlists extras and drops unknown keys', () => { + expect( + pickAllowedExtras({ + phase: 'pull_observations', + counts: 12, + formType: 'household', + screen: 'Sync', + success: true, + observationId: 'should-drop', + data: { secret: true }, + }), + ).toEqual({ + phase: 'pull_observations', + counts: 12, + formType: 'household', + screen: 'Sync', + success: true, + }); + }); + + it('joins webview args without persisting object dumps', () => { + const joined = joinLogArgs([ + 'saved', + { p_id: 'P1', name: 'secret' }, + '{"hh_id":"HH-1","name":"Amina","extra":true}', + ]); + expect(joined).toBe('saved [json] [json]'); + expect(redactText(joined, WEBVIEW_INFO_MESSAGE_MAX)).not.toContain('Amina'); + }); +}); diff --git a/formulus/src/diagnostics/classifyExit.ts b/formulus/src/diagnostics/classifyExit.ts new file mode 100644 index 000000000..e500d9ccb --- /dev/null +++ b/formulus/src/diagnostics/classifyExit.ts @@ -0,0 +1,60 @@ +import type { DirtyExit, ProcessExitRecord, SessionHeartbeat } from './types'; + +const DIRTY_REASONS = new Set([ + 'REASON_CRASH', + 'REASON_CRASH_NATIVE', + 'REASON_ANR', + 'REASON_LOW_MEMORY', + 'REASON_EXCESSIVE_RESOURCE_USAGE', +]); + +/** SIGILL, SIGABRT, SIGBUS, SIGFPE, SIGSEGV */ +const CRASH_SIGNALS = new Set([4, 6, 7, 8, 11]); + +export function isDirtyExit(record: ProcessExitRecord): boolean { + if (DIRTY_REASONS.has(record.reason)) { + return true; + } + if (record.reason === 'REASON_SIGNALED') { + return CRASH_SIGNALS.has(record.status ?? -1); + } + return false; +} + +export function formatExitReason(record: ProcessExitRecord): string { + const description = record.description?.trim(); + if (description) { + return description; + } + return humanizeReason(record.reason); +} + +export function humanizeReason(reason: string): string { + return reason + .replace(/^REASON_/, '') + .toLowerCase() + .replace(/_/g, ' '); +} + +export function isDirtyHeartbeat(session: SessionHeartbeat | null): boolean { + if (!session) { + return false; + } + return session.cleanExit !== true && session.appState === 'active'; +} + +export function dirtyExitFromAei(record: ProcessExitRecord): DirtyExit { + return { + source: 'aei', + timestamp: new Date(record.timestamp).toISOString(), + reason: formatExitReason(record), + }; +} + +export function dirtyExitFromHeartbeat(session: SessionHeartbeat): DirtyExit { + return { + source: 'heartbeat', + timestamp: session.startedAt, + reason: 'the app closed unexpectedly', + }; +} diff --git a/formulus/src/diagnostics/consumeDirtyExit.ts b/formulus/src/diagnostics/consumeDirtyExit.ts new file mode 100644 index 000000000..34fbda76d --- /dev/null +++ b/formulus/src/diagnostics/consumeDirtyExit.ts @@ -0,0 +1,67 @@ +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { + dirtyExitFromAei, + dirtyExitFromHeartbeat, + isDirtyExit, + isDirtyHeartbeat, +} from './classifyExit'; +import { readLastExit } from './DiagnosticLog'; +import { getRecentNativeExits } from './nativeExits'; +import { beginSession } from './sessionHeartbeat'; +import type { DirtyExit, ProcessExitRecord } from './types'; + +export const SHOWN_EXIT_KEY = '@diagnostics_shown_exit_at'; + +async function alreadyShown(timestamp: string): Promise { + const shown = await AsyncStorage.getItem(SHOWN_EXIT_KEY); + return shown === timestamp; +} + +async function markShown(timestamp: string): Promise { + await AsyncStorage.setItem(SHOWN_EXIT_KEY, timestamp); +} + +function pickLastExit( + fromFile: ProcessExitRecord | null, + fromNative: ProcessExitRecord[], +): ProcessExitRecord | null { + if (fromFile) { + return fromFile; + } + return fromNative.length > 0 ? fromNative[fromNative.length - 1] : null; +} + +/** + * Starts a new heartbeat session and returns a dirty exit to show once. + */ +export async function consumePendingDirtyExit(): Promise { + const previousSession = await beginSession(); + const [fromFile, fromNative] = await Promise.all([ + readLastExit(), + getRecentNativeExits(5), + ]); + const lastExit = pickLastExit(fromFile, fromNative); + + if (lastExit) { + if (!isDirtyExit(lastExit)) { + return null; + } + const dirty = dirtyExitFromAei(lastExit); + if (await alreadyShown(dirty.timestamp)) { + return null; + } + await markShown(dirty.timestamp); + return dirty; + } + + if (isDirtyHeartbeat(previousSession) && previousSession) { + const dirty = dirtyExitFromHeartbeat(previousSession); + if (await alreadyShown(dirty.timestamp)) { + return null; + } + await markShown(dirty.timestamp); + return dirty; + } + + return null; +} diff --git a/formulus/src/diagnostics/exportDiagnostics.ts b/formulus/src/diagnostics/exportDiagnostics.ts new file mode 100644 index 000000000..94ab2a21a --- /dev/null +++ b/formulus/src/diagnostics/exportDiagnostics.ts @@ -0,0 +1,75 @@ +import DeviceInfo from 'react-native-device-info'; +import RNFS from 'react-native-fs'; +import { zip } from 'react-native-zip-archive'; +import { saveZipToDevice } from '../services/saveZipToDevice'; +import { serverConfigService } from '../services/ServerConfigService'; +import { appVersionService } from '../services/AppVersionService'; +import { + getEventsFilePath, + getExitsFilePath, + readFileIfExists, + readLastExit, + readRecentEvents, +} from './DiagnosticLog'; +import { formatExitReason } from './classifyExit'; +import { buildSummaryText, serverHostnameOnly } from './exportDiagnosticsText'; + +export { + DIAGNOSTICS_ZIP_FILES, + buildSummaryText, + serverHostnameOnly, +} from './exportDiagnosticsText'; + +export async function exportDiagnosticsZip(): Promise { + const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); + const workDir = `${RNFS.CachesDirectoryPath}/formulus-diagnostics-${stamp}`; + const zipName = `formulus-diagnostics-${stamp}.zip`; + const zipPath = `${RNFS.CachesDirectoryPath}/${zipName}`; + + if (await RNFS.exists(workDir)) { + await RNFS.unlink(workDir); + } + await RNFS.mkdir(workDir); + + try { + const events = await readFileIfExists(getEventsFilePath()); + const exits = await readFileIfExists(getExitsFilePath()); + await RNFS.writeFile(`${workDir}/events.ndjson`, events, 'utf8'); + await RNFS.writeFile(`${workDir}/exits.ndjson`, exits, 'utf8'); + + const [lastExit, recent, serverUrl, appVersion] = await Promise.all([ + readLastExit(), + readRecentEvents(40), + serverConfigService.getServerUrl(), + appVersionService.getFullVersion().catch(() => 'unknown'), + ]); + const breadcrumbs = recent + .filter(event => event.kind === 'breadcrumb') + .slice(0, 20) + .map(event => `${event.ts} ${event.message}`); + + const summary = buildSummaryText({ + deviceModel: DeviceInfo.getModel(), + systemName: DeviceInfo.getSystemName(), + systemVersion: DeviceInfo.getSystemVersion(), + appVersion, + serverHost: serverHostnameOnly(serverUrl), + lastExitReason: lastExit ? formatExitReason(lastExit) : null, + breadcrumbs, + }); + await RNFS.writeFile(`${workDir}/summary.txt`, summary, 'utf8'); + + if (await RNFS.exists(zipPath)) { + await RNFS.unlink(zipPath); + } + await zip(workDir, zipPath); + } finally { + if (await RNFS.exists(workDir)) { + await RNFS.unlink(workDir).catch(() => { + /* best-effort */ + }); + } + } + + await saveZipToDevice(zipPath, zipName); +} diff --git a/formulus/src/diagnostics/exportDiagnosticsText.ts b/formulus/src/diagnostics/exportDiagnosticsText.ts new file mode 100644 index 000000000..acddcabdf --- /dev/null +++ b/formulus/src/diagnostics/exportDiagnosticsText.ts @@ -0,0 +1,39 @@ +export const DIAGNOSTICS_ZIP_FILES = [ + 'events.ndjson', + 'exits.ndjson', + 'summary.txt', +] as const; + +export function serverHostnameOnly(serverUrl: string | null): string { + if (!serverUrl) { + return '(none)'; + } + try { + return new URL(serverUrl).hostname || '(none)'; + } catch { + return '(none)'; + } +} + +export function buildSummaryText(input: { + deviceModel: string; + systemName: string; + systemVersion: string; + appVersion: string; + serverHost: string; + lastExitReason: string | null; + breadcrumbs: string[]; +}): string { + const lines = [ + 'Formulus diagnostic summary', + `device: ${input.deviceModel}`, + `os: ${input.systemName} ${input.systemVersion}`, + `app: ${input.appVersion}`, + `server: ${input.serverHost}`, + `last dirty/exit reason: ${input.lastExitReason ?? '(none)'}`, + '', + 'recent breadcrumbs:', + ...(input.breadcrumbs.length > 0 ? input.breadcrumbs : ['(none)']), + ]; + return `${lines.join('\n')}\n`; +} diff --git a/formulus/src/diagnostics/index.ts b/formulus/src/diagnostics/index.ts new file mode 100644 index 000000000..48ce610e7 --- /dev/null +++ b/formulus/src/diagnostics/index.ts @@ -0,0 +1,14 @@ +export { logger, persistWebViewConsole, webViewTag } from './logger'; +export { installErrorHandlers } from './installErrorHandlers'; +export { consumePendingDirtyExit } from './consumeDirtyExit'; +export { updateAppState } from './sessionHeartbeat'; +export { + exportDiagnosticsZip, + DIAGNOSTICS_ZIP_FILES, +} from './exportDiagnostics'; +export { + readRecentEvents, + readLastExit, + clearDiagnosticFiles, +} from './DiagnosticLog'; +export type { DiagnosticEvent, DirtyExit, ProcessExitRecord } from './types'; diff --git a/formulus/src/diagnostics/installErrorHandlers.ts b/formulus/src/diagnostics/installErrorHandlers.ts new file mode 100644 index 000000000..7af1e60a5 --- /dev/null +++ b/formulus/src/diagnostics/installErrorHandlers.ts @@ -0,0 +1,58 @@ +import { appendEvent } from './DiagnosticLog'; +import { redactText } from './redact'; + +type ErrorUtilsLike = { + getGlobalHandler?: () => + | ((error: Error, isFatal?: boolean) => void) + | undefined; + setGlobalHandler: ( + handler: (error: Error, isFatal?: boolean) => void, + ) => void; +}; + +function persistFatal( + kind: 'js_fatal' | 'js_unhandled', + message: string, +): void { + void appendEvent({ + ts: new Date().toISOString(), + kind, + level: 'error', + tag: 'js', + message: redactText(message), + }).catch(() => { + /* ignore */ + }); +} + +export function installErrorHandlers(): void { + const errorUtils = (globalThis as { ErrorUtils?: ErrorUtilsLike }).ErrorUtils; + if (errorUtils?.setGlobalHandler) { + const previous = errorUtils.getGlobalHandler?.(); + errorUtils.setGlobalHandler((error, isFatal) => { + persistFatal( + 'js_fatal', + error instanceof Error ? error.message : String(error), + ); + previous?.(error, isFatal); + }); + } + + const target = globalThis as unknown as { + addEventListener?: ( + type: string, + listener: (event: { reason?: unknown }) => void, + ) => void; + }; + if (typeof target.addEventListener === 'function') { + target.addEventListener('unhandledrejection', event => { + const reason = event?.reason; + persistFatal( + 'js_unhandled', + reason instanceof Error + ? reason.message + : String(reason ?? 'rejection'), + ); + }); + } +} diff --git a/formulus/src/diagnostics/logger.ts b/formulus/src/diagnostics/logger.ts new file mode 100644 index 000000000..9501d888c --- /dev/null +++ b/formulus/src/diagnostics/logger.ts @@ -0,0 +1,162 @@ +import { appendEvent } from './DiagnosticLog'; +import { + joinLogArgs, + pickAllowedExtras, + redactText, + WEBVIEW_INFO_MESSAGE_MAX, +} from './redact'; +import type { AllowedLogExtras, LogLevel } from './types'; + +const WEBVIEW_INFO_WINDOW_MS = 10_000; +const WEBVIEW_INFO_MAX_PER_WINDOW = 20; + +let webViewInfoTimes: number[] = []; + +type ConsoleLike = { + debug: (...args: unknown[]) => void; + info: (...args: unknown[]) => void; + warn: (...args: unknown[]) => void; + error: (...args: unknown[]) => void; +}; + +let consoleSink: ConsoleLike = console; +let persistEnabled = true; + +export function configureLogger(options: { + console?: ConsoleLike; + persist?: boolean; +}): void { + if (options.console) { + consoleSink = options.console; + } + if (options.persist != null) { + persistEnabled = options.persist; + } +} + +export function resetLoggerForTests(): void { + consoleSink = console; + persistEnabled = true; + webViewInfoTimes = []; +} + +function formatPrefix(level: LogLevel, tag: string, message: string): string { + return `[${tag}] ${message}`; +} + +async function persist( + level: LogLevel | 'breadcrumb', + tag: string, + message: string, + extras?: AllowedLogExtras, + breadcrumb?: { category: string; action: string }, +): Promise { + if (!persistEnabled) { + return; + } + try { + await appendEvent({ + ts: new Date().toISOString(), + kind: breadcrumb ? 'breadcrumb' : 'log', + level, + tag, + message: redactText(message), + extras: pickAllowedExtras(extras), + category: breadcrumb?.category, + action: breadcrumb?.action, + }); + } catch { + // Persistence must never break the caller. + } +} + +function emit( + level: LogLevel, + tag: string, + message: string, + extras?: AllowedLogExtras, +): void { + const line = formatPrefix(level, tag, message); + if (extras) { + consoleSink[level](line, extras); + } else { + consoleSink[level](line); + } + if (level !== 'debug') { + void persist(level, tag, message, extras); + } +} + +export const logger = { + debug(tag: string, message: string, extras?: AllowedLogExtras): void { + if (typeof __DEV__ !== 'undefined' && __DEV__) { + emit('debug', tag, message, extras); + } + }, + + info(tag: string, message: string, extras?: AllowedLogExtras): void { + emit('info', tag, message, extras); + }, + + warn(tag: string, message: string, extras?: AllowedLogExtras): void { + emit('warn', tag, message, extras); + }, + + error(tag: string, message: string, extras?: AllowedLogExtras): void { + emit('error', tag, message, extras); + }, + + async breadcrumb( + category: string, + action: string, + extras?: AllowedLogExtras, + ): Promise { + const message = `${category}.${action}`; + if (typeof __DEV__ !== 'undefined' && __DEV__) { + consoleSink.info(`[breadcrumb] ${message}`, extras ?? ''); + } + await persist('breadcrumb', category, message, extras, { + category, + action, + }); + }, +}; + +function allowWebViewInfo(): boolean { + const now = Date.now(); + webViewInfoTimes = webViewInfoTimes.filter( + t => now - t < WEBVIEW_INFO_WINDOW_MS, + ); + if (webViewInfoTimes.length >= WEBVIEW_INFO_MAX_PER_WINDOW) { + return false; + } + webViewInfoTimes.push(now); + return true; +} + +export function persistWebViewConsole( + tag: string, + level: string, + args: unknown[], +): void { + const joined = joinLogArgs(args); + if (level === 'debug') { + return; + } + if (level === 'warn' || level === 'error') { + logger[level](tag, joined); + return; + } + if (!allowWebViewInfo()) { + return; + } + logger.info(tag, redactText(joined, WEBVIEW_INFO_MESSAGE_MAX)); +} + +export function webViewTag(appName?: string): string { + const name = (appName ?? '').toLowerCase(); + if (name.includes('formplayer')) { + return 'webview:formplayer'; + } + return name ? `webview:${name}` : 'webview'; +} diff --git a/formulus/src/diagnostics/memoryFs.ts b/formulus/src/diagnostics/memoryFs.ts new file mode 100644 index 000000000..4dbca846a --- /dev/null +++ b/formulus/src/diagnostics/memoryFs.ts @@ -0,0 +1,41 @@ +import type { DiagnosticFs } from './types'; + +/** In-memory filesystem for unit tests. */ +export function createMemoryFs( + initial: Record = {}, +): DiagnosticFs { + const files = new Map(Object.entries(initial)); + const dirs = new Set(); + + return { + async exists(path: string) { + return files.has(path) || dirs.has(path); + }, + async readFile(path: string) { + if (!files.has(path)) { + throw new Error(`ENOENT: ${path}`); + } + return files.get(path) as string; + }, + async writeFile(path: string, contents: string) { + files.set(path, contents); + }, + async appendFile(path: string, contents: string) { + files.set(path, `${files.get(path) ?? ''}${contents}`); + }, + async unlink(path: string) { + files.delete(path); + dirs.delete(path); + }, + async mkdir(path: string) { + dirs.add(path); + }, + async stat(path: string) { + const contents = files.get(path); + if (contents == null) { + throw new Error(`ENOENT: ${path}`); + } + return { size: contents.length }; + }, + }; +} diff --git a/formulus/src/diagnostics/nativeExits.ts b/formulus/src/diagnostics/nativeExits.ts new file mode 100644 index 000000000..7558d98e4 --- /dev/null +++ b/formulus/src/diagnostics/nativeExits.ts @@ -0,0 +1,23 @@ +import { NativeModules, Platform } from 'react-native'; +import type { ProcessExitRecord } from './types'; + +type DiagnosticsNative = { + getRecentExits: (max: number) => Promise; +}; + +export async function getRecentNativeExits( + max: number = 5, +): Promise { + if (Platform.OS !== 'android') { + return []; + } + const mod = NativeModules.DiagnosticsModule as DiagnosticsNative | undefined; + if (!mod?.getRecentExits) { + return []; + } + try { + return await mod.getRecentExits(max); + } catch { + return []; + } +} diff --git a/formulus/src/diagnostics/paths.ts b/formulus/src/diagnostics/paths.ts new file mode 100644 index 000000000..3fc05ecd1 --- /dev/null +++ b/formulus/src/diagnostics/paths.ts @@ -0,0 +1,25 @@ +export const DIAGNOSTICS_DIR_NAME = 'diagnostics'; +export const EVENTS_FILE_NAME = 'events.ndjson'; +export const EXITS_FILE_NAME = 'exits.ndjson'; +export const SESSION_FILE_NAME = 'session.json'; +export const MAX_LOG_BYTES = 256 * 1024; + +export function diagnosticsDir(documentDirectoryPath: string): string { + return `${documentDirectoryPath}/${DIAGNOSTICS_DIR_NAME}`; +} + +export function eventsPath(documentDirectoryPath: string): string { + return `${diagnosticsDir(documentDirectoryPath)}/${EVENTS_FILE_NAME}`; +} + +export function exitsPath(documentDirectoryPath: string): string { + return `${diagnosticsDir(documentDirectoryPath)}/${EXITS_FILE_NAME}`; +} + +export function sessionPath(documentDirectoryPath: string): string { + return `${diagnosticsDir(documentDirectoryPath)}/${SESSION_FILE_NAME}`; +} + +export function backupPath(filePath: string): string { + return `${filePath}.1`; +} diff --git a/formulus/src/diagnostics/redact.ts b/formulus/src/diagnostics/redact.ts new file mode 100644 index 000000000..b1305115f --- /dev/null +++ b/formulus/src/diagnostics/redact.ts @@ -0,0 +1,105 @@ +import { ALLOWED_EXTRA_KEYS, AllowedLogExtras } from './types'; + +export const DEFAULT_MESSAGE_MAX = 500; +export const WEBVIEW_INFO_MESSAGE_MAX = 200; + +const BEARER_RE = /bearer\s+[a-z0-9._\-+=/]+/gi; +const COOKIE_RE = /(?:cookie|set-cookie)\s*[:=]\s*[^;\s]+/gi; +const FILE_URI_RE = /file:\/\/[^\s"'`]+/gi; +const EMAIL_RE = /[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/gi; +const LATLON_RE = + /\b-?(?:[1-8]?\d(?:\.\d+)?|90(?:\.0+)?),\s*-?(?:1[0-7]\d(?:\.\d+)?|180(?:\.0+)?|[1-9]?\d(?:\.\d+)?)\b/g; +const JSON_BLOB_RE = /\{[^{}]{40,}\}/g; + +export function pickAllowedExtras( + extras?: Record | AllowedLogExtras | null, +): AllowedLogExtras | undefined { + if (!extras || typeof extras !== 'object') { + return undefined; + } + const out: AllowedLogExtras = {}; + for (const key of ALLOWED_EXTRA_KEYS) { + if (!(key in extras)) { + continue; + } + const value = (extras as Record)[key]; + if ( + key === 'counts' && + typeof value === 'number' && + Number.isFinite(value) + ) { + out.counts = value; + } else if (key === 'success' && typeof value === 'boolean') { + out.success = value; + } else if ( + (key === 'phase' || key === 'formType' || key === 'screen') && + typeof value === 'string' + ) { + out[key] = value; + } + } + return Object.keys(out).length > 0 ? out : undefined; +} + +export function redactText( + input: unknown, + maxLength: number = DEFAULT_MESSAGE_MAX, +): string { + let text = stringifyUnknown(input); + text = text.replace(BEARER_RE, 'Bearer [redacted]'); + text = text.replace(COOKIE_RE, 'cookie=[redacted]'); + text = text.replace(FILE_URI_RE, 'file://[redacted]'); + text = text.replace(EMAIL_RE, '[email]'); + text = text.replace(LATLON_RE, '[latlon]'); + text = text.replace(JSON_BLOB_RE, '[json]'); + if (text.length > maxLength) { + return `${text.slice(0, maxLength)}…`; + } + return text; +} + +export function joinLogArgs(args: unknown[]): string { + return args + .map(arg => { + if (arg == null) { + return String(arg); + } + if (typeof arg === 'string') { + return looksLikeJsonObject(arg) ? '[json]' : arg; + } + if (typeof arg === 'object') { + return '[json]'; + } + return String(arg); + }) + .filter(part => part.length > 0) + .join(' '); +} + +function looksLikeJsonObject(value: string): boolean { + const trimmed = value.trim(); + return ( + (trimmed.startsWith('{') && trimmed.endsWith('}') && trimmed.length > 40) || + (trimmed.startsWith('[') && trimmed.endsWith(']') && trimmed.length > 40) + ); +} + +function stringifyUnknown(input: unknown): string { + if (input == null) { + return ''; + } + if (typeof input === 'string') { + return input; + } + if (input instanceof Error) { + return input.message || input.name; + } + if (typeof input === 'object') { + try { + return JSON.stringify(input); + } catch { + return String(input); + } + } + return String(input); +} diff --git a/formulus/src/diagnostics/sessionHeartbeat.ts b/formulus/src/diagnostics/sessionHeartbeat.ts new file mode 100644 index 000000000..99329ded2 --- /dev/null +++ b/formulus/src/diagnostics/sessionHeartbeat.ts @@ -0,0 +1,29 @@ +import { readSession, writeSession } from './DiagnosticLog'; +import { logger } from './logger'; +import type { SessionHeartbeat } from './types'; + +export async function beginSession(): Promise { + const previous = await readSession(); + const next: SessionHeartbeat = { + startedAt: new Date().toISOString(), + appState: 'active', + cleanExit: false, + }; + await writeSession(next); + await logger.breadcrumb('session', 'start'); + return previous; +} + +export async function updateAppState(appState: string): Promise { + const current = (await readSession()) ?? { + startedAt: new Date().toISOString(), + appState: 'active', + cleanExit: false, + }; + const cleanExit = appState === 'background' || appState === 'inactive'; + await writeSession({ + ...current, + appState, + cleanExit, + }); +} diff --git a/formulus/src/diagnostics/types.ts b/formulus/src/diagnostics/types.ts new file mode 100644 index 000000000..f3fc7d250 --- /dev/null +++ b/formulus/src/diagnostics/types.ts @@ -0,0 +1,67 @@ +export type LogLevel = 'debug' | 'info' | 'warn' | 'error'; + +export type AllowedLogExtras = { + phase?: string; + counts?: number; + formType?: string; + screen?: string; + success?: boolean; +}; + +export const ALLOWED_EXTRA_KEYS = [ + 'phase', + 'counts', + 'formType', + 'screen', + 'success', +] as const; + +export type DiagnosticEventKind = + | 'log' + | 'breadcrumb' + | 'js_fatal' + | 'js_unhandled' + | 'exit'; + +export type DiagnosticEvent = { + ts: string; + kind: DiagnosticEventKind; + level?: LogLevel | 'breadcrumb'; + tag?: string; + message: string; + extras?: AllowedLogExtras; + category?: string; + action?: string; +}; + +export type ProcessExitRecord = { + timestamp: number; + reason: string; + status?: number; + importance?: number; + pssKb?: number; + rssKb?: number; + description?: string; +}; + +export type SessionHeartbeat = { + startedAt: string; + appState: string; + cleanExit: boolean; +}; + +export type DirtyExit = { + source: 'aei' | 'heartbeat'; + timestamp: string; + reason: string; +}; + +export type DiagnosticFs = { + exists(path: string): Promise; + readFile(path: string): Promise; + writeFile(path: string, contents: string): Promise; + appendFile(path: string, contents: string): Promise; + unlink(path: string): Promise; + mkdir(path: string): Promise; + stat(path: string): Promise<{ size: number }>; +}; diff --git a/formulus/src/hooks/useForms.ts b/formulus/src/hooks/useForms.ts index 5f6ec172a..8844764d6 100644 --- a/formulus/src/hooks/useForms.ts +++ b/formulus/src/hooks/useForms.ts @@ -1,4 +1,4 @@ -import { useState, useEffect, useCallback } from 'react'; +import { useState, useEffect, useCallback, useRef } from 'react'; import { FormService, FormSpec } from '../services/FormService'; interface UseFormsResult { @@ -6,7 +6,7 @@ interface UseFormsResult { loading: boolean; error: string | null; refresh: () => Promise; - getObservationCount: (formId: string) => number; + getObservationCount: (formId: string) => number | undefined; observationCounts: Record; } @@ -17,6 +17,7 @@ export const useForms = (): UseFormsResult => { const [observationCounts, setObservationCounts] = useState< Record >({}); + const cancelledRef = useRef(false); const loadForms = useCallback(async () => { try { @@ -24,36 +25,52 @@ export const useForms = (): UseFormsResult => { setError(null); const formService = await FormService.getInstance(); const formSpecs = formService.getFormSpecs(); + if (cancelledRef.current) { + return; + } setForms(formSpecs); + setLoading(false); - const counts: Record = {}; for (const form of formSpecs) { + if (cancelledRef.current) { + return; + } try { - const observations = await formService.getObservationsByFormType( - form.id, - ); - counts[form.id] = observations.length; + const page = await formService.listObservationsPage({ + page: 1, + pageSize: 1, + formType: form.id, + }); + if (cancelledRef.current) { + return; + } + setObservationCounts(prev => ({ + ...prev, + [form.id]: page.total, + })); } catch (err) { console.error( `Failed to load observations for form ${form.id}:`, err, ); - counts[form.id] = 0; + if (!cancelledRef.current) { + setObservationCounts(prev => ({ ...prev, [form.id]: 0 })); + } } } - setObservationCounts(counts); } catch (err) { console.error('Failed to load forms:', err); - setError(err instanceof Error ? err.message : 'Failed to load forms'); - } finally { - setLoading(false); + if (!cancelledRef.current) { + setError(err instanceof Error ? err.message : 'Failed to load forms'); + setLoading(false); + } } }, []); useEffect(() => { - let cancelled = false; + cancelledRef.current = false; const timer = setTimeout(() => { - if (!cancelled) { + if (!cancelledRef.current) { void loadForms(); } }, 0); @@ -64,14 +81,16 @@ export const useForms = (): UseFormsResult => { }); }); return () => { - cancelled = true; + cancelledRef.current = true; clearTimeout(timer); }; }, [loadForms]); const getObservationCount = useCallback( - (formId: string): number => { - return observationCounts[formId] || 0; + (formId: string): number | undefined => { + return Object.prototype.hasOwnProperty.call(observationCounts, formId) + ? observationCounts[formId] + : undefined; }, [observationCounts], ); diff --git a/formulus/src/hooks/useObservations.ts b/formulus/src/hooks/useObservations.ts index 5af32df83..2afaaa38d 100644 --- a/formulus/src/hooks/useObservations.ts +++ b/formulus/src/hooks/useObservations.ts @@ -1,140 +1,132 @@ -import { useState, useEffect, useCallback, useMemo } from 'react'; +import { useState, useEffect, useCallback } from 'react'; import { FormService } from '../services/FormService'; -import { Observation } from '../database/models/Observation'; -import { isObservationFullySynced } from '../utils/observationSyncStatus'; -import { SortOption, FilterOption } from '../components/common/FilterBar'; +import type { SyncStatus } from '../components/common/SyncStatusButtons'; +import { logger } from '../diagnostics/logger'; +import type { + ObservationListPage, + ObservationListRow, +} from '../database/observationListQuery'; +import { OBSERVATION_LIST_PAGE_SIZE } from '../database/observationListQuery'; interface UseObservationsResult { - observations: Observation[]; + rows: ObservationListRow[]; + total: number; + totalPages: number; + page: number; + setPage: (page: number) => void; loading: boolean; error: string | null; refresh: () => Promise; searchQuery: string; setSearchQuery: (query: string) => void; - sortOption: SortOption; - setSortOption: (option: SortOption) => void; - filterOption: FilterOption; - setFilterOption: (option: FilterOption) => void; - filteredAndSorted: Observation[]; + selectedFormType: string | null; + setSelectedFormType: (formType: string | null) => void; + syncStatus: SyncStatus; + setSyncStatus: (status: SyncStatus) => void; } export const useObservations = (): UseObservationsResult => { - const [observations, setObservations] = useState([]); - const [loading, setLoading] = useState(true); + const [rows, setRows] = useState([]); + const [total, setTotal] = useState(0); + const [totalPages, setTotalPages] = useState(1); + const [page, setPageState] = useState(1); + const [loading, setLoading] = useState(true); const [error, setError] = useState(null); - const [searchQuery, setSearchQuery] = useState(''); - const [sortOption, setSortOption] = useState('date-desc'); - const [filterOption, setFilterOption] = useState('all'); + const [searchQuery, setSearchQueryState] = useState(''); + const [selectedFormType, setSelectedFormTypeState] = useState( + null, + ); + const [syncStatus, setSyncStatusState] = useState('all'); - const loadObservations = useCallback(async () => { + const loadPage = useCallback(async () => { + const started = Date.now(); + logger.info('observations', 'load start', { + screen: 'Observations', + phase: 'start', + counts: page, + }); try { setLoading(true); setError(null); const formService = await FormService.getInstance(); - const formSpecs = formService.getFormSpecs(); - const allObservations: Observation[] = []; - for (const formSpec of formSpecs) { - try { - const formObservations = await formService.getObservationsByFormType( - formSpec.id, - ); - allObservations.push(...formObservations); - } catch (err) { - console.error( - `Failed to load observations for form ${formSpec.id}:`, - err, - ); - } + const result: ObservationListPage = + await formService.listObservationsPage({ + page, + pageSize: OBSERVATION_LIST_PAGE_SIZE, + formType: selectedFormType, + syncStatus, + search: searchQuery, + }); + setRows(result.rows); + setTotal(result.total); + setTotalPages(result.totalPages); + if (page > result.totalPages) { + setPageState(result.totalPages); } - - setObservations(allObservations); + logger.info('observations', `load done in ${Date.now() - started}ms`, { + screen: 'Observations', + phase: 'done', + counts: result.total, + success: true, + }); } catch (err) { - console.error('Failed to load observations:', err); + logger.error( + 'observations', + err instanceof Error ? err.message : 'load failed', + { screen: 'Observations', phase: 'done', success: false }, + ); setError( err instanceof Error ? err.message : 'Failed to load observations', ); } finally { setLoading(false); } - }, []); + }, [page, searchQuery, selectedFormType, syncStatus]); useEffect(() => { - let cancelled = false; const timer = setTimeout(() => { - if (!cancelled) { - void loadObservations(); - } + void loadPage(); }, 0); - return () => { - cancelled = true; - clearTimeout(timer); - }; - }, [loadObservations]); - - const filteredAndSorted = useMemo(() => { - let filtered = observations.filter(obs => !obs.deleted); + return () => clearTimeout(timer); + }, [loadPage]); - if (searchQuery.trim()) { - const query = searchQuery.toLowerCase(); - filtered = filtered.filter(obs => { - try { - const data = - typeof obs.data === 'string' ? JSON.parse(obs.data) : obs.data; - const dataStr = JSON.stringify(data).toLowerCase(); - return ( - obs.observationId.toLowerCase().includes(query) || - obs.formType.toLowerCase().includes(query) || - dataStr.includes(query) - ); - } catch { - return ( - obs.observationId.toLowerCase().includes(query) || - obs.formType.toLowerCase().includes(query) - ); - } - }); - } + const setPage = useCallback((nextPage: number) => { + setPageState(nextPage); + setLoading(true); + }, []); - if (filterOption !== 'all') { - filtered = filtered.filter(obs => { - const synced = isObservationFullySynced(obs); - return filterOption === 'synced' ? synced : !synced; - }); - } + const setSearchQuery = useCallback((query: string) => { + setSearchQueryState(query); + setPageState(1); + setLoading(true); + }, []); - filtered.sort((a, b) => { - switch (sortOption) { - case 'date-desc': - return b.createdAt.getTime() - a.createdAt.getTime(); - case 'date-asc': - return a.createdAt.getTime() - b.createdAt.getTime(); - case 'form-type': - return a.formType.localeCompare(b.formType); - case 'sync-status': { - const aSynced = isObservationFullySynced(a); - const bSynced = isObservationFullySynced(b); - if (aSynced === bSynced) return 0; - return aSynced ? 1 : -1; - } - default: - return 0; - } - }); + const setSelectedFormType = useCallback((formType: string | null) => { + setSelectedFormTypeState(formType); + setPageState(1); + setLoading(true); + }, []); - return filtered; - }, [observations, searchQuery, sortOption, filterOption]); + const setSyncStatus = useCallback((status: SyncStatus) => { + setSyncStatusState(status); + setPageState(1); + setLoading(true); + }, []); return { - observations, + rows, + total, + totalPages, + page, + setPage, loading, error, - refresh: loadObservations, + refresh: loadPage, searchQuery, setSearchQuery, - sortOption, - setSortOption, - filterOption, - setFilterOption, - filteredAndSorted, + selectedFormType, + setSelectedFormType, + syncStatus, + setSyncStatus, }; }; diff --git a/formulus/src/locales/en.json b/formulus/src/locales/en.json index cbbcaf572..81ea728ad 100644 --- a/formulus/src/locales/en.json +++ b/formulus/src/locales/en.json @@ -82,11 +82,22 @@ "forms.emptyMessage": "No forms have been downloaded yet. Go to the Sync screen to download forms from the server.", "forms.emptySearchTitle": "No Forms Found", "forms.emptySearchMessage": "Try adjusting your search.", + "forms.colFormType": "Form type", + "forms.colObservationCount": "# observations", + "forms.colNewObservation": "New observation", "observations.title": "Observations", "observations.count_one": "{{count}} observation", "observations.count_other": "{{count}} observations", "observations.loading": "Loading observations...", - "observations.searchPlaceholder": "Search observations...", + "observations.searchPlaceholder": "Search form, author, or ID…", + "observations.colForm": "Form", + "observations.colCreated": "Created", + "observations.colSync": "Sync", + "observations.colAuthor": "Author", + "observations.colId": "ID", + "observations.pagerPrev": "Previous page", + "observations.pagerNext": "Next page", + "observations.pagerPage": "Page {{page}}", "observations.allForms": "All Forms", "observations.errorTitle": "Error Loading Observations", "observations.emptyFound": "No Observations Found", @@ -190,6 +201,20 @@ "help.exportFailed": "Export failed", "help.exportAttachmentsFailed": "Could not export attachments.", "help.exportObservationsFailed": "Could not export observations.", + "help.diagnostics.title": "Diagnostics", + "help.diagnostics.hint": "Local crash and sync logs only. No observation data. Download before uninstalling the app.", + "help.diagnostics.noExit": "No unexpected close recorded.", + "help.diagnostics.lastExit": "Last exit: {{reason}} ({{when}})", + "help.diagnostics.download": "Download diagnostic log", + "help.diagnostics.clear": "Clear diagnostic log", + "help.diagnostics.clearTitle": "Clear diagnostic log?", + "help.diagnostics.clearMessage": "This deletes the on-device diagnostic files. It does not change observations.", + "help.diagnostics.ok": "OK", + "help.diagnostics.saveLog": "Save diagnostic log", + "help.diagnostics.unexpectedTitle": "Unexpected close", + "help.diagnostics.unexpectedMessage": "Last time you used Formulus, it closed unexpectedly due to: {{reason}}", + "help.diagnostics.exportFailed": "Could not export the diagnostic log.", + "help.diagnostics.emptyEvents": "No diagnostic events yet.", "signature.noData": "No signature data captured. Please try again.", "signature.required": "Please provide a signature before saving.", "media.selectImageTitle": "Select image", diff --git a/formulus/src/locales/fr.json b/formulus/src/locales/fr.json index 9e72251d5..ce2b9ac3c 100644 --- a/formulus/src/locales/fr.json +++ b/formulus/src/locales/fr.json @@ -82,11 +82,22 @@ "forms.emptyMessage": "Aucun formulaire n'a encore été téléchargé. Allez à l'écran Synchroniser pour télécharger les formulaires depuis le serveur.", "forms.emptySearchTitle": "Aucun formulaire trouvé", "forms.emptySearchMessage": "Essayez d'ajuster votre recherche.", + "forms.colFormType": "Type de formulaire", + "forms.colObservationCount": "# observations", + "forms.colNewObservation": "Nouvelle observation", "observations.title": "Observations", "observations.count_one": "{{count}} observation", "observations.count_other": "{{count}} observations", "observations.loading": "Chargement des observations...", - "observations.searchPlaceholder": "Rechercher des observations...", + "observations.searchPlaceholder": "Rechercher formulaire, auteur ou ID…", + "observations.colForm": "Formulaire", + "observations.colCreated": "Créé", + "observations.colSync": "Sync", + "observations.colAuthor": "Auteur", + "observations.colId": "ID", + "observations.pagerPrev": "Page précédente", + "observations.pagerNext": "Page suivante", + "observations.pagerPage": "Page {{page}}", "observations.allForms": "Tous les formulaires", "observations.errorTitle": "Erreur de chargement des observations", "observations.emptyFound": "Aucune observation trouvée", @@ -190,6 +201,20 @@ "help.exportFailed": "Échec de l'export", "help.exportAttachmentsFailed": "Impossible d'exporter les pièces jointes.", "help.exportObservationsFailed": "Impossible d'exporter les observations.", + "help.diagnostics.title": "Diagnostic", + "help.diagnostics.hint": "Journaux locaux de plantage et de synchronisation uniquement. Aucune donnée d'observation. Téléchargez-les avant de désinstaller l'application.", + "help.diagnostics.noExit": "Aucune fermeture inattendue enregistrée.", + "help.diagnostics.lastExit": "Dernière sortie : {{reason}} ({{when}})", + "help.diagnostics.download": "Télécharger le journal de diagnostic", + "help.diagnostics.clear": "Effacer le journal de diagnostic", + "help.diagnostics.clearTitle": "Effacer le journal de diagnostic ?", + "help.diagnostics.clearMessage": "Cela supprime les fichiers de diagnostic sur l'appareil. Les observations ne sont pas modifiées.", + "help.diagnostics.ok": "OK", + "help.diagnostics.saveLog": "Enregistrer le journal", + "help.diagnostics.unexpectedTitle": "Fermeture inattendue", + "help.diagnostics.unexpectedMessage": "La dernière fois que vous avez utilisé Formulus, il s'est fermé de façon inattendue en raison de : {{reason}}", + "help.diagnostics.exportFailed": "Impossible d'exporter le journal de diagnostic.", + "help.diagnostics.emptyEvents": "Aucun événement de diagnostic pour le moment.", "signature.noData": "Aucune signature capturée. Veuillez réessayer.", "signature.required": "Veuillez fournir une signature avant d'enregistrer.", "media.selectImageTitle": "Sélectionner une image", diff --git a/formulus/src/locales/pt.json b/formulus/src/locales/pt.json index f1657c588..44e04745b 100644 --- a/formulus/src/locales/pt.json +++ b/formulus/src/locales/pt.json @@ -82,11 +82,22 @@ "forms.emptyMessage": "Ainda não foram descarregados formulários. Vá ao ecrã Sincronizar para descarregar formulários do servidor.", "forms.emptySearchTitle": "Nenhum formulário encontrado", "forms.emptySearchMessage": "Tente ajustar a pesquisa.", + "forms.colFormType": "Tipo de formulário", + "forms.colObservationCount": "# observações", + "forms.colNewObservation": "Nova observação", "observations.title": "Observações", "observations.count_one": "{{count}} observação", "observations.count_other": "{{count}} observações", "observations.loading": "A carregar observações...", - "observations.searchPlaceholder": "Pesquisar observações...", + "observations.searchPlaceholder": "Pesquisar formulário, autor ou ID…", + "observations.colForm": "Formulário", + "observations.colCreated": "Criado", + "observations.colSync": "Sincronização", + "observations.colAuthor": "Autor", + "observations.colId": "ID", + "observations.pagerPrev": "Página anterior", + "observations.pagerNext": "Página seguinte", + "observations.pagerPage": "Página {{page}}", "observations.allForms": "Todos os formulários", "observations.errorTitle": "Erro ao carregar observações", "observations.emptyFound": "Nenhuma observação encontrada", @@ -190,6 +201,20 @@ "help.exportFailed": "Exportação falhou", "help.exportAttachmentsFailed": "Não foi possível exportar anexos.", "help.exportObservationsFailed": "Não foi possível exportar observações.", + "help.diagnostics.title": "Diagnóstico", + "help.diagnostics.hint": "Apenas registos locais de falhas e sincronização. Sem dados de observação. Descarregue antes de desinstalar a aplicação.", + "help.diagnostics.noExit": "Nenhum fecho inesperado registado.", + "help.diagnostics.lastExit": "Última saída: {{reason}} ({{when}})", + "help.diagnostics.download": "Descarregar registo de diagnóstico", + "help.diagnostics.clear": "Limpar registo de diagnóstico", + "help.diagnostics.clearTitle": "Limpar o registo de diagnóstico?", + "help.diagnostics.clearMessage": "Isto apaga os ficheiros de diagnóstico no dispositivo. Não altera observações.", + "help.diagnostics.ok": "OK", + "help.diagnostics.saveLog": "Guardar registo de diagnóstico", + "help.diagnostics.unexpectedTitle": "Fecho inesperado", + "help.diagnostics.unexpectedMessage": "Da última vez que usou o Formulus, fechou inesperadamente devido a: {{reason}}", + "help.diagnostics.exportFailed": "Não foi possível exportar o registo de diagnóstico.", + "help.diagnostics.emptyEvents": "Ainda não há eventos de diagnóstico.", "signature.noData": "Nenhuma assinatura capturada. Tente novamente.", "signature.required": "Forneça uma assinatura antes de guardar.", "media.selectImageTitle": "Selecionar imagem", diff --git a/formulus/src/screens/FormsScreen.tsx b/formulus/src/screens/FormsScreen.tsx index ee9ba01ae..37d3e5acf 100644 --- a/formulus/src/screens/FormsScreen.tsx +++ b/formulus/src/screens/FormsScreen.tsx @@ -1,10 +1,8 @@ -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useCallback } from 'react'; import { View, Text, StyleSheet, - FlatList, - RefreshControl, ActivityIndicator, Alert, TouchableOpacity, @@ -12,11 +10,14 @@ import { import { SafeAreaView } from 'react-native-safe-area-context'; import Icon from '@react-native-vector-icons/material-design-icons'; import { useForms } from '../hooks/useForms'; -import { FormCard, EmptyState, Input as ODEInput } from '../components/common'; +import { + FormListTable, + EmptyState, + Input as ODEInput, +} from '../components/common'; import { openFormplayerFromNative } from '../webview/FormulusMessageHandlers'; import { useFocusEffect } from '@react-navigation/native'; import colors from '../theme/colors'; -import { FormSpec } from '../services'; import { useAppTheme } from '../contexts/AppThemeContext'; import { useScreenShellStyle } from '../hooks/useScreenShellStyle'; import { @@ -49,8 +50,7 @@ const FormsScreen: React.FC = () => { borderColor: themeColors.divider as string, }, ]; - const { forms, loading, error, refresh, getObservationCount } = useForms(); - const [refreshing, setRefreshing] = useState(false); + const { forms, loading, error, refresh, observationCounts } = useForms(); const [searchQuery, setSearchQuery] = useState(''); const [showSearch, setShowSearch] = useState(false); @@ -72,40 +72,23 @@ const FormsScreen: React.FC = () => { }, [refresh]), ); - const handleRefresh = async () => { - setRefreshing(true); - try { - await refresh(); - } finally { - setRefreshing(false); - } - }; - - const handleFormPress = async (formId: string) => { - try { - const result = await openFormplayerFromNative(formId, {}, {}); - if ( - result.status === 'form_submitted' || - result.status === 'form_updated' - ) { - await refresh(); + const handleCreate = useCallback( + async (formId: string) => { + try { + const result = await openFormplayerFromNative(formId, {}, {}); + if ( + result.status === 'form_submitted' || + result.status === 'form_updated' + ) { + await refresh(); + } + } catch (err) { + console.error('Error opening form:', err); + Alert.alert(t('common.error'), t('forms.openError')); } - } catch (err) { - console.error('Error opening form:', err); - Alert.alert(t('common.error'), t('forms.openError')); - } - }; - - const renderForm = ({ item }: { item: FormSpec }) => { - const observationCount = getObservationCount(item.id); - return ( - handleFormPress(item.id)} - /> - ); - }; + }, + [refresh, t], + ); if (loading && forms.length === 0) { return ( @@ -246,18 +229,10 @@ const FormsScreen: React.FC = () => { } /> ) : ( - item.id} - contentContainerStyle={styles.listContent} - refreshControl={ - - } + )} @@ -269,9 +244,6 @@ const styles = StyleSheet.create({ container: { flex: 1, }, - listTransparent: { - backgroundColor: colors.neutral.transparent, - }, header: { flexDirection: 'row', justifyContent: 'space-between', @@ -329,10 +301,6 @@ const styles = StyleSheet.create({ marginLeft: odeSpacing.xs, }, clearIcon: {}, - listContent: { - // Same gap as between cards: paddingTop + first card marginTop = 16. - paddingVertical: odeSpacing.xs, - }, loadingContainer: { flex: 1, justifyContent: 'center', diff --git a/formulus/src/screens/HelpScreen.tsx b/formulus/src/screens/HelpScreen.tsx index bddbf84af..2388d3dd8 100644 --- a/formulus/src/screens/HelpScreen.tsx +++ b/formulus/src/screens/HelpScreen.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useCallback, useState } from 'react'; import { View, Text, @@ -25,15 +25,31 @@ import logo from '../../assets/images/logo.png'; import { attachmentExportService } from '../services/AttachmentExportService'; import { observationExportService } from '../services/ObservationExportService'; import { useTranslation } from 'react-i18next'; +import { useFocusEffect } from '@react-navigation/native'; +import { useConfirmModal } from '../contexts/ConfirmModalContext'; +import { + clearDiagnosticFiles, + exportDiagnosticsZip, + logger, + readLastExit, + readRecentEvents, +} from '../diagnostics'; +import { formatExitReason } from '../diagnostics/classifyExit'; +import type { DiagnosticEvent, ProcessExitRecord } from '../diagnostics'; const FORUM_URL = 'https://forum.opendataensemble.org'; const EMAIL_URL = 'mailto:hello@opendataensemble.org'; const GH_URL = 'https://github.com/OpenDataEnsemble'; +const DIAGNOSTIC_EVENT_PREVIEW_LIMIT = 50; const HelpScreen: React.FC = () => { const { t } = useTranslation(); const [exportingAttachments, setExportingAttachments] = useState(false); const [exportingObservations, setExportingObservations] = useState(false); + const [exportingDiagnostics, setExportingDiagnostics] = useState(false); + const [events, setEvents] = useState([]); + const [lastExit, setLastExit] = useState(null); + const { showConfirm } = useConfirmModal(); const { themeColors, resolvedMode } = useAppTheme(); const shellStyle = useScreenShellStyle(); const isDark = resolvedMode === 'dark'; @@ -65,6 +81,52 @@ const HelpScreen: React.FC = () => { } }; + const refreshDiagnostics = useCallback(async () => { + const [recent, exit] = await Promise.all([ + readRecentEvents(DIAGNOSTIC_EVENT_PREVIEW_LIMIT), + readLastExit(), + ]); + setEvents(recent); + setLastExit(exit); + }, []); + + useFocusEffect( + useCallback(() => { + void logger.breadcrumb('screen', 'help', { screen: 'Help' }); + void refreshDiagnostics(); + }, [refreshDiagnostics]), + ); + + const onExportDiagnostics = async () => { + setExportingDiagnostics(true); + try { + await exportDiagnosticsZip(); + } catch (e) { + const message = + e instanceof Error ? e.message : t('help.diagnostics.exportFailed'); + Alert.alert(t('help.exportFailed'), message); + } finally { + setExportingDiagnostics(false); + } + }; + + const onClearDiagnostics = () => { + showConfirm({ + title: t('help.diagnostics.clearTitle'), + message: t('help.diagnostics.clearMessage'), + buttons: [ + { text: t('help.diagnostics.ok'), onPress: () => undefined }, + { + text: t('help.diagnostics.clear'), + variant: 'danger', + onPress: () => { + void clearDiagnosticFiles().then(() => refreshDiagnostics()); + }, + }, + ], + }); + }; + const onExportObservations = async () => { setExportingObservations(true); try { @@ -294,6 +356,94 @@ const HelpScreen: React.FC = () => { )} + + + + {t('help.diagnostics.title')} + + + {t('help.diagnostics.hint')} + + + {lastExit + ? t('help.diagnostics.lastExit', { + reason: formatExitReason(lastExit), + when: new Date(lastExit.timestamp).toLocaleString(), + }) + : t('help.diagnostics.noExit')} + + [ + styles.exportButton, + { + marginTop: odeSpacing.sm, + opacity: exportingDiagnostics ? 0.55 : pressed ? 0.85 : 1, + backgroundColor: themeColors.surface as string, + borderColor: themeColors.divider as string, + }, + ]}> + {exportingDiagnostics ? ( + + ) : ( + + {t('help.diagnostics.download')} + + )} + + [ + styles.exportButton, + { + marginTop: odeSpacing.sm, + opacity: pressed ? 0.85 : 1, + backgroundColor: themeColors.surface as string, + borderColor: themeColors.divider as string, + }, + ]}> + + {t('help.diagnostics.clear')} + + + + {events.length === 0 + ? t('help.diagnostics.emptyEvents') + : events + .map( + event => + `${event.ts} ${event.level ?? event.kind} ${event.tag ?? ''} ${event.message}`, + ) + .join('\n')} + + @@ -388,6 +538,12 @@ const styles = StyleSheet.create({ fontSize: odeTypography.bodySm, fontWeight: '600', }, + eventLog: { + fontFamily: 'monospace', + fontSize: 11, + lineHeight: 16, + marginTop: odeSpacing.sm, + }, }); export default HelpScreen; diff --git a/formulus/src/screens/ObservationDetailScreen.tsx b/formulus/src/screens/ObservationDetailScreen.tsx index 58f6e6798..82fc9f07d 100644 --- a/formulus/src/screens/ObservationDetailScreen.tsx +++ b/formulus/src/screens/ObservationDetailScreen.tsx @@ -61,21 +61,7 @@ const ObservationDetailScreen: React.FC = ({ try { setLoading(true); const formService = await FormService.getInstance(); - - const formSpecs = formService.getFormSpecs(); - let foundObservation: Observation | null = null; - - for (const formSpec of formSpecs) { - const observations = await formService.getObservationsByFormType( - formSpec.id, - ); - const obs = observations.find(o => o.observationId === observationId); - if (obs) { - foundObservation = obs; - setFormName(formSpec.name); - break; - } - } + const foundObservation = await formService.getObservation(observationId); if (!foundObservation) { Alert.alert(t('common.error'), t('observations.notFound')); @@ -83,6 +69,8 @@ const ObservationDetailScreen: React.FC = ({ return; } + const spec = formService.getFormSpecById(foundObservation.formType); + setFormName(spec?.name || foundObservation.formType); setObservation(foundObservation); } catch (error) { console.error('Error loading observation:', error); diff --git a/formulus/src/screens/ObservationsScreen.tsx b/formulus/src/screens/ObservationsScreen.tsx index 9ed426335..661232c87 100644 --- a/formulus/src/screens/ObservationsScreen.tsx +++ b/formulus/src/screens/ObservationsScreen.tsx @@ -1,12 +1,9 @@ -import React, { useState, useMemo } from 'react'; +import React, { useCallback, useEffect, useRef, useState } from 'react'; import { View, Text, StyleSheet, - FlatList, - RefreshControl, ActivityIndicator, - Alert, TouchableOpacity, } from 'react-native'; import { Input as ODEInput } from '../components/common'; @@ -14,22 +11,18 @@ import { SafeAreaView } from 'react-native-safe-area-context'; import Icon from '@react-native-vector-icons/material-design-icons'; import { useObservations } from '../hooks/useObservations'; import { - ObservationCard, EmptyState, FormTypeSelector, SyncStatusButtons, - SyncStatus, + ObservationListTable, + ObservationPager, } from '../components/common'; -import { isObservationFullySynced } from '../utils/observationSyncStatus'; -import { openFormplayerFromNative } from '../webview/FormulusMessageHandlers'; import { FormService } from '../services/FormService'; import { useFocusEffect, useNavigation } from '@react-navigation/native'; import { StackNavigationProp } from '@react-navigation/stack'; import { MainAppStackParamList } from '../types/NavigationTypes'; -import { Observation } from '../database/models/Observation'; import colors from '../theme/colors'; import { useAppTheme } from '../contexts/AppThemeContext'; -import { useConfirmModal } from '../contexts/ConfirmModalContext'; import { useScreenShellStyle } from '../hooks/useScreenShellStyle'; import { odeSpacing, @@ -39,6 +32,8 @@ import { odeScreenHeaderHeight, } from '../theme/odeDesign'; import { useTranslation } from 'react-i18next'; +import { logger } from '../diagnostics/logger'; +import type { ObservationListRow } from '../database/observationListQuery'; type ObservationsScreenNavigationProp = StackNavigationProp< MainAppStackParamList, @@ -56,9 +51,6 @@ const ObservationsScreen: React.FC = () => { const clearIconColor = isDark ? (colors.neutral[300] as string) : (colors.neutral[600] as string); - const _headerBg = isDark - ? (colors.neutral[900] as string) - : (colors.neutral[50] as string); const filtersContainerStyle = [ styles.filtersContainer, { @@ -75,27 +67,39 @@ const ObservationsScreen: React.FC = () => { }, ]; const navigation = useNavigation(); - const observationsHook = useObservations(); const { - filteredAndSorted, + rows, + total, + totalPages, + page, + setPage, loading, error, refresh, searchQuery, setSearchQuery, - } = observationsHook; - const [refreshing, setRefreshing] = useState(false); + selectedFormType, + setSelectedFormType, + syncStatus, + setSyncStatus, + } = useObservations(); const [formNames, setFormNames] = useState>({}); const [formTypes, setFormTypes] = useState<{ id: string; name: string }[]>( [], ); - const [selectedFormType, setSelectedFormType] = useState(null); - const [syncStatus, setSyncStatus] = useState('all'); const [showSearch, setShowSearch] = useState(false); - const { showConfirm } = useConfirmModal(); + const refreshRef = useRef(refresh); + const skipFocusRefresh = useRef(true); + + useEffect(() => { + refreshRef.current = refresh; + }, [refresh]); useFocusEffect( React.useCallback(() => { + void logger.breadcrumb('screen', 'observations', { + screen: 'Observations', + }); const loadFormData = async () => { try { const formService = await FormService.getInstance(); @@ -112,142 +116,27 @@ const ObservationsScreen: React.FC = () => { console.error('Failed to load form data:', err); } }; - loadFormData(); - refresh(); - }, [refresh]), + void loadFormData(); + // First focus: useObservations already loads. Later focuses (back from + // detail) refresh without tying this effect to the loadPage identity. + if (skipFocusRefresh.current) { + skipFocusRefresh.current = false; + return; + } + void refreshRef.current(); + }, []), ); - const finalFiltered = useMemo(() => { - let filtered = filteredAndSorted; - - if (selectedFormType) { - filtered = filtered.filter(obs => obs.formType === selectedFormType); - } + const showSubtitle = total > 0; - if (syncStatus !== 'all') { - filtered = filtered.filter(obs => { - const synced = isObservationFullySynced(obs); - return syncStatus === 'synced' ? synced : !synced; + const handleRowPress = useCallback( + (row: ObservationListRow) => { + navigation.navigate('ObservationDetail', { + observationId: row.observationId, }); - } - - return filtered; - }, [filteredAndSorted, selectedFormType, syncStatus]); - - const showSubtitle = finalFiltered.length > 0; - - const handleRefresh = async () => { - setRefreshing(true); - try { - await refresh(); - } finally { - setRefreshing(false); - } - }; - - const handleObservationPress = (observation: Observation) => { - navigation.navigate('ObservationDetail', { - observationId: observation.observationId, - }); - }; - - const handleEditObservation = async (observation: Observation) => { - try { - const result = await openFormplayerFromNative( - observation.formType, - {}, - typeof observation.data === 'string' - ? JSON.parse(observation.data) - : observation.data, - observation.observationId, - ); - if ( - result.status === 'form_submitted' || - result.status === 'form_updated' - ) { - await refresh(); - } - } catch (err) { - console.error('Error editing observation:', err); - Alert.alert(t('common.error'), t('observations.editError')); - } - }; - - const handleDeleteObservation = async (observation: Observation) => { - showConfirm({ - title: t('observations.deleteTitle'), - message: t('observations.deleteMessage'), - buttons: [ - { text: t('common.cancel'), onPress: () => {}, variant: 'tertiary' }, - { - text: t('observations.delete'), - variant: 'danger', - onPress: async () => { - try { - const formService = await FormService.getInstance(); - await formService.deleteObservation(observation.observationId); - await refresh(); - } catch (err) { - console.error('Error deleting observation:', err); - Alert.alert(t('common.error'), t('observations.deleteError')); - } - }, - }, - ], - }); - }; - - const renderObservation = ({ item }: { item: Observation }) => { - return ( - handleObservationPress(item)} - onEdit={() => handleEditObservation(item)} - onDelete={() => handleDeleteObservation(item)} - /> - ); - }; - - if (loading && filteredAndSorted.length === 0) { - return ( - - - - - - {t('observations.loading')} - - - - - ); - } - - if (error && filteredAndSorted.length === 0) { - return ( - - - - - - ); - } + }, + [navigation], + ); return ( @@ -280,7 +169,7 @@ const ObservationsScreen: React.FC = () => { {showSubtitle && ( - {t('observations.count', { count: finalFiltered.length })} + {t('observations.count', { count: total })} )} @@ -346,7 +235,23 @@ const ObservationsScreen: React.FC = () => { - {finalFiltered.length === 0 ? ( + {loading ? ( + + + + {t('observations.loading')} + + + ) : error && rows.length === 0 ? ( + + ) : rows.length === 0 ? ( { } /> ) : ( - item.observationId} - contentContainerStyle={styles.listContent} - refreshControl={ - - } - /> + + + + )} @@ -384,9 +288,6 @@ const styles = StyleSheet.create({ container: { flex: 1, }, - listTransparent: { - backgroundColor: colors.neutral.transparent, - }, header: { flexDirection: 'row', justifyContent: 'space-between', @@ -455,9 +356,9 @@ const styles = StyleSheet.create({ alignSelf: 'stretch', minHeight: 44, }, - listContent: { - // Same gap as between cards: paddingTop + first card marginTop = 16. - paddingVertical: odeSpacing.xs, + tableSection: { + flex: 1, + minHeight: 0, }, loadingContainer: { flex: 1, diff --git a/formulus/src/screens/SyncScreen.tsx b/formulus/src/screens/SyncScreen.tsx index a65a78e3d..3f09030bf 100644 --- a/formulus/src/screens/SyncScreen.tsx +++ b/formulus/src/screens/SyncScreen.tsx @@ -12,6 +12,7 @@ import { } from 'react-native'; import { SafeAreaView } from 'react-native-safe-area-context'; import { useFocusEffect } from '@react-navigation/native'; +import { logger } from '../diagnostics/logger'; import Icon from '@react-native-vector-icons/material-design-icons'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { formatRelativeTime } from '../utils/dateUtils'; @@ -23,6 +24,14 @@ import { getUserInfo, getUserFacingSyncErrorMessage, } from '../api/synkronus/Auth'; + +function isSyncCancelledError(error: unknown): boolean { + return ( + error instanceof Error && + (error.message === 'Sync cancelled' || + error.message === 'Sync cancelled by user') + ); +} import { isRepositoryResetRequiredError, type RepositoryResetRequiredError, @@ -281,7 +290,9 @@ const SyncScreen = () => { ); } else { syncError = getUserFacingSyncErrorMessage(error); - Alert.alert(t('sync.failed'), syncError); + if (!isSyncCancelledError(error)) { + Alert.alert(t('sync.failed'), syncError); + } } } finally { finishSync(syncError); @@ -357,7 +368,9 @@ const SyncScreen = () => { } else { const errorMessage = getUserFacingSyncErrorMessage(error); finishSync(errorMessage); - Alert.alert(t('sync.operationFailed'), errorMessage); + if (!isSyncCancelledError(error)) { + Alert.alert(t('sync.operationFailed'), errorMessage); + } } } finally { setActiveOperation(null); @@ -495,6 +508,7 @@ const SyncScreen = () => { // Refresh pending count and bundle status whenever the Sync screen gains focus useFocusEffect( useCallback(() => { + void logger.breadcrumb('screen', 'sync', { screen: 'Sync' }); updatePendingUploads(); updatePendingObservations(); refreshUserRole(); diff --git a/formulus/src/services/AppConfigService.ts b/formulus/src/services/AppConfigService.ts index cbe3f0a91..1269d27ae 100644 --- a/formulus/src/services/AppConfigService.ts +++ b/formulus/src/services/AppConfigService.ts @@ -83,9 +83,6 @@ class AppConfigService { try { const exists = await RNFS.exists(APP_CONFIG_PATH); if (!exists) { - console.log( - '[AppConfigService] No app.config.json found — using ODE defaults.', - ); this.config = null; this.loaded = true; return; @@ -104,9 +101,6 @@ class AppConfigService { } this.config = parsed; - console.log( - `[AppConfigService] Loaded config for "${parsed.name}" v${parsed.version}`, - ); } catch (err) { // Deliberately not latching `loaded` here. A read that throws mid-bundle // extraction is transient, and latching would pin the config to null — diff --git a/formulus/src/services/AppVersionService.ts b/formulus/src/services/AppVersionService.ts index 12bafcf1e..cbd90c349 100644 --- a/formulus/src/services/AppVersionService.ts +++ b/formulus/src/services/AppVersionService.ts @@ -31,7 +31,6 @@ export class AppVersionService { try { this.cachedVersion = DeviceInfo.getVersion(); - console.log('AppVersionService: App version:', this.cachedVersion); return this.cachedVersion; } catch (error) { console.error('AppVersionService: Error getting app version:', error); @@ -54,7 +53,6 @@ export class AppVersionService { try { this.cachedBuildNumber = DeviceInfo.getBuildNumber(); - console.log('AppVersionService: Build number:', this.cachedBuildNumber); return this.cachedBuildNumber; } catch (error) { console.error('AppVersionService: Error getting build number:', error); diff --git a/formulus/src/services/ClientIdService.ts b/formulus/src/services/ClientIdService.ts index 0de44d994..b1ccf07f9 100644 --- a/formulus/src/services/ClientIdService.ts +++ b/formulus/src/services/ClientIdService.ts @@ -41,7 +41,6 @@ export class ClientIdService { const deviceId = await DeviceInfo.getUniqueId(); this.cachedClientId = `formulus-${deviceId}`; - console.log('ClientIdService: Generated client ID:', this.cachedClientId); return this.cachedClientId; } catch (error) { console.error('ClientIdService: Error getting device ID:', error); diff --git a/formulus/src/services/ExtensionService.ts b/formulus/src/services/ExtensionService.ts index ba340c5ae..a65f7aa75 100644 --- a/formulus/src/services/ExtensionService.ts +++ b/formulus/src/services/ExtensionService.ts @@ -93,21 +93,10 @@ export class ExtensionService { renderers: {}, }; - console.log( - `[ExtensionService] Loading extensions for app: ${customAppPath}, form: ${formName || 'none'}`, - ); - // Load app-level extensions const appExtPath = `${customAppPath}/forms/ext.json`; - console.log( - `[ExtensionService] Loading app-level ext.json from: ${appExtPath}`, - ); const appLevelExt = await this.loadExtensionFile(appExtPath); if (appLevelExt) { - console.log(`[ExtensionService] App-level extensions loaded:`, { - functionCount: Object.keys(appLevelExt.functions || {}).length, - functionNames: Object.keys(appLevelExt.functions || {}), - }); this.mergeExtension(result, appLevelExt); } else { console.warn( @@ -118,31 +107,12 @@ export class ExtensionService { // Load form-level extensions (higher precedence) if (formName) { const formExtPath = `${customAppPath}/forms/${formName}/ext.json`; - console.log( - `[ExtensionService] Loading form-level ext.json from: ${formExtPath}`, - ); const formLevelExt = await this.loadExtensionFile(formExtPath); if (formLevelExt) { - console.log(`[ExtensionService] Form-level extensions loaded:`, { - functionCount: Object.keys(formLevelExt.functions || {}).length, - functionNames: Object.keys(formLevelExt.functions || {}), - }); this.mergeExtension(result, formLevelExt); } } - console.log(`[ExtensionService] Final merged extensions:`, { - definitionKeys: Object.keys(result.definitions), - functionKeys: Object.keys(result.functions), - functionDetails: Object.entries(result.functions).map(([k, v]) => ({ - key: k, - name: v.name, - module: v.module, - export: v.export, - })), - rendererKeys: Object.keys(result.renderers), - }); - return result; } @@ -155,23 +125,12 @@ export class ExtensionService { try { const exists = await RNFS.exists(filePath); if (!exists) { - console.log(`[ExtensionService] File does not exist: ${filePath}`); return null; } const content = await RNFS.readFile(filePath, 'utf8'); const rawExtension = JSON.parse(content) as Record; - console.log(`[ExtensionService] Loaded ext.json from ${filePath}:`, { - hasSchemas: !!rawExtension.schemas, - hasDefinitions: !!rawExtension.definitions, - hasFunctions: !!rawExtension.functions, - functionKeys: rawExtension.functions - ? Object.keys(rawExtension.functions) - : [], - hasRenderers: !!rawExtension.renderers, - }); - // Normalize the extension format to match ExtensionDefinition interface const extension: ExtensionDefinition = { // Handle both "schemas.definitions" and direct "definitions" @@ -220,20 +179,6 @@ export class ExtensionService { : undefined, }; - console.log(`[ExtensionService] Normalized extension:`, { - definitionKeys: Object.keys(extension.definitions || {}), - functionKeys: Object.keys(extension.functions || {}), - functionDetails: extension.functions - ? Object.entries(extension.functions).map(([k, v]) => ({ - key: k, - name: v.name, - module: v.module, - export: v.export, - })) - : [], - rendererKeys: Object.keys(extension.renderers || {}), - }); - // Validate structure this.validateExtension(extension, filePath); diff --git a/formulus/src/services/FormService.ts b/formulus/src/services/FormService.ts index e0ab0537e..10726ad19 100644 --- a/formulus/src/services/FormService.ts +++ b/formulus/src/services/FormService.ts @@ -10,6 +10,11 @@ import { SHARED_CHOICE_SCHEMA_ID, type SharedChoiceSchemaDoc, } from '../utils/sharedChoiceSchema'; +import { logger } from '../diagnostics/logger'; +import type { + ObservationListPage, + ObservationListQuery, +} from '../database/observationListQuery'; /** * Interface representing a form type @@ -37,20 +42,12 @@ export class FormService { SharedChoiceSchemaDoc | null >(); - private constructor() { - console.log( - 'FormService: Instance created - use await getInstance() to access singleton instance', - ); - } + private constructor() {} private async _initialize(): Promise { - console.log('FormService: Starting initialization...'); try { const specs = await this.getFormspecsFromStorage(); this.formSpecs = specs; - console.log( - `FormService: ${specs.length} form specs loaded successfully`, - ); } catch (error) { console.error( 'Failed to load default form types during FormService construction:', @@ -86,9 +83,6 @@ export class FormService { doc.$id = SHARED_CHOICE_SCHEMA_ID; } this.sharedChoiceSchemaByDir.set(formsDir, doc); - console.log( - `FormService: loaded shared choice defs (${Object.keys(doc.$defs).length} lists) from ${filePath}`, - ); return doc; } catch (error) { console.warn( @@ -105,10 +99,8 @@ export class FormService { formsParentDir: string, ): Promise { if (!formDir.isDirectory()) { - console.log('Skipping non-directory:', formDir.name); return null; } - console.log('Loading form spec:', formDir.path); let schema: unknown; try { const filePath = formDir.path + '/schema.json'; @@ -209,12 +201,6 @@ export class FormService { await RNFS.mkdir(rootFormsDir); } - console.log( - `🟢🟢🟢 [FormService] Successfully loaded ${allFormSpecs.length} form specs`, - ); - console.log( - `🟢 [FormService] Form IDs: ${allFormSpecs.map(f => f.id).join(', ')}`, - ); return allFormSpecs; } catch (error) { console.error( @@ -235,7 +221,6 @@ export class FormService { } if (!FormService.initializationPromise) { - console.log('FormService: Starting initialization...'); FormService.initializationPromise = FormService.instance ._initialize() .catch(error => { @@ -272,13 +257,9 @@ export class FormService { * This should be called after app bundle updates */ public async invalidateCache(): Promise { - console.log('FormService: Invalidating cache and reloading form specs...'); try { const specs = await this.getFormspecsFromStorage(); this.formSpecs = specs; - console.log( - `FormService: Cache invalidated, ${specs.length} form specs reloaded`, - ); // Notify all subscribers that cache has been invalidated this.cacheInvalidationCallbacks.forEach(callback => { @@ -307,15 +288,8 @@ export class FormService { */ public getFormSpecById(id: string): FormSpec | undefined { const found = this.formSpecs.find(formSpec => formSpec.id === id); - if (found) { - console.log( - 'FormService: Found form spec for', - id, - 'sending schema and uiSchema', - ); - } else { + if (!found) { console.warn('FormService: Form spec not found for', id); - console.debug('FormService: Form specs:', this.formSpecs); } return found; } @@ -332,6 +306,25 @@ export class FormService { return await localRepo.getObservationsByFormType(formTypeId); } + public async getActiveObservations(): Promise { + const localRepo = databaseService.getLocalRepo(); + return localRepo.getActiveObservations(); + } + + public async getObservation( + observationId: string, + ): Promise { + const localRepo = databaseService.getLocalRepo(); + return localRepo.getObservation(observationId); + } + + public async listObservationsPage( + query: ObservationListQuery, + ): Promise { + const localRepo = databaseService.getLocalRepo(); + return localRepo.listObservationsPage(query); + } + /** * Get observations with optional WHERE clause filtering (for dynamic choice lists). * Filters by data.field = 'value' conditions. age_from_dob() is handled in formplayer. @@ -392,14 +385,13 @@ export class FormService { formVersion: '1.0', // Default version }; - console.debug('Observation input: ', input); if (input.formType === undefined) { throw new Error('Form type is required to save observation'); } if (input.data === undefined) { throw new Error('Data is required to save observation'); } - console.log('Saving observation of type: ' + input.formType); + logger.info('forms', 'saving observation', { formType }); const localRepo = databaseService.getLocalRepo(); return await localRepo.saveObservation(input); } @@ -419,14 +411,12 @@ export class FormService { data, }; - console.debug('Observation update input: ', input); if (input.observationId === undefined) { throw new Error('Observation ID is required to update observation'); } if (input.data === undefined) { throw new Error('Data is required to update observation'); } - console.log('Updating observation with ID: ' + input.observationId); const localRepo = databaseService.getLocalRepo(); await localRepo.updateObservation(input); return input.observationId; @@ -438,8 +428,6 @@ export class FormService { */ public async debugDatabase(): Promise { try { - console.log('=== DATABASE DEBUG INFO ==='); - // Get the local repository const localRepo = databaseService.getLocalRepo(); if (!localRepo) { @@ -448,23 +436,17 @@ export class FormService { } // Log some test observations - console.log('Creating test observations...'); // Create a test observation with person form type - const testId1 = await localRepo.saveObservation({ + await localRepo.saveObservation({ formType: 'person', data: { test: 'data1' }, }); - console.log('Created test observation 1:', testId1); - // Create another test observation with a different form type - const testId2 = await localRepo.saveObservation({ + await localRepo.saveObservation({ formType: 'test_form', data: { test: 'data2' }, }); - console.log('Created test observation 2:', testId2); - - console.log('=== END DEBUG INFO ==='); } catch (error) { console.error('Error debugging database:', error); } diff --git a/formulus/src/services/GeolocationService.ts b/formulus/src/services/GeolocationService.ts index 2196e0114..24b46e358 100644 --- a/formulus/src/services/GeolocationService.ts +++ b/formulus/src/services/GeolocationService.ts @@ -229,7 +229,6 @@ export class GeolocationService { Geolocation.getCurrentPosition( position => { const location = this.convertToObservationGeolocation(position); - console.debug('Got location for observation:', location); resolve(location); }, error => { diff --git a/formulus/src/services/NotificationService.ts b/formulus/src/services/NotificationService.ts index 8aba6004d..e5d4c1593 100644 --- a/formulus/src/services/NotificationService.ts +++ b/formulus/src/services/NotificationService.ts @@ -13,6 +13,10 @@ import { shouldShowSyncProgressCurrentItem, } from '../sync/syncProgressUi'; import { i18n } from '../i18n/instance'; +import { logger } from '../diagnostics/logger'; + +/** White-on-transparent status-bar glyph — not the adaptive launcher icon. */ +const SYNC_SMALL_ICON = 'ic_stat_formulus'; class NotificationService { private syncNotificationId = 'sync_progress'; @@ -20,6 +24,13 @@ class NotificationService { private isConfigured = false; private foregroundServiceRunning = false; + private androidDefaults() { + return { + channelId: this.channelId, + smallIcon: SYNC_SMALL_ICON, + }; + } + async configure() { if (this.isConfigured) return; await notifee.requestPermission(); @@ -67,7 +78,7 @@ class NotificationService { title, body, android: { - channelId: this.channelId, + ...this.androidDefaults(), ongoing: true, progress: { max: 100, @@ -85,12 +96,13 @@ class NotificationService { if (Platform.OS !== 'android' || this.foregroundServiceRunning) return; await this.configure(); + await logger.breadcrumb('fgs', 'start'); await notifee.displayNotification({ id: this.syncNotificationId, title: 'Syncing…', body: 'Starting…', android: { - channelId: this.channelId, + ...this.androidDefaults(), asForegroundService: true, foregroundServiceTypes: [ AndroidForegroundServiceType.FOREGROUND_SERVICE_TYPE_DATA_SYNC, @@ -105,6 +117,7 @@ class NotificationService { async stopForegroundService() { if (Platform.OS !== 'android' || !this.foregroundServiceRunning) return; this.foregroundServiceRunning = false; + await logger.breadcrumb('fgs', 'stop'); try { await notifee.stopForegroundService(); } catch (e) { @@ -151,7 +164,7 @@ class NotificationService { title: `Sync completed @ ${timeString}`, body: 'All data synchronized successfully', android: { - channelId: this.channelId, + ...this.androidDefaults(), autoCancel: true, ongoing: false, pressAction: { id: 'default' }, @@ -163,7 +176,7 @@ class NotificationService { title: 'Sync failed', body: error || 'An error occurred during synchronization', android: { - channelId: this.channelId, + ...this.androidDefaults(), autoCancel: true, ongoing: false, pressAction: { id: 'default' }, @@ -179,7 +192,7 @@ class NotificationService { title: 'Sync canceled', body: 'Synchronization was canceled', android: { - channelId: this.channelId, + ...this.androidDefaults(), autoCancel: true, ongoing: false, pressAction: { id: 'default' }, diff --git a/formulus/src/services/ObservationIndexService.ts b/formulus/src/services/ObservationIndexService.ts index a52ed56c8..1bdb0535d 100644 --- a/formulus/src/services/ObservationIndexService.ts +++ b/formulus/src/services/ObservationIndexService.ts @@ -20,6 +20,7 @@ import { database } from '../database/database'; import { ObservationModel } from '../database/models/ObservationModel'; import AppConfigService from './AppConfigService'; import type { ObservationIndexDef } from '../types/AppConfig'; +import { logger } from '../diagnostics/logger'; type SqlArg = string | number | boolean | null; type SqlStatement = [string, SqlArg[]]; @@ -105,9 +106,9 @@ export function computeDefsSignature(defs: ObservationIndexDef[]): string { /** * Yield to the event loop so React Native can paint. * - * A rebuild walks every observation and parses its JSON once per definition. - * Left as one uninterrupted loop that work blocks the JS thread, which freezes - * the UI — including whatever spinner is meant to show the rebuild running. + * Map-join still parses JSON on the JS thread. Left as one uninterrupted loop + * that work blocks the UI — including whatever spinner is meant to show the + * rebuild running. */ function yieldToUi(): Promise { return new Promise(resolve => setTimeout(resolve, 0)); @@ -117,15 +118,137 @@ function yieldToUi(): Promise { * Observations processed per index write. * * Used by a full rebuild and by `incrementalReindexMany` (sync pull). Each - * observation produces a DELETE plus one INSERT per matching definition, and - * none of that is released until `unsafeExecute` returns. Two hundred rows - * keeps the statement list in the low thousands even with a generous - * `observationIndexes` config — small enough for a Blackview-class tablet, - * large enough that a first-time pull of a few thousand observations is - * tens of writes rather than one giant flush. + * batch is one `DELETE … IN (…)` plus a few multi-row INSERTs (map-join), + * flushed in a single `unsafeExecute`. Two hundred rows keeps a Blackview- + * class tablet responsive between yields without turning a first-time pull + * into one giant write. */ export const INDEX_WRITE_BATCH_SIZE = 200; +/** SQLite default max is 999 binds. Six columns per index row → 120 is safe. */ +const INDEX_INSERT_ROW_CHUNK = 120; + +/** Leave headroom under the 999-bind limit (`ids` + generation). */ +const INDEX_DELETE_ID_CHUNK = 400; + +export type PreparedIndexRow = { + id: string; + observationId: string; + indexKey: string; + generation: number; + valueText: string | null; + valueNum: number | null; +}; + +function parseObservationData( + dataJson: string | unknown, +): Record | null { + if (dataJson && typeof dataJson === 'object' && !Array.isArray(dataJson)) { + return dataJson as Record; + } + if (typeof dataJson !== 'string') { + return null; + } + try { + const parsed = JSON.parse(dataJson) as unknown; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record; + } + return null; + } catch { + return null; + } +} + +/** + * Parse `data` once and emit every EAV row for this observation. + * + * Desktop's rebuild does the same map step (there, in parallel with rayon). + * Hermes is single-threaded, so the win here is avoiding one JSON.parse per + * index definition — that was the dominant cost on a 500-row pull page. + * Pull already has parsed objects; pass those through to skip parse entirely. + */ +export function extractIndexRows( + observationId: string, + formType: string, + dataJson: string | unknown, + defs: ObservationIndexDef[], + generation: number, +): { rows: PreparedIndexRow[]; nonScalarKeys: string[] } { + const data = parseObservationData(dataJson); + if (!data) { + return { rows: [], nonScalarKeys: [] }; + } + const rows: PreparedIndexRow[] = []; + const nonScalarKeys: string[] = []; + for (const def of defs) { + if (!formTypeMatches(formType, def.formTypes)) { + continue; + } + const val = data[jsonPathToKey(def.path)]; + if (val == null) { + continue; + } + const columns = scalarToColumns(val, def.valueType); + if (!columns) { + nonScalarKeys.push(def.key); + continue; + } + rows.push({ + id: `${observationId}:${def.key}:${generation}`, + observationId, + indexKey: def.key, + generation, + valueText: columns.valueText, + valueNum: columns.valueNum, + }); + } + return { rows, nonScalarKeys }; +} + +export function deleteIndexSqls( + observationIds: string[], + generation: number, +): SqlStatement[] { + if (observationIds.length === 0) { + return []; + } + const sqls: SqlStatement[] = []; + for (let i = 0; i < observationIds.length; i += INDEX_DELETE_ID_CHUNK) { + const part = observationIds.slice(i, i + INDEX_DELETE_ID_CHUNK); + const placeholders = part.map(() => '?').join(','); + sqls.push([ + `DELETE FROM observation_index WHERE observation_id IN (${placeholders}) AND index_generation = ?`, + [...part, generation], + ]); + } + return sqls; +} + +export function insertIndexSqls(rows: PreparedIndexRow[]): SqlStatement[] { + const sqls: SqlStatement[] = []; + for (let i = 0; i < rows.length; i += INDEX_INSERT_ROW_CHUNK) { + const part = rows.slice(i, i + INDEX_INSERT_ROW_CHUNK); + const values = part.map(() => '(?, ?, ?, ?, ?, ?)').join(', '); + const args: SqlArg[] = []; + for (const row of part) { + args.push( + row.id, + row.observationId, + row.indexKey, + row.generation, + row.valueText, + row.valueNum, + ); + } + sqls.push([ + `INSERT INTO observation_index (id, observation_id, index_key, index_generation, value_text, value_num) VALUES ${values}`, + args, + ]); + } + return sqls; +} + export interface IndexRebuildProgress { /** Observations processed so far. */ current: number; @@ -133,15 +256,6 @@ export interface IndexRebuildProgress { total: number; } -function extractScalar(dataJson: string, path: string): unknown { - try { - const data = JSON.parse(dataJson) as Record; - return data[jsonPathToKey(path)]; - } catch { - return undefined; - } -} - export class ObservationIndexService { private static instance: ObservationIndexService; private readonly db: Database; @@ -246,7 +360,7 @@ export class ObservationIndexService { private warnOnce(token: string, message: string): void { if (this.warnedOnce.has(token)) return; this.warnedOnce.add(token); - console.warn(message); + logger.warn('index', message); } getInitialRebuildFinished(): boolean { @@ -284,12 +398,17 @@ export class ObservationIndexService { if (this.initialRebuildPromise) return this.initialRebuildPromise; this.initialRebuildPromise = (async () => { try { + logger.info('index', 'ensureInitialRebuild start', { + phase: 'ensure', + }); await AppConfigService.getInstance() .loadConfig() .catch(err => { - console.warn( - '[ObservationIndexService] loadConfig before initial rebuild failed:', - err, + logger.warn( + 'index', + err instanceof Error + ? err.message + : 'loadConfig before initial rebuild failed', ); }); @@ -318,26 +437,52 @@ export class ObservationIndexService { Boolean(status.lastRebuildAt) && signatureMatches; + logger.info( + 'index', + `ensureInitialRebuild obs=${observationCount} indexRows=${indexCount} stamped=${Boolean(status.lastRebuildAt)} sigMatch=${signatureMatches}`, + { phase: 'ensure', counts: observationCount }, + ); + if (skipBecausePopulated) { + logger.info('index', 'ensureInitialRebuild skip: populated', { + phase: 'skip', + counts: indexCount, + success: true, + }); this.initialRebuildFinished = true; return; } if (skipBecauseEmptyInstall) { + logger.info('index', 'ensureInitialRebuild skip: empty install', { + phase: 'skip', + counts: 0, + success: true, + }); this.initialRebuildFinished = true; return; } if (!signatureMatches && Boolean(status.lastRebuildAt)) { - console.log( - '[ObservationIndexService] index definitions changed or a previous rebuild did not complete — rebuilding', + logger.info( + 'index', + 'index definitions changed or a previous rebuild did not complete — rebuilding', + { phase: 'rebuild', counts: observationCount }, ); + } else { + logger.info('index', 'ensureInitialRebuild running full rebuild', { + phase: 'rebuild', + counts: observationCount, + }); } await this.rebuildAllIndexes(); this.initialRebuildFinished = true; } catch (err) { - console.warn('[ObservationIndexService] initial rebuild failed:', err); + logger.warn( + 'index', + err instanceof Error ? err.message : 'initial rebuild failed', + ); // Allow a future caller to retry. this.initialRebuildPromise = null; } @@ -411,6 +556,7 @@ export class ObservationIndexService { 'SELECT COUNT(*) AS cnt FROM observations', ); const total = totalRows[0]?.cnt ?? 0; + await logger.breadcrumb('index', 'rebuild_start', { counts: total }); let processed = 0; let cursor: string | null = null; @@ -431,24 +577,36 @@ export class ObservationIndexService { ); if (!batch.length) break; - const sqls: SqlStatement[] = []; + const mapped: PreparedIndexRow[] = []; for (const obs of batch) { - this.collectReindexStatements( + const extracted = extractIndexRows( obs.id, obs.form_type ?? '', obs.data, defs, gen, - sqls, ); + for (const key of extracted.nonScalarKeys) { + this.warnOnce( + `nonscalar:${key}`, + `[ObservationIndexService] index "${key}" holds a non-scalar value; it is not indexed, so only any() filters can match it`, + ); + } + mapped.push(...extracted.rows); } await this.db.write(async () => { - await this.flush(sqls); + await this.flush(insertIndexSqls(mapped)); }); cursor = batch[batch.length - 1].id; processed += batch.length; options?.onProgress?.({ current: processed, total }); + if (processed === total || processed % 1000 === 0) { + logger.info('index', `rebuild ${processed}/${total}`, { + phase: 'rebuild', + counts: processed, + }); + } await yieldToUi(); } @@ -471,6 +629,7 @@ export class ObservationIndexService { }); const metaAfter = await this.getStatus(); + await logger.breadcrumb('index', 'rebuild_done', { counts: processed }); return { generation: gen, lastRebuildAt: metaAfter.lastRebuildAt, @@ -506,40 +665,69 @@ export class ObservationIndexService { * * The signature is left alone: this is additive work against the current * definitions. A crash mid-loop is safe because the pull cursor is persisted - * only after this returns, so the page is re-applied and these statements - * are INSERT OR REPLACE. + * only after this returns, so the page is re-applied. Each batch deletes the + * observation's current-generation rows first, then INSERT (not OR REPLACE). */ async incrementalReindexMany( - rows: Array<{ observationId: string; formType: string; dataJson: string }>, + rows: Array<{ + observationId: string; + formType: string; + dataJson: string | unknown; + }>, onProgress?: (progress: IndexRebuildProgress) => void, + isCancelled?: () => boolean, ): Promise { const defs = this.getIndexDefs(); if (!defs.length || rows.length === 0) return; const total = rows.length; onProgress?.({ current: 0, total }); + const generation = await this.readActiveGeneration(); + let mapMs = 0; + let writeMs = 0; + let eavRows = 0; for ( let offset = 0; offset < rows.length; offset += INDEX_WRITE_BATCH_SIZE ) { + if (isCancelled?.()) { + throw new Error('Sync cancelled'); + } const batch = rows.slice(offset, offset + INDEX_WRITE_BATCH_SIZE); - await this.db.write(async () => { - const generation = await this.readActiveGeneration(); - const sqls: SqlStatement[] = []; - for (const r of batch) { - this.collectReindexStatements( - r.observationId, - r.formType, - r.dataJson, - defs, - generation, - sqls, + const mapStarted = Date.now(); + const mapped: PreparedIndexRow[] = []; + const ids: string[] = []; + for (const r of batch) { + ids.push(r.observationId); + const extracted = extractIndexRows( + r.observationId, + r.formType, + r.dataJson, + defs, + generation, + ); + for (const key of extracted.nonScalarKeys) { + this.warnOnce( + `nonscalar:${key}`, + `[ObservationIndexService] index "${key}" holds a non-scalar value; it is not indexed, so only any() filters can match it`, ); } + mapped.push(...extracted.rows); + } + eavRows += mapped.length; + mapMs += Date.now() - mapStarted; + + const sqls: SqlStatement[] = [ + ...deleteIndexSqls(ids, generation), + ...insertIndexSqls(mapped), + ]; + const writeStarted = Date.now(); + await this.db.write(async () => { await this.flush(sqls); }); + writeMs += Date.now() - writeStarted; onProgress?.({ current: Math.min(offset + batch.length, total), total, @@ -548,6 +736,11 @@ export class ObservationIndexService { await yieldToUi(); } } + logger.info( + 'sync', + `index many map=${mapMs}ms write=${writeMs}ms rows=${total} eav=${eavRows}`, + { phase: 'index', counts: total }, + ); } /** @@ -571,31 +764,21 @@ export class ObservationIndexService { generation: number, out: SqlStatement[], ): void { - out.push([ - 'DELETE FROM observation_index WHERE observation_id = ? AND index_generation = ?', - [observationId, generation], - ]); - for (const def of defs) { - if (!formTypeMatches(formType, def.formTypes)) continue; - const val = extractScalar(dataJson, def.path); - if (val == null) continue; - const columns = scalarToColumns(val, def.valueType); - if (!columns) { - this.warnOnce( - `nonscalar:${def.key}`, - `[ObservationIndexService] index "${def.key}" holds a non-scalar value; it is not indexed, so only any() filters can match it`, - ); - continue; - } - const { valueText, valueNum } = columns; - const rowId = `${observationId}:${def.key}:${generation}`; - out.push([ - `INSERT OR REPLACE INTO observation_index - (id, observation_id, index_key, index_generation, value_text, value_num) - VALUES (?, ?, ?, ?, ?, ?)`, - [rowId, observationId, def.key, generation, valueText, valueNum], - ]); + out.push(...deleteIndexSqls([observationId], generation)); + const extracted = extractIndexRows( + observationId, + formType, + dataJson, + defs, + generation, + ); + for (const key of extracted.nonScalarKeys) { + this.warnOnce( + `nonscalar:${key}`, + `[ObservationIndexService] index "${key}" holds a non-scalar value; it is not indexed, so only any() filters can match it`, + ); } + out.push(...insertIndexSqls(extracted.rows)); } private collectSqliteIndexStatements( diff --git a/formulus/src/services/QRSettingsService.ts b/formulus/src/services/QRSettingsService.ts index 7ef60cc4a..b450ab55b 100644 --- a/formulus/src/services/QRSettingsService.ts +++ b/formulus/src/services/QRSettingsService.ts @@ -46,8 +46,6 @@ export class QRSettingsService { // Save credentials to Keychain await Keychain.setGenericPassword(settings.username, settings.password); - - console.log('Settings updated successfully from QR code'); } catch (error) { console.error('Failed to update settings:', error); throw new Error('Failed to save settings'); diff --git a/formulus/src/services/SyncService.ts b/formulus/src/services/SyncService.ts index 140380372..95c69bebf 100644 --- a/formulus/src/services/SyncService.ts +++ b/formulus/src/services/SyncService.ts @@ -25,6 +25,7 @@ import { normalizeAppBundleVersion, } from '../utils/appBundleVersion'; import { i18n } from '../i18n/instance'; +import { logger } from '../diagnostics/logger'; type SyncStatusCallback = (status: string) => void; type SyncProgressDetailCallback = (progress: SyncProgress) => void; @@ -68,13 +69,19 @@ export class SyncService { notificationService .showSyncProgress(progress) .catch(error => - console.warn('Failed to show sync progress notification:', error), + logger.warn( + 'sync', + error instanceof Error + ? error.message + : 'Failed to show sync progress notification', + ), ); } public cancelSync(): void { if (this.canCancel) { this.shouldCancel = true; + logger.info('sync', 'cancel requested'); this.updateStatus('Cancelling sync...'); } } @@ -127,7 +134,8 @@ export class SyncService { if (isUnauthorizedError(error)) { // Prevent infinite retry loops if (this.autoLoginRetryCount >= 1) { - console.error( + logger.error( + 'sync', 'Auto-login retry limit reached. Please login manually in Settings.', ); throw new Error( @@ -136,8 +144,9 @@ export class SyncService { } this.autoLoginRetryCount++; - console.log( - `🚨 401 Unauthorized error detected during ${operationName}, attempting auto-login...`, + logger.info( + 'sync', + `401 during ${operationName}, attempting auto-login`, ); this.updateStatus('Session expired, re-authenticating...'); @@ -145,18 +154,16 @@ export class SyncService { // Attempt auto-login const userInfo = await autoLogin(); if (userInfo) { - console.log(`Auto-login successful, retrying ${operationName}...`); + logger.info('sync', `auto-login ok, retrying ${operationName}`); this.updateStatus(`Retrying ${operationName}...`); // Clear API cache to force new token usage synkronusApi.clearTokenCache(); - console.log( - `🔄 API cache cleared, retrying ${operationName} with new token...`, - ); // Retry the operation once (protected by retry count check above) try { const result = await operation(); - console.log( - `✅ ${operationName} succeeded after auto-login retry`, + logger.info( + 'sync', + `${operationName} succeeded after auto-login retry`, ); // Reset retry count on successful retry this.autoLoginRetryCount = 0; @@ -177,7 +184,7 @@ export class SyncService { } } catch (autoLoginError: unknown) { const loginError = autoLoginError as HttpError; - console.error('Auto-login failed:', loginError); + logger.error('sync', loginError?.message || 'Auto-login failed'); // Reset retry count on failure this.autoLoginRetryCount = 0; throw new Error( @@ -208,10 +215,16 @@ export class SyncService { notificationService .clearAllSyncNotifications() .catch(error => - console.warn('Failed to clear stale notifications:', error), + logger.warn( + 'sync', + error instanceof Error + ? error.message + : 'Failed to clear stale notifications', + ), ); try { + await logger.breadcrumb('sync', 'start'); await notificationService.startForegroundService(); const syncOptions: SynkronusSyncOptions = { @@ -234,12 +247,9 @@ export class SyncService { const repoGenStorage = (await AsyncStorage.getItem('@repository_generation')) ?? '(missing)'; - console.log( - '[RepositoryGeneration] SyncService: observations sync done', - { - finalObservationDataVersion: finalVersion, - repositoryGenerationStorage: repoGenStorage, - }, + logger.info( + 'sync', + `observations sync done @ ${finalVersion} gen=${repoGenStorage}`, ); this.updateProgress({ @@ -251,22 +261,27 @@ export class SyncService { await AsyncStorage.setItem('@last_seen_version', finalVersion.toString()); this.updateStatus(`Sync completed @ data version ${finalVersion}`); - console.log( - 'Sync completed successfully, showing completion notification...', - ); + await logger.breadcrumb('sync', 'end', { success: true }); + logger.info('sync', `completed @ data version ${finalVersion}`); // Don't let notification service block sync completion notificationService .showSyncComplete(true) - .then(() => console.log('Sync completion notification shown')) .catch(error => - console.warn('Failed to show sync completion notification:', error), + logger.warn( + 'sync', + error instanceof Error + ? error.message + : 'Failed to show sync completion notification', + ), ); - console.log('Returning final version:', finalVersion); return finalVersion; } catch (error) { - console.error('Sync failed', error); + logger.error( + 'sync', + error instanceof Error ? error.message : 'Sync failed', + ); if ( error instanceof Error && error.message === 'Sync cancelled' && @@ -275,9 +290,11 @@ export class SyncService { notificationService .showSyncCanceled() .catch(notifError => - console.warn( - 'Failed to show sync canceled notification:', - notifError, + logger.warn( + 'sync', + notifError instanceof Error + ? notifError.message + : 'Failed to show sync canceled notification', ), ); throw error; @@ -289,7 +306,12 @@ export class SyncService { notificationService .showSyncComplete(false, errorMessage) .catch(notifError => - console.warn('Failed to show sync failure notification:', notifError), + logger.warn( + 'sync', + notifError instanceof Error + ? notifError.message + : 'Failed to show sync failure notification', + ), ); throw error; @@ -314,16 +336,16 @@ export class SyncService { () => synkronusApi.getManifest(), 'check for updates', ); - console.log( - '[AppBundle] getManifest response (check for updates)', - manifest, + logger.info( + 'sync', + `app bundle check local vs server version ${String(manifest.version)}`, ); const serverVersion = normalizeAppBundleVersion(manifest.version); if (!isNumericAppBundleVersionString(serverVersion)) { - console.warn( - '[AppBundle] manifest.version is not numeric; treating check as failed:', - manifest.version, + logger.warn( + 'sync', + `manifest.version is not numeric: ${String(manifest.version)}`, ); return null; } @@ -350,7 +372,10 @@ export class SyncService { return { localVersion, serverVersion, updateAvailable }; } catch (error) { - console.warn('Failed to check for updates', error); + logger.warn( + 'sync', + error instanceof Error ? error.message : 'Failed to check for updates', + ); return null; } } @@ -416,6 +441,7 @@ export class SyncService { phase: 'index_rebuild', indeterminate: true, }); + await logger.breadcrumb('index', 'rebuild_start'); await ObservationIndexService.getInstance().rebuildForBundleUpdate( ({ current, total }) => { this.updateProgress({ @@ -426,6 +452,7 @@ export class SyncService { }); }, ); + await logger.breadcrumb('index', 'rebuild_done'); const syncTime = new Date().toLocaleTimeString(); await AsyncStorage.setItem('@lastSync', syncTime); @@ -439,7 +466,10 @@ export class SyncService { appEvents.emit('bundleUpdated'); } catch (error) { - console.error('App sync failed', error); + logger.error( + 'sync', + error instanceof Error ? error.message : 'App sync failed', + ); const message = await getUserFacingAppBundleUpdateErrorMessage(error); this.updateStatus(message); if (error instanceof Error && message === error.message) { @@ -471,7 +501,10 @@ export class SyncService { 'download app bundle', ); } catch (error) { - console.error('Download failed', error); + logger.error( + 'sync', + error instanceof Error ? error.message : 'Download failed', + ); throw error; } } diff --git a/formulus/src/services/__tests__/FormService.test.ts b/formulus/src/services/__tests__/FormService.test.ts index 3989fd4b8..78b86ca4c 100644 --- a/formulus/src/services/__tests__/FormService.test.ts +++ b/formulus/src/services/__tests__/FormService.test.ts @@ -91,6 +91,8 @@ jest.mock('../../database/DatabaseService', () => ({ getLocalRepo: jest.fn(() => ({ getObservationsByFormId: mockGetObservationsByFormId, getObservationsByFormType: mockGetObservationsByFormType, + getObservation: jest.fn(), + listObservationsPage: jest.fn(), queryObservations: mockQueryObservations, deleteObservation: mockDeleteObservation, saveObservation: mockSaveObservation, diff --git a/formulus/src/sync/__tests__/syncProgressUi.test.ts b/formulus/src/sync/__tests__/syncProgressUi.test.ts index 83cd07933..3e597b222 100644 --- a/formulus/src/sync/__tests__/syncProgressUi.test.ts +++ b/formulus/src/sync/__tests__/syncProgressUi.test.ts @@ -70,7 +70,7 @@ describe('syncProgressUi', () => { ).toBe('Preparing data for search'); }); - it('uses the index-rebuild title during an observation pull', () => { + it('uses the index-rebuild title for a full rebuild (bundle update path)', () => { expect( getSyncProgressCardTitle( { diff --git a/formulus/src/sync/syncConstants.ts b/formulus/src/sync/syncConstants.ts index defb0a67b..74bc632c1 100644 --- a/formulus/src/sync/syncConstants.ts +++ b/formulus/src/sync/syncConstants.ts @@ -4,3 +4,25 @@ * "last write wins" strategy in WatermelonDBRepo.applyServerChanges. */ export const LAST_WRITE_WON_TAG = 'last_write_won'; + +/** + * Records requested per `syncPull` page. + * + * Synkronus defaults to 50 when `limit` is omitted (OpenAPI max 500; service + * cap 1000). Typical observation JSON is ~0.5–2 KB, so 500 rows is about + * 0.25–1 MB per page (two pages in memory while the next HTTP fetch overlaps + * apply). That halves the round-trips of 250 on a 20k first pull without + * crossing the documented API max. SQLite is still one writer — pages are + * applied sequentially. + */ +export const PULL_PAGE_SIZE = 500; + +/** + * Attachment files pulled at once during sync. + * + * Each download is native I/O (RNFS) and pays a full HTTP/TLS round trip, so a + * small pool hides latency on photo-heavy first syncs. Four stays under typical + * field-WiFi and tablet connection limits; unbounded `Promise.all` does not. + * Observation apply stays sequential — this pool is downloads only. + */ +export const ATTACHMENT_DOWNLOAD_CONCURRENCY = 4; diff --git a/formulus/src/utils/__tests__/dateUtils.test.ts b/formulus/src/utils/__tests__/dateUtils.test.ts new file mode 100644 index 000000000..9f7cf4069 --- /dev/null +++ b/formulus/src/utils/__tests__/dateUtils.test.ts @@ -0,0 +1,13 @@ +import { formatDateTimeShort } from '../dateUtils'; + +describe('formatDateTimeShort', () => { + test('formats a valid local date without locale APIs', () => { + const date = new Date(2024, 5, 3, 9, 7); + expect(formatDateTimeShort(date)).toBe('2024-06-03 09:07'); + }); + + test('returns an em dash for invalid input', () => { + expect(formatDateTimeShort(null)).toBe('—'); + expect(formatDateTimeShort('not-a-date')).toBe('—'); + }); +}); diff --git a/formulus/src/utils/dateUtils.ts b/formulus/src/utils/dateUtils.ts index be774caaf..411865c6e 100644 --- a/formulus/src/utils/dateUtils.ts +++ b/formulus/src/utils/dateUtils.ts @@ -1,3 +1,24 @@ +function pad2(value: number): string { + return value < 10 ? `0${value}` : String(value); +} + +/** + * Local date-time for dense tables. Avoid Date#toLocaleString — on Hermes + * the first ICU locale load can stall the JS thread for 1–2s. + */ +export function formatDateTimeShort(date: Date | string | null): string { + if (!date) { + return '—'; + } + const dateObj = typeof date === 'string' ? new Date(date) : date; + if (Number.isNaN(dateObj.getTime())) { + return '—'; + } + return `${dateObj.getFullYear()}-${pad2(dateObj.getMonth() + 1)}-${pad2( + dateObj.getDate(), + )} ${pad2(dateObj.getHours())}:${pad2(dateObj.getMinutes())}`; +} + export const formatRelativeTime = (date: Date | string | null): string => { if (!date) { return 'Never'; diff --git a/formulus/src/webview/FormulusMessageHandlers.ts b/formulus/src/webview/FormulusMessageHandlers.ts index 8efb1a1c5..900142373 100644 --- a/formulus/src/webview/FormulusMessageHandlers.ts +++ b/formulus/src/webview/FormulusMessageHandlers.ts @@ -11,6 +11,7 @@ import * as Keychain from 'react-native-keychain'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { Alert, Platform } from 'react-native'; import { i18n } from '../i18n/instance'; +import { logger } from '../diagnostics/logger'; import * as ImagePicker from 'react-native-image-picker'; import { check, @@ -251,12 +252,10 @@ export const rejectFormOperation = (operationId: string, error: Error) => { export function createFormulusMessageHandlers(): FormulusMessageHandlers { return { - onInitForm: (payload: unknown) => { + onInitForm: (_payload: unknown) => { // TODO: implement init form logic - console.log('FormulusMessageHandlers: onInitForm called', payload); }, onGetVersion: async (): Promise => { - console.log('FormulusMessageHandlers: onGetVersion handler invoked.'); // Return the bridge interface contract version so custom apps can do // meaningful compatibility checks (see isCompatibleVersion). return FORMULUS_INTERFACE_VERSION; @@ -266,16 +265,9 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { finalData: Record; }) => { const { formType, finalData } = data; - console.log( - 'FormulusMessageHandlers: onSubmitObservation handler invoked.', - { formType, finalData }, - ); // Use the active FormplayerModal's handleSubmission method if available if (activeFormplayerModalRef) { - console.log( - 'FormulusMessageHandlers: Delegating to FormplayerModal.handleSubmission', - ); return await activeFormplayerModalRef.handleSubmission({ formType, finalData, @@ -298,9 +290,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { // Formplayer uses updateObservation for existing rows; submitObservation for new. // Route updates through the modal too so the operation promise resolves and the UI closes. if (activeFormplayerModalRef) { - console.log( - 'FormulusMessageHandlers: Delegating to FormplayerModal.handleSubmission (update)', - ); return await activeFormplayerModalRef.handleSubmission({ formType: data.formType, finalData: data.finalData, @@ -317,7 +306,7 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { ); }, onRequestCamera: async (fieldId: string): Promise => { - console.log('Request camera handler called', fieldId); + logger.debug('bridge', 'camera requested'); return new Promise(resolve => { try { @@ -347,18 +336,10 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }, }; - console.log( - 'Launching image picker with camera and gallery options, options:', - options, - ); - // Common response handler for both camera and gallery // eslint-disable-next-line @typescript-eslint/no-explicit-any const handleImagePickerResponse = (response: any) => { - console.log('Camera response received:', response); - if (response.didCancel) { - console.log('User cancelled camera'); resolve({ fieldId, status: 'cancelled', @@ -396,36 +377,16 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { const imageGuid = generateGUID(); const guidFilename = `${imageGuid}.jpg`; - console.log( - 'Photo captured, processing for persistent storage:', - { - imageGuid, - guidFilename, - tempUri: asset.uri, - size: asset.fileSize, - }, - ); - const attachmentsDirectory = `${RNFS.DocumentDirectoryPath}/attachments`; const draftDirectory = `${attachmentsDirectory}/draft`; const draftFilePath = `${draftDirectory}/${guidFilename}`; - console.log('Copying camera image to draft attachment storage:', { - source: asset.uri, - draftPath: draftFilePath, - }); - Promise.all([ RNFS.mkdir(attachmentsDirectory), RNFS.mkdir(draftDirectory), ]) .then(() => RNFS.copyFile(asset.uri, draftFilePath)) .then(() => { - console.log( - 'Image saved to draft attachments:', - draftFilePath, - ); - const webViewUrl = `file://${draftFilePath}`; resolve({ @@ -537,15 +498,12 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }); }, onRequestQrcode: async (fieldId: string): Promise => { - console.log('Request QR code handler called', fieldId); - const promise = qrcodeRequestCoordinator.request(fieldId); try { appEvents.emit('openQRScanner', { fieldId, onResult: (result: unknown) => { - console.log('QR scan result received:', result); qrcodeRequestCoordinator.settle(result); }, }); @@ -563,7 +521,7 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { return promise; }, onRequestSignature: async (fieldId: string): Promise => { - console.log('Request signature handler called', fieldId); + logger.debug('bridge', 'signature requested'); return new Promise(resolve => { try { @@ -572,8 +530,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { fieldId, // eslint-disable-next-line @typescript-eslint/no-explicit-any onResult: async (result: any) => { - console.log('Signature capture result received:', result); - try { // If the result contains base64 data, save it to file and return URI if ( @@ -617,7 +573,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }, }; - console.log('Signature saved to file:', filePath); resolve(updatedResult); } else { // Return result as-is if no base64 data or if it's an error/cancellation @@ -649,13 +604,13 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { onRequestLocation: async ( payload: string | { fieldId?: string }, ): Promise => { + logger.debug('bridge', 'location requested'); const fieldId = typeof payload === 'string' ? payload : typeof payload?.fieldId === 'string' ? payload.fieldId : ''; - console.log('Request location handler called', fieldId); // eslint-disable-next-line no-async-promise-executor return new Promise(async (resolve, reject) => { @@ -681,7 +636,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }, }; - console.log('Location captured successfully:', locationResult); resolve(locationResult); } else { throw new Error('Unable to get current location'); @@ -803,7 +757,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }; ImagePicker.launchCamera(options, async response => { if (response.didCancel) { - console.log('Video recording cancelled'); reject({ fieldId, status: 'cancelled', @@ -882,7 +835,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }, }; - console.log('Video recorded successfully:', videoResult); resolve(videoResult); } catch (fileError) { console.error('Error saving video file:', fileError); @@ -914,7 +866,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }, onRequestFile: async (fieldId: string) => { - console.log('Request file handler called (v12 API)', fieldId); try { const [result] = await pick({ type: [types.allFiles], @@ -922,8 +873,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { allowMultiSelection: false, }); - console.log('File selected:', result); - const originalName = typeof result.name === 'string' && result.name.trim().length > 0 ? result.name.trim() @@ -1002,17 +951,18 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { } }, - onLaunchIntent: (fieldId: string, intentSpec: Record) => { + onLaunchIntent: ( + _fieldId: string, + _intentSpec: Record, + ) => { // TODO: implement launch intent logic - console.log('Launch intent handler called', fieldId, intentSpec); }, onCallSubform: ( - fieldId: string, - formType: string, - options: Record, + _fieldId: string, + _formType: string, + _options: Record, ) => { // TODO: implement call subform logic - console.log('Call subform handler called', fieldId, formType, options); }, onRequestAudio: async (fieldId: string) => { // Lazy-load NitroSound only when audio is requested (avoids console error on startup when disabled) @@ -1075,10 +1025,7 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }, }; } catch (error) { - console.log('Audio recording error:', error); - // Check if this is a user cancellation or permission error - console.log('Audio recording error:', error); if (typeof error === 'object' && error !== null) { const err = error as Record; @@ -1121,17 +1068,14 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { }; } }, - onRequestBiometric: (fieldId: string) => { + onRequestBiometric: (_fieldId: string) => { // TODO: implement biometric request logic - console.log('Request biometric handler called', fieldId); }, onRequestConnectivityStatus: () => { // TODO: implement connectivity status logic - console.log('Request connectivity status handler called'); }, onRequestSyncStatus: () => { // TODO: implement sync status logic - console.log('Request sync status handler called'); }, onPersistObservation: async ( data: { @@ -1213,12 +1157,11 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { } }, onRunLocalModel: ( - fieldId: string, - modelId: string, - input: Record, + _fieldId: string, + _modelId: string, + _input: Record, ) => { // TODO: implement run local model logic - console.log('Run local model handler called', fieldId, modelId, input); }, onGetAvailableForms: async (): Promise => { try { @@ -1365,11 +1308,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { isDraft?: boolean, includeDeleted?: boolean, ) => { - console.log( - 'FormulusMessageHandlers: onGetObservations handler invoked.', - { formType, isDraft, includeDeleted }, - ); - // Extract the actual formType string value let formTypeString: string; if (typeof formType === 'string') { @@ -1379,9 +1317,6 @@ export function createFormulusMessageHandlers(): FormulusMessageHandlers { formType !== null && 'formType' in formType ) { - console.debug( - 'FormulusMessageHandlers: onGetObservations received formType as object, extracting string value', - ); formTypeString = formType.formType; } else { console.error( diff --git a/formulus/src/webview/FormulusWebViewHandler.ts b/formulus/src/webview/FormulusWebViewHandler.ts index e1bfc8a1f..a89911d0a 100644 --- a/formulus/src/webview/FormulusWebViewHandler.ts +++ b/formulus/src/webview/FormulusWebViewHandler.ts @@ -7,6 +7,7 @@ */ import { WebViewMessageEvent, WebView } from 'react-native-webview'; +import { persistWebViewConsole, webViewTag } from '../diagnostics/logger'; import { createFormulusMessageHandlers } from './FormulusMessageHandlers'; import { FormInitData } from './FormulusInterfaceDefinition'; @@ -221,6 +222,11 @@ export class FormulusWebViewMessageManager { // If logging fails, silently ignore to prevent cascading errors // This can happen if console methods are not properly available } + persistWebViewConsole( + webViewTag(this.appName), + String(logLevel), + logArgs, + ); } else if (type === 'console') { // Keep existing handler for type === 'console' as fallback // Handle console messages from WebView if type is exactly 'console' and level is in payload @@ -237,6 +243,11 @@ export class FormulusWebViewMessageManager { } else { console.log(`${this.logPrefix} [WebView]`, ...args); } + persistWebViewConsole( + webViewTag(this.appName), + String(level ?? 'log'), + Array.isArray(args) ? args : [args], + ); } else { this.handleIncomingAction(type, payload, messageId); } From 0652d8bcfa96a31e5ba3dc1cc38dc516da575342 Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Sun, 16 Aug 2026 18:31:01 +0200 Subject: [PATCH 31/32] fix(formulus): Include bundled sqlite package (jsi) to prevent dependency on system sqlite --- .../formulus/MainApplication.kt | 4 + formulus/src/contexts/SyncContext.tsx | 14 ++- .../installWatermelonLogBridge.test.ts | 57 +++++++++ .../__tests__/probeSqliteEngine.test.ts | 117 +++++++++++++++++ formulus/src/database/database.ts | 22 +++- .../database/installWatermelonLogBridge.ts | 72 +++++++++++ formulus/src/database/probeSqliteEngine.ts | 118 ++++++++++++++++++ formulus/src/locales/en.json | 1 + formulus/src/locales/fr.json | 1 + formulus/src/locales/pt.json | 1 + formulus/src/screens/SyncScreen.tsx | 32 ++--- formulus/src/services/SyncService.ts | 18 +-- .../__tests__/SyncService.autoLogin.test.ts | 39 ++++++ 13 files changed, 469 insertions(+), 27 deletions(-) create mode 100644 formulus/src/database/__tests__/installWatermelonLogBridge.test.ts create mode 100644 formulus/src/database/__tests__/probeSqliteEngine.test.ts create mode 100644 formulus/src/database/installWatermelonLogBridge.ts create mode 100644 formulus/src/database/probeSqliteEngine.ts diff --git a/formulus/android/app/src/main/java/org/opendataensemble/formulus/MainApplication.kt b/formulus/android/app/src/main/java/org/opendataensemble/formulus/MainApplication.kt index 163ecce95..758adb56e 100644 --- a/formulus/android/app/src/main/java/org/opendataensemble/formulus/MainApplication.kt +++ b/formulus/android/app/src/main/java/org/opendataensemble/formulus/MainApplication.kt @@ -7,6 +7,7 @@ import com.facebook.react.ReactApplication import com.facebook.react.ReactHost import com.facebook.react.ReactNativeApplicationEntryPoint.loadReactNative import com.facebook.react.defaults.DefaultReactHost.getDefaultReactHost +import com.nozbe.watermelondb.jsi.WatermelonDBJSIPackage import org.opendataensemble.formulus.UserAppPackage class MainApplication : Application(), ReactApplication { @@ -17,6 +18,9 @@ class MainApplication : Application(), ReactApplication { packageList = PackageList(this).packages.apply { // Packages that cannot be autolinked yet can be added manually here add(UserAppPackage()) + // JSI is a separate Gradle project from Watermelon's autolinked Java + // adapter. Without this, jsi: true silently falls back to system SQLite. + add(WatermelonDBJSIPackage()) }, ) } diff --git a/formulus/src/contexts/SyncContext.tsx b/formulus/src/contexts/SyncContext.tsx index ce6571a3f..31ab836c9 100644 --- a/formulus/src/contexts/SyncContext.tsx +++ b/formulus/src/contexts/SyncContext.tsx @@ -6,6 +6,7 @@ import React, { ReactNode, } from 'react'; import { syncService as syncServiceInstance } from '../services/SyncService'; +import { i18n } from '../i18n/instance'; export type { SyncProgress, SyncProgressPhase } from '../sync/syncProgress'; export type { SyncProgressReporter } from '../sync/syncProgress'; @@ -69,12 +70,19 @@ export const SyncProvider: React.FC = ({ children }) => { const cancelSync = useCallback(() => { syncServiceInstance.cancelSync(); + // Keep isActive until the in-flight syncObservations/updateAppBundle + // finally-block clears isSyncing. Flipping it here re-enabled the Sync + // button while the service was still busy, and the next tap threw + // "Sync already in progress". setSyncState(prev => ({ ...prev, - isActive: false, canCancel: false, - error: 'Sync cancelled by user', - progress: undefined, + progress: prev.progress + ? { + ...prev.progress, + details: i18n.t('sync.progress.cancelling'), + } + : prev.progress, })); }, []); diff --git a/formulus/src/database/__tests__/installWatermelonLogBridge.test.ts b/formulus/src/database/__tests__/installWatermelonLogBridge.test.ts new file mode 100644 index 000000000..248f8e5cc --- /dev/null +++ b/formulus/src/database/__tests__/installWatermelonLogBridge.test.ts @@ -0,0 +1,57 @@ +import watermelonLogger from '@nozbe/watermelondb/utils/common/logger'; +import { + configureDiagnosticLog, + readRecentEvents, +} from '../../diagnostics/DiagnosticLog'; +import { configureLogger, resetLoggerForTests } from '../../diagnostics/logger'; +import { createMemoryFs } from '../../diagnostics/memoryFs'; +import { + installWatermelonLogBridge, + resetWatermelonLogBridgeForTests, +} from '../installWatermelonLogBridge'; + +describe('installWatermelonLogBridge', () => { + beforeEach(() => { + resetWatermelonLogBridgeForTests(); + resetLoggerForTests(); + configureLogger({ persist: true }); + configureDiagnosticLog({ + fs: createMemoryFs(), + documentDirectoryPath: '/docs', + }); + }); + + afterEach(() => { + resetWatermelonLogBridgeForTests(); + }); + + it('persists Watermelon warn and error into the diagnostic log', async () => { + installWatermelonLogBridge(); + watermelonLogger.warn('JSI SQLiteAdapter not available… falling back'); + watermelonLogger.error(new Error('Failed to initialize JSI')); + await new Promise(resolve => setTimeout(resolve, 0)); + const events = await readRecentEvents(10); + expect( + events.some( + e => + e.tag === 'watermelon' && + e.level === 'warn' && + e.message.includes('JSI SQLiteAdapter not available'), + ), + ).toBe(true); + expect( + events.some( + e => + e.tag === 'watermelon' && + e.level === 'error' && + e.message.includes('Failed to initialize JSI'), + ), + ).toBe(true); + }); + + it('is idempotent', () => { + installWatermelonLogBridge(); + installWatermelonLogBridge(); + expect(() => watermelonLogger.warn('once')).not.toThrow(); + }); +}); diff --git a/formulus/src/database/__tests__/probeSqliteEngine.test.ts b/formulus/src/database/__tests__/probeSqliteEngine.test.ts new file mode 100644 index 000000000..7230cfd43 --- /dev/null +++ b/formulus/src/database/__tests__/probeSqliteEngine.test.ts @@ -0,0 +1,117 @@ +import { + describeSqliteEngine, + probeSqliteEngine, + sqliteEngineLogLevel, +} from '../probeSqliteEngine'; + +function mockDb(options: { + dispatcher?: string; + versionRows?: Record[]; + jsonRows?: Record[] | Error; + jsiBinding?: boolean; +}) { + let queryCount = 0; + const unsafeFetchRaw = jest.fn(async () => { + queryCount += 1; + if (queryCount === 1) { + return options.versionRows ?? [{ sqlite_version: '3.46.0' }]; + } + if (options.jsonRows instanceof Error) { + throw options.jsonRows; + } + return options.jsonRows ?? [{ json_ok: '1' }]; + }); + const db = { + adapter: { + underlyingAdapter: { + _dispatcherType: options.dispatcher ?? 'jsi', + initializingPromise: Promise.resolve(), + }, + }, + get: jest.fn(() => ({ + query: jest.fn(() => ({ unsafeFetchRaw })), + })), + }; + if (options.jsiBinding) { + ( + globalThis as { nativeWatermelonCreateAdapter?: () => unknown } + ).nativeWatermelonCreateAdapter = () => ({}); + } else { + delete (globalThis as { nativeWatermelonCreateAdapter?: () => unknown }) + .nativeWatermelonCreateAdapter; + } + return db; +} + +describe('probeSqliteEngine', () => { + afterEach(() => { + delete (globalThis as { nativeWatermelonCreateAdapter?: () => unknown }) + .nativeWatermelonCreateAdapter; + }); + + it('describes a healthy bundled engine', () => { + const report = { + dispatcher: 'jsi', + sqliteVersion: '3.46.0', + jsonExtract: true, + jsiBinding: true, + }; + expect(describeSqliteEngine(report)).toBe( + 'sqlite engine dispatcher=jsi version=3.46.0 json_extract=ok jsiBinding=yes', + ); + expect(sqliteEngineLogLevel(report)).toBe('info'); + }); + + it('treats a system-sqlite fallback as a warning even when JSON1 exists', () => { + expect( + sqliteEngineLogLevel({ + dispatcher: 'asynchronous', + sqliteVersion: '3.32.2', + jsonExtract: true, + jsiBinding: false, + }), + ).toBe('warn'); + }); + + it('treats missing json_extract as an error', () => { + expect( + sqliteEngineLogLevel({ + dispatcher: 'asynchronous', + sqliteVersion: '3.22.0', + jsonExtract: false, + jsiBinding: false, + }), + ).toBe('error'); + expect( + describeSqliteEngine({ + dispatcher: 'asynchronous', + sqliteVersion: '3.22.0', + jsonExtract: false, + jsiBinding: false, + }), + ).toContain('json_extract=missing'); + }); + + it('probes dispatcher, version, json_extract, and JSI binding', async () => { + const db = mockDb({ dispatcher: 'jsi', jsiBinding: true }); + const report = await probeSqliteEngine(db as never); + expect(report).toEqual({ + dispatcher: 'jsi', + sqliteVersion: '3.46.0', + jsonExtract: true, + jsiBinding: true, + }); + }); + + it('records json_extract as missing when the probe query fails', async () => { + const db = mockDb({ + dispatcher: 'asynchronous', + jsonRows: new Error('no such function: json_extract'), + jsiBinding: false, + }); + const report = await probeSqliteEngine(db as never); + expect(report.jsonExtract).toBe(false); + expect(report.dispatcher).toBe('asynchronous'); + expect(report.jsiBinding).toBe(false); + }); +}); diff --git a/formulus/src/database/database.ts b/formulus/src/database/database.ts index cc4472eb2..e2e295e6c 100644 --- a/formulus/src/database/database.ts +++ b/formulus/src/database/database.ts @@ -6,6 +6,12 @@ import { schemaMigrations, unsafeExecuteSql, } from '@nozbe/watermelondb/Schema/migrations'; +import { logger } from '../diagnostics/logger'; +import { installWatermelonLogBridge } from './installWatermelonLogBridge'; +import { logSqliteEngine } from './probeSqliteEngine'; + +// Capture Watermelon's JSI-fallback warn before SQLiteAdapter runs initializeJSI. +installWatermelonLogBridge(); // Define migrations const migrations = schemaMigrations({ @@ -109,11 +115,14 @@ const adapter = new SQLiteAdapter({ dbName: 'formulus', // Configure migrations migrations: migrations, - // Optional synchronous mode for development + // Requests the bundled JSI SQLite. Confirm with logSqliteEngine — Android + // still falls back to system SQLite if WatermelonDBJSIPackage is missing. jsi: true, - // Optional onSetUpError callback onSetUpError: error => { - console.error('Database setup error:', error); + logger.error( + 'db', + error instanceof Error ? error.message : 'Database setup error', + ); }, }); @@ -125,3 +134,10 @@ export const database = new Database({ // Add more models as needed ], }); + +void logSqliteEngine(database).catch(error => { + logger.warn( + 'db', + error instanceof Error ? error.message : 'sqlite engine probe failed', + ); +}); diff --git a/formulus/src/database/installWatermelonLogBridge.ts b/formulus/src/database/installWatermelonLogBridge.ts new file mode 100644 index 000000000..abab5de96 --- /dev/null +++ b/formulus/src/database/installWatermelonLogBridge.ts @@ -0,0 +1,72 @@ +import watermelonLogger from '@nozbe/watermelondb/utils/common/logger'; +import { joinLogArgs } from '../diagnostics/redact'; +import { logger } from '../diagnostics/logger'; + +type WatermelonLogger = { + silent: boolean; + warn: (...messages: unknown[]) => void; + error: (...messages: unknown[]) => void; +}; + +const wmLogger = watermelonLogger as WatermelonLogger; + +let installed = false; +let originalWarn: WatermelonLogger['warn'] | undefined; +let originalError: WatermelonLogger['error'] | undefined; + +function formatWatermelonArgs(messages: unknown[]): string { + const normalized = messages.map(message => + message instanceof Error ? message.message || String(message) : message, + ); + return joinLogArgs(normalized); +} + +function persist(level: 'warn' | 'error', messages: unknown[]): void { + if (wmLogger.silent) { + return; + } + const text = formatWatermelonArgs(messages); + if (!text) { + return; + } + logger[level]('watermelon', text); +} + +/** + * WatermelonDB logs JSI fallback and native errors to console only. Mirror + * warn/error into the Formulus diagnostic log so field exports include them. + * + * Must run before `new SQLiteAdapter({ jsi: true })`, which is when the + * fallback warning is emitted. + */ +export function installWatermelonLogBridge(): void { + if (installed) { + return; + } + originalWarn = wmLogger.warn.bind(wmLogger); + originalError = wmLogger.error.bind(wmLogger); + wmLogger.warn = (...messages: unknown[]) => { + originalWarn?.(...messages); + persist('warn', messages); + }; + wmLogger.error = (...messages: unknown[]) => { + originalError?.(...messages); + persist('error', messages); + }; + installed = true; +} + +export function resetWatermelonLogBridgeForTests(): void { + if (!installed) { + return; + } + if (originalWarn) { + wmLogger.warn = originalWarn; + } + if (originalError) { + wmLogger.error = originalError; + } + originalWarn = undefined; + originalError = undefined; + installed = false; +} diff --git a/formulus/src/database/probeSqliteEngine.ts b/formulus/src/database/probeSqliteEngine.ts new file mode 100644 index 000000000..266d4cc9c --- /dev/null +++ b/formulus/src/database/probeSqliteEngine.ts @@ -0,0 +1,118 @@ +import { Database, Q } from '@nozbe/watermelondb'; +import { logger } from '../diagnostics/logger'; + +export type SqliteEngineReport = { + dispatcher: string; + sqliteVersion: string | null; + jsonExtract: boolean; + jsiBinding: boolean; +}; + +type SqliteAdapterLike = { + _dispatcherType?: string; + initializingPromise?: Promise; +}; + +function getUnderlyingAdapter(db: Database): SqliteAdapterLike | undefined { + return (db.adapter as { underlyingAdapter?: SqliteAdapterLike }) + .underlyingAdapter; +} + +function hasJsiBinding(): boolean { + return ( + typeof (globalThis as { nativeWatermelonCreateAdapter?: unknown }) + .nativeWatermelonCreateAdapter === 'function' + ); +} + +async function rawQuery( + db: Database, + sql: string, +): Promise[]> { + return (await db + .get('observations') + .query(Q.unsafeSqlQuery(sql)) + .unsafeFetchRaw()) as Record[]; +} + +function readText( + row: Record | undefined, + key: string, +): string | null { + const value = row?.[key]; + if (value == null) { + return null; + } + return String(value); +} + +export function describeSqliteEngine(report: SqliteEngineReport): string { + return `sqlite engine dispatcher=${report.dispatcher} version=${ + report.sqliteVersion ?? 'unknown' + } json_extract=${report.jsonExtract ? 'ok' : 'missing'} jsiBinding=${ + report.jsiBinding ? 'yes' : 'no' + }`; +} + +export function sqliteEngineLogLevel( + report: SqliteEngineReport, +): 'info' | 'warn' | 'error' { + if (!report.jsonExtract) { + return 'error'; + } + if (report.dispatcher !== 'jsi') { + return 'warn'; + } + return 'info'; +} + +export async function probeSqliteEngine( + db: Database, +): Promise { + const adapter = getUnderlyingAdapter(db); + if (adapter?.initializingPromise) { + await adapter.initializingPromise; + } + + const dispatcher = adapter?._dispatcherType ?? 'unknown'; + const jsiBinding = hasJsiBinding(); + let sqliteVersion: string | null = null; + let jsonExtract = false; + + try { + const rows = await rawQuery( + db, + 'SELECT sqlite_version() AS sqlite_version', + ); + sqliteVersion = readText(rows[0], 'sqlite_version'); + } catch (error) { + logger.warn( + 'db', + error instanceof Error ? error.message : 'sqlite_version() probe failed', + ); + } + + try { + const rows = await rawQuery( + db, + `SELECT json_extract('{"a":1}', '$.a') AS json_ok`, + ); + jsonExtract = readText(rows[0], 'json_ok') === '1'; + } catch { + jsonExtract = false; + } + + return { dispatcher, sqliteVersion, jsonExtract, jsiBinding }; +} + +export async function logSqliteEngine( + db: Database, +): Promise { + const report = await probeSqliteEngine(db); + const level = sqliteEngineLogLevel(report); + logger[level]('db', describeSqliteEngine(report), { + phase: 'sqlite_probe', + success: report.jsonExtract && report.dispatcher === 'jsi', + }); + return report; +} diff --git a/formulus/src/locales/en.json b/formulus/src/locales/en.json index 81ea728ad..79cbdb64d 100644 --- a/formulus/src/locales/en.json +++ b/formulus/src/locales/en.json @@ -170,6 +170,7 @@ "sync.progress.uploadComplete": "Upload complete", "sync.progress.countOf": "{{current}} of {{total}}", "sync.progress.inProgress": "In progress…", + "sync.progress.cancelling": "Cancelling…", "settings.switchServerTitle": "Switch server?", "settings.switchServerWipeMessage": "Switching servers will wipe all local data for the previous server.", "settings.switchServerPendingMessage": "Unsynced observations: {{observations}}\nUnsynced attachments: {{attachments}}\n\nSync is recommended before switching.", diff --git a/formulus/src/locales/fr.json b/formulus/src/locales/fr.json index ce2b9ac3c..2ffdbc788 100644 --- a/formulus/src/locales/fr.json +++ b/formulus/src/locales/fr.json @@ -170,6 +170,7 @@ "sync.progress.uploadComplete": "Envoi terminé", "sync.progress.countOf": "{{current}} sur {{total}}", "sync.progress.inProgress": "En cours…", + "sync.progress.cancelling": "Annulation…", "settings.switchServerTitle": "Changer de serveur ?", "settings.switchServerWipeMessage": "Changer de serveur effacera toutes les données locales de l'ancien serveur.", "settings.switchServerPendingMessage": "Observations non synchronisées : {{observations}}\nPièces jointes non synchronisées : {{attachments}}\n\nIl est recommandé de synchroniser avant de changer.", diff --git a/formulus/src/locales/pt.json b/formulus/src/locales/pt.json index 44e04745b..599f1ebad 100644 --- a/formulus/src/locales/pt.json +++ b/formulus/src/locales/pt.json @@ -170,6 +170,7 @@ "sync.progress.uploadComplete": "Envio concluído", "sync.progress.countOf": "{{current}} de {{total}}", "sync.progress.inProgress": "Em curso…", + "sync.progress.cancelling": "A cancelar…", "settings.switchServerTitle": "Mudar de servidor?", "settings.switchServerWipeMessage": "Mudar de servidor irá apagar todos os dados locais do servidor anterior.", "settings.switchServerPendingMessage": "Observações por sincronizar: {{observations}}\nAnexos por sincronizar: {{attachments}}\n\nRecomenda-se sincronizar antes de mudar.", diff --git a/formulus/src/screens/SyncScreen.tsx b/formulus/src/screens/SyncScreen.tsx index 3f09030bf..15ab664fe 100644 --- a/formulus/src/screens/SyncScreen.tsx +++ b/formulus/src/screens/SyncScreen.tsx @@ -266,7 +266,7 @@ const SyncScreen = () => { ); const handleSync = useCallback(async () => { - if (syncState.isActive) return; + if (syncState.isActive || syncService.getIsSyncing()) return; let syncError: string | undefined; @@ -277,7 +277,9 @@ const SyncScreen = () => { await syncService.syncObservations(true); await refreshAfterOperation(); } catch (error) { - if (isRepositoryResetRequiredError(error)) { + if (isSyncCancelledError(error)) { + // Cancel is requested, not a failure — do not Alert or paint the card red. + } else if (isRepositoryResetRequiredError(error)) { syncError = getUserFacingSyncErrorMessage(error); runRepositoryResetRecovery( error, @@ -290,9 +292,7 @@ const SyncScreen = () => { ); } else { syncError = getUserFacingSyncErrorMessage(error); - if (!isSyncCancelledError(error)) { - Alert.alert(t('sync.failed'), syncError); - } + Alert.alert(t('sync.failed'), syncError); } } finally { finishSync(syncError); @@ -321,12 +321,16 @@ const SyncScreen = () => { const fs = await formService.FormService.getInstance(); await fs.invalidateCache(); } catch (error) { - const errorMessage = (error as Error).message; - finishSync(errorMessage); - if (errorMessage.includes('401')) { - Alert.alert(t('sync.authErrorTitle'), t('sync.sessionExpired')); + if (isSyncCancelledError(error)) { + finishSync(); } else { - Alert.alert(t('sync.updateFailed'), errorMessage); + const errorMessage = (error as Error).message; + finishSync(errorMessage); + if (errorMessage.includes('401')) { + Alert.alert(t('sync.authErrorTitle'), t('sync.sessionExpired')); + } else { + Alert.alert(t('sync.updateFailed'), errorMessage); + } } } finally { setActiveOperation(null); @@ -365,12 +369,12 @@ const SyncScreen = () => { }, t('sync.operationFailed'), ); + } else if (isSyncCancelledError(error)) { + finishSync(); } else { const errorMessage = getUserFacingSyncErrorMessage(error); finishSync(errorMessage); - if (!isSyncCancelledError(error)) { - Alert.alert(t('sync.operationFailed'), errorMessage); - } + Alert.alert(t('sync.operationFailed'), errorMessage); } } finally { setActiveOperation(null); @@ -384,7 +388,7 @@ const SyncScreen = () => { ]); const handleCustomAppUpdate = useCallback(async () => { - if (syncState.isActive) return; + if (syncState.isActive || syncService.getIsSyncing()) return; const userInfo = await getUserInfo(); if (!userInfo) { diff --git a/formulus/src/services/SyncService.ts b/formulus/src/services/SyncService.ts index 95c69bebf..83f1f84e8 100644 --- a/formulus/src/services/SyncService.ts +++ b/formulus/src/services/SyncService.ts @@ -278,15 +278,19 @@ export class SyncService { return finalVersion; } catch (error) { - logger.error( - 'sync', - error instanceof Error ? error.message : 'Sync failed', - ); - if ( + const cancelled = error instanceof Error && error.message === 'Sync cancelled' && - this.shouldCancel - ) { + this.shouldCancel; + if (cancelled) { + logger.info('sync', 'cancel observed, aborting'); + } else { + logger.error( + 'sync', + error instanceof Error ? error.message : 'Sync failed', + ); + } + if (cancelled) { notificationService .showSyncCanceled() .catch(notifError => diff --git a/formulus/src/services/__tests__/SyncService.autoLogin.test.ts b/formulus/src/services/__tests__/SyncService.autoLogin.test.ts index 9b45dbc6a..f0d71b61d 100644 --- a/formulus/src/services/__tests__/SyncService.autoLogin.test.ts +++ b/formulus/src/services/__tests__/SyncService.autoLogin.test.ts @@ -346,4 +346,43 @@ describe('SyncService - Auto-Login Integration', () => { expect(result).toBe(true); // Update available (version changed from '0') }); }); + + describe('cancel while a pull is in flight', () => { + test('rejects a second start until the cancelled run has finished', async () => { + let sawCancel = (_options: { isCancelled?: () => boolean }) => {}; + const started = new Promise<{ isCancelled?: () => boolean }>(resolve => { + sawCancel = resolve; + }); + + (isUnauthorizedError as jest.Mock).mockReturnValue(false); + (synkronusApi.syncObservations as jest.Mock).mockImplementation( + (_include: boolean, options: { isCancelled?: () => boolean }) => { + sawCancel(options); + return new Promise((_resolve, reject) => { + const id = setInterval(() => { + if (options.isCancelled?.()) { + clearInterval(id); + reject(new Error('Sync cancelled')); + } + }, 5); + }); + }, + ); + + const first = syncService.syncObservations(true); + await started; + expect(syncService.getIsSyncing()).toBe(true); + + syncService.cancelSync(); + await expect(syncService.syncObservations(true)).rejects.toThrow( + 'Sync already in progress', + ); + + await expect(first).rejects.toThrow('Sync cancelled'); + expect(syncService.getIsSyncing()).toBe(false); + + (synkronusApi.syncObservations as jest.Mock).mockResolvedValueOnce(7); + await expect(syncService.syncObservations(true)).resolves.toBe(7); + }); + }); }); From 0abf4aef96684bd08e30a4fe1df466c9adae3c3c Mon Sep 17 00:00:00 2001 From: Emil Rossing Date: Wed, 19 Aug 2026 17:01:42 +0200 Subject: [PATCH 32/32] chore: prep for v1.3.0 --- desktop/package.json | 2 +- desktop/src-tauri/Cargo.lock | 2 +- desktop/src-tauri/Cargo.toml | 2 +- desktop/src-tauri/tauri.conf.json | 2 +- desktop/src/lib/synkConstants.ts | 2 +- formulus/android/app/build.gradle | 4 ++-- formulus/ios/Formulus.xcodeproj/project.pbxproj | 8 ++++---- formulus/package-lock.json | 4 ++-- formulus/package.json | 2 +- synkronus-cli/internal/cmd/version.go | 2 +- synkronus-cli/versioninfo.json | 12 ++++++------ synkronus-portal/package.json | 2 +- 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/desktop/package.json b/desktop/package.json index 4b05504b8..15f9bab27 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,7 +1,7 @@ { "name": "ode-desktop", "private": true, - "version": "1.2.1", + "version": "1.3.0", "packageManager": "pnpm@10.33.2", "type": "module", "scripts": { diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index a7fb8b707..76cc24848 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "odedesktop" -version = "1.2.1" +version = "1.3.0" dependencies = [ "arrow", "chrono", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 625ba7b35..f562e59e1 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "odedesktop" -version = "1.2.1" +version = "1.3.0" description = "ODE Desktop" authors = ["OpenDataEnsemble.org"] edition = "2024" diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index 6675ea986..4fc2191da 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "ODE Desktop", - "version": "1.2.1", + "version": "1.3.0", "identifier": "org.opendataensemble.custodian", "build": { "beforeDevCommand": "pnpm dev", diff --git a/desktop/src/lib/synkConstants.ts b/desktop/src/lib/synkConstants.ts index e29860d88..eeb9acb0a 100644 --- a/desktop/src/lib/synkConstants.ts +++ b/desktop/src/lib/synkConstants.ts @@ -1,2 +1,2 @@ /** Must match Synkronus OpenAPI `x-ode-version` (semver). */ -export const SYNKRONUS_CLIENT_VERSION = '1.2.1'; +export const SYNKRONUS_CLIENT_VERSION = '1.3.0'; diff --git a/formulus/android/app/build.gradle b/formulus/android/app/build.gradle index 989202ff8..207054cdf 100644 --- a/formulus/android/app/build.gradle +++ b/formulus/android/app/build.gradle @@ -103,8 +103,8 @@ android { applicationId = "org.opendataensemble.formulus" minSdk = rootProject.ext.minSdkVersion targetSdk = rootProject.ext.targetSdkVersion - versionCode = 25 - versionName = "1.2.1" + versionCode = 35 + versionName = "1.3.0" buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", (findProperty("newArchEnabled") ?: "false").toString() diff --git a/formulus/ios/Formulus.xcodeproj/project.pbxproj b/formulus/ios/Formulus.xcodeproj/project.pbxproj index 6fa0d3b8d..778eef91c 100644 --- a/formulus/ios/Formulus.xcodeproj/project.pbxproj +++ b/formulus/ios/Formulus.xcodeproj/project.pbxproj @@ -318,7 +318,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_INCLUDE_ALL_APPICON_ASSETS = YES; CLANG_ENABLE_MODULES = YES; - CURRENT_PROJECT_VERSION = 24; + CURRENT_PROJECT_VERSION = 35; DEVELOPMENT_TEAM = 57WY3GA5K7; ENABLE_BITCODE = NO; INFOPLIST_FILE = Formulus/Info.plist; @@ -327,7 +327,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.2.1; + MARKETING_VERSION = 1.3.0; OTHER_LDFLAGS = ( "$(inherited)", "-ObjC", @@ -353,7 +353,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_INCLUDE_ALL_APPICON_ASSETS = YES; CLANG_ENABLE_MODULES = YES; - CURRENT_PROJECT_VERSION = 24; + CURRENT_PROJECT_VERSION = 35; DEVELOPMENT_TEAM = 57WY3GA5K7; INFOPLIST_FILE = Formulus/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 15.1; @@ -361,7 +361,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.2.1; + MARKETING_VERSION = 1.3.0; OTHER_LDFLAGS = ( "$(inherited)", "-ObjC", diff --git a/formulus/package-lock.json b/formulus/package-lock.json index 337527fc3..596974d7c 100644 --- a/formulus/package-lock.json +++ b/formulus/package-lock.json @@ -1,12 +1,12 @@ { "name": "formulus-app", - "version": "1.2.1", + "version": "1.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "formulus-app", - "version": "1.2.1", + "version": "1.3.0", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/formulus/package.json b/formulus/package.json index cd6063598..401e6ded3 100644 --- a/formulus/package.json +++ b/formulus/package.json @@ -1,6 +1,6 @@ { "name": "formulus-app", - "version": "1.2.1", + "version": "1.3.0", "packageManager": "pnpm@10.33.2", "type": "module", "private": true, diff --git a/synkronus-cli/internal/cmd/version.go b/synkronus-cli/internal/cmd/version.go index c8987a3f0..b82797977 100644 --- a/synkronus-cli/internal/cmd/version.go +++ b/synkronus-cli/internal/cmd/version.go @@ -11,7 +11,7 @@ import ( var ( // Version is the CLI version, set during build - Version = "1.2.1" + Version = "1.3.0" // BuildDate is the date when the CLI was built BuildDate = "unknown" // CommitHash is the git commit hash diff --git a/synkronus-cli/versioninfo.json b/synkronus-cli/versioninfo.json index 997ae840d..446d7adda 100644 --- a/synkronus-cli/versioninfo.json +++ b/synkronus-cli/versioninfo.json @@ -2,14 +2,14 @@ "FixedFileInfo": { "FileVersion": { "Major": 1, - "Minor": 2, - "Patch": 1, + "Minor": 3, + "Patch": 0, "Build": 0 }, "ProductVersion": { "Major": 1, - "Minor": 2, - "Patch": 1, + "Minor": 3, + "Patch": 0, "Build": 0 }, "FileFlagsMask": "3f", @@ -22,14 +22,14 @@ "Comments": "Synkronus CLI Tool", "CompanyName": "OpenDataEnsemble", "FileDescription": "Synkronus CLI - A command-line interface for the Synkronus API", - "FileVersion": "1.2.1.0", + "FileVersion": "1.3.0.0", "InternalName": "synk", "LegalCopyright": "© 2025 OpenDataEnsemble", "LegalTrademarks": "", "OriginalFilename": "synk.exe", "PrivateBuild": "", "ProductName": "Synkronus CLI", - "ProductVersion": "1.2.1.0", + "ProductVersion": "1.3.0.0", "SpecialBuild": "" }, "VarFileInfo": { diff --git a/synkronus-portal/package.json b/synkronus-portal/package.json index 9a7a58110..879934f46 100644 --- a/synkronus-portal/package.json +++ b/synkronus-portal/package.json @@ -1,7 +1,7 @@ { "name": "synkronus-portal", "private": true, - "version": "1.2.1", + "version": "1.3.0", "packageManager": "pnpm@10.33.2", "license": "MIT", "type": "module",