diff --git a/README.md b/README.md index e8f8d2f..4703aa8 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ make check - Train/Validation 중복·누락과 template leakage - 외국인등록번호·여권번호·전화번호 형태의 개인정보 패턴 - 6대 Workflow Runtime의 의존성·완료 증빙·HR Gate와 E2E 경로 18건 -- 전체 자동화 테스트 25개 +- 전체 자동화 테스트 27개 ## 저장소 구성 @@ -188,6 +188,7 @@ make check - [Intent 라벨 기준](fowoco-knowledge/docs/INTENT_DATA.md) - [Agent 연동 계약](fowoco-knowledge/docs/AGENT_INTEGRATION.md) - [6대 Workflow 공통 실행 계약](fowoco-knowledge/docs/WORKFLOW_RUNTIME.md) +- [범용 문서·Excel 처리 계약](fowoco-knowledge/docs/DOCUMENT_PROCESSING_CONTRACT.md) - [공식 데이터 파이프라인](fowoco-knowledge/docs/OFFICIAL_DATA_PIPELINE.md) - [E-9 신고·연장 Workflow](fowoco-knowledge/docs/E9_REPORTING_WORKFLOWS.md) - [모델 서빙 실행 방법](fowoco-knowledge/hr-intent-service/README.md) diff --git a/fowoco-knowledge/CHANGELOG.md b/fowoco-knowledge/CHANGELOG.md index 4336578..a8d9b27 100644 --- a/fowoco-knowledge/CHANGELOG.md +++ b/fowoco-knowledge/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- PDF·이미지·HWP·HWPX·XLSX·CSV 공통 Document IR과 Field Evidence 계약 추가 +- 공식 Template과 회사 전용 Tenant Template의 소유권·승격·격리 정책 추가 +- Excel 수식·cache·날짜·숨김·병합 Cell과 행별 오류 보존 규칙 및 6개 Fixture 추가 - 6대 Workflow의 Master/Subflow 관계, 공통 상태·이벤트·단계 의존성·완료 증빙 계약 추가 - Workflow별 정상·정보 누락·수동 검토 경로를 고정한 Runtime E2E Fixture 18건 추가 - 외부기관 처리와 민감 판단의 HR Gate, 병렬 READY 조건, 복합 Intent 분해 정책 검증 추가 diff --git a/fowoco-knowledge/README.md b/fowoco-knowledge/README.md index e16ef84..f6e4931 100644 --- a/fowoco-knowledge/README.md +++ b/fowoco-knowledge/README.md @@ -24,6 +24,8 @@ Slot·공식 출처·Guardrail을 조회하고, 데이터 계약과 교차참조 [`docs/MODEL_ARTIFACT_POLICY.md`](docs/MODEL_ARTIFACT_POLICY.md)를 따릅니다. 여섯 대표 업무의 단계 의존성·완료 증빙·HR 승인 계약은 [`docs/WORKFLOW_RUNTIME.md`](docs/WORKFLOW_RUNTIME.md)를 기준으로 공유합니다. +문서 Parser와 OCR이 공통으로 반환할 IR, Template 격리, Excel 정규화 기준은 +[`docs/DOCUMENT_PROCESSING_CONTRACT.md`](docs/DOCUMENT_PROCESSING_CONTRACT.md)를 따릅니다. ## 데이터 기준 diff --git a/fowoco-knowledge/data/evaluation/document_ir_cases.jsonl b/fowoco-knowledge/data/evaluation/document_ir_cases.jsonl new file mode 100644 index 0000000..72a31d5 --- /dev/null +++ b/fowoco-knowledge/data/evaluation/document_ir_cases.jsonl @@ -0,0 +1,6 @@ +{"contract_version":"1.0.0","document_ref":"DOC-PDF-001","company_ref":"COMPANY-DEMO","source_file":{"file_name":"contract.pdf","format":"PDF","media_type":"application/pdf","sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","bytes":12000},"role":"EVIDENCE","document_type":"STANDARD_EMPLOYMENT_CONTRACT","processing_status":"REVIEW_REQUIRED","parser_route":"KNOWN_TEMPLATE","template":{"scope":"OFFICIAL","template_id":"TPL-CONTRACT-001","owner_company_ref":null,"version":"2026.1","visible_to_other_companies":true},"fields":[{"field_key":"contract.end_date","data_type":"DATE","raw_value":"2027. 01. 31","normalized_value":"2027-01-31","extractor":{"name":"pdf-text-locator","version":"1.0.0","mode":"PARSER"},"confidence":1.0,"review_status":"UNREVIEWED","evidence":{"status":"AVAILABLE","source_text":"계약 종료일 2027. 01. 31","page":1,"bounding_box":[120,340,180,24]}}],"errors":[],"requires_human_review":true,"automatic_persistence_allowed":false} +{"contract_version":"1.0.0","document_ref":"DOC-IMAGE-001","company_ref":"COMPANY-DEMO","source_file":{"file_name":"passport.jpg","format":"IMAGE","media_type":"image/jpeg","sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","bytes":320000},"role":"EVIDENCE","document_type":"PASSPORT_COPY","processing_status":"REVIEW_REQUIRED","parser_route":"SIMILAR_TEMPLATE","template":{"scope":"TENANT","template_id":"TPL-COMPANY-DEMO-PASSPORT-001","owner_company_ref":"COMPANY-DEMO","version":"1","visible_to_other_companies":false},"fields":[{"field_key":"passport.expiry_date","data_type":"DATE","raw_value":"15 SEP 2028","normalized_value":"2028-09-15","extractor":{"name":"clova-template-ocr","version":"2026.08","mode":"OCR"},"confidence":0.91,"review_status":"UNREVIEWED","evidence":{"status":"AVAILABLE","source_text":"15 SEP 2028","page":1,"bounding_box":[420,510,145,31]}}],"errors":[{"code":"SIMILAR_LAYOUT","scope":"DOCUMENT","location":null,"recoverable":true,"message":"등록 양식과 위치가 달라 HR 확인이 필요합니다."}],"requires_human_review":true,"automatic_persistence_allowed":false} +{"contract_version":"1.0.0","document_ref":"DOC-HWP-001","company_ref":"COMPANY-DEMO","source_file":{"file_name":"employment-guide.hwp","format":"HWP","media_type":"application/x-hwp","sha256":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","bytes":48000},"role":"REFERENCE","document_type":"STANDARD_EMPLOYMENT_CONTRACT","processing_status":"REVIEW_REQUIRED","parser_route":"KNOWN_TEMPLATE","template":{"scope":"OFFICIAL","template_id":"TPL-HWP-CONTRACT-GUIDE-001","owner_company_ref":null,"version":"2026.1","visible_to_other_companies":true},"fields":[{"field_key":"contract.wage","data_type":"MONEY","raw_value":null,"normalized_value":null,"extractor":{"name":"hwp-structure-parser","version":"1.0.0","mode":"PARSER"},"confidence":null,"review_status":"UNAVAILABLE","evidence":{"status":"UNAVAILABLE","unavailable_reason":"참고 문서이며 실제 계약 금액 칸이 비어 있습니다."}}],"errors":[],"requires_human_review":true,"automatic_persistence_allowed":false} +{"contract_version":"1.0.0","document_ref":"DOC-HWPX-001","company_ref":"COMPANY-DEMO","source_file":{"file_name":"standard-contract.hwpx","format":"HWPX","media_type":"application/hwp+zip","sha256":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd","bytes":56000},"role":"FORM_TEMPLATE","document_type":"STANDARD_EMPLOYMENT_CONTRACT","processing_status":"REVIEW_REQUIRED","parser_route":"KNOWN_TEMPLATE","template":{"scope":"OFFICIAL","template_id":"TPL-HWPX-CONTRACT-001","owner_company_ref":null,"version":"2026.1","visible_to_other_companies":true},"fields":[{"field_key":"worker.display_name","data_type":"STRING","raw_value":"{{worker_name}}","normalized_value":"worker.display_name","extractor":{"name":"hwpx-xml-locator","version":"1.1.0","mode":"RULE"},"confidence":1.0,"review_status":"UNREVIEWED","evidence":{"status":"AVAILABLE","source_text":"{{worker_name}}","page":1,"bounding_box":[100,140,210,22]}}],"errors":[],"requires_human_review":true,"automatic_persistence_allowed":false} +{"contract_version":"1.0.0","document_ref":"DOC-XLSX-001","company_ref":"COMPANY-DEMO","source_file":{"file_name":"worker-import.xlsx","format":"XLSX","media_type":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","sha256":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","bytes":41000},"role":"EVIDENCE","document_type":"WORKER_IMPORT_SHEET","processing_status":"REVIEW_REQUIRED","parser_route":"KNOWN_TEMPLATE","template":{"scope":"TENANT","template_id":"TPL-COMPANY-DEMO-IMPORT-001","owner_company_ref":"COMPANY-DEMO","version":"3","visible_to_other_companies":false},"fields":[{"field_key":"worker.stay_expiry_date","data_type":"DATE","raw_value":46389,"normalized_value":"2027-01-02","extractor":{"name":"poi-xlsx-parser","version":"1.0.0","mode":"PARSER"},"confidence":1.0,"review_status":"UNREVIEWED","evidence":{"status":"AVAILABLE","source_text":"1/2/27","sheet_name":"근로자","cell_range":"C2"},"value_metadata":{"formula":null,"cached_value":46389,"recalculated":false,"original_number_format":"m/d/yy"}},{"field_key":"worker.monthly_wage","data_type":"MONEY","raw_value":"=D2+E2","normalized_value":2500000,"extractor":{"name":"poi-xlsx-parser","version":"1.0.0","mode":"PARSER"},"confidence":1.0,"review_status":"UNREVIEWED","evidence":{"status":"AVAILABLE","source_text":"=D2+E2","sheet_name":"근로자","cell_range":"F2"},"value_metadata":{"formula":"=D2+E2","cached_value":2500000,"recalculated":false,"original_number_format":"#,##0"}}],"spreadsheet":{"selected_sheet":"근로자","header_row":1,"hidden_rows":[5],"hidden_columns":["G"],"merged_ranges":["A1:B1"],"formula_policy":"PRESERVE_FORMULA_AND_CACHE","row_results":[{"row_number":2,"status":"VALID","error_codes":[]},{"row_number":3,"status":"REVIEW_REQUIRED","error_codes":["INVALID_DATE"]}]},"errors":[{"code":"INVALID_DATE","scope":"ROW","location":"근로자!C3","recoverable":true,"message":"3행 날짜를 확인해야 하며 2행은 계속 등록할 수 있습니다."}],"requires_human_review":true,"automatic_persistence_allowed":false} +{"contract_version":"1.0.0","document_ref":"DOC-CSV-001","company_ref":"COMPANY-DEMO","source_file":{"file_name":"unknown-workers.csv","format":"CSV","media_type":"text/csv","sha256":"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","bytes":8200},"role":"UNKNOWN","document_type":"UNKNOWN","processing_status":"REVIEW_REQUIRED","parser_route":"UNCLASSIFIED","template":{"scope":"NONE","template_id":null,"owner_company_ref":null,"version":null,"visible_to_other_companies":false},"fields":[{"field_key":"worker.display_name","data_type":"STRING","raw_value":null,"normalized_value":null,"extractor":{"name":"csv-detector","version":"1.0.0","mode":"PARSER"},"confidence":null,"review_status":"UNAVAILABLE","evidence":{"status":"UNAVAILABLE","unavailable_reason":"이름으로 확정할 Header를 찾지 못했습니다."}}],"spreadsheet":{"selected_sheet":"CSV","header_row":1,"hidden_rows":[],"hidden_columns":[],"merged_ranges":[],"formula_policy":"NOT_APPLICABLE","csv_detection":{"encoding":"CP949","delimiter":";","line_ending":"CRLF"},"row_results":[{"row_number":2,"status":"REJECTED","error_codes":["HEADER_NOT_MAPPED"]}]},"errors":[{"code":"HEADER_NOT_MAPPED","scope":"DOCUMENT","location":"row 1","recoverable":true,"message":"Header를 수동으로 연결해야 합니다."}],"requires_human_review":true,"automatic_persistence_allowed":false} diff --git a/fowoco-knowledge/data/evaluation/document_ir_manifest.yaml b/fowoco-knowledge/data/evaluation/document_ir_manifest.yaml new file mode 100644 index 0000000..38c2c17 --- /dev/null +++ b/fowoco-knowledge/data/evaluation/document_ir_manifest.yaml @@ -0,0 +1,9 @@ +dataset_id: FOWOCO-DOCUMENT-IR-GOLDEN +version: 1.0.0 +path: data/evaluation/document_ir_cases.jsonl +schema: schemas/document-ir.schema.json +field_schema: schemas/document-field-candidate.schema.json +record_count: 6 +sha256: b6e6d84d48d8248ddad7f8afd13ea23ac738a15c0723d4c0c7f0aa91f3e6eabc +required_formats: [PDF, IMAGE, HWP, HWPX, XLSX, CSV] +contains_real_personal_data: false diff --git a/fowoco-knowledge/docs/DOCUMENT_PROCESSING_CONTRACT.md b/fowoco-knowledge/docs/DOCUMENT_PROCESSING_CONTRACT.md new file mode 100644 index 0000000..7e63d17 --- /dev/null +++ b/fowoco-knowledge/docs/DOCUMENT_PROCESSING_CONTRACT.md @@ -0,0 +1,64 @@ +# 범용 문서·Excel 처리 계약 + +## 목표 + +PDF·이미지·HWP·HWPX·XLSX·CSV Parser가 제각각인 결과를 내더라도 Server와 Client는 +하나의 `Document IR`만 읽는다. 알려진 양식은 근거가 표시된 후보값을 만들고, 새 양식과 +처리 불가 문서는 업무 DB에 자동 반영하지 않고 HR 검토로 전환한다. + +## 공통 흐름 + +```text +업로드 + → 형식·암호화·용량 검사 + → 문서 역할·유형·Template 분류 + → Parser/OCR로 필드 후보와 원본 근거 생성 + → Schema·Tenant·민감도 검증 + → HR 수정·승인 + → 승인된 값만 업무에 반영 +``` + +## 지원 수준 + +| 형식 | 현재 계약 | 자동 처리 기준 | 수동 전환 기준 | +| --- | --- | --- | --- | +| PDF | 텍스트·페이지·좌표 | 승인된 공식/사업장 Template | 암호화·스캔 품질 저하·미분류 | +| 이미지 | OCR text·좌표·confidence | 승인된 OCR Template | 낮은 confidence·유사/미분류 layout | +| HWP | 구조·텍스트 후보 | 승인 Parser와 Template | 손상·암호화·지원하지 않는 control | +| HWPX | XML locator·Template field | 승인된 Template version | locator 불일치·layout fingerprint 변경 | +| XLSX | Sheet·Header·Cell·수식/cache | Header mapping과 행 검증 통과 | Sheet/Header 미탐지·행별 오류 | +| CSV | 인코딩·구분자·개행·행 | Header mapping 통과 | 인코딩/구분자 불명확·Header 미탐지 | + +이 표는 “모든 파일을 정확히 읽는다”는 보장이 아니다. 실제 Parser 지원 여부와 품질은 +AI·Server 구현 및 Smoke Test 결과로 별도 표시한다. + +## 필드 근거 + +모든 후보 필드는 다음 중 하나를 가져야 한다. + +- `AVAILABLE`: 원문 문자열과 페이지 또는 Sheet/Cell 위치 +- `UNAVAILABLE`: 값을 만들 수 없는 구체적인 이유 + +AI가 추측한 값만 있거나 좌표·Cell 근거가 없는 값은 승인 후보가 될 수 없다. HR의 +수정값은 원본 후보와 분리해 감사 가능한 상태로 저장한다. + +## Template 소유권 + +- `OFFICIAL`: FOWOCO가 공식 출처·locator test·HR 검토를 거쳐 전 사업장에 배포 +- `TENANT`: 한 회사가 검토·승격하며 `owner_company_ref`가 같은 회사에만 노출 +- AI가 발견한 유사 양식은 항상 `DRAFT`이며 자동으로 Active가 되지 않음 +- Active version은 수정하지 않고 새 version으로 승격 + +## Excel·CSV 규칙 + +- 숨김 Sheet/행/열과 병합 Cell을 제거하지 않고 메타데이터에 기록 +- 날짜 serial, 금액, 백분율, 문자형 숫자의 원본값과 정규화값을 함께 보존 +- 수식 문자열과 cached value를 보존하고 Parser가 임의 재계산하지 않음 +- 오류 행은 `REVIEW_REQUIRED` 또는 `REJECTED`로 분리하고 정상 행은 유지 +- 암호화 파일, 읽을 Sheet 없음, Header 미탐지는 전체 수동 처리 + +## 팀 공통 Contract Test + +`data/evaluation/document_ir_cases.jsonl`의 6개 비식별 Fixture를 AI·Server·Client가 +공유한다. 새 Parser가 추가돼도 반환 결과가 이 Schema를 통과해야 한다. 실제 파일 +미리보기·다운로드 구현은 Server 범위이며 이 계약은 결과 데이터 형식만 소유한다. diff --git a/fowoco-knowledge/knowledge/company_template_policy.yaml b/fowoco-knowledge/knowledge/company_template_policy.yaml new file mode 100644 index 0000000..d69365e --- /dev/null +++ b/fowoco-knowledge/knowledge/company_template_policy.yaml @@ -0,0 +1,26 @@ +version: 1.0.0 +scopes: + OFFICIAL: + owner: FOWOCO + visibility: ALL_COMPANIES + activation_requires: [source_verified, locator_test_passed, hr_approval] + TENANT: + owner: COMPANY + visibility: OWNER_COMPANY_ONLY + activation_requires: [company_id_bound, sample_reviewed, locator_test_passed, company_hr_approval] +lifecycle: [DRAFT, REVIEW_REQUIRED, ACTIVE, RETIRED] +promotion_rules: + ai_proposal_starts_as: DRAFT + active_requires_human: true + tenant_to_official_automatic_promotion: false + retired_template_new_processing: false +versioning: + immutable_active_version: true + layout_fingerprint_required: true + source_file_sha256_required: true + knowledge_version_required: true + parser_rule_version_required: true +tenant_isolation: + owner_company_ref_required: true + cross_company_search_forbidden: true + cross_company_fixture_expected_result: NOT_FOUND diff --git a/fowoco-knowledge/knowledge/document_roles.yaml b/fowoco-knowledge/knowledge/document_roles.yaml new file mode 100644 index 0000000..a979c90 --- /dev/null +++ b/fowoco-knowledge/knowledge/document_roles.yaml @@ -0,0 +1,27 @@ +version: 1.0.0 +roles: + EVIDENCE: 업무 사실과 상태를 확인하는 제출 문서 + FORM_TEMPLATE: 값을 채워 새 문서를 생성하는 승인된 양식 + REFERENCE: 사람이 내용을 참고하지만 자동 입력에는 사용하지 않는 자료 + GENERATED_OUTPUT: 승인된 Template과 값으로 생성한 결과물 + COMMUNICATION_ATTACHMENT: 근로자·HR 소통 과정의 첨부파일 + UNKNOWN: 역할을 분류하지 못해 수동 검토가 필요한 문서 +processing_states: [UPLOADED, SCANNING, PARSING, CLASSIFYING, REVIEW_REQUIRED, VERIFIED, FAILED] +parser_routes: + KNOWN_TEMPLATE: + next_state: REVIEW_REQUIRED + automatic_persistence_allowed: false + SIMILAR_TEMPLATE: + next_state: REVIEW_REQUIRED + automatic_persistence_allowed: false + UNCLASSIFIED: + next_state: REVIEW_REQUIRED + automatic_persistence_allowed: false + UNSUPPORTED: + next_state: FAILED + automatic_persistence_allowed: false +policy: + unknown_document_never_updates_business_data: true + field_candidates_require_evidence_or_unavailable_reason: true + verified_state_requires_hr_review: true + generated_output_never_auto_sent: true diff --git a/fowoco-knowledge/knowledge/document_type_catalog.yaml b/fowoco-knowledge/knowledge/document_type_catalog.yaml new file mode 100644 index 0000000..1433d76 --- /dev/null +++ b/fowoco-knowledge/knowledge/document_type_catalog.yaml @@ -0,0 +1,38 @@ +version: 1.0.0 +document_types: + - id: PASSPORT_COPY + display_name_ko: 여권 사본 + allowed_roles: [EVIDENCE] + formats: [PDF, IMAGE] + sensitivity: critical + required_review: true + - id: ALIEN_REGISTRATION_CARD_COPY + display_name_ko: 외국인등록증 사본 + allowed_roles: [EVIDENCE] + formats: [PDF, IMAGE] + sensitivity: critical + required_review: true + - id: STANDARD_EMPLOYMENT_CONTRACT + display_name_ko: 표준근로계약서 + allowed_roles: [EVIDENCE, FORM_TEMPLATE, REFERENCE, GENERATED_OUTPUT] + formats: [PDF, HWP, HWPX] + sensitivity: high + required_review: true + - id: PAYSLIP + display_name_ko: 급여명세서 + allowed_roles: [EVIDENCE] + formats: [PDF, IMAGE, XLSX] + sensitivity: critical + required_review: true + - id: WORKER_IMPORT_SHEET + display_name_ko: 근로자 일괄 등록표 + allowed_roles: [EVIDENCE, REFERENCE] + formats: [XLSX, CSV] + sensitivity: high + required_review: true + - id: UNKNOWN + display_name_ko: 분류되지 않은 문서 + allowed_roles: [UNKNOWN] + formats: [PDF, IMAGE, HWP, HWPX, XLSX, CSV] + sensitivity: high + required_review: true diff --git a/fowoco-knowledge/knowledge/manifest.yaml b/fowoco-knowledge/knowledge/manifest.yaml index a544adf..489acd7 100644 --- a/fowoco-knowledge/knowledge/manifest.yaml +++ b/fowoco-knowledge/knowledge/manifest.yaml @@ -27,6 +27,10 @@ files: multilingual_templates: knowledge/multilingual_templates.yaml guardrails: knowledge/guardrail_rules.yaml workflow_runtime: knowledge/workflow_runtime.yaml + document_roles: knowledge/document_roles.yaml + document_types: knowledge/document_type_catalog.yaml + company_template_policy: knowledge/company_template_policy.yaml + spreadsheet_normalization: knowledge/spreadsheet_normalization.yaml datasets: source_manifest: data/external/source_manifest.yaml processed_manifest: data/processed/manifest.yaml @@ -41,6 +45,8 @@ datasets: demo_expiry_renewal: data/demo/expiry_renewal_golden.yaml workflow_runtime_manifest: data/evaluation/workflow_runtime_manifest.yaml workflow_runtime_cases: data/evaluation/workflow_runtime_cases.jsonl + document_ir_manifest: data/evaluation/document_ir_manifest.yaml + document_ir_cases: data/evaluation/document_ir_cases.jsonl artifacts: intent_models: hr-intent-service/models/artifact-manifest.yaml intent_model_schema: schemas/model-artifact-manifest.schema.json diff --git a/fowoco-knowledge/knowledge/spreadsheet_normalization.yaml b/fowoco-knowledge/knowledge/spreadsheet_normalization.yaml new file mode 100644 index 0000000..93d74d0 --- /dev/null +++ b/fowoco-knowledge/knowledge/spreadsheet_normalization.yaml @@ -0,0 +1,32 @@ +version: 1.0.0 +sheet_selection: + ignore_fully_hidden_sheets: true + prefer_sheet_with_required_headers: true + multiple_candidate_sheets: REVIEW_REQUIRED +header_detection: + scan_first_rows: 30 + normalize_whitespace: true + duplicate_header: REVIEW_REQUIRED + merged_header: EXPAND_LABEL_WITH_SOURCE_RANGE +structure: + preserve_hidden_rows_and_columns: true + skip_fully_empty_rows: true + keep_source_row_number: true +values: + date_serial: PRESERVE_RAW_AND_NORMALIZE_ISO_DATE + currency: PRESERVE_RAW_AND_NORMALIZE_DECIMAL + percentage: PRESERVE_RAW_AND_NORMALIZE_RATIO + numeric_text: PRESERVE_RAW_AND_NORMALIZE_WHEN_UNAMBIGUOUS +formula: + preserve_formula: true + preserve_cached_value: true + recalculate: false + record_recalculated_flag: true +csv: + detect_encodings: [UTF-8, UTF-8-BOM, CP949] + detect_delimiters: [COMMA, SEMICOLON, TAB, PIPE] + preserve_line_ending: true +row_error_policy: + valid_rows_survive_other_row_errors: true + row_statuses: [VALID, REVIEW_REQUIRED, REJECTED] + full_import_failure_only_for: [UNSUPPORTED_FORMAT, ENCRYPTED_FILE, NO_READABLE_SHEET, HEADER_NOT_FOUND] diff --git a/fowoco-knowledge/schemas/document-field-candidate.schema.json b/fowoco-knowledge/schemas/document-field-candidate.schema.json new file mode 100644 index 0000000..0ba53a4 --- /dev/null +++ b/fowoco-knowledge/schemas/document-field-candidate.schema.json @@ -0,0 +1,98 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://fowoco.dev/schemas/document-field-candidate.schema.json", + "title": "Document field candidate", + "type": "object", + "additionalProperties": false, + "required": ["field_key", "data_type", "raw_value", "normalized_value", "extractor", "confidence", "review_status", "evidence"], + "properties": { + "field_key": {"type": "string", "pattern": "^[a-z][a-z0-9_.]+$"}, + "data_type": {"enum": ["STRING", "DATE", "DATETIME", "NUMBER", "MONEY", "PERCENTAGE", "BOOLEAN"]}, + "raw_value": {"type": ["string", "number", "boolean", "null"]}, + "normalized_value": {"type": ["string", "number", "boolean", "null"]}, + "extractor": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version", "mode"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "mode": {"enum": ["RULE", "OCR", "PARSER", "AI", "MANUAL"]} + } + }, + "confidence": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "review_status": {"enum": ["UNREVIEWED", "ACCEPTED", "CORRECTED", "REJECTED", "UNAVAILABLE"]}, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["status"], + "properties": { + "status": {"enum": ["AVAILABLE", "UNAVAILABLE"]}, + "source_text": {"type": "string"}, + "page": {"type": "integer", "minimum": 1}, + "sheet_name": {"type": "string", "minLength": 1}, + "cell_range": {"type": "string", "pattern": "^[A-Z]+[1-9][0-9]*(?::[A-Z]+[1-9][0-9]*)?$"}, + "bounding_box": { + "type": "array", + "prefixItems": [ + {"type": "number", "minimum": 0}, + {"type": "number", "minimum": 0}, + {"type": "number", "exclusiveMinimum": 0}, + {"type": "number", "exclusiveMinimum": 0} + ], + "minItems": 4, + "maxItems": 4 + }, + "unavailable_reason": {"type": "string", "minLength": 3} + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "AVAILABLE"}}}, + "then": { + "required": ["source_text"], + "anyOf": [ + {"required": ["page"]}, + {"required": ["sheet_name", "cell_range"]} + ] + } + }, + { + "if": {"properties": {"status": {"const": "UNAVAILABLE"}}}, + "then": { + "required": ["unavailable_reason"], + "properties": { + "source_text": false, + "page": false, + "sheet_name": false, + "cell_range": false, + "bounding_box": false + } + } + } + ] + }, + "value_metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "formula": {"type": ["string", "null"]}, + "cached_value": {"type": ["string", "number", "boolean", "null"]}, + "recalculated": {"type": "boolean"}, + "original_number_format": {"type": ["string", "null"]} + } + } + }, + "allOf": [ + { + "if": {"properties": {"review_status": {"const": "UNAVAILABLE"}}}, + "then": { + "properties": { + "raw_value": {"const": null}, + "normalized_value": {"const": null}, + "confidence": {"const": null}, + "evidence": {"properties": {"status": {"const": "UNAVAILABLE"}}} + } + } + } + ] +} diff --git a/fowoco-knowledge/schemas/document-ir.schema.json b/fowoco-knowledge/schemas/document-ir.schema.json new file mode 100644 index 0000000..6743965 --- /dev/null +++ b/fowoco-knowledge/schemas/document-ir.schema.json @@ -0,0 +1,153 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://fowoco.dev/schemas/document-ir.schema.json", + "title": "FOWOCO Document Intermediate Representation", + "type": "object", + "additionalProperties": false, + "required": ["contract_version", "document_ref", "company_ref", "source_file", "role", "document_type", "processing_status", "parser_route", "template", "fields", "errors", "requires_human_review", "automatic_persistence_allowed"], + "properties": { + "contract_version": {"const": "1.0.0"}, + "document_ref": {"type": "string", "pattern": "^DOC-[A-Z0-9-]+$"}, + "company_ref": {"type": "string", "pattern": "^COMPANY-[A-Z0-9-]+$"}, + "source_file": { + "type": "object", + "additionalProperties": false, + "required": ["file_name", "format", "media_type", "sha256", "bytes"], + "properties": { + "file_name": {"type": "string", "minLength": 1}, + "format": {"enum": ["PDF", "IMAGE", "HWP", "HWPX", "XLSX", "CSV"]}, + "media_type": {"type": "string", "minLength": 3}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "bytes": {"type": "integer", "minimum": 1} + } + }, + "role": {"enum": ["EVIDENCE", "FORM_TEMPLATE", "REFERENCE", "GENERATED_OUTPUT", "COMMUNICATION_ATTACHMENT", "UNKNOWN"]}, + "document_type": {"type": "string", "pattern": "^[A-Z][A-Z0-9_]+$"}, + "processing_status": {"enum": ["UPLOADED", "SCANNING", "PARSING", "CLASSIFYING", "REVIEW_REQUIRED", "VERIFIED", "FAILED"]}, + "parser_route": {"enum": ["KNOWN_TEMPLATE", "SIMILAR_TEMPLATE", "UNCLASSIFIED", "UNSUPPORTED"]}, + "template": { + "type": "object", + "additionalProperties": false, + "required": ["scope", "template_id", "owner_company_ref", "version", "visible_to_other_companies"], + "properties": { + "scope": {"enum": ["OFFICIAL", "TENANT", "NONE"]}, + "template_id": {"type": ["string", "null"]}, + "owner_company_ref": {"type": ["string", "null"], "pattern": "^COMPANY-[A-Z0-9-]+$"}, + "version": {"type": ["string", "null"]}, + "visible_to_other_companies": {"type": "boolean"} + }, + "allOf": [ + { + "if": {"properties": {"scope": {"const": "TENANT"}}}, + "then": { + "properties": { + "template_id": {"type": "string", "minLength": 1}, + "owner_company_ref": {"type": "string", "pattern": "^COMPANY-[A-Z0-9-]+$"}, + "version": {"type": "string", "minLength": 1}, + "visible_to_other_companies": {"const": false} + }, + "required": ["owner_company_ref", "template_id", "version"] + } + }, + { + "if": {"properties": {"scope": {"const": "OFFICIAL"}}}, + "then": { + "properties": { + "template_id": {"type": "string", "minLength": 1}, + "owner_company_ref": {"const": null}, + "version": {"type": "string", "minLength": 1}, + "visible_to_other_companies": {"const": true} + } + } + }, + { + "if": {"properties": {"scope": {"const": "NONE"}}}, + "then": { + "properties": { + "template_id": {"const": null}, + "owner_company_ref": {"const": null}, + "version": {"const": null}, + "visible_to_other_companies": {"const": false} + } + } + } + ] + }, + "fields": { + "type": "array", + "items": {"$ref": "https://fowoco.dev/schemas/document-field-candidate.schema.json"} + }, + "spreadsheet": { + "type": "object", + "additionalProperties": false, + "required": ["selected_sheet", "header_row", "hidden_rows", "hidden_columns", "merged_ranges", "formula_policy", "row_results"], + "properties": { + "selected_sheet": {"type": "string", "minLength": 1}, + "header_row": {"type": "integer", "minimum": 1}, + "hidden_rows": {"type": "array", "items": {"type": "integer", "minimum": 1}}, + "hidden_columns": {"type": "array", "items": {"type": "string", "pattern": "^[A-Z]+$"}}, + "merged_ranges": {"type": "array", "items": {"type": "string"}}, + "formula_policy": {"enum": ["PRESERVE_FORMULA_AND_CACHE", "NOT_APPLICABLE"]}, + "csv_detection": { + "type": "object", + "additionalProperties": false, + "required": ["encoding", "delimiter", "line_ending"], + "properties": { + "encoding": {"type": "string"}, + "delimiter": {"enum": [",", ";", "\\t", "|"]}, + "line_ending": {"enum": ["LF", "CRLF", "CR"]} + } + }, + "row_results": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["row_number", "status", "error_codes"], + "properties": { + "row_number": {"type": "integer", "minimum": 1}, + "status": {"enum": ["VALID", "REVIEW_REQUIRED", "REJECTED"]}, + "error_codes": {"type": "array", "uniqueItems": true, "items": {"type": "string"}} + } + } + } + } + }, + "errors": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "scope", "location", "recoverable", "message"], + "properties": { + "code": {"type": "string", "pattern": "^[A-Z][A-Z0-9_]+$"}, + "scope": {"enum": ["DOCUMENT", "PAGE", "SHEET", "ROW", "FIELD"]}, + "location": {"type": ["string", "null"]}, + "recoverable": {"type": "boolean"}, + "message": {"type": "string", "minLength": 3} + } + } + }, + "requires_human_review": {"type": "boolean"}, + "automatic_persistence_allowed": {"type": "boolean"} + }, + "allOf": [ + { + "if": {"properties": {"source_file": {"properties": {"format": {"enum": ["XLSX", "CSV"]}}}}}, + "then": {"required": ["spreadsheet"]} + }, + { + "if": {"properties": {"parser_route": {"enum": ["UNCLASSIFIED", "UNSUPPORTED"]}}}, + "then": { + "properties": { + "requires_human_review": {"const": true}, + "automatic_persistence_allowed": {"const": false} + } + } + }, + { + "if": {"properties": {"processing_status": {"const": "FAILED"}}}, + "then": {"properties": {"automatic_persistence_allowed": {"const": false}}} + } + ] +} diff --git a/fowoco-knowledge/src/fowoco_knowledge/repository.py b/fowoco-knowledge/src/fowoco_knowledge/repository.py index a9a6777..72ddc5c 100644 --- a/fowoco-knowledge/src/fowoco_knowledge/repository.py +++ b/fowoco-knowledge/src/fowoco_knowledge/repository.py @@ -87,6 +87,15 @@ def list_workflows(self) -> list[dict[str, Any]]: def list_case_templates(self) -> list[dict[str, Any]]: return self.load_yaml("knowledge/workflow_catalog.yaml")["case_templates"] + def list_document_types(self) -> list[dict[str, Any]]: + return self.load_yaml("knowledge/document_type_catalog.yaml")["document_types"] + + def get_document_type(self, document_type: str) -> dict[str, Any]: + for definition in self.list_document_types(): + if definition["id"] == document_type: + return definition + raise KeyError(f"Unknown document_type: {document_type}") + def get_case_template(self, case_template_id: str) -> dict[str, Any]: for template in self.list_case_templates(): if template["id"] == case_template_id: diff --git a/fowoco-knowledge/src/fowoco_knowledge/validation.py b/fowoco-knowledge/src/fowoco_knowledge/validation.py index fdc5099..a54d137 100644 --- a/fowoco-knowledge/src/fowoco_knowledge/validation.py +++ b/fowoco-knowledge/src/fowoco_knowledge/validation.py @@ -8,6 +8,7 @@ from typing import Any from jsonschema import Draft202012Validator +from referencing import Registry, Resource from .ingestion import file_sha256 from .repository import KnowledgeRepository @@ -114,6 +115,7 @@ def validate_all(self) -> list[str]: self._validate_processed_datasets() self._validate_workflow_schema() self._validate_workflow_runtime() + self._validate_document_contracts() self._validate_required_slot_contracts() self._validate_cross_references() self._validate_seed_data() @@ -420,6 +422,132 @@ def _validate_workflow_runtime(self) -> None: if profile_paths != required_paths: self.errors.append(f"workflow runtime fixtures: {profile_id} paths are incomplete") + def _validate_document_contracts(self) -> None: + roles = self.repository.load_yaml("knowledge/document_roles.yaml") + expected_roles = { + "EVIDENCE", + "FORM_TEMPLATE", + "REFERENCE", + "GENERATED_OUTPUT", + "COMMUNICATION_ATTACHMENT", + "UNKNOWN", + } + expected_states = { + "UPLOADED", + "SCANNING", + "PARSING", + "CLASSIFYING", + "REVIEW_REQUIRED", + "VERIFIED", + "FAILED", + } + if set(roles["roles"]) != expected_roles: + self.errors.append("document contract: role catalog mismatch") + if set(roles["processing_states"]) != expected_states: + self.errors.append("document contract: processing state catalog mismatch") + if any(route["automatic_persistence_allowed"] for route in roles["parser_routes"].values()): + self.errors.append("document contract: parser route cannot auto-persist") + + type_catalog = self.repository.load_yaml("knowledge/document_type_catalog.yaml") + document_types = {item["id"]: item for item in type_catalog["document_types"]} + if len(document_types) != len(type_catalog["document_types"]): + self.errors.append("document contract: duplicate document type") + for document_type, definition in document_types.items(): + if not set(definition["allowed_roles"]) <= expected_roles: + self.errors.append(f"document contract {document_type}: unknown allowed role") + + template_policy = self.repository.load_yaml("knowledge/company_template_policy.yaml") + if template_policy["scopes"]["TENANT"]["visibility"] != "OWNER_COMPANY_ONLY": + self.errors.append("document contract: tenant template must be company-only") + if not template_policy["tenant_isolation"]["cross_company_search_forbidden"]: + self.errors.append("document contract: cross-company template search must be blocked") + + spreadsheet_policy = self.repository.load_yaml("knowledge/spreadsheet_normalization.yaml") + if spreadsheet_policy["formula"]["recalculate"]: + self.errors.append("document contract: parser must not recalculate formulas") + if not spreadsheet_policy["row_error_policy"]["valid_rows_survive_other_row_errors"]: + self.errors.append("document contract: valid spreadsheet rows must be preserved") + + manifest = self.repository.load_yaml("data/evaluation/document_ir_manifest.yaml") + fixture_path = self.repository.root / manifest["path"] + if file_sha256(fixture_path) != manifest["sha256"]: + self.errors.append("document IR fixtures: checksum mismatch") + field_schema = self.repository.load_json(manifest["field_schema"]) + ir_schema = self.repository.load_json(manifest["schema"]) + registry = Registry().with_resource( + field_schema["$id"], Resource.from_contents(field_schema) + ) + validator = Draft202012Validator(ir_schema, registry=registry) + + fixtures: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + for line_number, raw_line in enumerate( + fixture_path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not raw_line.strip(): + continue + try: + document = json.loads(raw_line) + except json.JSONDecodeError as exc: + self.errors.append(f"document IR line {line_number}: invalid JSON ({exc})") + continue + fixtures.append(document) + for error in validator.iter_errors(document): + path = ".".join(str(item) for item in error.path) + self.errors.append(f"document IR line {line_number} [{path}]: {error.message}") + + document_ref = document.get("document_ref") + if document_ref in seen_ids: + self.errors.append(f"document IR line {line_number}: duplicate {document_ref}") + seen_ids.add(document_ref) + definition = document_types.get(document.get("document_type")) + if not definition: + self.errors.append(f"document IR line {line_number}: unknown document type") + else: + if document.get("role") not in definition["allowed_roles"]: + self.errors.append( + f"document IR line {line_number}: role not allowed for document type" + ) + if document.get("source_file", {}).get("format") not in definition["formats"]: + self.errors.append( + f"document IR line {line_number}: format not allowed for document type" + ) + + template = document.get("template", {}) + if template.get("scope") == "TENANT": + if template.get("owner_company_ref") != document.get("company_ref"): + self.errors.append( + f"document IR line {line_number}: tenant template owner mismatch" + ) + if template.get("visible_to_other_companies"): + self.errors.append( + f"document IR line {line_number}: tenant template visibility leak" + ) + if document.get("parser_route") in {"UNCLASSIFIED", "UNSUPPORTED"} and document.get( + "automatic_persistence_allowed" + ): + self.errors.append( + f"document IR line {line_number}: unclassified document cannot auto-persist" + ) + + if len(fixtures) != manifest["record_count"]: + self.errors.append("document IR fixtures: record count mismatch") + formats = {fixture["source_file"]["format"] for fixture in fixtures} + if formats != set(manifest["required_formats"]): + self.errors.append("document IR fixtures: required format coverage missing") + xlsx_fixture = next( + (fixture for fixture in fixtures if fixture["source_file"]["format"] == "XLSX"), + None, + ) + expected_row_statuses = {"VALID", "REVIEW_REQUIRED"} + actual_row_statuses = ( + {row["status"] for row in xlsx_fixture["spreadsheet"]["row_results"]} + if xlsx_fixture + else set() + ) + if actual_row_statuses != expected_row_statuses: + self.errors.append("document IR fixtures: XLSX partial row handling missing") + def _validate_required_slot_contracts(self) -> None: config = self.repository.load_yaml("knowledge/required_slots.yaml") schema = self.repository.load_json("schemas/required-slots.schema.json") diff --git a/fowoco-knowledge/tests/test_repository.py b/fowoco-knowledge/tests/test_repository.py index bc9c022..90709ba 100644 --- a/fowoco-knowledge/tests/test_repository.py +++ b/fowoco-knowledge/tests/test_repository.py @@ -21,6 +21,15 @@ def test_lists_mvp_workflows() -> None: } +def test_document_type_catalog_exposes_safe_unknown_fallback() -> None: + repository = KnowledgeRepository(ROOT) + + assert repository.get_document_type("PASSPORT_COPY")["sensitivity"] == "critical" + unknown = repository.get_document_type("UNKNOWN") + assert unknown["allowed_roles"] == ["UNKNOWN"] + assert unknown["required_review"] is True + + def test_compiled_context_is_cross_linked() -> None: context = KnowledgeRepository(ROOT).compile_context("WF-STY-001") assert context["intent"]["id"] == "EXPIRY_RENEWAL" diff --git a/fowoco-knowledge/tests/test_validation.py b/fowoco-knowledge/tests/test_validation.py index c2747bf..956e461 100644 --- a/fowoco-knowledge/tests/test_validation.py +++ b/fowoco-knowledge/tests/test_validation.py @@ -207,3 +207,34 @@ def test_six_workflow_runtime_profiles_have_three_e2e_paths_each() -> None: assert all(stage["completion_evidence"] for stage in profile["stages"]) assert all(not case["guardrails"]["automatic_external_submission"] for case in cases) assert all(not case["guardrails"]["automatic_completion"] for case in cases) + + +def test_document_ir_fixtures_cover_six_formats_and_never_auto_persist() -> None: + fixtures = [ + json.loads(line) + for line in (ROOT / "data/evaluation/document_ir_cases.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line.strip() + ] + + assert {fixture["source_file"]["format"] for fixture in fixtures} == { + "PDF", + "IMAGE", + "HWP", + "HWPX", + "XLSX", + "CSV", + } + assert all(not fixture["automatic_persistence_allowed"] for fixture in fixtures) + assert all( + field["evidence"]["status"] in {"AVAILABLE", "UNAVAILABLE"} + for fixture in fixtures + for field in fixture["fields"] + ) + tenant_templates = [fixture for fixture in fixtures if fixture["template"]["scope"] == "TENANT"] + assert all( + fixture["template"]["owner_company_ref"] == fixture["company_ref"] + and not fixture["template"]["visible_to_other_companies"] + for fixture in tenant_templates + )