Remove category from AI responsibilities

Author: jdav-dev

lib/archivist.ex

@@ -4,9 +4,7 @@ defmodule Archivist do
   """
 
   @callback extract_pdf_information(path :: Path.t()) ::
-              {:ok,
-               %{category: String.t(), date: String.t(), source: String.t(), title: String.t()}}
-              | :error
+              {:ok, %{date: String.t(), source: String.t(), title: String.t()}} | :error
 
   @callback init :: :ok | :error
 
@@ -24,7 +22,6 @@ defmodule Archivist do
 
       iex> Archivist.extract_pdf_information(pdf_path)
       {:ok, %{
-          category: "money",
           date: "2025-01-30",
           source: "abc-corp",
           title: "invoice-for-jan"

lib/archivist/system_calls.ex

@@ -8,131 +8,12 @@ defmodule Archivist.SystemCalls do
   @model "llama3.2"
   @num_ctx 8192
 
-  @categories [
-    "Vital Records and Identification",
-    "Financial Documents",
-    "Tax Records",
-    "Insurance Documents",
-    "Medical and Health Records",
-    "Property and Real Estate",
-    "Housing and Household",
-    "Vehicle and Transportation",
-    "Employment and Career",
-    "Legal and Estate Planning",
-    "Education and Professional Development",
-    "Family and Household Members",
-    "Warranties and Manuals",
-    "Memberships and Subscriptions",
-    "Travel and Leisure",
-    "Digital Assets and Online Accounts",
-    "Sentimental and Historical",
-    "Miscellaneous and Other"
-  ]
-
   @slug_length 25
 
   @system """
   You are a text classification and metadata extraction assistant.  You will be given text
   extracted from a PDF, and your job is to return the following information in valid JSON format:
 
-    - category (string)
-      - Must be exactly one of these: #{@categories |> Enum.map(&~s/"#{&1}"/) |> Enum.join(", ")}.
-      - This refers to the overall subject area or domain of the document.
-      - Below are the category explanations for reference:
-          - Vital Records and Identification
-            - Description: Documents that establish or verify an individual's identity and
-              significant life events.
-            - Examples: Birth certificates, marriage or divorce certificates, death certificates
-              (for family members), passports, Social Security cards (or equivalents), citizenship
-              or naturalization papers, name change documents.
-          - Financial Documents
-            - Description: Paperwork related to banking, credit, investments, and recurring
-              expenses.
-            - Examples: Bank statements, credit card statements, loan agreements (mortgage,
-              student, car), investment records (stocks, bonds, mutual funds, cryptocurrency),
-              budget worksheets, utility bills, subscription invoices.
-          - Tax Records
-            - Description: All documents needed for tax filing, verification, and historical
-              reference.
-            - Examples: Past tax returns, W-2/1099 forms (or international equivalents), receipts
-              for deductible expenses (charitable donations, medical, business), property tax
-              statements.
-          - Insurance Documents
-            - Description: Policies and claims information for various types of insurance.
-            - Examples: Health insurance policy details, life insurance contracts, auto or
-              homeowners policies, coverage schedules, claim forms, renewal notices.
-          - Medical and Health Records
-            - Description: Personal and family health documentation, including treatments and
-              prescriptions.
-            - Examples: Immunization records, physician or hospital visit summaries, lab test
-              results, prescription information, dental/vision care records, documentation of
-              chronic conditions.
-          - Property and Real Estate
-            - Description: Paperwork detailing real property ownership, transactions, and
-              improvements.
-            - Examples: Mortgage agreements, deeds and titles, closing documents, lease agreements
-              for rental properties, receipts for major renovations, HOA (Homeowners Association)
-              guidelines.
-          - Housing and Household
-            - Description: Day-to-day living documents and service agreements for your home.
-            - Examples: Rental lease agreements (if renting), utility contracts and bills
-              (electricity, water, internet), service or maintenance contracts (e.g., lawn care,
-              pest control), appliance manuals, home repair receipts.
-          - Vehicle and Transportation
-            - Description: Records associated with car ownership, maintenance, and usage.
-            - Examples: Vehicle titles, registration papers, auto insurance policies, maintenance
-              and service records, warranty details, driver's license copies, parking permits.
-          - Employment and Career
-            - Description: Information related to current and past employment, as well as
-              professional growth.
-            - Examples: Employment contracts, offer letters, pay stubs, performance evaluations,
-              benefits guides, separation or termination documents, professional certifications,
-              résumés/CVs.
-          - Legal and Estate Planning
-            - Description: Legally binding papers covering estates, end-of-life directives, and
-                other legal matters.
-            - Examples: Wills, trusts, power of attorney documents, living wills or advance
-                directives, guardianship papers, and court or legal settlement documents.
-          - Education and Professional Development
-            - Description: Records of academic achievements, certifications, and ongoing
-              education.
-            - Examples: Transcripts, diplomas, course certificates, scholarships or grant info,
-              professional licenses, continuing education credits, conference attendance records.
-          - Family and Household Members
-            - Description: Personal documents specific to each household member or dependent.
-            - Examples: Spouse or partner's documents (if kept separately), children's birth
-              certificates, school records, immunization details, childcare arrangements, pet
-              adoption or vaccination papers.
-          - Warranties and Manuals
-            - Description: Documentation for product guarantees and user guides.
-            - Examples: Warranty information for electronics or appliances, user manuals, extended
-              service contracts, purchase receipts for large items or equipment.
-          - Memberships and Subscriptions
-            - Description: Details on recurring membership-based services or organizations.
-            - Examples: Gym memberships, club or association memberships, magazine or streaming
-              subscriptions, loyalty or frequent flyer program statements, renewal notices.
-          - Travel and Leisure
-            - Description: Arrangements and records related to vacations, trips, and leisure
-              activities.
-            - Examples: Travel itineraries, flight tickets, hotel confirmations, visa
-              documentation, travel insurance policies, timeshare contracts, past trip expense
-              receipts.
-          - Digital Assets and Online Accounts
-            - Description: Information and credentials for online identities, cloud services, and
-              digital platforms.
-            - Examples: Password manager references (stored securely), domain registrations, cloud
-              storage subscriptions, digital payment account details (PayPal, etc.), important
-              email or social media account notes.
-          - Sentimental and Historical
-            - Description: Keepsakes and personal or family history items with emotional or
-              genealogical importance.
-            - Examples: Family photos, letters, journals, genealogy research, copies of heirlooms,
-              scrapbooks, memorabilia.
-          - Miscellaneous and Other
-            - Description: A catch-all for documents that do not neatly fit into other categories.
-            - Examples: Personal or hobby-related projects, unusual one-off contracts, event
-              memorabilia, or temporary items awaiting proper classification.
-
     - date (string)
       - Must be a valid ISO 8601 date in the format YYYY-MM-DD.
       - If the text contains multiple dates, choose the one most relevant to the document (e.g.,
@@ -156,19 +37,18 @@ defmodule Archivist.SystemCalls do
   Your output must follow exactly this JSON structure (example with placeholder values):
 
   ```
-  {"category": "money", "date": "2025-01-30", "source": "abc-corp", "title": "invoice-for-jan"}
+  {"date": "2025-01-30", "source": "abc-corp", "title": "invoice-for-jan"}
   ```
   """
 
   @format %{
     type: :object,
     properties: %{
-      category: %{type: :string, enum: @categories},
       date: %{type: :string, format: :date},
       source: %{type: :string},
       title: %{type: :string}
     },
-    required: [:category, :date, :source, :title]
+    required: [:date, :source, :title]
   }
 
   @impl Archivist
@@ -215,14 +95,12 @@ defmodule Archivist.SystemCalls do
            ),
          {:ok,
           %{
-            "category" => category,
             "date" => date,
             "source" => source,
             "title" => title
           }} <- JSON.decode(response) do
       {:ok,
        %{
-         category: category,
          date: date,
          source: Slug.slugify(source, truncate: @slug_length),
          title: Slug.slugify(title, truncate: @slug_length)

lib/archivist/worker.ex

@@ -56,13 +56,11 @@ defmodule Archivist.Worker do
   end
 
   defp move_file(path, extracted_info, state) do
-    category_path = Path.join(state.archive, extracted_info.category)
-
     archive_filename =
       "#{extracted_info.date}_#{extracted_info.source}_#{extracted_info.title}.pdf"
 
-    File.mkdir_p!(category_path)
-    archive_path = Path.join(category_path, archive_filename)
+    File.mkdir_p!(state.archive)
+    archive_path = Path.join(state.archive, archive_filename)
 
     Logger.info("Moving #{inspect(Path.basename(path))} to #{inspect(archive_path)}")
 
@@ -75,7 +73,7 @@ defmodule Archivist.Worker do
           [
             System.system_time(:second),
             Path.basename(path),
-            Path.join(extracted_info.category, archive_filename)
+            archive_filename
           ]
         ])
 

test/archivist/worker_test.exs

@@ -131,7 +131,6 @@ defmodule Archivist.WorkerTest do
       File.touch!(pdf_path)
 
       pdf_info = %{
-        category: "money",
         date: "2025-01-30",
         source: "abc-corp",
         title: "invoice-for-jan"
@@ -148,7 +147,7 @@ defmodule Archivist.WorkerTest do
 
       assert {:noreply, state, {:continue, :next_file}} == result
 
-      archive_path = Path.join([archive, "money", "2025-01-30_abc-corp_invoice-for-jan.pdf"])
+      archive_path = Path.join([archive, "2025-01-30_abc-corp_invoice-for-jan.pdf"])
       assert File.exists?(archive_path)
 
       assert info_log =~ "Archiving #{inspect(pdf_path)}"
@@ -203,15 +202,13 @@ defmodule Archivist.WorkerTest do
       File.touch!(pdf_path)
 
       pdf_info = %{
-        category: "money",
         date: "2025-01-30",
         source: "abc-corp",
         title: "invoice-for-jan"
       }
 
-      category_path = Path.join(archive, "money")
-      File.mkdir_p!(category_path)
-      archive_path = Path.join(category_path, "2025-01-30_abc-corp_invoice-for-jan.pdf")
+      File.mkdir_p!(archive)
+      archive_path = Path.join(archive, "2025-01-30_abc-corp_invoice-for-jan.pdf")
       File.touch!(archive_path)
 
       Archivist.Mock