From 08ba017c7e775f66a1b9d0a95f0d3b9d28460b16 Mon Sep 17 00:00:00 2001 From: Rick Barrette Date: Wed, 11 Nov 2026 21:09:12 -0500 Subject: [PATCH] Search inside documents: PDF text, Tesseract OCR for scans (in the image), optional Claude; nightly catch-up Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 2 +- README.md | 2 +- app/controllers/admin/settings_controller.rb | 6 +- app/jobs/document_text_job.rb | 26 +++++ app/models/document.rb | 6 + app/models/setting.rb | 1 + app/services/document_text.rb | 108 ++++++++++++++++++ app/views/admin/settings/edit.html.erb | 8 ++ config/recurring.yml | 3 + ...61111040000_add_pages_text_to_documents.rb | 9 ++ db/schema.rb | 6 +- docs/guide/administration.md | 1 + docs/guide/documents.md | 4 + test/integration/document_text_test.rb | 95 +++++++++++++++ 14 files changed, 271 insertions(+), 6 deletions(-) create mode 100644 app/jobs/document_text_job.rb create mode 100644 app/services/document_text.rb create mode 100644 db/migrate/20261111040000_add_pages_text_to_documents.rb create mode 100644 test/integration/document_text_test.rb diff --git a/Dockerfile b/Dockerfile index c710e49..543e6e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ WORKDIR /rails # Install base packages RUN apt-get update -qq && \ - apt-get install --no-install-recommends -y curl libjemalloc2 libvips sqlite3 && \ + apt-get install --no-install-recommends -y curl libjemalloc2 libvips sqlite3 tesseract-ocr tesseract-ocr-eng poppler-utils && \ rm -rf /var/lib/apt/lists /var/cache/apt/archives # Set production environment diff --git a/README.md b/README.md index 8d70291..9adeae9 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ To back up or move a single aircraft with all its files, use **Access & Sharing | AD import | Import ADs from an FAA DRS export (`.xlsx` recommended: its AD numbers link to each AD's DRS page, which the import keeps; `.csv` works without links). ADs are matched by number, so nothing is duplicated. The preview suggests Track, Not applicable or Skip for each AD, and flags superseded or non-current ADs, model mismatches, appliance or STC-only ADs, and repetitive actions. | | Inspections | Shared inspection types with hour, month, and day recurrence, plus history. Each says what overdue means (grounded, no IFR, no transponder, advisory); an annual can also count as the 100-hour, and the 100-hour's 10-hour overrun is credited (91.409(b)). One click adds the standard Part 91 set. | | Logbook | Maintenance entries with search and a link to a scanned document page. Records the signer's certificate kind and number and an inspection's result, and flags what 43.9 / 43.11 needs that's missing. Entries transcribed by Claude stay "unverified" until someone checks them against the scan. Entry templates (built-in pilot preventive maintenance per 43.3(g) and Part 43 Appendix A(c); admins add and edit their own under Admin → Logbook templates). Engine and propeller entries link the component and record its total time and time since overhaul (SMOH/SFOH/SNEW) at the entry, worked out from the entry's airframe total time; sub-components (an oil filter on the engine…) can be included with their own times. Print entries as a PDF on Avery 5163/8163 or 5164/8164 labels or half-letter/letter pages (boxed) to sign and put in the paper logbook. | -| Documents | Collections, multi-file documents with a page viewer, per-document share visibility, bulk upload, and expiration dates (registration, insurance…). | +| Documents | Collections, multi-file documents with a page viewer, per-document share visibility, bulk upload, and expiration dates (registration, insurance…). Their text is read for search: PDF text layers directly, scans by Tesseract OCR (in the image) or optionally Claude. | | Electronic signatures | Logbook entries can be signed electronically along the lines of AC 120-78B chapter 2: the signer reviews the entry, picks their certificate, re-enters their password (or signs in again with SSO) and confirms a "Certified by" statement. The signature records who, which certificate, the statement, the time and a SHA-256 fingerprint of the entry, sealed by the server; the signer gets an email. Signed entries are locked: a correction is a new signed entry that supersedes the original. Owners can invite an outside mechanic to sign one entry with their own account. Disabled accounts can't sign; their past signatures stay. | | Records integrity | Every change to maintenance records is kept (who, when, field by field) in a hash chain per aircraft (Activity → Record history). Nightly: a verified SQLite backup in `storage/backups` (how many are kept: Admin → Settings), then a check of the history chains and every signature; admins are emailed on a problem (Admin → About). Software versions are recorded. A records package (Access & Sharing) bundles every entry as a PDF with signatures, a signature register, the history, the status report and the `.sam.zip` for the FAA or a buyer, with a README (and a ready-to-run `viewer/docker-compose.yml`) so the recipient can browse the data in their own local copy of the app. Scans of paper logbooks can be marked as legacy records, with an "electronic records since" date per aircraft. | | Off-site backup | Nightly copy of the verified database snapshot (newest 30 kept) and every new uploaded file to Google Drive, using your own Google OAuth client and the `drive.file` scope (the app sees only its own files). Restore test from the admin page; failures emailed to admins; `bin/rails offsite:restore_files` brings files back after a restore. | diff --git a/app/controllers/admin/settings_controller.rb b/app/controllers/admin/settings_controller.rb index 4de8081..43de13e 100644 --- a/app/controllers/admin/settings_controller.rb +++ b/app/controllers/admin/settings_controller.rb @@ -2,18 +2,18 @@ class Admin::SettingsController < ApplicationController before_action :require_admin! def edit - @values = %w[max_upload_mb backups_kept offsite_backups_kept].index_with { Setting.integer(_1) } + @values = %w[max_upload_mb backups_kept offsite_backups_kept documents_claude_text].index_with { Setting.integer(_1) } end def update - submitted = params.fetch(:settings, {}).permit(:max_upload_mb, :backups_kept, :offsite_backups_kept).to_h + submitted = params.fetch(:settings, {}).permit(:max_upload_mb, :backups_kept, :offsite_backups_kept, :documents_claude_text).to_h records = Setting.update_integers(submitted) @errors = records.flat_map { _1.errors.full_messages } if @errors.empty? Rails.logger.info("[settings] #{current_user.email_address} changed #{submitted.inspect}") redirect_to edit_admin_settings_path, notice: "Settings saved." else - @values = %w[max_upload_mb backups_kept offsite_backups_kept].index_with { Setting.integer(_1) }.merge(submitted) + @values = %w[max_upload_mb backups_kept offsite_backups_kept documents_claude_text].index_with { Setting.integer(_1) }.merge(submitted) render :edit, status: :unprocessable_entity end end diff --git a/app/jobs/document_text_job.rb b/app/jobs/document_text_job.rb new file mode 100644 index 0000000..dd9536c --- /dev/null +++ b/app/jobs/document_text_job.rb @@ -0,0 +1,26 @@ +# Reads a document's text for search after its pages change; the nightly run +# catches up documents that were never read (or changed since). +class DocumentTextJob < ApplicationJob + queue_as :default + limits_concurrency to: 1, key: "document_text", duration: 2.hours if respond_to?(:limits_concurrency) + + BATCH = 25 + + def perform(document_id = nil) + if document_id + (doc = Document.find_by(id: document_id)) && read(doc) + else + Document.includes(pages_attachments: :blob).find_each.select { _1.pages_text_stale? }.first(BATCH).each { read(_1) } + end + end + + private + + def read(doc) + return unless doc.pages_text_stale? + result = DocumentText.new(doc).read + # Derived search text: written directly, so it isn't a record change. + doc.update_columns(pages_text: result.text.presence, pages_text_method: result.method, + pages_text_source: doc.pages_source_key, pages_text_at: Time.current) + end +end diff --git a/app/models/document.rb b/app/models/document.rb index 8328a1b..d06f398 100644 --- a/app/models/document.rb +++ b/app/models/document.rb @@ -31,6 +31,12 @@ class Document < ApplicationRecord def to_s = name def type_label = TYPES[doc_type] def expired?(today = Date.current) = expires_on.present? && today > expires_on + + # The page files the search text was read from; when they change, it's read again. + def pages_source_key = pages.map(&:blob_id).sort.join(",") + def pages_text_stale? = pages_text_source.to_s != pages_source_key + + after_commit(on: %i[create update]) { DocumentTextJob.perform_later(id) if pages.attached? && pages_text_stale? } def expiry_warning_days = EXPIRY_WARNING_DAYS.fetch(doc_type, 30) # nil visibility inherits from the collection; uncollected docs default to private. diff --git a/app/models/setting.rb b/app/models/setting.rb index 029e17c..4f5ad2c 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -9,6 +9,7 @@ class Setting < ApplicationRecord "max_upload_mb" => { default: 50, in: 1..1024, label: "Maximum upload size (MB)" }, "backups_kept" => { default: 14, in: 1..365, label: "Database backups kept" }, "offsite_backups_kept" => { default: 30, in: 1..365, label: "Database copies kept in Google Drive" }, + "documents_claude_text" => { default: 0, in: 0..1, label: "Read scanned documents with Claude" }, "logbook_templates_seeded" => { default: 0, in: 0..1, label: "Built-in logbook templates added" } # internal flag }.freeze diff --git a/app/services/document_text.rb b/app/services/document_text.rb new file mode 100644 index 0000000..d53e55a --- /dev/null +++ b/app/services/document_text.rb @@ -0,0 +1,108 @@ +require "open3" +require "anthropic" + +# Reads the text on a document's pages so search can find it: a PDF's own +# text where it has one, otherwise OCR. OCR is Tesseract (free, local, good on +# print, poor on handwriting) unless an admin turned on reading scans with +# Claude (Settings) and an Anthropic API key is set. +class DocumentText + MAX_CHARS = 300_000 + MAX_PAGES = 40 # per file, for OCR + SCANNED = 30 # fewer characters a page than this: treat a PDF as scanned + CLAUDE_TYPES = %w[image/jpeg image/png image/gif image/webp].freeze + + Result = Data.define(:text, :method) + + def self.tesseract? = system("which tesseract >/dev/null 2>&1") + def self.pdftoppm? = system("which pdftoppm >/dev/null 2>&1") + def self.claude? = Setting.integer("documents_claude_text") == 1 && ENV["ANTHROPIC_API_KEY"].present? + + def initialize(document) + @document = document + end + + def read + texts = [] + methods = [] + @document.pages.each do |page| + text, method = read_page(page) + texts << text if text.present? + methods << method + end + method = methods.include?("claude") ? "claude" : (methods.include?("ocr") ? "ocr" : (methods.include?("text") ? "text" : "none")) + Result.new(text: texts.join("\n\n").squeeze(" \n").first(MAX_CHARS), method: method) + end + + private + + def read_page(page) + type = page.blob.content_type.to_s + page.blob.open do |file| + if type == "application/pdf" then read_pdf(file.path, page.blob.byte_size) + elsif type.start_with?("text/") then [ File.read(file.path, MAX_CHARS).to_s.scrub, "text" ] + elsif type.start_with?("image/") then scan_image(file.path, type, page.blob.byte_size) + else [ nil, "none" ] + end + end + rescue StandardError => e + Rails.logger.warn("[document text] #{@document.id} #{page.filename}: #{e.class}: #{e.message}") + [ nil, "none" ] + end + + def read_pdf(path, bytes) + reader = PDF::Reader.new(path) + pages = reader.pages.first(500).map { _1.text.to_s } + text = pages.join("\n").squeeze(" ") + return [ text, "text" ] if pages.empty? || text.gsub(/\s/, "").length >= SCANNED * pages.size + scan_pdf(path, bytes) # little or no text layer: it's a scan + end + + def scan_pdf(path, bytes) + return claude([ { data: File.binread(path), media_type: "application/pdf" } ]) if self.class.claude? && bytes <= LogbookTranscriber::MAX_PDF_BYTES + return [ nil, "none" ] unless self.class.pdftoppm? && self.class.tesseract? + Dir.mktmpdir("sam-ocr") do |dir| + _, status = Open3.capture2e("timeout", "300", "pdftoppm", "-r", "150", "-png", "-f", "1", "-l", MAX_PAGES.to_s, path, File.join(dir, "p")) + return [ nil, "none" ] unless status.success? + text = Dir[File.join(dir, "p*.png")].sort.map { tesseract(_1) }.compact.join("\n") + [ text, "ocr" ] + end + end + + def scan_image(path, type, bytes) + if self.class.claude? && CLAUDE_TYPES.include?(type) && bytes <= LogbookTranscriber::MAX_IMAGE_BYTES + claude([ { data: File.binread(path), media_type: type } ]) + elsif self.class.tesseract? + [ tesseract(path), "ocr" ] + else + [ nil, "none" ] + end + end + + def tesseract(path) + out, status = Open3.capture2("timeout", "120", "tesseract", path, "stdout", "-l", "eng", err: File::NULL) + status.success? ? out : nil + end + + # Claude reads the scan (handwriting too). Uses the same model and settings + # as logbook transcription. + def claude(files) + blocks = files.map do |f| + kind = f[:media_type] == "application/pdf" ? "document" : "image" + { type: kind, source: { type: "base64", media_type: f[:media_type], data: Base64.strict_encode64(f[:data]) } } + end + stream = LogbookTranscriber.client.beta.messages.stream( + model: LogbookTranscriber::MODEL, + max_tokens: 64_000, + output_config: { effort: :low }, + fallbacks: :default, + betas: [ LogbookTranscriber::FALLBACK_BETA ], + messages: [ { role: "user", content: blocks + [ { type: "text", text: "Transcribe all the text in this document, in reading order, as plain text. Include handwriting. Output only the text." } ] } ] + ) + message = stream.accumulated_message + return [ nil, "none" ] if message.stop_reason == :refusal + [ message.content.select { _1.type == :text }.map(&:text).join, "claude" ] + rescue Anthropic::Errors::APIError => e + Rails.logger.warn("[document text] Claude: #{e.class}: #{e.message}") + [ nil, "none" ] + end +end diff --git a/app/views/admin/settings/edit.html.erb b/app/views/admin/settings/edit.html.erb index 5aa398a..1a12d48 100644 --- a/app/views/admin/settings/edit.html.erb +++ b/app/views/admin/settings/edit.html.erb @@ -38,5 +38,13 @@

With off-site backup to Google Drive, this many of the newest nightly database copies are kept there. Uploaded files are copied once and kept.

+

Document search

+
+ <%= label_tag "settings[documents_claude_text]", "Read scanned documents with", class: "label" %> + <%= select_tag "settings[documents_claude_text]", options_for_select([ [ "Tesseract (free, on this server)", 0 ], [ "Claude (better on handwriting; uses the Anthropic API)", 1 ] ], @values["documents_claude_text"].to_i), + id: "settings_documents_claude_text", class: "input" %> +

Text in documents is read so search can find it. PDFs that contain text are read directly either way. Claude is used only when an ANTHROPIC_API_KEY is set<%= " (it isn't)" if ENV["ANTHROPIC_API_KEY"].blank? %>, and costs API credit per page.

+
+
<%= f.submit "Save Settings", class: "btn-primary" %><%= link_to "Cancel", admin_about_path, class: "btn-secondary" %>
<% end %> diff --git a/config/recurring.yml b/config/recurring.yml index 30fc1d3..ac507dc 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -25,6 +25,9 @@ production: ad_watch: class: AdWatchJob schedule: every sunday at 10am + document_text: + class: DocumentTextJob + schedule: every day at 2am records_integrity: class: RecordsIntegrityJob schedule: every day at 3:30am diff --git a/db/migrate/20261111040000_add_pages_text_to_documents.rb b/db/migrate/20261111040000_add_pages_text_to_documents.rb new file mode 100644 index 0000000..0361a27 --- /dev/null +++ b/db/migrate/20261111040000_add_pages_text_to_documents.rb @@ -0,0 +1,9 @@ +class AddPagesTextToDocuments < ActiveRecord::Migration[8.1] + def change + # Text read from the document's pages, for search (derived, not a record). + add_column :documents, :pages_text, :text + add_column :documents, :pages_text_method, :string # text | ocr | claude | none + add_column :documents, :pages_text_source, :string # the page files it was read from + add_column :documents, :pages_text_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index e9450a8..647c5e7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_11_11_030000) do +ActiveRecord::Schema[8.1].define(version: 2026_11_11_040000) do create_table "active_storage_attachments", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false @@ -404,6 +404,10 @@ ActiveRecord::Schema[8.1].define(version: 2026_11_11_030000) do t.datetime "updated_at", null: false t.date "expires_on" t.boolean "legacy_record", default: false, null: false + t.text "pages_text" + t.string "pages_text_method" + t.string "pages_text_source" + t.datetime "pages_text_at" t.index ["aircraft_id"], name: "index_documents_on_aircraft_id" t.index ["document_collection_id"], name: "index_documents_on_document_collection_id" end diff --git a/docs/guide/administration.md b/docs/guide/administration.md index 57da0b9..6355a09 100644 --- a/docs/guide/administration.md +++ b/docs/guide/administration.md @@ -43,6 +43,7 @@ If single sign-on is broken and `PASSWORD_LOGIN=false` is set, remove that setti - **Uploads**: the largest file anyone can upload. A proxy in front of the app may have its own smaller limit. - **Backups**: how many nightly database copies to keep, on the server and in Google Drive. +- **Document search**: read scanned documents with Tesseract (free, on the server) or Claude (better on handwriting, needs `ANTHROPIC_API_KEY`, costs API credit). PDFs with text are read directly either way. ## About diff --git a/docs/guide/documents.md b/docs/guide/documents.md index 6d54fd9..08ad091 100644 --- a/docs/guide/documents.md +++ b/docs/guide/documents.md @@ -6,6 +6,10 @@ The **Documents** tab keeps scans and files: registration, airworthiness certifi - **Bulk upload**: many files at once. Each becomes its own document named after the file; files with the same name are combined, in order, as pages of one document. - **+ Collection** groups documents (for example *Engine logbook* or *Annual 2026*). Collections start collapsed, showing how many documents they hold; click one to open it. +## Searching inside documents + +The text on each document's pages is read in the background so **Search** can find it: a PDF's own text when it has one, otherwise OCR of the scan (Tesseract, on the server). Printed text reads well; handwriting much less so. An admin can switch scans to Claude (Admin → Settings → Document search), which reads handwriting far better but uses the Anthropic API. New uploads are read within a minute or so; existing documents are caught up overnight. + ## Expiry Give a registration, insurance policy or similar an expiration date and it shows on the airworthiness status as it approaches. An expired registration grounds the aircraft (91.203(a)(2)). diff --git a/test/integration/document_text_test.rb b/test/integration/document_text_test.rb new file mode 100644 index 0000000..ba42ebf --- /dev/null +++ b/test/integration/document_text_test.rb @@ -0,0 +1,95 @@ +require "test_helper" +require "prawn" + +# Text inside documents, read for search: PDF text, OCR for scans, or Claude. +class DocumentTextTest < ActionDispatch::IntegrationTest + class FakeClaude + Message = Struct.new(:stop_reason, :content, :model) + Text = Struct.new(:type, :text) + attr_reader :requests + + def initialize(text) = (@text = text; @requests = []) + def beta = self + def messages = self + + def stream(**params) + @requests << params + Struct.new(:accumulated_message).new(Message.new(:end_turn, [ Text.new(:text, @text) ], "claude-opus-5-5")) + end + end + + setup do + @owner = create_user("owner@example.com") + @a = create_aircraft(owner: @owner, tail_number: "N12345") + @dir = Dir.mktmpdir + end + + teardown { FileUtils.rm_rf(@dir) } + + def text_pdf(text) + path = File.join(@dir, "text-#{SecureRandom.hex(3)}.pdf") + Prawn::Document.generate(path) { |pdf| pdf.font_size(28) { pdf.text text } } + path + end + + # A page with no text layer: a picture of text, as a scanner makes. + def scan_png(text) + pdf = text_pdf(text) + system("pdftoppm", "-r", "150", "-png", "-singlefile", pdf, File.join(@dir, "scan")) or skip "pdftoppm not installed" + File.join(@dir, "scan.png") + end + + def document(path, type) + doc = @a.documents.create!(name: File.basename(path), doc_type: "INVOICE") + doc.pages.attach(io: File.open(path, "rb"), filename: File.basename(path), content_type: type) + doc + end + + test "a PDF's own text is read, and search finds it" do + doc = document(text_pdf("Magneto timing set to 22 degrees BTC"), "application/pdf") + perform_enqueued_jobs { DocumentTextJob.perform_now(doc.id) } + doc.reload + assert_equal "text", doc.pages_text_method + assert_includes doc.pages_text, "Magneto timing" + refute doc.pages_text_stale? + assert_no_difference("RecordVersion.count") { DocumentTextJob.perform_now(doc.id) } # derived text isn't a record change + + sign_in @owner + get search_path(q: "timing set") + assert_select "h2", text: /Documents/ + assert_select "mark", text: /timing set/i + end + + test "a scan is read with Tesseract, and a scanned PDF too" do + skip "tesseract not installed" unless DocumentText.tesseract? + image = document(scan_png("ALTERNATOR REPLACED"), "image/png") + DocumentTextJob.perform_now(image.id) + assert_equal "ocr", image.reload.pages_text_method + assert_match(/ALTERNATOR/i, image.pages_text) + + scanned_pdf = File.join(@dir, "scanned.pdf") + png = scan_png("VACUUM PUMP INSPECTED") + Prawn::Document.generate(scanned_pdf) { |pdf| pdf.image png, fit: [ 500, 700 ] } + doc = document(scanned_pdf, "application/pdf") + DocumentTextJob.perform_now(doc.id) + assert_equal "ocr", doc.reload.pages_text_method + assert_match(/VACUUM/i, doc.pages_text) + end + + test "with Claude turned on and a key set, scans go to Claude; the nightly run catches up" do + Setting.create!(key: "documents_claude_text", value: "1") + claude = FakeClaude.new("Starter replaced, Sky example S/N 10000000") + LogbookTranscriber.client = claude + ENV["ANTHROPIC_API_KEY"] = "test-key" + doc = document(scan_png("anything"), "image/png") + doc.update_columns(pages_text_source: nil) + DocumentTextJob.perform_now # nightly: finds documents not read yet + assert_equal [ "claude", "Starter replaced, Sky example S/N 10000000" ], doc.reload.slice(:pages_text_method, :pages_text).values + request = claude.requests.sole + assert_equal :default, request[:fallbacks] + assert_equal "image", request[:messages].first[:content].first[:type] + ensure + ENV.delete("ANTHROPIC_API_KEY") + LogbookTranscriber.client = nil + end +end