mirror of
https://github.com/rickbarrette/simple-aircraft-manager-rails.git
synced 2026-11-17 23:50:36 -05:00
Search inside documents: PDF text, Tesseract OCR for scans (in the image), optional Claude; nightly catch-up
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2953b9a18d
commit
08ba017c7e
14 files changed
+271
-6
No files matched your search
@@ -2,18 +2,18 @@ class Admin::SettingsController < ApplicationController
|
||||
before_action :require_admin!
|
||||
|
||||
def edit
|
||||
@values = %w[max_upload_mb backups_kept offsite_backups_kept].index_with { Setting.integer(_1) }
|
||||
@values = %w[max_upload_mb backups_kept offsite_backups_kept documents_claude_text].index_with { Setting.integer(_1) }
|
||||
end
|
||||
|
||||
def update
|
||||
submitted = params.fetch(:settings, {}).permit(:max_upload_mb, :backups_kept, :offsite_backups_kept).to_h
|
||||
submitted = params.fetch(:settings, {}).permit(:max_upload_mb, :backups_kept, :offsite_backups_kept, :documents_claude_text).to_h
|
||||
records = Setting.update_integers(submitted)
|
||||
@errors = records.flat_map { _1.errors.full_messages }
|
||||
if @errors.empty?
|
||||
Rails.logger.info("[settings] #{current_user.email_address} changed #{submitted.inspect}")
|
||||
redirect_to edit_admin_settings_path, notice: "Settings saved."
|
||||
else
|
||||
@values = %w[max_upload_mb backups_kept offsite_backups_kept].index_with { Setting.integer(_1) }.merge(submitted)
|
||||
@values = %w[max_upload_mb backups_kept offsite_backups_kept documents_claude_text].index_with { Setting.integer(_1) }.merge(submitted)
|
||||
render :edit, status: :unprocessable_entity
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Reads a document's text for search after its pages change; the nightly run
|
||||
# catches up documents that were never read (or changed since).
|
||||
class DocumentTextJob < ApplicationJob
|
||||
queue_as :default
|
||||
limits_concurrency to: 1, key: "document_text", duration: 2.hours if respond_to?(:limits_concurrency)
|
||||
|
||||
BATCH = 25
|
||||
|
||||
def perform(document_id = nil)
|
||||
if document_id
|
||||
(doc = Document.find_by(id: document_id)) && read(doc)
|
||||
else
|
||||
Document.includes(pages_attachments: :blob).find_each.select { _1.pages_text_stale? }.first(BATCH).each { read(_1) }
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def read(doc)
|
||||
return unless doc.pages_text_stale?
|
||||
result = DocumentText.new(doc).read
|
||||
# Derived search text: written directly, so it isn't a record change.
|
||||
doc.update_columns(pages_text: result.text.presence, pages_text_method: result.method,
|
||||
pages_text_source: doc.pages_source_key, pages_text_at: Time.current)
|
||||
end
|
||||
end
|
||||
@@ -31,6 +31,12 @@ class Document < ApplicationRecord
|
||||
def to_s = name
|
||||
def type_label = TYPES[doc_type]
|
||||
def expired?(today = Date.current) = expires_on.present? && today > expires_on
|
||||
|
||||
# The page files the search text was read from; when they change, it's read again.
|
||||
def pages_source_key = pages.map(&:blob_id).sort.join(",")
|
||||
def pages_text_stale? = pages_text_source.to_s != pages_source_key
|
||||
|
||||
after_commit(on: %i[create update]) { DocumentTextJob.perform_later(id) if pages.attached? && pages_text_stale? }
|
||||
def expiry_warning_days = EXPIRY_WARNING_DAYS.fetch(doc_type, 30)
|
||||
|
||||
# nil visibility inherits from the collection; uncollected docs default to private.
|
||||
|
||||
@@ -9,6 +9,7 @@ class Setting < ApplicationRecord
|
||||
"max_upload_mb" => { default: 50, in: 1..1024, label: "Maximum upload size (MB)" },
|
||||
"backups_kept" => { default: 14, in: 1..365, label: "Database backups kept" },
|
||||
"offsite_backups_kept" => { default: 30, in: 1..365, label: "Database copies kept in Google Drive" },
|
||||
"documents_claude_text" => { default: 0, in: 0..1, label: "Read scanned documents with Claude" },
|
||||
"logbook_templates_seeded" => { default: 0, in: 0..1, label: "Built-in logbook templates added" } # internal flag
|
||||
}.freeze
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
require "open3"
|
||||
require "anthropic"
|
||||
|
||||
# Reads the text on a document's pages so search can find it: a PDF's own
|
||||
# text where it has one, otherwise OCR. OCR is Tesseract (free, local, good on
|
||||
# print, poor on handwriting) unless an admin turned on reading scans with
|
||||
# Claude (Settings) and an Anthropic API key is set.
|
||||
class DocumentText
|
||||
MAX_CHARS = 300_000
|
||||
MAX_PAGES = 40 # per file, for OCR
|
||||
SCANNED = 30 # fewer characters a page than this: treat a PDF as scanned
|
||||
CLAUDE_TYPES = %w[image/jpeg image/png image/gif image/webp].freeze
|
||||
|
||||
Result = Data.define(:text, :method)
|
||||
|
||||
def self.tesseract? = system("which tesseract >/dev/null 2>&1")
|
||||
def self.pdftoppm? = system("which pdftoppm >/dev/null 2>&1")
|
||||
def self.claude? = Setting.integer("documents_claude_text") == 1 && ENV["ANTHROPIC_API_KEY"].present?
|
||||
|
||||
def initialize(document)
|
||||
@document = document
|
||||
end
|
||||
|
||||
def read
|
||||
texts = []
|
||||
methods = []
|
||||
@document.pages.each do |page|
|
||||
text, method = read_page(page)
|
||||
texts << text if text.present?
|
||||
methods << method
|
||||
end
|
||||
method = methods.include?("claude") ? "claude" : (methods.include?("ocr") ? "ocr" : (methods.include?("text") ? "text" : "none"))
|
||||
Result.new(text: texts.join("\n\n").squeeze(" \n").first(MAX_CHARS), method: method)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def read_page(page)
|
||||
type = page.blob.content_type.to_s
|
||||
page.blob.open do |file|
|
||||
if type == "application/pdf" then read_pdf(file.path, page.blob.byte_size)
|
||||
elsif type.start_with?("text/") then [ File.read(file.path, MAX_CHARS).to_s.scrub, "text" ]
|
||||
elsif type.start_with?("image/") then scan_image(file.path, type, page.blob.byte_size)
|
||||
else [ nil, "none" ]
|
||||
end
|
||||
end
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn("[document text] #{@document.id} #{page.filename}: #{e.class}: #{e.message}")
|
||||
[ nil, "none" ]
|
||||
end
|
||||
|
||||
def read_pdf(path, bytes)
|
||||
reader = PDF::Reader.new(path)
|
||||
pages = reader.pages.first(500).map { _1.text.to_s }
|
||||
text = pages.join("\n").squeeze(" ")
|
||||
return [ text, "text" ] if pages.empty? || text.gsub(/\s/, "").length >= SCANNED * pages.size
|
||||
scan_pdf(path, bytes) # little or no text layer: it's a scan
|
||||
end
|
||||
|
||||
def scan_pdf(path, bytes)
|
||||
return claude([ { data: File.binread(path), media_type: "application/pdf" } ]) if self.class.claude? && bytes <= LogbookTranscriber::MAX_PDF_BYTES
|
||||
return [ nil, "none" ] unless self.class.pdftoppm? && self.class.tesseract?
|
||||
Dir.mktmpdir("sam-ocr") do |dir|
|
||||
_, status = Open3.capture2e("timeout", "300", "pdftoppm", "-r", "150", "-png", "-f", "1", "-l", MAX_PAGES.to_s, path, File.join(dir, "p"))
|
||||
return [ nil, "none" ] unless status.success?
|
||||
text = Dir[File.join(dir, "p*.png")].sort.map { tesseract(_1) }.compact.join("\n")
|
||||
[ text, "ocr" ]
|
||||
end
|
||||
end
|
||||
|
||||
def scan_image(path, type, bytes)
|
||||
if self.class.claude? && CLAUDE_TYPES.include?(type) && bytes <= LogbookTranscriber::MAX_IMAGE_BYTES
|
||||
claude([ { data: File.binread(path), media_type: type } ])
|
||||
elsif self.class.tesseract?
|
||||
[ tesseract(path), "ocr" ]
|
||||
else
|
||||
[ nil, "none" ]
|
||||
end
|
||||
end
|
||||
|
||||
def tesseract(path)
|
||||
out, status = Open3.capture2("timeout", "120", "tesseract", path, "stdout", "-l", "eng", err: File::NULL)
|
||||
status.success? ? out : nil
|
||||
end
|
||||
|
||||
# Claude reads the scan (handwriting too). Uses the same model and settings
|
||||
# as logbook transcription.
|
||||
def claude(files)
|
||||
blocks = files.map do |f|
|
||||
kind = f[:media_type] == "application/pdf" ? "document" : "image"
|
||||
{ type: kind, source: { type: "base64", media_type: f[:media_type], data: Base64.strict_encode64(f[:data]) } }
|
||||
end
|
||||
stream = LogbookTranscriber.client.beta.messages.stream(
|
||||
model: LogbookTranscriber::MODEL,
|
||||
max_tokens: 64_000,
|
||||
output_config: { effort: :low },
|
||||
fallbacks: :default,
|
||||
betas: [ LogbookTranscriber::FALLBACK_BETA ],
|
||||
messages: [ { role: "user", content: blocks + [ { type: "text", text: "Transcribe all the text in this document, in reading order, as plain text. Include handwriting. Output only the text." } ] } ]
|
||||
)
|
||||
message = stream.accumulated_message
|
||||
return [ nil, "none" ] if message.stop_reason == :refusal
|
||||
[ message.content.select { _1.type == :text }.map(&:text).join, "claude" ]
|
||||
rescue Anthropic::Errors::APIError => e
|
||||
Rails.logger.warn("[document text] Claude: #{e.class}: #{e.message}")
|
||||
[ nil, "none" ]
|
||||
end
|
||||
end
|
||||
@@ -38,5 +38,13 @@
|
||||
<p class="hint">With off-site backup to Google Drive, this many of the newest nightly database copies are kept there. Uploaded files are copied once and kept.</p>
|
||||
</div>
|
||||
|
||||
<h2 class="card-title pt-2">Document search</h2>
|
||||
<div>
|
||||
<%= label_tag "settings[documents_claude_text]", "Read scanned documents with", class: "label" %>
|
||||
<%= select_tag "settings[documents_claude_text]", options_for_select([ [ "Tesseract (free, on this server)", 0 ], [ "Claude (better on handwriting; uses the Anthropic API)", 1 ] ], @values["documents_claude_text"].to_i),
|
||||
id: "settings_documents_claude_text", class: "input" %>
|
||||
<p class="hint">Text in documents is read so search can find it. PDFs that contain text are read directly either way. Claude is used only when an ANTHROPIC_API_KEY is set<%= " (it isn't)" if ENV["ANTHROPIC_API_KEY"].blank? %>, and costs API credit per page.</p>
|
||||
</div>
|
||||
|
||||
<div class="flex gap-2"><%= f.submit "Save Settings", class: "btn-primary" %><%= link_to "Cancel", admin_about_path, class: "btn-secondary" %></div>
|
||||
<% end %>
|
||||
Reference in new issue
Block a user