Search inside documents: PDF text, Tesseract OCR for scans (in the image), optional Claude; nightly catch-up

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rickyandClaude Opus 5.5 committed 2026-11-11 21:09:12 -05:00
1 parent 2953b9a18d
commit 08ba017c7e
14 files changed
+271 -6

No files matched your search

+3 -3
View File
@@ -2,18 +2,18 @@ class Admin::SettingsController < ApplicationController
before_action :require_admin!
def edit
@values = %w[max_upload_mb backups_kept offsite_backups_kept].index_with { Setting.integer(_1) }
@values = %w[max_upload_mb backups_kept offsite_backups_kept documents_claude_text].index_with { Setting.integer(_1) }
end
def update
submitted = params.fetch(:settings, {}).permit(:max_upload_mb, :backups_kept, :offsite_backups_kept).to_h
submitted = params.fetch(:settings, {}).permit(:max_upload_mb, :backups_kept, :offsite_backups_kept, :documents_claude_text).to_h
records = Setting.update_integers(submitted)
@errors = records.flat_map { _1.errors.full_messages }
if @errors.empty?
Rails.logger.info("[settings] #{current_user.email_address} changed #{submitted.inspect}")
redirect_to edit_admin_settings_path, notice: "Settings saved."
else
@values = %w[max_upload_mb backups_kept offsite_backups_kept].index_with { Setting.integer(_1) }.merge(submitted)
@values = %w[max_upload_mb backups_kept offsite_backups_kept documents_claude_text].index_with { Setting.integer(_1) }.merge(submitted)
render :edit, status: :unprocessable_entity
end
end
+26
View File
@@ -0,0 +1,26 @@
# Reads a document's text for search after its pages change; the nightly run
# catches up documents that were never read (or changed since).
class DocumentTextJob < ApplicationJob
queue_as :default
limits_concurrency to: 1, key: "document_text", duration: 2.hours if respond_to?(:limits_concurrency)
BATCH = 25
def perform(document_id = nil)
if document_id
(doc = Document.find_by(id: document_id)) && read(doc)
else
Document.includes(pages_attachments: :blob).find_each.select { _1.pages_text_stale? }.first(BATCH).each { read(_1) }
end
end
private
def read(doc)
return unless doc.pages_text_stale?
result = DocumentText.new(doc).read
# Derived search text: written directly, so it isn't a record change.
doc.update_columns(pages_text: result.text.presence, pages_text_method: result.method,
pages_text_source: doc.pages_source_key, pages_text_at: Time.current)
end
end
+6
View File
@@ -31,6 +31,12 @@ class Document < ApplicationRecord
def to_s = name
def type_label = TYPES[doc_type]
def expired?(today = Date.current) = expires_on.present? && today > expires_on
# The page files the search text was read from; when they change, it's read again.
def pages_source_key = pages.map(&:blob_id).sort.join(",")
def pages_text_stale? = pages_text_source.to_s != pages_source_key
after_commit(on: %i[create update]) { DocumentTextJob.perform_later(id) if pages.attached? && pages_text_stale? }
def expiry_warning_days = EXPIRY_WARNING_DAYS.fetch(doc_type, 30)
# nil visibility inherits from the collection; uncollected docs default to private.
+1
View File
@@ -9,6 +9,7 @@ class Setting < ApplicationRecord
"max_upload_mb" => { default: 50, in: 1..1024, label: "Maximum upload size (MB)" },
"backups_kept" => { default: 14, in: 1..365, label: "Database backups kept" },
"offsite_backups_kept" => { default: 30, in: 1..365, label: "Database copies kept in Google Drive" },
"documents_claude_text" => { default: 0, in: 0..1, label: "Read scanned documents with Claude" },
"logbook_templates_seeded" => { default: 0, in: 0..1, label: "Built-in logbook templates added" } # internal flag
}.freeze
+108
View File
@@ -0,0 +1,108 @@
require "open3"
require "anthropic"
# Reads the text on a document's pages so search can find it: a PDF's own
# text where it has one, otherwise OCR. OCR is Tesseract (free, local, good on
# print, poor on handwriting) unless an admin turned on reading scans with
# Claude (Settings) and an Anthropic API key is set.
class DocumentText
MAX_CHARS = 300_000
MAX_PAGES = 40 # per file, for OCR
SCANNED = 30 # fewer characters a page than this: treat a PDF as scanned
CLAUDE_TYPES = %w[image/jpeg image/png image/gif image/webp].freeze
Result = Data.define(:text, :method)
def self.tesseract? = system("which tesseract >/dev/null 2>&1")
def self.pdftoppm? = system("which pdftoppm >/dev/null 2>&1")
def self.claude? = Setting.integer("documents_claude_text") == 1 && ENV["ANTHROPIC_API_KEY"].present?
def initialize(document)
@document = document
end
def read
texts = []
methods = []
@document.pages.each do |page|
text, method = read_page(page)
texts << text if text.present?
methods << method
end
method = methods.include?("claude") ? "claude" : (methods.include?("ocr") ? "ocr" : (methods.include?("text") ? "text" : "none"))
Result.new(text: texts.join("\n\n").squeeze(" \n").first(MAX_CHARS), method: method)
end
private
def read_page(page)
type = page.blob.content_type.to_s
page.blob.open do |file|
if type == "application/pdf" then read_pdf(file.path, page.blob.byte_size)
elsif type.start_with?("text/") then [ File.read(file.path, MAX_CHARS).to_s.scrub, "text" ]
elsif type.start_with?("image/") then scan_image(file.path, type, page.blob.byte_size)
else [ nil, "none" ]
end
end
rescue StandardError => e
Rails.logger.warn("[document text] #{@document.id} #{page.filename}: #{e.class}: #{e.message}")
[ nil, "none" ]
end
def read_pdf(path, bytes)
reader = PDF::Reader.new(path)
pages = reader.pages.first(500).map { _1.text.to_s }
text = pages.join("\n").squeeze(" ")
return [ text, "text" ] if pages.empty? || text.gsub(/\s/, "").length >= SCANNED * pages.size
scan_pdf(path, bytes) # little or no text layer: it's a scan
end
def scan_pdf(path, bytes)
return claude([ { data: File.binread(path), media_type: "application/pdf" } ]) if self.class.claude? && bytes <= LogbookTranscriber::MAX_PDF_BYTES
return [ nil, "none" ] unless self.class.pdftoppm? && self.class.tesseract?
Dir.mktmpdir("sam-ocr") do |dir|
_, status = Open3.capture2e("timeout", "300", "pdftoppm", "-r", "150", "-png", "-f", "1", "-l", MAX_PAGES.to_s, path, File.join(dir, "p"))
return [ nil, "none" ] unless status.success?
text = Dir[File.join(dir, "p*.png")].sort.map { tesseract(_1) }.compact.join("\n")
[ text, "ocr" ]
end
end
def scan_image(path, type, bytes)
if self.class.claude? && CLAUDE_TYPES.include?(type) && bytes <= LogbookTranscriber::MAX_IMAGE_BYTES
claude([ { data: File.binread(path), media_type: type } ])
elsif self.class.tesseract?
[ tesseract(path), "ocr" ]
else
[ nil, "none" ]
end
end
def tesseract(path)
out, status = Open3.capture2("timeout", "120", "tesseract", path, "stdout", "-l", "eng", err: File::NULL)
status.success? ? out : nil
end
# Claude reads the scan (handwriting too). Uses the same model and settings
# as logbook transcription.
def claude(files)
blocks = files.map do |f|
kind = f[:media_type] == "application/pdf" ? "document" : "image"
{ type: kind, source: { type: "base64", media_type: f[:media_type], data: Base64.strict_encode64(f[:data]) } }
end
stream = LogbookTranscriber.client.beta.messages.stream(
model: LogbookTranscriber::MODEL,
max_tokens: 64_000,
output_config: { effort: :low },
fallbacks: :default,
betas: [ LogbookTranscriber::FALLBACK_BETA ],
messages: [ { role: "user", content: blocks + [ { type: "text", text: "Transcribe all the text in this document, in reading order, as plain text. Include handwriting. Output only the text." } ] } ]
)
message = stream.accumulated_message
return [ nil, "none" ] if message.stop_reason == :refusal
[ message.content.select { _1.type == :text }.map(&:text).join, "claude" ]
rescue Anthropic::Errors::APIError => e
Rails.logger.warn("[document text] Claude: #{e.class}: #{e.message}")
[ nil, "none" ]
end
end
+8
View File
@@ -38,5 +38,13 @@
<p class="hint">With off-site backup to Google Drive, this many of the newest nightly database copies are kept there. Uploaded files are copied once and kept.</p>
</div>
<h2 class="card-title pt-2">Document search</h2>
<div>
<%= label_tag "settings[documents_claude_text]", "Read scanned documents with", class: "label" %>
<%= select_tag "settings[documents_claude_text]", options_for_select([ [ "Tesseract (free, on this server)", 0 ], [ "Claude (better on handwriting; uses the Anthropic API)", 1 ] ], @values["documents_claude_text"].to_i),
id: "settings_documents_claude_text", class: "input" %>
<p class="hint">Text in documents is read so search can find it. PDFs that contain text are read directly either way. Claude is used only when an ANTHROPIC_API_KEY is set<%= " (it isn't)" if ENV["ANTHROPIC_API_KEY"].blank? %>, and costs API credit per page.</p>
</div>
<div class="flex gap-2"><%= f.submit "Save Settings", class: "btn-primary" %><%= link_to "Cancel", admin_about_path, class: "btn-secondary" %></div>
<% end %>