Blackstone oil analysis, .sam.zip export/import, AI logbook transcription

- Oil analysis: Blackstone PDF import (20 elements, oil properties, universal
  averages), results stored per element, oil type, "leave out of trends",
  the original app's statuses; AvLab fixes (multi-line comments, unrelated
  labels no longer steal values). Existing records migrate.
- Aircraft archives in the original app's .sam.zip format (schema v2): export
  from Access & Sharing, background import from the Fleet page, rake tasks.
  Archives move both ways between this app and the original.
- Scanned logbook pages (images or PDFs) upload as a document; with
  ANTHROPIC_API_KEY set, Claude transcribes them into logbook entries.
- Fix: deleting a component or aircraft failed when AD compliances or
  inspection records pointed at a component.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rickyandClaude Opus 5.5 committed 2026-11-06 21:05:19 -05:00
1 parent 2d068e1a63
commit 1c6c2f88c0
54 files changed
+2794 -286

No files matched your search

+43
View File
@@ -0,0 +1,43 @@
# Aircraft archives (.sam.zip): a whole aircraft's records and files in one ZIP,
# in the format of the original Simple Aircraft Manager (manifest schema v2), so
# archives move both ways between this app and the original.
#
# manifest.json all records, with archive-local IDs linking them
# attachments/… pictures, document pages, squawk files, track logs…
#
# Records carry a few extra keys for data only this app has (all squawk
# attachments, engine monitor files, oil report PDFs and lab averages, newer AD
# fields). The original ignores keys it doesn't know inside records, but rejects
# unknown top-level keys, so none are added there.
module AircraftArchive
class Error < StandardError; end
SCHEMA_VERSION = 2
TOP_LEVEL_KEYS = %w[
schema_version exported_at source_instance aircraft component_types components document_collections
documents document_images logbook_entries squawks inspection_types inspection_records ads ad_compliances
consumable_records major_records notes oil_analysis_reports flight_logs features
].freeze
OPTIONAL_KEYS = %w[features].freeze # and flight_logs before schema v2
MANIFEST_MAX_BYTES = 50.megabytes
FILE_MAX_BYTES = 512.megabytes
ARCHIVE_MAX_BYTES = 10.gigabytes # uncompressed
MAX_RATIO = 100 # compression ratio above this looks like a zip bomb
# Leading bytes per extension; text formats have none to check.
MAGIC = {
"jpg" => [ "\xFF\xD8\xFF".b ], "jpeg" => [ "\xFF\xD8\xFF".b ], "png" => [ "\x89PNG\r\n\x1A\n".b ],
"gif" => [ "GIF8".b ], "webp" => [ "RIFF".b ], "bmp" => [ "BM".b ], "tif" => [ "II*\x00".b, "MM\x00*".b ],
"tiff" => [ "II*\x00".b, "MM\x00*".b ], "pdf" => [ "%PDF-".b ]
}.freeze
TEXT_EXTENSIONS = %w[txt csv gpx kml xml].freeze
EXTENSIONS = (MAGIC.keys + TEXT_EXTENSIONS).freeze
def self.extension(path) = File.extname(path.to_s).delete(".").downcase
def self.content_matches?(data, ext)
return EXTENSIONS.include?(ext) unless MAGIC.key?(ext)
MAGIC[ext].any? { data.byteslice(0, _1.bytesize) == _1 }
end
end
+191
View File
@@ -0,0 +1,191 @@
require "zip"
module AircraftArchive
# Writes an aircraft and everything recorded for it to a .sam.zip.
#
# AircraftArchive::Exporter.new(aircraft).write_to(path)
class Exporter
def initialize(aircraft, source: nil)
@aircraft = aircraft
@source = source
@files = {} # archive path → attachment
@missing = 0
end
attr_reader :missing # files that were attached but missing from storage
def filename = "#{@aircraft.tail_number.to_s.downcase.gsub(/[^a-z0-9-]/, '')}-#{Date.current.iso8601}.sam.zip"
def write_to(path)
manifest = build_manifest
Zip::OutputStream.open(path) do |zip|
zip.put_next_entry("manifest.json")
zip.write(JSON.pretty_generate(manifest))
@files.each do |archive_path, attachment|
zip.put_next_entry(archive_path)
attachment.blob.download { zip.write(_1) }
end
end
path
end
def build_manifest
a = @aircraft
components = a.components.includes(:component_type).to_a
component_ids = components.map(&:id)
inspection_types = (a.inspection_types.to_a + InspectionType.joins(:components).where(components: { id: component_ids }).to_a +
InspectionType.where(id: a.inspection_records.select(:inspection_type_id)).to_a).uniq
ads = (a.airworthiness_directives.to_a + AirworthinessDirective.joins(:components).where(components: { id: component_ids }).to_a +
AirworthinessDirective.where(id: a.ad_compliances.select(:airworthiness_directive_id)).to_a).uniq
{
schema_version: SCHEMA_VERSION,
exported_at: Time.current.iso8601,
source_instance: @source.to_s,
aircraft: aircraft_hash(a),
component_types: components.map(&:component_type).uniq.map { { id: id(_1), name: _1.name, consumable: _1.consumable } },
components: components.map { component_hash(_1) },
document_collections: a.document_collections.map do |c|
{ id: id(c), aircraft_id: id(a), name: c.name, description: c.description, visibility: c.visibility, starred: c.starred, components: [] }
end,
documents: a.documents.map do |d|
{ id: id(d), aircraft_id: id(a), collection_id: id(d.document_collection_id), doc_type: d.doc_type, name: d.name,
description: d.description, visibility: d.visibility, components: [] }
end,
document_images: a.documents.includes(pages_attachments: :blob).flat_map do |d|
d.pages.map { |page| { id: id(page), document_id: id(d), notes: "", image: file(page, "health/documents") } }
end,
logbook_entries: a.logbook_entries.map { logbook_hash(_1) },
squawks: a.squawks.includes(:reported_by, attachments_attachments: :blob).map { squawk_hash(_1) },
inspection_types: inspection_types.map { inspection_type_hash(_1, component_ids) },
inspection_records: a.inspection_records.map do |r|
{ id: id(r), date: r.date&.iso8601, aircraft_hours: dec(r.aircraft_hours), inspection_type_id: id(r.inspection_type_id),
logbook_entry_id: id(r.logbook_entry_id), aircraft_id: id(a), documents: [], component: [ id(r.component_id) ].compact,
notes: r.notes }
end,
ads: ads.map { ad_hash(_1, component_ids) },
ad_compliances: a.ad_compliances.map do |c|
{ id: id(c), ad_id: id(c.airworthiness_directive_id), date_complied: c.date_complied&.iso8601, compliance_notes: c.compliance_notes,
permanent: c.permanent, next_due_at_time: dec(c.next_due_at_time), aircraft_hours_at_compliance: dec(c.aircraft_hours_at_compliance),
logbook_entry_id: id(c.logbook_entry_id), inspection_record_id: id(c.inspection_record_id), aircraft_id: id(a),
component_id: id(c.component_id), not_applicable: c.not_applicable }
end,
consumable_records: a.consumable_records.map do |r|
{ id: id(r), record_type: r.record_type, aircraft_id: id(a), date: r.date&.iso8601, quantity_added: dec(r.quantity_added),
level_after: dec(r.level_after), consumable_type: r.consumable_type, flight_hours: dec(r.flight_hours), notes: r.notes,
excluded_from_averages: r.excluded_from_averages, flight_log_id: id(r.flight_log_id) }
end,
major_records: a.major_records.map do |r|
{ id: id(r), aircraft_id: id(a), record_type: r.record_type, title: r.title, description: r.description,
date_performed: r.date_performed&.iso8601, performed_by: r.performed_by, component_id: id(r.component_id),
form_337_document_id: id(r.form_337_document_id), stc_number: r.stc_number, stc_holder: r.stc_holder,
stc_document_id: id(r.stc_document_id), logbook_entry_id: id(r.logbook_entry_id), aircraft_hours: dec(r.aircraft_hours),
notes: r.notes, has_ica: r.has_ica, ica_notes: r.ica_notes }
end,
notes: a.notes.includes(:user).map do |n|
{ id: id(n), aircraft_id: id(a), text: n.text, public: n.public, added_by_display: n.user&.display_name,
added_timestamp: n.created_at&.iso8601, edited_at: n.edited_at&.iso8601 }
end,
oil_analysis_reports: a.oil_analyses.includes(report_attachment: :blob).map { oil_hash(_1) },
flight_logs: a.flight_logs.includes(:user, track_log_attachment: :blob, engine_data_attachment: :blob).map { flight_hash(_1) },
features: [ { feature: "airworthiness_enforcement", enabled: a.airworthiness_enforcement } ]
}
end
private
def id(record_or_id)
value = record_or_id.respond_to?(:id) ? record_or_id.id : record_or_id
value&.to_s
end
def dec(value) = value&.to_s
# Queue an attachment for the archive and return its archive path. Each file
# gets its own folder so original filenames survive. A file missing from
# storage is left out (and counted) rather than failing the export.
def file(attachable, folder)
attachment = attachable.is_a?(ActiveStorage::Attachment) ? attachable : attachable&.attachment
return nil unless attachment
unless attachment.blob.service.exist?(attachment.blob.key)
@missing += 1
return nil
end
name = attachment.filename.sanitized.presence || "file"
path = "attachments/#{folder}/#{attachment.blob.key}/#{name}"
@files[path] = attachment
path
end
def aircraft_hash(a)
{ id: id(a), tail_number: a.tail_number, make: a.make, model: a.model, serial_number: a.serial_number,
description: a.description, purchased: a.purchased&.iso8601, status: a.status, tach_time: dec(a.tach_time),
tach_time_offset: dec(a.tach_time_offset), hobbs_time: dec(a.hobbs_time), hobbs_time_offset: dec(a.hobbs_time_offset),
picture: file(a.picture, "aircraft_pictures"), icao_type: a.icao_type }
end
def component_hash(c)
{ id: id(c), aircraft_id: id(c.aircraft_id), parent_component_id: id(c.parent_component_id), component_type_id: id(c.component_type_id),
manufacturer: c.manufacturer, model: c.model, serial_number: c.serial_number, install_location: c.install_location,
notes: c.notes, status: c.status, date_in_service: c.date_in_service&.iso8601, hours_in_service: dec(c.hours_in_service),
hours_since_overhaul: dec(c.hours_since_overhaul), overhaul_date: c.overhaul_date&.iso8601, tbo_hours: c.tbo_hours,
tbo_days: c.tbo_days, inspection_hours: c.inspection_hours, inspection_days: c.inspection_days,
replacement_hours: c.replacement_hours, replacement_days: c.replacement_days, tbo_critical: c.tbo_critical,
on_condition: c.on_condition, inspection_critical: c.inspection_critical, replacement_critical: c.replacement_critical }
end
def logbook_hash(e)
{ id: id(e), aircraft_id: id(e.aircraft_id), log_type: e.log_type, date: e.date&.iso8601, text: e.text,
signoff_person: e.signoff_person, signoff_location: e.signoff_location, log_image_id: id(e.document_id),
aircraft_hours_at_entry: dec(e.aircraft_hours_at_entry), component_hours: {}, entry_type: e.entry_type,
page_number: e.page_number, components: [], related_documents: [] }
end
def squawk_hash(s)
paths = s.attachments.map { file(_1, "health/squawks") }
{ id: id(s), aircraft_id: id(s.aircraft_id), component_id: id(s.component_id), priority: s.priority,
issue_reported: s.issue_reported, attachment: paths.first, created_at: s.created_at&.iso8601,
reported_by_display: s.reported_by&.display_name, resolved: s.resolved, notes: s.notes, logbook_entries: [],
attachments: paths, resolved_at: s.resolved_at&.iso8601 }
end
def inspection_type_hash(t, component_ids)
{ id: id(t), name: t.name, recurring: t.recurring, required: t.required, recurring_hours: dec(t.recurring_hours),
recurring_days: t.recurring_days, recurring_months: t.recurring_months,
applicable_aircraft: t.aircraft_ids.include?(@aircraft.id) ? [ id(@aircraft) ] : [],
applicable_component: (t.component_ids & component_ids).map { id(_1) } }
end
def ad_hash(ad, component_ids)
{ id: id(ad), name: ad.name, short_description: ad.short_description, required_action: ad.required_action,
compliance_type: ad.compliance_type, trigger_condition: ad.trigger_condition, recurring: ad.recurring,
recurring_hours: dec(ad.recurring_hours), recurring_months: ad.recurring_months, recurring_days: ad.recurring_days,
bulletin_type: ad.bulletin_type, mandatory: ad.mandatory,
document_id: (id(ad.document_id) if ad.document&.aircraft_id == @aircraft.id), on_inspection_type: [],
applicable_aircraft: ad.aircraft_ids.include?(@aircraft.id) ? [ id(@aircraft) ] : [],
applicable_component: (ad.component_ids & component_ids).map { id(_1) },
effective_date: ad.effective_date&.iso8601, supersedes: ad.supersedes, superseded_by: ad.superseded_by,
docket_number: ad.docket_number, applicable_models: ad.applicable_models, product_type: ad.product_type,
document_url: ad.document_url }
end
def oil_hash(o)
{ id: id(o), aircraft_id: id(o.aircraft_id), component_id: id(o.component_id), sample_date: o.sample_date&.iso8601,
analysis_date: o.analysis_date&.iso8601, lab: o.lab.to_s, lab_number: o.sample_number.to_s, oil_type: o.oil_type.to_s,
oil_hours: dec(o.oil_hours), engine_hours: dec(o.engine_hours), oil_added_quarts: dec(o.oil_added),
elements_ppm: o.elements, oil_properties: o.oil_properties.presence, lab_comments: o.lab_comments.to_s,
status: o.status, notes: o.notes.to_s, excluded_from_averages: o.excluded_from_averages,
filter_hours: dec(o.filter_hours), element_averages: o.element_averages, report: file(o.report, "oil_reports") }
end
def flight_hash(f)
{ id: id(f), aircraft_id: id(f.aircraft_id), date: f.date&.iso8601, tach_time: dec(f.tach_time), tach_out: dec(f.tach_out),
tach_in: dec(f.tach_in), hobbs_time: dec(f.hobbs_time), hobbs_out: dec(f.hobbs_out), hobbs_in: dec(f.hobbs_in),
departure_location: f.departure_location.to_s, destination_location: f.destination_location.to_s, route: f.route.to_s,
oil_added: dec(f.oil_added), oil_added_type: f.oil_added_type.to_s, oil_level_after: dec(f.oil_level_after),
fuel_added: dec(f.fuel_added), fuel_added_type: f.fuel_added_type.to_s, fuel_level_after: dec(f.fuel_level_after),
track_log: file(f.track_log, "track_logs"), notes: f.notes.to_s, created_at: f.created_at&.iso8601,
engine_data: file(f.engine_data, "engine_data"), pilot_display: f.user&.display_name }
end
end
end
+479
View File
@@ -0,0 +1,479 @@
require "zip"
module AircraftArchive
# Reads a .sam.zip (from this app or the original Simple Aircraft Manager) and
# creates the aircraft with all its records and files.
#
# preview = AircraftArchive::Importer.check(path) # fast safety checks
# AircraftArchive::Importer.new(path, owner:, tail_number:).run
#
# Archive IDs are only used to link records inside the archive; everything
# gets new IDs here. Component types, inspection types and ADs are shared
# across aircraft, so existing ones with the same name are reused.
#
# No single transaction wraps the import (SQLite would block other writers for
# its whole length). If it fails part-way, the new aircraft and anything else
# it created are removed again.
class Importer
Preview = Struct.new(:tail_number, :make, :model, :source, :exported_at, :counts, keyword_init: true)
RECORD_SECTIONS = %w[components document_collections documents document_images logbook_entries squawks inspection_types
inspection_records ads ad_compliances consumable_records major_records notes oil_analysis_reports flight_logs].freeze
# Safety and format checks that don't import anything. Raises Error.
def self.check(path)
manifest = with_zip(path) { |zip| validate_zip(zip) }
a = manifest["aircraft"]
Preview.new(tail_number: a["tail_number"], make: a["make"], model: a["model"], source: manifest["source_instance"].presence,
exported_at: (Time.zone.parse(manifest["exported_at"].to_s) rescue nil),
counts: RECORD_SECTIONS.to_h { [ _1, Array(manifest[_1]).size ] })
end
def self.with_zip(path)
Zip.validate_entry_sizes = true # a lying size header raises instead of inflating forever
Zip::File.open(path) { |zip| yield zip }
rescue Zip::Error => e
raise Error, "Not a valid .sam.zip archive (#{e.message})"
end
def self.validate_zip(zip)
uncompressed = compressed = 0
zip.each do |entry|
raise Error, "The archive contains a link (#{entry.name}), which isn't allowed." if entry.symlink?
raise Error, "The archive contains an unsafe path: #{entry.name}" unless safe_name?(entry.name)
uncompressed += entry.size
compressed += entry.compressed_size
raise Error, "The archive is too large once unpacked." if uncompressed > ARCHIVE_MAX_BYTES
end
raise Error, "The archive is compressed suspiciously well (over #{MAX_RATIO}:1)." if compressed.positive? && uncompressed / compressed > MAX_RATIO
entry = zip.find_entry("manifest.json") or raise Error, "The archive has no manifest.json. Is it a .sam.zip export?"
raise Error, "manifest.json is too large." if entry.size > MANIFEST_MAX_BYTES
manifest = begin
JSON.parse(entry.get_input_stream.read.force_encoding("UTF-8"))
rescue JSON::ParserError => e
raise Error, "manifest.json couldn't be read (#{e.message.truncate(120)})"
end
raise Error, "manifest.json isn't a JSON object." unless manifest.is_a?(Hash)
version = manifest["schema_version"]
unless version.is_a?(Integer) && version.between?(1, SCHEMA_VERSION)
raise Error, "Unsupported archive version #{version.inspect} (this app reads versions 1–#{SCHEMA_VERSION})."
end
required = TOP_LEVEL_KEYS - OPTIONAL_KEYS - (version < 2 ? %w[flight_logs] : [])
missing = required - manifest.keys
raise Error, "manifest.json is missing #{missing.to_sentence}." if missing.any?
raise Error, "manifest.json has no aircraft tail number." unless manifest["aircraft"].is_a?(Hash) && manifest["aircraft"]["tail_number"].present?
RECORD_SECTIONS.each { |key| raise Error, "manifest.json #{key} must be a list." unless manifest[key].nil? || manifest[key].is_a?(Array) }
manifest
end
# Relative, no "..", valid UTF-8 (zip entry names arrive as raw bytes).
def self.safe_name?(name)
name = name.to_s.dup.force_encoding(Encoding::UTF_8)
return false if name.blank? || !name.valid_encoding? || name.start_with?("/", "\\") || name.match?(/\A[A-Za-z]:/)
name.unicode_normalize(:nfc).tr("\\", "/").split("/").exclude?("..")
end
attr_reader :counts, :warnings, :aircraft
def initialize(path, owner:, tail_number: nil, progress: nil)
@path = path
@owner = owner
@tail_number = tail_number
@progress = progress || ->(_counts) { }
@counts = Hash.new(0)
@warnings = []
@map = Hash.new { |h, k| h[k] = {} } # section → archive id → new record id
@created_blobs = []
@created_shared = [] # component types / inspection types / ADs made by this import
end
def run
self.class.with_zip(@path) do |zip|
@zip = zip
@manifest = self.class.validate_zip(zip)
import_all
end
{ aircraft: @aircraft, counts: @counts, warnings: @warnings }
rescue StandardError
clean_up
raise
end
private
def m(key) = Array(@manifest[key]).select { _1.is_a?(Hash) }
def import_all
import_aircraft
import_component_types
import_components
import_documents
import_logbook
import_squawks
import_inspections
import_ads
import_consumables
import_major_records
import_notes
import_oil_analyses
import_flight_logs
@aircraft.roles.find_or_create_by!(user: @owner, role: "owner") if @owner
@aircraft.log_event!("aircraft", "Aircraft imported from a .sam.zip archive",
notes: [ @manifest["source_instance"].presence && "Source: #{@manifest['source_instance']}", summary ].compact.join(". "),
user: @owner)
end
def summary = @counts.select { _2.positive? }.map { |k, n| "#{n} #{k.tr('_', ' ')}" }.join(", ")
def tick(section, n = 1)
@counts[section] += n
@progress.call(@counts) if (@counts.values.sum % 25).zero?
end
# ---- records ----
def import_aircraft
a = @manifest["aircraft"]
@aircraft = Aircraft.new(
tail_number: (@tail_number.presence || a["tail_number"]).to_s.strip.upcase,
make: a["make"].to_s, model: a["model"].to_s, serial_number: a["serial_number"].to_s, description: a["description"].to_s,
purchased: date(a["purchased"]), status: Aircraft::STATUSES.key?(a["status"]) ? a["status"] : "AVAILABLE",
tach_time: num(a["tach_time"] || a["flight_time"]) || 0, tach_time_offset: num(a["tach_time_offset"]) || 0,
hobbs_time: num(a["hobbs_time"]) || 0, hobbs_time_offset: num(a["hobbs_time_offset"]) || 0,
icao_type: a["icao_type"].to_s.upcase.presence&.then { _1.match?(/\A[A-Z0-9]{1,4}\z/) ? _1 : nil }
)
feature = m("features").find { _1["feature"] == "airworthiness_enforcement" }
@aircraft.airworthiness_enforcement = feature["enabled"] if feature && !feature["enabled"].nil? && @aircraft.has_attribute?(:airworthiness_enforcement)
@aircraft.save!(validate: false)
attach(@aircraft.picture, a["picture"], "aircraft picture") unless a["_missing"]
end
def import_component_types
m("component_types").each do |data|
name = data["name"].to_s.strip
next if name.blank?
type = ComponentType.where("LOWER(name) = ?", name.downcase).first ||
ComponentType.create!(name: name, consumable: data["consumable"] == true).tap { @created_shared << _1 }
@map[:component_type][data["id"].to_s] = type.id
end
end
def import_components
by_id = m("components").index_by { _1["id"].to_s }
ordered = []
visit = lambda do |data, seen = []|
return if ordered.include?(data) || seen.include?(data)
parent = by_id[data["parent_component_id"].to_s]
visit.(parent, seen + [ data ]) if parent
ordered << data
end
by_id.each_value { visit.(_1) }
ordered.each do |data|
type_id = @map[:component_type][data["component_type_id"].to_s]
next warn("Component #{data['manufacturer']} #{data['model']}: unknown component type, skipped.") unless type_id
c = @aircraft.components.new(
component_type_id: type_id, parent_component_id: @map[:component][data["parent_component_id"].to_s],
manufacturer: data["manufacturer"].to_s, model: data["model"].to_s, serial_number: data["serial_number"].to_s,
install_location: data["install_location"].to_s, notes: data["notes"].to_s,
status: Component::STATUSES.key?(data["status"]) ? data["status"] : "SPARE",
date_in_service: date(data["date_in_service"]) || Date.new(1900, 1, 1), hours_in_service: num(data["hours_in_service"]) || 0,
hours_since_overhaul: num(data["hours_since_overhaul"]) || 0, overhaul_date: date(data["overhaul_date"]),
**data.slice(*%w[tbo_hours tbo_days inspection_hours inspection_days replacement_hours replacement_days]).symbolize_keys,
**data.slice(*%w[tbo_critical on_condition inspection_critical replacement_critical]).transform_values { _1 == true }.symbolize_keys
)
c.save!(validate: false)
@map[:component][data["id"].to_s] = c.id
tick("components")
end
end
def import_documents
m("document_collections").each do |data|
c = @aircraft.document_collections.new(name: data["name"].to_s.presence || "Untitled", description: data["description"].to_s,
visibility: visibility(data["visibility"]) || "private", starred: data["starred"] == true)
c.save!(validate: false)
@map[:collection][data["id"].to_s] = c.id
end
documents = {}
m("documents").each do |data|
d = @aircraft.documents.new(name: data["name"].to_s.presence || "Untitled", description: data["description"].to_s,
doc_type: data["doc_type"].presence, visibility: visibility(data["visibility"]),
document_collection_id: @map[:collection][data["collection_id"].to_s])
d.doc_type = "OTHER" unless Document::TYPES.key?(d.doc_type)
d.save!(validate: false)
@map[:document][data["id"].to_s] = d.id
documents[d.id] = d
tick("documents")
end
m("document_images").each do |data|
document = documents[@map[:document][data["document_id"].to_s]] or next
next if data["_missing"]
tick("document_pages") if attach(document.pages, data["image"], "document #{document.name}")
end
end
def import_logbook
linked = 0
m("logbook_entries").each do |data|
e = @aircraft.logbook_entries.new(
log_type: LogbookEntry::LOG_TYPES.key?(data["log_type"]) ? data["log_type"] : "AC",
entry_type: LogbookEntry::ENTRY_TYPES.key?(data["entry_type"]) ? data["entry_type"] : LogbookEntry::ENTRY_TYPES.keys.first,
date: date(data["date"]) || Date.new(1900, 1, 1), text: data["text"].to_s.presence || "—",
signoff_person: data["signoff_person"], signoff_location: data["signoff_location"],
aircraft_hours_at_entry: num(data["aircraft_hours_at_entry"]), page_number: data["page_number"],
document_id: @map[:document][data["log_image_id"].to_s]
)
e.save!(validate: false)
@map[:logbook][data["id"].to_s] = e.id
linked += 1 if Array(data["components"]).any? || data["component_hours"].present?
tick("logbook_entries")
end
warn("#{linked} logbook #{linked == 1 ? 'entry was' : 'entries were'} linked to components; those links aren't kept here.") if linked.positive?
end
def import_squawks
m("squawks").each do |data|
s = @aircraft.squawks.new(
priority: Squawk::PRIORITIES.key?(data["priority"]) ? data["priority"] : 3, issue_reported: data["issue_reported"].to_s.presence || "—",
notes: data["notes"], resolved: data["resolved"] == true, component_id: @map[:component][data["component_id"].to_s],
created_at: time(data["created_at"]) || Time.current
)
s.resolved_at = time(data["resolved_at"]) || (s.created_at if s.resolved)
s.save!(validate: false)
paths = Array(data["attachments"]).presence || [ data["attachment"] ].compact
paths.each { attach(s.attachments, _1, "squawk “#{s.issue_reported.truncate(40)}”") } unless data["_missing"]
tick("squawks")
end
end
def import_inspections
m("inspection_types").each do |data|
name = data["name"].to_s.strip
next if name.blank?
type = InspectionType.where("LOWER(name) = ?", name.downcase).first
if type
warn("Inspection type “#{name}” already existed here and was reused.") if type.recurring != (data["recurring"] == true)
else
type = InspectionType.new(name: name, recurring: data["recurring"] == true, required: data["required"] != false,
recurring_hours: num(data["recurring_hours"]), recurring_days: data["recurring_days"],
recurring_months: data["recurring_months"])
type.save!(validate: false)
@created_shared << type
end
type.aircraft << @aircraft if Array(data["applicable_aircraft"]).any? && !type.aircraft.include?(@aircraft)
components(data["applicable_component"]).each { |c| type.components << c unless type.components.include?(c) }
@map[:inspection_type][data["id"].to_s] = type.id
end
m("inspection_records").each do |data|
type_id = @map[:inspection_type][data["inspection_type_id"].to_s]
next warn("An inspection record from #{data['date']} has no inspection type and was skipped.") unless type_id
r = @aircraft.inspection_records.new(
inspection_type_id: type_id, date: date(data["date"]) || Date.new(1900, 1, 1), aircraft_hours: num(data["aircraft_hours"]),
logbook_entry_id: @map[:logbook][data["logbook_entry_id"].to_s], notes: data["notes"],
component_id: @map[:component][Array(data["component"]).first.to_s]
)
r.save!(validate: false)
@map[:inspection_record][data["id"].to_s] = r.id
tick("inspection_records")
end
end
def import_ads
m("ads").each do |data|
name = data["name"].to_s.strip
next if name.blank?
ad = AirworthinessDirective.where("LOWER(name) = ?", name.downcase).first
unless ad
ad = AirworthinessDirective.new(
name: name, short_description: data["short_description"].to_s, required_action: data["required_action"].to_s,
compliance_type: AirworthinessDirective::COMPLIANCE_TYPES.key?(data["compliance_type"]) ? data["compliance_type"] : "standard",
trigger_condition: data["trigger_condition"].to_s, recurring: data["recurring"] == true,
recurring_hours: num(data["recurring_hours"]), recurring_months: data["recurring_months"], recurring_days: data["recurring_days"],
bulletin_type: AirworthinessDirective::BULLETIN_TYPES.key?(data["bulletin_type"]) ? data["bulletin_type"] : "ad",
mandatory: data["mandatory"] != false, document_id: @map[:document][data["document_id"].to_s],
effective_date: date(data["effective_date"]), supersedes: data["supersedes"], superseded_by: data["superseded_by"],
docket_number: data["docket_number"], applicable_models: data["applicable_models"], product_type: data["product_type"],
document_url: data["document_url"]
)
ad.save!(validate: false)
@created_shared << ad
end
ad.aircraft << @aircraft if Array(data["applicable_aircraft"]).any? && !ad.aircraft.include?(@aircraft)
components(data["applicable_component"]).each { |c| ad.components << c unless ad.components.include?(c) }
@map[:ad][data["id"].to_s] = ad.id
tick("ads")
end
m("ad_compliances").each do |data|
ad_id = @map[:ad][data["ad_id"].to_s] or next
c = @aircraft.ad_compliances.new(
airworthiness_directive_id: ad_id, date_complied: date(data["date_complied"]) || Date.new(1900, 1, 1),
compliance_notes: data["compliance_notes"].to_s, permanent: data["permanent"] == true,
next_due_at_time: num(data["next_due_at_time"]) || 0, aircraft_hours_at_compliance: num(data["aircraft_hours_at_compliance"]),
logbook_entry_id: @map[:logbook][data["logbook_entry_id"].to_s],
inspection_record_id: @map[:inspection_record][data["inspection_record_id"].to_s],
component_id: @map[:component][data["component_id"].to_s], not_applicable: data["not_applicable"] == true
)
c.save!(validate: false)
tick("ad_compliances")
end
end
def import_consumables
@consumable_flight_links = {}
m("consumable_records").each do |data|
next unless ConsumableRecord::TYPES.key?(data["record_type"])
r = @aircraft.consumable_records.new(
record_type: data["record_type"], date: date(data["date"]) || Date.new(1900, 1, 1), quantity_added: num(data["quantity_added"]) || 0,
level_after: num(data["level_after"]), consumable_type: data["consumable_type"].to_s, flight_hours: num(data["flight_hours"]) || 0,
notes: data["notes"].to_s, excluded_from_averages: data["excluded_from_averages"] == true
)
r.save!(validate: false)
@consumable_flight_links[r.id] = data["flight_log_id"].to_s if data["flight_log_id"]
tick("consumable_records")
end
end
def import_major_records
m("major_records").each do |data|
r = @aircraft.major_records.new(
record_type: MajorRecord::RECORD_TYPES.key?(data["record_type"]) ? data["record_type"] : "repair",
title: data["title"].to_s.presence || "—", description: data["description"].to_s, date_performed: date(data["date_performed"]),
performed_by: data["performed_by"].to_s, component_id: @map[:component][data["component_id"].to_s],
form_337_document_id: @map[:document][data["form_337_document_id"].to_s], stc_number: data["stc_number"].to_s,
stc_holder: data["stc_holder"].to_s, stc_document_id: @map[:document][data["stc_document_id"].to_s],
logbook_entry_id: @map[:logbook][data["logbook_entry_id"].to_s], aircraft_hours: num(data["aircraft_hours"]),
notes: data["notes"].to_s, has_ica: data["has_ica"] == true, ica_notes: data["ica_notes"]
)
r.save!(validate: false)
tick("major_records")
end
end
def import_notes
m("notes").each do |data|
at = time(data["added_timestamp"]) || Time.current
n = @aircraft.notes.new(text: data["text"].to_s.presence || "—", public: data["public"] == true, created_at: at,
edited_at: time(data["edited_at"]))
n.text = "#{n.text}\n\n— #{data['added_by_display']}" if data["added_by_display"].present?
n.save!(validate: false)
tick("notes")
end
end
def import_oil_analyses
m("oil_analysis_reports").each do |data|
o = @aircraft.oil_analyses.new(
component_id: @map[:component][data["component_id"].to_s], sample_date: date(data["sample_date"]) || Date.new(1900, 1, 1),
analysis_date: date(data["analysis_date"]), lab: data["lab"].presence, sample_number: data["lab_number"].presence,
oil_type: data["oil_type"].presence, oil_hours: num(data["oil_hours"]), engine_hours: num(data["engine_hours"]),
filter_hours: num(data["filter_hours"]), oil_added: num(data["oil_added_quarts"]),
elements: data["elements_ppm"].is_a?(Hash) ? data["elements_ppm"] : {},
element_averages: data["element_averages"].is_a?(Hash) ? data["element_averages"] : {},
oil_properties: data["oil_properties"].is_a?(Hash) ? data["oil_properties"] : {},
lab_comments: data["lab_comments"].presence, notes: data["notes"].presence,
status: OilAnalysis::STATUSES.key?(data["status"]) ? data["status"] : nil,
excluded_from_averages: data["excluded_from_averages"] == true
)
o.save!(validate: false)
attach(o.report, data["report"], "oil analysis #{o.sample_date}") if data["report"]
tick("oil_analyses")
end
end
def import_flight_logs
m("flight_logs").each do |data|
f = @aircraft.flight_logs.new(
date: date(data["date"]) || Date.new(1900, 1, 1), tach_time: num(data["tach_time"]) || 0, tach_out: num(data["tach_out"]),
tach_in: num(data["tach_in"]), hobbs_time: num(data["hobbs_time"]), hobbs_out: num(data["hobbs_out"]),
hobbs_in: num(data["hobbs_in"]), departure_location: data["departure_location"].to_s,
destination_location: data["destination_location"].to_s, route: data["route"].to_s, oil_added: num(data["oil_added"]),
oil_added_type: data["oil_added_type"].to_s, oil_level_after: num(data["oil_level_after"]), fuel_added: num(data["fuel_added"]),
fuel_added_type: data["fuel_added_type"].to_s, fuel_level_after: num(data["fuel_level_after"]), notes: data["notes"].to_s,
created_at: time(data["created_at"]) || Time.current
)
f.save!(validate: false)
@map[:flight_log][data["id"].to_s] = f.id
attach(f.track_log, data["track_log"], "flight #{f.date} track log") if data["track_log"] && !data["_missing"]
attach(f.engine_data, data["engine_data"], "flight #{f.date} engine data") if data["engine_data"]
tick("flight_logs")
end
(@consumable_flight_links || {}).each do |record_id, archive_flight_id|
flight_id = @map[:flight_log][archive_flight_id] or next
ConsumableRecord.where(id: record_id).update_all(flight_log_id: flight_id)
end
end
# ---- files ----
# Copy one archive file into storage and attach it. Problems with a single
# file become warnings; the record is kept without it.
def attach(target, archive_path, what)
return false if archive_path.blank?
blob = upload(archive_path.to_s)
target.attach(blob)
true
rescue Error => e
warn("#{what.upcase_first}: #{e.message}")
false
end
def upload(archive_path)
raise Error, "unsafe file path #{archive_path}" unless self.class.safe_name?(archive_path)
entry = @zip.find_entry(archive_path) or raise Error, "file #{File.basename(archive_path)} isn't in the archive"
raise Error, "#{File.basename(archive_path)} is too large" if entry.size > FILE_MAX_BYTES
ext = AircraftArchive.extension(archive_path)
raise Error, "#{File.basename(archive_path)} isn't an allowed file type" unless EXTENSIONS.include?(ext)
Tempfile.create([ "sam-import", ".#{ext}" ], binmode: true) do |tmp|
entry.get_input_stream { |input| IO.copy_stream(input, tmp) }
tmp.rewind
head = tmp.read(16).to_s
raise Error, "#{File.basename(archive_path)} doesn't contain what its name says (.#{ext})" unless AircraftArchive.content_matches?(head, ext)
tmp.rewind
ActiveStorage::Blob.create_and_upload!(io: tmp, filename: File.basename(archive_path)).tap { @created_blobs << _1 }
end
end
# ---- helpers ----
def warn(message)
@warnings << message
nil
end
def components(ids) = Array(ids).filter_map { @map[:component][_1.to_s] }.then { Component.where(id: _1) }
def visibility(value) = DocumentCollection::VISIBILITIES.key?(value) ? value : nil
def date(value)
Date.iso8601(value.to_s[0, 10]) if value.present?
rescue Date::Error
nil
end
def time(value)
Time.zone.parse(value.to_s) if value.present?
rescue ArgumentError
nil
end
def num(value)
return nil if value.blank?
BigDecimal(value.to_s)
rescue ArgumentError
nil
end
# Undo a failed import: the aircraft (and its records, via dependent
# destroys), shared records only this import created, and uploaded files.
def clean_up
@aircraft&.destroy if @aircraft&.persisted?
@created_shared.reverse_each { |record| record.destroy if record.class.exists?(record.id) }
@created_blobs.each { |blob| blob.purge if ActiveStorage::Blob.exists?(blob.id) }
rescue StandardError => e
Rails.logger.error("[aircraft import] clean-up failed: #{e.class}: #{e.message}")
end
end
end
+91
View File
@@ -0,0 +1,91 @@
require "anthropic"
# Reads scanned logbook pages with Claude and returns the entries on them.
# One call handles one batch: several page images, or one PDF.
#
# LogbookTranscriber.new.transcribe([ { data: bytes, media_type: "image/jpeg" }, … ], carried: [...])
#
# `carried` are entries already read from page 0 of this batch (the last page of
# the previous one), so an entry continuing onto the next page is completed
# rather than duplicated.
class LogbookTranscriber
class Error < StandardError; end
MODEL = ENV.fetch("LOGBOOK_AI_MODEL", "claude-opus-5-5")
EFFORT = ENV.fetch("LOGBOOK_AI_EFFORT", "high") # handwriting rewards a careful read
MAX_TOKENS = 64_000
PROMPT = File.read(File.join(__dir__, "logbook_transcriber/prompt.txt")).freeze
SCHEMA = JSON.parse(File.read(File.join(__dir__, "logbook_transcriber/schema.json"))).freeze
FALLBACK_BETA = "server-side-fallback-2026-07-01"
IMAGE_TYPES = %w[image/jpeg image/png image/gif image/webp].freeze
MAX_IMAGE_BYTES = 5.megabytes # the API's per-image limit
MAX_PDF_BYTES = 30.megabytes # request limit is 32 MB
Result = Struct.new(:entries, :non_logbook_pages, :unparseable_pages, :model, keyword_init: true)
class << self
attr_writer :client # tests swap in a fake
def client = @client ||= Anthropic::Client.new
end
def initialize(client: self.class.client, model: MODEL, effort: EFFORT)
@client = client
@model = model
@effort = effort
end
def transcribe(pages, carried: [])
stream = @client.beta.messages.stream(
model: @model,
max_tokens: MAX_TOKENS,
system_: PROMPT,
output_config: { effort: @effort.to_sym, format_: { type: :json_schema, schema: SCHEMA } },
# If a safety classifier declines, the API retries on its default substitute model.
fallbacks: :default,
betas: [ FALLBACK_BETA ],
messages: [ { role: "user", content: content(pages, carried) } ]
)
message = stream.accumulated_message
case message.stop_reason
when :refusal then raise Error, "Claude declined to read these pages"
when :max_tokens then raise Error, "the transcription was too long to finish; try fewer pages per file"
end
text = message.content.select { _1.type == :text }.map(&:text).join
data = JSON.parse(text)
Result.new(entries: Array(data["entries"]), non_logbook_pages: Array(data["non_logbook_pages"]),
unparseable_pages: Array(data["unparseable_pages"]), model: message.model.to_s)
rescue JSON::ParserError
raise Error, "Claude's answer wasn't valid JSON"
rescue Anthropic::Errors::RateLimitError
raise Error, "the Anthropic API is rate limiting requests; try again in a few minutes"
rescue Anthropic::Errors::AuthenticationError, Anthropic::Errors::PermissionDeniedError
raise Error, "the Anthropic API key was rejected"
rescue Anthropic::Errors::APIStatusError => e
raise Error, "the Anthropic API returned an error (#{e.status})"
rescue Anthropic::Errors::APIConnectionError
raise Error, "couldn't reach the Anthropic API"
end
private
def content(pages, carried)
blocks = pages.each_with_index.flat_map do |page, i|
source = { type: "base64", media_type: page[:media_type], data: Base64.strict_encode64(page[:data]) }
if page[:media_type] == "application/pdf"
[ { type: "text", text: "This PDF holds the logbook pages; page indices count its pages from 0." },
{ type: "document", source: source } ]
else
[ { type: "text", text: "Page #{i}:" }, { type: "image", source: source } ]
end
end
if carried.any?
context = carried.map do |e|
"- #{e['date'] || 'undated'} (#{e['log_type']}, signed #{e['signoff_person'] || 'unsigned'}): #{e['text'].to_s.truncate(400)}"
end
blocks << { type: "text", text: "Entries already read from page 0 (it was the last page of the previous batch):\n#{context.join("\n")}" }
end
blocks << { type: "text", text: "Extract every logbook entry from these pages." }
end
end
@@ -0,0 +1,36 @@
You are an expert aviation maintenance logbook analyst with deep knowledge of FAA regulations,
aircraft maintenance records, and standard logbook formats.
You will get numbered pages of an aircraft logbook (page 0 is the first page in this request).
Extract every logbook entry visible.
Rules:
- A single entry may span two consecutive pages (look for "continued" or entries that cut off mid-sentence).
- Several separate entries may appear on one page (separated by dates, rules, or spacing).
- Some pages are not logbook entries: FAA Form 337 (major repair/alteration), yellow tags (8130-3),
weight and balance records, equipment lists, AD lists, or other administrative forms. List them in
non_logbook_pages instead of extracting entries from them.
- Dates: convert MM/DD/YYYY, MM/DD/YY, or written dates to YYYY-MM-DD. If the year is ambiguous, use the
surrounding entries. Use null only if the date is truly unreadable.
- Signoffs: include the mechanic's name and certificate number (e.g. "A&P 123456", IA number) or the repair
station number.
Entry types:
- MAINTENANCE: repairs, part replacements, AD compliance, alterations, STCs, 337s referenced
- INSPECTION: annual, 100-hour, phase checks, pre-buy, progressive, conditional inspections
- FLIGHT: test flights, ferry flights, return-to-service flights
- HOURS_UPDATE: an hours entry with no specific work described
- OTHER: administrative entries, certifications
Fields:
- log_type: AC (airframe), ENG (engine), PROP (propeller), or OTHER
- text: the entry text verbatim, or a close paraphrase where wording is unclear; write [?] for illegible words
- aircraft_hours: the tach/total time the entry records (a number), or null if none is written
- signoff_person / signoff_location: a string, or null
- page_start / page_end: page indices within this request
- confidence: high (clearly legible), medium (some guesswork), low (mostly illegible)
- unparseable_pages: pages too illegible to extract anything from
Overlapping pages: when the request says entries were already read from page 0, return each of those
entries exactly once, completed with any text that continues onto later pages, and do not return any
other entry that lies wholly on page 0.
@@ -0,0 +1,30 @@
{
"type": "object",
"properties": {
"entries": {
"type": "array",
"items": {
"type": "object",
"properties": {
"date": { "type": ["string", "null"], "description": "YYYY-MM-DD, or null if unreadable" },
"log_type": { "type": "string", "enum": ["AC", "ENG", "PROP", "OTHER"] },
"entry_type": { "type": "string", "enum": ["MAINTENANCE", "INSPECTION", "FLIGHT", "HOURS_UPDATE", "OTHER"] },
"text": { "type": "string" },
"aircraft_hours": { "type": ["number", "null"] },
"signoff_person": { "type": ["string", "null"] },
"signoff_location": { "type": ["string", "null"] },
"page_start": { "type": "integer" },
"page_end": { "type": "integer" },
"confidence": { "type": "string", "enum": ["high", "medium", "low"] }
},
"required": ["date", "log_type", "entry_type", "text", "aircraft_hours", "signoff_person", "signoff_location",
"page_start", "page_end", "confidence"],
"additionalProperties": false
}
},
"non_logbook_pages": { "type": "array", "items": { "type": "integer" } },
"unparseable_pages": { "type": "array", "items": { "type": "integer" } }
},
"required": ["entries", "non_logbook_pages", "unparseable_pages"],
"additionalProperties": false
}
+22 -186
View File
@@ -1,207 +1,43 @@
require "pdf-reader"
# Reads an AvLab (Aviation Laboratories) oil analysis report PDF. A report
# holds the current sample plus up to two previous ones, each with its dates,
# hours, lab verdict, comments and a ppm table (value, lab average beneath).
# Reads an oil analysis lab report PDF into sample records. Supported labs:
# - Aviation Laboratories (AvLab): the current sample plus up to two previous
# ones, each with its own verdict, comments and lab averages.
# - Blackstone Laboratories: a grid of up to ~5 samples (newest first) with 20
# elements, oil properties, and the lab's universal averages.
#
# The PDF's text layer is messy: it hides the cell color codes ("#00CC00") as
# larger text on top of the results. So this works from positioned characters,
# drops the color-code layer by font size, rebuilds words and lines, and maps
# each result to its element column by x position.
# Every sample comes back as a hash of OilAnalysis attributes plus :current.
class OilReportParser
class Error < StandardError; end
LAB = "AvLab".freeze
COLOR_CODE_FONT_SIZE = 8.5 # the hidden color codes are 9pt; real text is 7–8pt
COLUMNS = {
"Iron" => "iron", "Copper" => "copper", "Nickel" => "nickel", "Chromium" => "chromium",
"Silver" => "silver", "Magnesium" => "magnesium", "Aluminum" => "aluminum", "Lead" => "lead",
"Silicon" => "silicon", "Titanium" => "titanium", "Tin" => "tin", "Moly." => "molybdenum"
}.freeze
LABELS = {
"Sample Date:" => :sample_date, "Analysis Date:" => :analysis_date, "Sample Number:" => :sample_number,
"TSN/TSO:" => :engine_hours, "Oil Hours:" => :oil_hours, "Filter Hours:" => :filter_hours,
"Oil Added:" => :oil_added
}.freeze
LABEL_MAX_X = 108 # left-hand labels end before here…
VALUE_X = 105..160 # …and their values sit in this column
Char = Struct.new(:text, :x, :endx, :y, :page)
Word = Struct.new(:text, :x, :endx, :y, :page) do
def center = (x + endx) / 2.0
end
Line = Struct.new(:y, :page, :words) do
def text = words.map(&:text).join(" ")
end
Report = Struct.new(:lab, :tail_number, :engine_serial, :engine_model, :samples, keyword_init: true)
Report = Struct.new(:lab, :tail_number, :engine_serial, :engine_model, :oil_type, :samples, keyword_init: true)
def self.from_pdf(io)
reader = PDF::Reader.new(io)
chars = reader.pages.each_with_index.flat_map do |page, index|
receiver = PDF::Reader::PageTextReceiver.new
page.walk(receiver)
# Keep overlapping glyphs: a real "0" under a hidden one would otherwise be dropped.
receiver.runs(merge: false, skip_overlapping: false).filter_map do |run|
next if run.font_size >= COLOR_CODE_FONT_SIZE || run.text.strip.empty?
Char.new(run.text, run.x, run.endx, run.y, index)
end
receivers = reader.pages.map { |page| PDF::Reader::PageTextReceiver.new.tap { page.walk(_1) } }
text = receivers.flat_map { |r| r.runs(merge: true, skip_overlapping: false).map(&:text) }.join(" ")
if text.match?(/blackstone-labs\.com|BLACKSTONE LABORATORIES/i)
Blackstone.new(receivers.first.runs(merge: true, skip_overlapping: false)).parse
elsif text.match?(/OIL ANALYSIS RESULTS IN PARTS PER MILLION|AVIATION LABORATORIES|avlab\.com/i)
Avlab.from_receivers(receivers)
else
raise Error, "Not a recognized lab report (AvLab and Blackstone PDFs are supported)"
end
new(chars).parse
rescue PDF::Reader::MalformedPDFError, PDF::Reader::UnsupportedFeatureError => e
raise Error, "Couldn't read the PDF (#{e.message})"
end
def initialize(chars)
@lines = build_lines(chars)
# "1,903" → 1903, "<0.5" → 0.5, "5 qts" → 5, ".3" → 0.3; nil for "-", "N/A"…
def self.number(text)
digits = text.to_s.delete(",").strip.sub(/\A[<>]\s*/, "")[/\A\d*\.?\d+/] or return nil
BigDecimal(digits.sub(/\A\./, "0."))
end
def parse
unless @lines.any? { _1.text.include?("OIL ANALYSIS RESULTS IN PARTS PER MILLION") }
raise Error, "Not an AvLab oil analysis report"
end
samples = sample_blocks.map { parse_sample(_1) }.select { _1[:sample_date] }
raise Error, "No samples found in the report" if samples.empty?
Report.new(lab: LAB, tail_number: after("Tail No.:"), engine_serial: after("Engine S/N:"),
engine_model: after("Engine Model:", words: 3), samples: samples)
end
private
# Characters → lines (same baseline ±1pt) → words (glyphs closer than a space).
def build_lines(chars)
lines = []
chars.sort_by { [ _1.page, -_1.y ] }.each do |c|
line = lines.reverse_each.find { _1.page == c.page && (_1.y - c.y).abs <= 1.0 }
line ? line.words << c : lines << Line.new(c.y, c.page, [ c ])
end
lines.each do |line|
line.words = line.words.sort_by(&:x).each_with_object([]) do |c, words|
last = words.last
if last && c.x - last.endx <= 1.6
last.text += c.text
last.endx = c.endx
else
words << Word.new(c.text.dup, c.x, c.endx, c.y, c.page)
end
end
end
end
def header_lines
first = @lines.find { _1.words.first&.text.in?(%w[CURRENT PREVIOUS]) }
first ? @lines.take_while { _1 != first } : @lines
end
# The word(s) following a label on the same line, e.g. "Tail No.:" → "N12345".
def after(label, words: 1)
parts = label.split
header_lines.each do |line|
texts = line.words.map(&:text)
texts.each_cons(parts.size).with_index do |window, i|
next unless window == parts
found = texts[i + parts.size, words]&.take_while { !_1.end_with?(":") }
return found.join(" ").presence if found&.any?
end
end
nil
end
# Lines from each "CURRENT SAMPLE" / "PREVIOUS SAMPLE n" heading to the next.
def sample_blocks
starts = @lines.each_index.select { |i| @lines[i].words.first&.text.in?(%w[CURRENT PREVIOUS]) && @lines[i].words[1]&.text == "SAMPLE" }
starts.each_with_index.map { |s, n| @lines[s...(starts[n + 1] || @lines.size)] }
end
def parse_sample(block)
heading = block.first
verdict = heading.words.select { _1.x > 140 && _1.x < 470 }.map(&:text).join(" ")
fields = labeled_values(block)
comments = block.find { _1.words.first&.text == "Comments:" }&.words&.drop(1)&.map(&:text)&.join(" ")
{
current: heading.words.first.text == "CURRENT",
sample_date: date(fields[:sample_date]),
analysis_date: date(fields[:analysis_date]),
sample_number: fields[:sample_number],
engine_hours: number(fields[:engine_hours]),
oil_hours: number(fields[:oil_hours]),
filter_hours: number(fields[:filter_hours]),
oil_added: number(fields[:oil_added]),
status: status(verdict),
lab_comments: [ (verdict unless status(verdict) == "normal"), comments ].compact_blank.join("\n").presence
}.merge(results(block))
end
# Left-hand "Label:" lines and the values beside them. A value isn't always on
# the label's baseline, so each value goes to the nearest label above or below.
def labeled_values(block)
labels = block.filter_map do |line|
text = line.words.select { _1.x < LABEL_MAX_X - 20 || (_1.x < LABEL_MAX_X && _1.text.end_with?(":")) }.map(&:text).join(" ")
[ LABELS[text], line.y ] if LABELS[text]
end
values = block.drop(1).flat_map { |line| line.words.select { VALUE_X.cover?(_1.x) } } # skip the heading
values.each_with_object({}) do |word, found|
key, = labels.min_by { (_2 - word.y).abs }
found[key] ||= word.text if key
end
end
# The ppm row under the element headings, and the (average) row under that.
def results(block)
header = block.find { |line| COLUMNS.keys.count { line.words.map(&:text).include?(_1) } >= 6 } or return {}
columns = header.words.filter_map { |w| [ COLUMNS[w.text], w.center ] if COLUMNS[w.text] }
below = block.select { _1.y < header.y && _1.y > header.y - 25 }
below.each_with_object({}) do |line, out|
# Results start right of the label column ("Cylinder Type: steel" shares these lines).
merge_less_than(line.words.select { _1.x >= VALUE_X.end }).each do |word|
element, = columns.min_by { (_2 - word.center).abs }
if word.text.start_with?("(")
out[:"#{element}_avg"] = ppm(word.text.delete("()"))
else
out[element.to_sym] = ppm(word.text)
end
end
end
end
# "<" and "0.1" are drawn as separate words.
def merge_less_than(words)
words.each_with_object([]) do |w, out|
if out.last&.text == "<"
out.last.text = "< #{w.text}"
out.last.endx = w.endx
else
out << w.dup
end
end
end
# "N/A" → nil; "< 0.1" (below detection) → 0; ".3" → 0.3
def ppm(text)
return nil if text.blank? || text == "N/A"
return 0 if text.start_with?("<")
number(text)
end
def number(text)
BigDecimal(text.to_s.strip.sub(/\A\./, "0."))
rescue ArgumentError, TypeError
nil
end
def date(text)
# M/D/YYYY
def self.date(text)
Date.strptime(text, "%m/%d/%Y") if text
rescue Date::Error
nil
end
def status(verdict)
case verdict.upcase
when /ABNORMAL|HIGH|CRITICAL|SEVERE|PROBLEM/ then "high"
when /APPEARS NORMAL/ then "normal"
else "elevated"
end
end
end
+208
View File
@@ -0,0 +1,208 @@
class OilReportParser
# Aviation Laboratories (AvLab) report. Each sample block has its dates,
# hours, the lab's verdict, comments and a ppm table (value, lab average beneath).
#
# The PDF's text layer is messy: it hides the cell color codes ("#00CC00") as
# larger text on top of the results. So this works from positioned characters,
# drops the color-code layer by font size, rebuilds words and lines, and maps
# each result to its element column by x position.
class Avlab
LAB = "AvLab".freeze
COLOR_CODE_FONT_SIZE = 8.5 # the hidden color codes are 9pt; real text is 7–8pt
WORD_GAP = 1.0 # glyphs closer than this are one word (spaces run 1.3pt+)
COLUMNS = {
"Iron" => "iron", "Copper" => "copper", "Nickel" => "nickel", "Chromium" => "chromium",
"Silver" => "silver", "Magnesium" => "magnesium", "Aluminum" => "aluminum", "Lead" => "lead",
"Silicon" => "silicon", "Titanium" => "titanium", "Tin" => "tin", "Moly." => "molybdenum"
}.freeze
LABELS = {
"Sample Date:" => :sample_date, "Analysis Date:" => :analysis_date, "Sample Number:" => :sample_number,
"TSN/TSO:" => :engine_hours, "Oil Hours:" => :oil_hours, "Filter Hours:" => :filter_hours,
"Oil Added:" => :oil_added
}.freeze
LABEL_MAX_X = 108 # left-hand labels end before here…
VALUE_X = 105..160 # …and their values sit in this column
Char = Struct.new(:text, :x, :endx, :y, :page)
Word = Struct.new(:text, :x, :endx, :y, :page) do
def center = (x + endx) / 2.0
end
Line = Struct.new(:y, :page, :words) do
def text = words.map(&:text).join(" ")
end
def self.from_receivers(receivers)
chars = receivers.each_with_index.flat_map do |receiver, index|
# Keep overlapping glyphs: a real "0" under a hidden one would otherwise be dropped.
receiver.runs(merge: false, skip_overlapping: false).filter_map do |run|
next if run.font_size >= COLOR_CODE_FONT_SIZE || run.text.strip.empty?
Char.new(run.text, run.x, run.endx, run.y, index)
end
end
new(chars).parse
end
def initialize(chars)
@lines = build_lines(chars)
end
def parse
unless @lines.any? { _1.text.include?("OIL ANALYSIS RESULTS IN PARTS PER MILLION") }
raise Error, "Not an AvLab oil analysis report"
end
samples = sample_blocks.map { parse_sample(_1) }.select { _1[:sample_date] }
raise Error, "No samples found in the report" if samples.empty?
Report.new(lab: LAB, tail_number: after("Tail No.:"), engine_serial: after("Engine S/N:"),
engine_model: after("Engine Model:", words: 3), samples: samples)
end
private
# Characters → lines (same baseline ±1pt) → words (glyphs closer than a space).
def build_lines(chars)
lines = []
chars.sort_by { [ _1.page, -_1.y ] }.each do |c|
line = lines.reverse_each.find { _1.page == c.page && (_1.y - c.y).abs <= 1.0 }
line ? line.words << c : lines << Line.new(c.y, c.page, [ c ])
end
lines.each do |line|
line.words = line.words.sort_by(&:x).each_with_object([]) do |c, words|
last = words.last
if last && c.x - last.endx <= WORD_GAP
last.text += c.text
last.endx = c.endx
else
words << Word.new(c.text.dup, c.x, c.endx, c.y, c.page)
end
end
end
end
def header_lines
first = @lines.find { _1.words.first&.text.in?(%w[CURRENT PREVIOUS]) }
first ? @lines.take_while { _1 != first } : @lines
end
# The word(s) following a label on the same line, e.g. "Tail No.:" → "N12345".
def after(label, words: 1)
parts = label.split
header_lines.each do |line|
texts = line.words.map(&:text)
texts.each_cons(parts.size).with_index do |window, i|
next unless window == parts
found = texts[i + parts.size, words]&.take_while { !_1.end_with?(":") }
return found.join(" ").presence if found&.any?
end
end
nil
end
# Lines from each "CURRENT SAMPLE" / "PREVIOUS SAMPLE n" heading to the next.
def sample_blocks
starts = @lines.each_index.select { |i| @lines[i].words.first&.text.in?(%w[CURRENT PREVIOUS]) && @lines[i].words[1]&.text == "SAMPLE" }
starts.each_with_index.map { |s, n| @lines[s...(starts[n + 1] || @lines.size)] }
end
def parse_sample(block)
heading = block.first
verdict = heading.words.select { _1.x > 140 && _1.x < 470 }.map(&:text).join(" ")
fields = labeled_values(block)
comments = comments(block)
elements, averages = results(block)
{
current: heading.words.first.text == "CURRENT",
sample_date: OilReportParser.date(fields[:sample_date]),
analysis_date: OilReportParser.date(fields[:analysis_date]),
sample_number: fields[:sample_number],
engine_hours: OilReportParser.number(fields[:engine_hours]),
oil_hours: OilReportParser.number(fields[:oil_hours]),
filter_hours: OilReportParser.number(fields[:filter_hours]),
oil_added: OilReportParser.number(fields[:oil_added]),
status: status(verdict),
lab_comments: [ (verdict unless status(verdict) == "normal"), comments ].compact_blank.join("\n").presence,
elements: elements,
element_averages: averages
}
end
# Left-hand "Label:" lines and the values beside them. A value isn't always on
# the label's baseline, so each value goes to the nearest label above or below.
# Every label counts, including ones we don't keep (e.g. "Filter Wt. (mgs):"),
# so their values don't drift onto a neighbouring label.
def labeled_values(block)
labels = block.filter_map do |line|
text = line.words.select { _1.x < LABEL_MAX_X - 20 || (_1.x < LABEL_MAX_X && _1.text.end_with?(":")) }.map(&:text).join(" ")
[ LABELS[text], line.y ] if text.end_with?(":")
end
values = block.drop(1).flat_map { |line| line.words.select { VALUE_X.cover?(_1.x) } } # skip the heading
values.each_with_object({}) do |word, found|
key, = labels.min_by { (_2 - word.y).abs }
found[key] ||= word.text if key
end
end
# "Comments: …" and the lines that continue it at the same left margin.
def comments(block)
start = block.index { _1.words.first&.text == "Comments:" } or return nil
first = block[start]
margin = first.words.first.x
lines = [ first.words.drop(1) ]
block[(start + 1)..].each_with_index do |line, i|
previous = block[start + i]
break unless (line.words.first.x - margin).abs <= 3 && previous.y - line.y <= 14 && line.page == previous.page
lines << line.words
end
lines.flatten.map(&:text).join(" ").presence
end
# The ppm row under the element headings, and the (average) row under that.
def results(block)
elements, averages = {}, {}
header = block.find { |line| COLUMNS.keys.count { line.words.map(&:text).include?(_1) } >= 6 } or return [ elements, averages ]
columns = header.words.filter_map { |w| [ COLUMNS[w.text], w.center ] if COLUMNS[w.text] }
below = block.select { _1.y < header.y && _1.y > header.y - 25 }
below.each do |line|
# Results start right of the label column ("Cylinder Type: steel" shares these lines).
merge_less_than(line.words.select { _1.x >= VALUE_X.end }).each do |word|
element, = columns.min_by { (_2 - word.center).abs }
if word.text.start_with?("(")
averages[element] = ppm(word.text.delete("()"))
else
elements[element] = ppm(word.text)
end
end
end
[ elements.compact, averages.compact ]
end
# "<" and "0.1" are drawn as separate words.
def merge_less_than(words)
words.each_with_object([]) do |w, out|
if out.last&.text == "<"
out.last.text = "< #{w.text}"
out.last.endx = w.endx
else
out << w.dup
end
end
end
# "N/A" → nil; "< 0.1" (below detection) → 0; ".3" → 0.3
def ppm(text)
return nil if text.blank? || text == "N/A"
return 0 if text.start_with?("<")
OilReportParser.number(text)
end
def status(verdict)
case verdict.upcase
when /ABNORMAL|HIGH|CRITICAL|SEVERE|PROBLEM/ then "action_required"
when /APPEARS NORMAL/ then "normal"
else "monitor"
end
end
end
end
@@ -0,0 +1,114 @@
class OilReportParser
# Blackstone Laboratories report: one page with a header (lab number, unit ID,
# oil type), the lab's comments, then a grid with one column per sample (newest
# on the left, dated across the top) and one row per element / oil property.
# Two extra columns hold averages: this unit's own (skipped) and the lab's
# universal averages for the engine type (kept as the comparison average).
#
# Numbers are right-aligned under their sample dates, so values are matched to
# a sample by their right edge.
class Blackstone
LAB = "Blackstone".freeze
ELEMENTS = %w[aluminum chromium iron copper lead tin molybdenum nickel manganese silver titanium
potassium boron silicon sodium calcium magnesium phosphorus zinc barium].freeze
PROPERTIES = {
/\ASUS Viscosity/ => "viscosity_sus_210f", /\AcSt Viscosity/ => "viscosity_cst_100c",
/\AFlashpoint/ => "flashpoint_f", /\AFuel %/ => "fuel_percent", /\AAntifreeze %/ => "antifreeze_percent",
/\AWater %/ => "water_percent", /\AInsolubles %/ => "insolubles_percent",
/\ATBN\z/ => "total_base_number", /\ATAN\z/ => "total_acid_number"
}.freeze
LABEL_MAX_X = 130 # row labels sit in the left margin
ROW_SLOP = 3.5 # a row's values can sit a couple of points off its label's baseline
COLUMN_SLOP = 8 # how far a value's right edge may be from its date's
DATE = %r{\A\d{1,2}/\d{1,2}/\d{4}\z}
# A US registration in the "unit ID" field; anything else (owners often put an
# engine name there) isn't treated as a tail number.
TAIL = /\AN[1-9][0-9A-Z]{0,4}\z/i
Run = Struct.new(:text, :x, :endx, :y)
def initialize(runs)
# The vertical side captions ("UNIT INFORMATION"…) are single letters at x≈43.
@runs = runs.filter_map { Run.new(_1.text.strip, _1.x, _1.endx, _1.y) if _1.x >= 48 && _1.text.strip.present? }
end
def parse
date_label = label("Sample Date") or raise Error, "No sample dates found in the Blackstone report"
dates = row(date_label).select { DATE.match?(_1.text) }.sort_by(&:x)
raise Error, "No sample dates found in the Blackstone report" if dates.empty?
@columns = dates.map(&:endx)
@universal_x = (@runs.find { _1.text == "UNIVERSAL" }&.x || 530) - 10
oil_hours, engine_hours, oil_added = [ "MI/HR on Oil", "MI/HR on Unit", "Make Up Oil Added" ].map { values(label(_1)) }
elements, universal = ELEMENTS.each_with_object([ {}, {} ]) do |el, (by_el, avg)|
by_el[el] = values(label(el.upcase))
avg[el] = universal_value(label(el.upcase))
end
properties = PROPERTIES.to_h { |pattern, key| [ key, values(label(pattern)) ] }
universal = universal.transform_values { OilReportParser.number(_1) }.compact
lab_number = after("LAB NUMBER:")
samples = dates.each_with_index.map do |date, i|
{
current: i.zero?,
sample_date: OilReportParser.date(date.text),
sample_number: (lab_number if i.zero?),
oil_hours: OilReportParser.number(oil_hours[i]),
engine_hours: OilReportParser.number(engine_hours[i]),
oil_added: OilReportParser.number(oil_added[i]),
status: nil, # Blackstone gives comments, not a rating
lab_comments: (comments if i.zero?),
elements: elements.transform_values { OilReportParser.number(_1[i]) }.compact,
element_averages: universal,
oil_properties: properties.transform_values { OilReportParser.number(_1[i]) }.compact
}
end.select { _1[:sample_date] }
unit = after("UNIT ID:")
Report.new(lab: LAB, tail_number: (unit if TAIL.match?(unit.to_s)), engine_serial: nil,
engine_model: after("MAKE/MODEL:"), oil_type: after("OIL TYPE & GRADE:"), samples: samples)
end
private
# The left-margin run that labels a row.
def label(match)
@runs.find { _1.x < LABEL_MAX_X && (match.is_a?(Regexp) ? match.match?(_1.text) : _1.text == match) }
end
# Runs on the same row as a label, to its right.
def row(label_run)
return [] unless label_run
@runs.select { (_1.y - label_run.y).abs <= ROW_SLOP && _1.x > label_run.endx }
end
# The row's value text for each sample column (nil where blank).
def values(label_run)
found = Array.new(@columns.size)
row(label_run).each do |run|
next if run.x >= @universal_x
i = @columns.each_index.min_by { (@columns[_1] - run.endx).abs }
found[i] ||= run.text if (@columns[i] - run.endx).abs <= COLUMN_SLOP
end
found
end
def universal_value(label_run) = row(label_run).find { _1.x >= @universal_x }&.text
# The value printed after a header label, e.g. "UNIT ID:" → "N12345".
def after(text)
label_run = @runs.find { _1.text == text } or return nil
@runs.select { (_1.y - label_run.y).abs <= 2 && _1.x > label_run.endx }.min_by(&:x)&.text
end
# The comment paragraph between the contact block and the results grid.
# It starts with who it's addressed to ("OWNER:"), which is dropped.
def comments
contact = @runs.select { _1.text.match?(/\A(PHONE|FAX|ALT PHONE|EMAIL):/) }
top = contact.map(&:y).min or return nil
grid = label("MI/HR on Oil")&.y || 0
lines = @runs.select { _1.y < top - 2 && _1.y > grid + 6 }.sort_by { [ -_1.y, _1.x ] }
lines.map(&:text).join(" ").sub(/\A[A-Z]+:\s*/, "").squeeze(" ").presence
end
end
end