diff --git a/.env.example b/.env.example index d81426a..f4e58eb 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,5 @@ FORCE_SSL=false # SMTP_USERNAME= # SMTP_PASSWORD= # MAIL_FROM=aircraft@example.com +# Optional: Claude transcribes scanned logbook pages +# ANTHROPIC_API_KEY= diff --git a/Gemfile b/Gemfile index e2a9d87..148e16b 100644 --- a/Gemfile +++ b/Gemfile @@ -75,3 +75,5 @@ gem "caxlsx", "~> 4.5", group: :test gem "webmock", "~> 3.34", group: :test gem "pdf-reader", "~> 2.16" +gem "rubyzip", ">= 2.4", "< 3" # aircraft archives (.sam.zip) +gem "anthropic", "~> 2.6" # AI logbook transcription (Claude) diff --git a/Gemfile.lock b/Gemfile.lock index bdfac22..ee8bf3d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,6 +79,11 @@ GEM addressable (2.9.2) public_suffix (>= 2.0.2, < 8.0) afm (1.0.1) + anthropic (2.6.0) + base64 (~> 0.1) + connection_pool (>= 2, < 4) + opentelemetry-api (>= 1.2.4, < 2) + standardwebhooks (~> 1.0) ast (2.4.3) base64 (0.3.0) bcrypt (3.1.22) @@ -208,6 +213,8 @@ GEM racc (~> 1.4) nokogiri (1.19.7-x86_64-linux-musl) racc (~> 1.4) + opentelemetry-api (1.8.0) + logger ostruct (0.6.3) parallel (1.28.0) parser (3.3.13.0) @@ -355,6 +362,7 @@ GEM net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) ostruct + standardwebhooks (1.0.1) stimulus-rails (1.3.4) railties (>= 6.0.0) tailwindcss-rails (4.7.0) @@ -415,6 +423,7 @@ PLATFORMS x86_64-linux-musl DEPENDENCIES + anthropic (~> 2.6) bcrypt (~> 3.1.7) bootsnap brakeman @@ -432,6 +441,7 @@ DEPENDENCIES rails (~> 8.1.0) roo (~> 2.10) rubocop-rails-omakase + rubyzip (>= 2.4, < 3) selenium-webdriver solid_cable solid_cache @@ -461,6 +471,7 @@ CHECKSUMS activesupport (8.1.4.1) sha256=16ac4f341ca65c85a1fcdb8ac73f68891aae9e3c4a6f9c3a27daed8b00017ed7 addressable (2.9.2) sha256=9117c7e7d6ee07230e75ce41fb9401ae55354dc654fcd0e862ba41d539e9a610 afm (1.0.1) sha256=4fc98c6ea9cad60c25ae12bad517513b0d6fdbedc099e25be88bf47977b7d879 + anthropic (2.6.0) sha256=0bbc1cf9b71c82dc72b552ccd0d5e253b76a55fd2d80f8ea4666a9d836a492cb ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b bcrypt (3.1.22) sha256=1f0072e88c2d705d94aff7f2c5cb02eb3f1ec4b8368671e19112527489f29032 @@ -525,6 +536,7 @@ CHECKSUMS nokogiri (1.19.7-x86_64-darwin) sha256=508e81e8c5f84aaa240dce981bf1c43faae60808726f0ce7024ee92effc5cafe nokogiri (1.19.7-x86_64-linux-gnu) sha256=25865ccb462f714e712fdf8906fde07849cb6704a808e3d629d75baa5af5dd3b nokogiri (1.19.7-x86_64-linux-musl) sha256=464d1f3dfb0d53cf54c0c6a7e23e8e91cf68ec3cafc2b27dc4a7682f2d9bf546 + opentelemetry-api (1.8.0) sha256=3af51183daf0f56a164bc1579782245be70a40678566b9a393cbe5af28ea87c6 ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912 parallel (1.28.0) sha256=33e6de1484baf2524792d178b0913fc8eb94c628d6cfe45599ad4458c638c970 parser (3.3.13.0) sha256=e02f9b63c2fe05b0c68f692ba4df89b3cb113b663964f75d988fbebfd6c3deca @@ -574,6 +586,7 @@ CHECKSUMS sqlite3 (2.10.2-x86_64-linux-gnu) sha256=ca92731ab464df9cae503fa1a439a549e2724f6953c8c05a7769b59366e38f15 sqlite3 (2.10.2-x86_64-linux-musl) sha256=b9f8fc62f3e0eb8966b41de251b492b4c8e9d1b23559beb6e150ad7f8d590941 sshkit (1.25.1) sha256=be3f10b9d6eb0b44d5eaba3f7cbe41bc6bb894bce4339688ac20124391455b78 + standardwebhooks (1.0.1) sha256=35db391599d9318747e1e75e77b96236b3fc25241b92387c9a8d31bd974f3d7d stimulus-rails (1.3.4) sha256=765676ffa1f33af64ce026d26b48e8ffb2e0b94e0f50e9119e11d6107d67cb06 tailwindcss-rails (4.7.0) sha256=9b09bd677d7cd226fc8ff3f7d2d91c6dc6c3da75dd74dc4cab5f94711f69ba41 tailwindcss-ruby (4.3.3) sha256=ee0a64030749862deb501acab4c4aaf5adbee13865746a33299d46d7b5d0952a diff --git a/README.md b/README.md index abfdbcd..f00e04f 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,8 @@ The app is a standard Rails app. Run `bin/rails db:prepare && bin/thrust bin/rai | `FORCE_SSL` | `true` (default) redirects to HTTPS and sets secure cookies. Use `false` for plain HTTP. | | `APP_HOST` | Host used in password-reset email links. | | `SMTP_ADDRESS`, `SMTP_PORT`, `SMTP_USERNAME`, `SMTP_PASSWORD`, `MAIL_FROM` | Optional. They enable password-reset emails. Without them an admin resets passwords from **Users**. | +| `ANTHROPIC_API_KEY` | Optional. Lets Claude transcribe scanned logbook pages into entries (**Logbook → Import scans**). Without it, scans are only uploaded. | +| `LOGBOOK_AI_MODEL`, `LOGBOOK_AI_EFFORT` | Optional. The Claude model and effort used for transcription (default `claude-opus-5-5`, `high`). | ### Backups @@ -68,6 +70,8 @@ rsync -a --exclude '*.sqlite3*' storage/ backup/ # uploaded files Alternatively, stop the container and copy the volume. +To back up or move a single aircraft with all its files, use **Access & Sharing → Download .sam.zip** (or `bin/rails "sam:export[N12345]"`). Import it from **Fleet → Import** (or `bin/rails "sam:import[file.sam.zip,owner@example.com]"`). The archive format is the original Simple Aircraft Manager's, so archives move between the two apps. + ## Features | Area | Notes | @@ -124,7 +128,11 @@ Differences from the original: ### Not yet ported -AI logbook transcription, oil-analysis PDF import, the MCP server for AI agents, OIDC single sign-on, invitation codes, aircraft import/export, the plugin system, and the desktop builds. +The MCP server for AI agents, OIDC single sign-on, invitation codes, per-aircraft feature switches, the plugin system, and the desktop builds. + +### Added in this port + +AirTrail sync and export, an airport database, FAA DRS AD import, spreadsheet flight import, engine monitor data (CGR-30P/C) with interactive charts, AvLab and Blackstone oil analysis import with trends, multiple squawk attachments with a photo viewer, AI logbook transcription with Claude (images and PDFs), and an installable PWA. ## License diff --git a/app/controllers/aircraft_exports_controller.rb b/app/controllers/aircraft_exports_controller.rb new file mode 100644 index 0000000..88d66db --- /dev/null +++ b/app/controllers/aircraft_exports_controller.rb @@ -0,0 +1,19 @@ +# Download an aircraft as a .sam.zip archive (owners and admins). +class AircraftExportsController < ApplicationController + include AircraftScoped + before_action :require_owner! + + EXPORT_DIR = Rails.root.join("tmp/exports") + + def show + exporter = AircraftArchive::Exporter.new(@aircraft, source: request.host) + # A plain file (not a Tempfile) so it's still there while send_file streams it; + # older exports are swept on the next one. + FileUtils.mkdir_p(EXPORT_DIR) + Dir[EXPORT_DIR.join("*.zip")].each { |old| File.delete(old) if File.mtime(old) < 1.hour.ago } + path = EXPORT_DIR.join("#{SecureRandom.hex(12)}.zip").to_s + exporter.write_to(path) + log_event("aircraft", "Aircraft exported (.sam.zip)") + send_file path, filename: exporter.filename, type: "application/zip", disposition: "attachment" + end +end diff --git a/app/controllers/aircraft_imports_controller.rb b/app/controllers/aircraft_imports_controller.rb new file mode 100644 index 0000000..2167967 --- /dev/null +++ b/app/controllers/aircraft_imports_controller.rb @@ -0,0 +1,39 @@ +# Upload a .sam.zip (from this app or the original Simple Aircraft Manager) to +# create an aircraft with all its records. The archive is checked right away; +# the import itself runs in the background (AircraftImportJob). +class AircraftImportsController < ApplicationController + before_action :set_import, only: :show + + def new; end + + def create + file = params[:archive] + return redirect_to(new_aircraft_import_path, alert: "Choose a .sam.zip file.") if file.blank? + + preview = AircraftArchive::Importer.check(file.tempfile.path) + tail = params[:tail_number].to_s.strip.upcase.presence || preview.tail_number.to_s.upcase + if Aircraft.where("UPPER(tail_number) = ?", tail).exists? + @preview = preview + @conflict = tail + flash.now[:alert] = "An aircraft #{tail} already exists. Choose another tail number for the imported one." + return render :new, status: :unprocessable_entity + end + + import = AircraftImport.create!(user: current_user, filename: file.original_filename, tail_number: tail, source: preview.source) + import.archive.attach(io: file.tempfile.tap(&:rewind), filename: file.original_filename.presence || "archive.sam.zip", content_type: "application/zip") + AircraftImportJob.perform_later(import.id) + redirect_to aircraft_import_path(import) + rescue AircraftArchive::Error => e + redirect_to new_aircraft_import_path, alert: "That archive can't be imported: #{e.message}" + end + + def show; end + + private + + # Users see their own imports; admins see all. + def set_import + scope = current_user.admin? ? AircraftImport.all : AircraftImport.where(user: current_user) + @import = scope.find(params[:id]) + end +end diff --git a/app/controllers/logbook_imports_controller.rb b/app/controllers/logbook_imports_controller.rb new file mode 100644 index 0000000..e42eb85 --- /dev/null +++ b/app/controllers/logbook_imports_controller.rb @@ -0,0 +1,55 @@ +# Upload scanned logbook pages (images or PDFs) as a document and, when an +# Anthropic API key is configured, have Claude transcribe them into logbook +# entries in the background (LogbookTranscriptionJob). +class LogbookImportsController < ApplicationController + include AircraftScoped + before_action :require_owner! + + def new + @collections = @aircraft.document_collections.ordered + end + + def create + files = Array(params[:pages]).compact_blank + return redirect_to(new_aircraft_logbook_import_path(@aircraft), alert: "Choose the scanned pages to upload.") if files.empty? + + # Page order follows the file names ("page-002" before "page-010"). + files = files.sort_by { |f| f.original_filename.to_s.downcase.scan(/\d+|\D+/).map { _1.match?(/\d/) ? [ 0, _1.to_i ] : [ 1, _1 ] } } + collection = find_or_build_collection + document = @aircraft.documents.new(name: params[:document_name].presence || collection.name, doc_type: doc_type, + document_collection: collection) + document.pages.attach(files) + unless collection.valid? && document.valid? + @collections = @aircraft.document_collections.ordered + flash.now[:alert] = (collection.errors.full_messages + document.errors.full_messages).to_sentence + return render :new, status: :unprocessable_entity + end + + import = nil + ActiveRecord::Base.transaction do + collection.save! + document.save! + import = @aircraft.logbook_imports.create!(user: current_user, document: document, log_type: params[:log_type], + transcribe: params[:transcribe] == "1" && LogbookImport.ai_available?) + end + log_event("document", "Logbook pages uploaded: #{document.name} (#{files.size} #{'page'.pluralize(files.size)})") + LogbookTranscriptionJob.perform_later(import.id) + redirect_to aircraft_logbook_import_path(@aircraft, import) + end + + def show + @import = @aircraft.logbook_imports.find(params[:id]) + end + + private + + def find_or_build_collection + if params[:collection_id].present? && params[:collection_id] != "new" + @aircraft.document_collections.find(params[:collection_id]) + else + @aircraft.document_collections.new(name: params[:collection_name].presence || "Logbooks", visibility: "maintenance") + end + end + + def doc_type = Document::TYPES.key?(params[:doc_type]) ? params[:doc_type] : "LOG" +end diff --git a/app/controllers/oil_analyses_controller.rb b/app/controllers/oil_analyses_controller.rb index b6758d3..77000ef 100644 --- a/app/controllers/oil_analyses_controller.rb +++ b/app/controllers/oil_analyses_controller.rb @@ -31,8 +31,8 @@ class OilAnalysesController < ApplicationController end end - # Create records from a lab report PDF. AvLab reports include the two previous - # samples too, so samples already on file are skipped rather than duplicated. + # Create records from a lab report PDF (AvLab or Blackstone). Reports repeat + # earlier samples, so samples already on file are skipped rather than duplicated. def import file = params[:report] return redirect_to(new_aircraft_oil_analysis_path(@aircraft), alert: "Choose a lab report PDF.") if file.blank? @@ -50,7 +50,7 @@ class OilAnalysesController < ApplicationController OilAnalysis.transaction do report.samples.each do |sample| record = existing_sample(sample) || - @aircraft.oil_analyses.create!(sample.except(:current).merge(lab: report.lab, component: engine)).tap { created << _1 } + @aircraft.oil_analyses.create!(sample.except(:current).merge(lab: report.lab, oil_type: report.oil_type, component: engine)).tap { created << _1 } current_record = record if sample[:current] end end @@ -108,9 +108,10 @@ class OilAnalysesController < ApplicationController def analysis_params params.require(:oil_analysis).permit( - :component_id, :sample_date, :analysis_date, :lab, :sample_number, :engine_hours, :oil_hours, - :filter_hours, :oil_added, :status, :lab_comments, :notes, :report, - *OilAnalysis::ELEMENTS.keys.flat_map { [ _1, :"#{_1}_avg" ] } + :component_id, :sample_date, :analysis_date, :lab, :sample_number, :oil_type, :engine_hours, :oil_hours, + :filter_hours, :oil_added, :status, :lab_comments, :notes, :report, :excluded_from_averages, + elements: OilAnalysis::ELEMENTS.keys, element_averages: OilAnalysis::ELEMENTS.keys, + oil_properties: OilAnalysis::PROPERTIES.keys ).tap { |p| p[:component_id] = scoped_id(@aircraft.components, p[:component_id]) if p.key?(:component_id) } end end diff --git a/app/helpers/oil_analyses_helper.rb b/app/helpers/oil_analyses_helper.rb index 201d291..6ef217b 100644 --- a/app/helpers/oil_analyses_helper.rb +++ b/app/helpers/oil_analyses_helper.rb @@ -1,9 +1,18 @@ module OilAnalysesHelper - STATUS_BADGES = { "normal" => "badge-green", "elevated" => "badge-orange", "high" => "badge-red" }.freeze + STATUS_BADGES = { "normal" => "badge-green", "monitor" => "badge-orange", "action_required" => "badge-red" }.freeze TREND_COLORS = %w[#0ea5e9 #f97316 #22c55e #a855f7 #eab308 #ec4899 #64748b].freeze + # The lab's verdict; Blackstone reports don't rate samples, so there may be none. def oil_status_badge(analysis) - tag.span OilAnalysis::STATUSES[analysis.status], class: STATUS_BADGES[analysis.status] + return tag.span("Not rated", class: "badge-gray") unless analysis.status + tag.span analysis.status_label, class: STATUS_BADGES[analysis.status] + end + + def oil_property(analysis, key) + value = analysis.property(key) or return "—" + _, unit = OilAnalysis::PROPERTIES.fetch(key) + number = number_with_precision(value, precision: 2, strip_insignificant_zeros: true) + unit.present? ? "#{number} #{unit}" : number end def ppm(value) @@ -28,8 +37,9 @@ module OilAnalysesHelper "#{sign}#{ppm(delta)}" end + # Samples marked "exclude from averages" (e.g. right after an overhaul) are left out. def oil_trend_series(analyses) - chronological = analyses.sort_by(&:sample_date) + chronological = analyses.reject(&:excluded_from_averages?).sort_by(&:sample_date) OilAnalysis::TREND_ELEMENTS.filter_map do |el| data = chronological.filter_map { |a| [ a.sample_date, a.value(el).to_f ] if a.value(el) } { name: OilAnalysis.element_label(el), data: data } if data.size >= 2 diff --git a/app/jobs/aircraft_import_job.rb b/app/jobs/aircraft_import_job.rb new file mode 100644 index 0000000..ebaa113 --- /dev/null +++ b/app/jobs/aircraft_import_job.rb @@ -0,0 +1,24 @@ +# Imports a staged .sam.zip (see AircraftArchive::Importer) and records the outcome. +class AircraftImportJob < ApplicationJob + queue_as :default + + def perform(import_id) + import = AircraftImport.find_by(id: import_id) or return + return unless import.status == "queued" + import.update!(status: "running", started_at: Time.current) + + result = import.archive.open do |file| + AircraftArchive::Importer.new(file.path, owner: import.user, tail_number: import.tail_number, + progress: ->(counts) { import.update_columns(counts: counts, updated_at: Time.current) }).run + end + import.update!(status: "done", finished_at: Time.current, aircraft: result[:aircraft], + counts: result[:counts], warnings: result[:warnings].first(200)) + rescue AircraftArchive::Error => e + import&.update!(status: "failed", finished_at: Time.current, error: e.message) + rescue StandardError => e + import&.update!(status: "failed", finished_at: Time.current, error: "Import failed: #{e.class}: #{e.message}".truncate(1000)) + Rails.logger.error("[aircraft import] #{e.class}: #{e.message}\n#{e.backtrace&.first(10)&.join("\n")}") + ensure + import&.archive&.purge_later if import&.reload&.status.in?(%w[done failed]) + end +end diff --git a/app/jobs/logbook_transcription_job.rb b/app/jobs/logbook_transcription_job.rb new file mode 100644 index 0000000..0aa147a --- /dev/null +++ b/app/jobs/logbook_transcription_job.rb @@ -0,0 +1,162 @@ +# Transcribes an uploaded logbook document into LogbookEntry records with +# Claude (LogbookTranscriber). +# +# Images go up in batches of a few pages. Consecutive batches overlap by one +# page so an entry running onto the next page is read whole: entries seen on a +# batch's last page are held back and handed to the next batch, which returns +# them completed. Each PDF goes up as one request. +class LogbookTranscriptionJob < ApplicationJob + queue_as :default + + class_attribute :batch_pages, default: 6 + BATCH_BYTES = 18.megabytes # base64 grows this by a third; requests cap at 32 MB + + def perform(import_id) + @import = LogbookImport.find_by(id: import_id) or return + return unless @import.status == "queued" + + @document = @import.document + pages = @document ? @document.pages.attachments.includes(:blob).sort_by(&:id) : [] + @import.update!(status: "running", started_at: Time.current, pages_total: pages.size, model: (LogbookTranscriber::MODEL if @import.transcribe)) + transcribe(pages) if @import.transcribe && pages.any? + finish + rescue StandardError => e + @import&.update!(status: "failed", finished_at: Time.current, error: "#{e.class}: #{e.message}".truncate(1000)) + Rails.logger.error("[logbook import] #{e.class}: #{e.message}\n#{e.backtrace&.first(10)&.join("\n")}") + end + + private + + def transcribe(pages) + @transcriber = LogbookTranscriber.new + @last_date = nil + readable = pages.each_with_index.filter_map do |attachment, index| + type = attachment.blob.content_type + if type == "application/pdf" + next warn("Page #{index + 1} (#{attachment.filename}) is over #{LogbookTranscriber::MAX_PDF_BYTES / 1.megabyte} MB, too large to transcribe.") if attachment.blob.byte_size > LogbookTranscriber::MAX_PDF_BYTES + elsif !LogbookTranscriber::IMAGE_TYPES.include?(type) + next warn("Page #{index + 1} (#{attachment.filename}) isn't a JPEG, PNG, GIF, WebP or PDF, so it wasn't transcribed.") + elsif attachment.blob.byte_size > LogbookTranscriber::MAX_IMAGE_BYTES + next warn("Page #{index + 1} (#{attachment.filename}) is over 5 MB, too large to transcribe. Scan it at a lower resolution.") + end + { attachment: attachment, number: index + 1, pdf: type == "application/pdf" } + end + tick(pages.size - readable.size) + + readable.slice_when { |a, b| a[:pdf] || b[:pdf] }.each do |run| + run.first[:pdf] ? transcribe_pdf(run.first) : transcribe_images(run) + end + end + + def transcribe_pdf(page) + result = @transcriber.transcribe([ payload(page) ]) + note_skipped(result, ->(i) { "#{page[:attachment].filename} page #{i + 1}" }) + result.entries.sort_by { _1["page_start"].to_i }.each { save(_1, page[:number]) } + rescue LogbookTranscriber::Error => e + warn("#{page[:attachment].filename} wasn't transcribed: #{e.message}.") + ensure + tick(1) + end + + def transcribe_images(pages) + batches = [] + start = 0 + while start < pages.size + batch = [ pages[start] ] + bytes = pages[start][:attachment].blob.byte_size + while (nxt = pages[start + batch.size]) && batch.size < batch_pages && bytes + nxt[:attachment].blob.byte_size <= BATCH_BYTES + batch << nxt + bytes += nxt[:attachment].blob.byte_size + end + batches << batch + break if start + batch.size >= pages.size + start += [ batch.size - 1, 1 ].max # overlap by one page + end + + held = [] # entries read from the previous batch's last page + batches.each_with_index do |batch, n| + last_batch = n == batches.size - 1 + begin + result = @transcriber.transcribe(batch.map { payload(_1) }, carried: held.map { _1[:entry] }) + rescue LogbookTranscriber::Error => e + held.each { save(_1[:entry], _1[:number]) } # keep what the previous batch read + held = [] + pages = batch.size == 1 ? "Page #{batch.first[:number]} wasn't" : "Pages #{batch.first[:number]}–#{batch.last[:number]} weren't" + warn("#{pages} transcribed: #{e.message}.") + tick(n.zero? ? batch.size : batch.size - 1) + next + end + note_skipped(result, ->(i) { "page #{batch[i]&.dig(:number) || '?'}" }) + + entries = result.entries.sort_by { _1["page_start"].to_i } + held = [] + entries.each do |entry| + page = batch[entry["page_start"].to_i.clamp(0, batch.size - 1)] + if !last_batch && entry["page_end"].to_i >= batch.size - 1 + held << { entry: entry, number: page[:number] } # may continue on the next batch + else + save(entry, page[:number]) + end + end + tick(n.zero? ? batch.size : batch.size - 1) + end + held.each { save(_1[:entry], _1[:number]) } + end + + def payload(page) + { data: page[:attachment].blob.download, media_type: page[:attachment].blob.content_type } + end + + def save(entry, page_number) + date = parse_date(entry["date"]) + text = entry["text"].to_s.strip.presence || "[no text read]" + unless date + text = "[date unreadable] #{text}" + warn("An entry on page #{page_number} had no readable date#{@last_date ? "; it was given the previous entry's date (#{@last_date})" : ' and was dated today'}. Please check it.") + end + date ||= @last_date || Date.current + @last_date = date + log_type = @import.log_type || (LogbookEntry::LOG_TYPES.key?(entry["log_type"]) ? entry["log_type"] : "AC") + @import.aircraft.logbook_entries.create!( + date: date, text: text, log_type: log_type, + entry_type: LogbookEntry::ENTRY_TYPES.key?(entry["entry_type"]) ? entry["entry_type"] : "OTHER", + signoff_person: entry["signoff_person"].presence, signoff_location: entry["signoff_location"].presence, + aircraft_hours_at_entry: entry["aircraft_hours"].is_a?(Numeric) && entry["aircraft_hours"] >= 0 ? entry["aircraft_hours"] : nil, + document: @document, page_number: page_number + ) + @import.increment!(:entries_created) + warn("The #{date} entry on page #{page_number} was hard to read. Please check it against the scan.") if entry["confidence"] == "low" + end + + def note_skipped(result, label) + forms = result.non_logbook_pages.map { label.(_1) } + unreadable = result.unparseable_pages.map { label.(_1) } + warn("Not logbook entries (forms, tags…), so nothing was transcribed from: #{forms.uniq.to_sentence}.") if forms.any? + warn("Too hard to read: #{unreadable.uniq.to_sentence}.") if unreadable.any? + end + + def parse_date(value) + date = Date.iso8601(value.to_s) if value.present? + date if date && date.year > 1900 && date <= Date.current + 1.year + rescue Date::Error + nil + end + + def warn(message) + @import.warnings = @import.warnings + [ message ] + @import.save! + nil + end + + def tick(pages) + @import.update_columns(pages_done: [ @import.pages_done + pages, @import.pages_total ].min, updated_at: Time.current) if pages.positive? + end + + def finish + @import.update!(status: "done", finished_at: Time.current, pages_done: @import.pages_total) + if @import.transcribe + @import.aircraft.log_event!("logbook", "Logbook pages transcribed (#{@import.entries_created} entries)", + notes: @document&.name, user: @import.user) + end + end +end diff --git a/app/models/aircraft.rb b/app/models/aircraft.rb index 9151234..039d6f1 100644 --- a/app/models/aircraft.rb +++ b/app/models/aircraft.rb @@ -16,6 +16,7 @@ class Aircraft < ApplicationRecord has_many :components, dependent: :destroy has_many :squawks, dependent: :destroy has_many :logbook_entries, dependent: :destroy + has_many :logbook_imports, dependent: :destroy has_many :document_collections, dependent: :destroy has_many :documents, dependent: :destroy has_many :inspection_records, dependent: :destroy diff --git a/app/models/aircraft_import.rb b/app/models/aircraft_import.rb new file mode 100644 index 0000000..c907c44 --- /dev/null +++ b/app/models/aircraft_import.rb @@ -0,0 +1,19 @@ +# One upload of an aircraft archive (.sam.zip), imported by AircraftImportJob. +class AircraftImport < ApplicationRecord + belongs_to :user, optional: true + belongs_to :aircraft, optional: true + has_one_attached :archive # staged upload; purged once imported + + STATUSES = %w[queued running done failed].freeze + + validates :status, inclusion: { in: STATUSES } + + def in_progress? = status.in?(%w[queued running]) + def done? = status == "done" + def failed? = status == "failed" + + # "12 components, 140 logbook entries, …" for whatever was imported. + def summary + counts.select { |_, n| n.to_i.positive? }.map { |key, n| "#{n} #{key.to_s.humanize(capitalize: false).then { n == 1 ? _1.singularize : _1 }}" }.to_sentence + end +end diff --git a/app/models/component.rb b/app/models/component.rb index b20d817..71f1f92 100644 --- a/app/models/component.rb +++ b/app/models/component.rb @@ -7,6 +7,10 @@ class Component < ApplicationRecord has_many :child_components, class_name: "Component", foreign_key: :parent_component_id, dependent: :destroy has_many :squawks, dependent: :nullify has_many :oil_analyses, dependent: :nullify + # Records about this component outlive it (their foreign keys would otherwise block deleting it). + has_many :ad_compliances, dependent: :nullify + has_many :inspection_records, dependent: :nullify + has_many :major_records, dependent: :nullify has_and_belongs_to_many :inspection_types has_and_belongs_to_many :airworthiness_directives diff --git a/app/models/logbook_import.rb b/app/models/logbook_import.rb new file mode 100644 index 0000000..1cbe274 --- /dev/null +++ b/app/models/logbook_import.rb @@ -0,0 +1,22 @@ +# Scanned logbook pages uploaded as a document, and (optionally) their +# transcription into logbook entries by Claude. See LogbookTranscriptionJob. +class LogbookImport < ApplicationRecord + belongs_to :aircraft + belongs_to :user, optional: true + belongs_to :document, optional: true + + STATUSES = %w[queued running done failed].freeze + + validates :status, inclusion: { in: STATUSES } + validates :log_type, inclusion: { in: LogbookEntry::LOG_TYPES.keys }, allow_nil: true + + normalizes :log_type, with: ->(v) { v.presence } + + # Transcription needs an Anthropic API key in the environment. + def self.ai_available? = ENV["ANTHROPIC_API_KEY"].present? + + def in_progress? = status.in?(%w[queued running]) + def done? = status == "done" + def failed? = status == "failed" + def progress_percent = pages_total.zero? ? 0 : (100.0 * pages_done / pages_total).round +end diff --git a/app/models/oil_analysis.rb b/app/models/oil_analysis.rb index f85fc7c..a69dec1 100644 --- a/app/models/oil_analysis.rb +++ b/app/models/oil_analysis.rb @@ -1,8 +1,14 @@ # A spectrographic oil analysis lab report for one oil sample. +# +# Results are stored per element ({ "iron" => 44.0, … } in ppm), with the lab's +# average for similar engines alongside, so any lab's element list fits (AvLab +# tests 12 elements, Blackstone 20). Oil properties (viscosity, flash point, +# fuel/water %) are stored the same way. class OilAnalysis < ApplicationRecord include UploadValidation - # Element → [label, what it usually indicates]. + # Element → [label, what it usually indicates]. The first twelve are the + # wear-metal set every lab reports; the rest are mostly oil additives. ELEMENTS = { "iron" => [ "Iron", "Cylinders, cam, lifters, gears" ], "copper" => [ "Copper", "Bearings, bushings, oil cooler" ], @@ -15,9 +21,30 @@ class OilAnalysis < ApplicationRecord "silicon" => [ "Silicon", "Dirt (air filter), sealant" ], "titanium" => [ "Titanium", "Hardened parts" ], "tin" => [ "Tin", "Bearings, piston pin bushings" ], - "molybdenum" => [ "Moly", "Rings, additive" ] + "molybdenum" => [ "Moly", "Rings, additive" ], + "manganese" => [ "Manganese", "Steel alloys (valves, gears)" ], + "potassium" => [ "Potassium", "Coolant, additive" ], + "boron" => [ "Boron", "Additive" ], + "sodium" => [ "Sodium", "Additive, dirt" ], + "calcium" => [ "Calcium", "Detergent additive" ], + "phosphorus" => [ "Phosphorus", "Anti-wear additive" ], + "zinc" => [ "Zinc", "Anti-wear additive, brass" ], + "barium" => [ "Barium", "Additive" ] }.freeze - STATUSES = { "normal" => "Normal", "elevated" => "Elevated", "high" => "High" }.freeze + COMMON_ELEMENTS = ELEMENTS.keys.first(12).freeze + # Property → [label, unit]. + PROPERTIES = { + "viscosity_sus_210f" => [ "Viscosity @ 210°F", "SUS" ], + "viscosity_cst_100c" => [ "Viscosity @ 100°C", "cSt" ], + "flashpoint_f" => [ "Flash point", "°F" ], + "fuel_percent" => [ "Fuel", "%" ], + "antifreeze_percent" => [ "Antifreeze", "%" ], + "water_percent" => [ "Water", "%" ], + "insolubles_percent" => [ "Insolubles", "%" ], + "total_base_number" => [ "TBN", "" ], + "total_acid_number" => [ "TAN", "" ] + }.freeze + STATUSES = { "normal" => "Normal", "monitor" => "Monitor", "action_required" => "Action required" }.freeze # A reading this far above the lab's average for similar engines gets flagged. FLAG_RATIO = 1.5 # Wear metals worth charting together (lead is on its own scale: it comes from fuel). @@ -28,20 +55,42 @@ class OilAnalysis < ApplicationRecord has_one_attached :report validates :sample_date, presence: true - validates :status, inclusion: { in: STATUSES.keys } - validates(*ELEMENTS.keys.flat_map { [ _1, :"#{_1}_avg" ] }, :engine_hours, :oil_hours, :filter_hours, :oil_added, - numericality: { greater_than_or_equal_to: 0 }, allow_nil: true) + validates :status, inclusion: { in: STATUSES.keys }, allow_nil: true + validates :engine_hours, :oil_hours, :filter_hours, :oil_added, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true + validate :results_are_numbers validate :component_belongs_to_aircraft validates_uploads :report - normalizes :lab, :sample_number, with: ->(v) { v.strip.presence } + normalizes :lab, :sample_number, :oil_type, with: ->(v) { v.strip.presence } + normalizes :status, with: ->(v) { v.presence } scope :recent_first, -> { order(sample_date: :desc, created_at: :desc) } def self.element_label(el) = ELEMENTS.fetch(el)[0] - def value(el) = self[el] - def average(el) = self[:"#{el}_avg"] + # Elements any of these samples has a reading for, in display order. + def self.elements_in(analyses) = ELEMENTS.keys.select { |el| analyses.any? { _1.value(el) } } + + def elements = super || {} + def element_averages = super || {} + def oil_properties = super || {} + + # Accept form or parser input: blanks dropped, numbers kept as numbers. + def elements=(values) + super(clean(values, ELEMENTS, :elements)) + end + + def element_averages=(values) + super(clean(values, ELEMENTS, :element_averages)) + end + + def oil_properties=(values) + super(clean(values, PROPERTIES, :oil_properties)) + end + + def value(el) = elements[el] + def average(el) = element_averages[el] + def property(key) = oil_properties[key] # Above the lab average by FLAG_RATIO (needs an average to compare against). def flagged?(el) @@ -50,10 +99,33 @@ class OilAnalysis < ApplicationRecord end def flagged_elements = ELEMENTS.keys.select { flagged?(_1) } - def any_results? = ELEMENTS.keys.any? { value(_1) } + def any_results? = elements.any? + def status_label = STATUSES[status] private + def clean(values, allowed, attribute) + (@invalid_results ||= {})[attribute] = [] + values.to_h.each_with_object({}) do |(key, raw), out| + key = key.to_s + next unless allowed.key?(key) + next if raw.blank? + number = raw.is_a?(Numeric) ? raw.to_f : Float(raw.to_s.strip, exception: false) + if number && number >= 0 + out[key] = number + else + @invalid_results[attribute] << key + end + end + end + + def results_are_numbers + (@invalid_results || {}).each do |attribute, keys| + labels = keys.map { (ELEMENTS[_1] || PROPERTIES[_1]).first } + errors.add(attribute, "must be numbers of 0 or more (#{labels.to_sentence})") if keys.any? + end + end + def component_belongs_to_aircraft errors.add(:component, "must be on this aircraft") if component && component.aircraft_id != aircraft_id end diff --git a/app/services/aircraft_archive.rb b/app/services/aircraft_archive.rb new file mode 100644 index 0000000..06c1780 --- /dev/null +++ b/app/services/aircraft_archive.rb @@ -0,0 +1,43 @@ +# Aircraft archives (.sam.zip): a whole aircraft's records and files in one ZIP, +# in the format of the original Simple Aircraft Manager (manifest schema v2), so +# archives move both ways between this app and the original. +# +# manifest.json all records, with archive-local IDs linking them +# attachments/… pictures, document pages, squawk files, track logs… +# +# Records carry a few extra keys for data only this app has (all squawk +# attachments, engine monitor files, oil report PDFs and lab averages, newer AD +# fields). The original ignores keys it doesn't know inside records, but rejects +# unknown top-level keys, so none are added there. +module AircraftArchive + class Error < StandardError; end + + SCHEMA_VERSION = 2 + TOP_LEVEL_KEYS = %w[ + schema_version exported_at source_instance aircraft component_types components document_collections + documents document_images logbook_entries squawks inspection_types inspection_records ads ad_compliances + consumable_records major_records notes oil_analysis_reports flight_logs features + ].freeze + OPTIONAL_KEYS = %w[features].freeze # and flight_logs before schema v2 + + MANIFEST_MAX_BYTES = 50.megabytes + FILE_MAX_BYTES = 512.megabytes + ARCHIVE_MAX_BYTES = 10.gigabytes # uncompressed + MAX_RATIO = 100 # compression ratio above this looks like a zip bomb + + # Leading bytes per extension; text formats have none to check. + MAGIC = { + "jpg" => [ "\xFF\xD8\xFF".b ], "jpeg" => [ "\xFF\xD8\xFF".b ], "png" => [ "\x89PNG\r\n\x1A\n".b ], + "gif" => [ "GIF8".b ], "webp" => [ "RIFF".b ], "bmp" => [ "BM".b ], "tif" => [ "II*\x00".b, "MM\x00*".b ], + "tiff" => [ "II*\x00".b, "MM\x00*".b ], "pdf" => [ "%PDF-".b ] + }.freeze + TEXT_EXTENSIONS = %w[txt csv gpx kml xml].freeze + EXTENSIONS = (MAGIC.keys + TEXT_EXTENSIONS).freeze + + def self.extension(path) = File.extname(path.to_s).delete(".").downcase + + def self.content_matches?(data, ext) + return EXTENSIONS.include?(ext) unless MAGIC.key?(ext) + MAGIC[ext].any? { data.byteslice(0, _1.bytesize) == _1 } + end +end diff --git a/app/services/aircraft_archive/exporter.rb b/app/services/aircraft_archive/exporter.rb new file mode 100644 index 0000000..40c7f11 --- /dev/null +++ b/app/services/aircraft_archive/exporter.rb @@ -0,0 +1,191 @@ +require "zip" + +module AircraftArchive + # Writes an aircraft and everything recorded for it to a .sam.zip. + # + # AircraftArchive::Exporter.new(aircraft).write_to(path) + class Exporter + def initialize(aircraft, source: nil) + @aircraft = aircraft + @source = source + @files = {} # archive path → attachment + @missing = 0 + end + + attr_reader :missing # files that were attached but missing from storage + + def filename = "#{@aircraft.tail_number.to_s.downcase.gsub(/[^a-z0-9-]/, '')}-#{Date.current.iso8601}.sam.zip" + + def write_to(path) + manifest = build_manifest + Zip::OutputStream.open(path) do |zip| + zip.put_next_entry("manifest.json") + zip.write(JSON.pretty_generate(manifest)) + @files.each do |archive_path, attachment| + zip.put_next_entry(archive_path) + attachment.blob.download { zip.write(_1) } + end + end + path + end + + def build_manifest + a = @aircraft + components = a.components.includes(:component_type).to_a + component_ids = components.map(&:id) + inspection_types = (a.inspection_types.to_a + InspectionType.joins(:components).where(components: { id: component_ids }).to_a + + InspectionType.where(id: a.inspection_records.select(:inspection_type_id)).to_a).uniq + ads = (a.airworthiness_directives.to_a + AirworthinessDirective.joins(:components).where(components: { id: component_ids }).to_a + + AirworthinessDirective.where(id: a.ad_compliances.select(:airworthiness_directive_id)).to_a).uniq + + { + schema_version: SCHEMA_VERSION, + exported_at: Time.current.iso8601, + source_instance: @source.to_s, + aircraft: aircraft_hash(a), + component_types: components.map(&:component_type).uniq.map { { id: id(_1), name: _1.name, consumable: _1.consumable } }, + components: components.map { component_hash(_1) }, + document_collections: a.document_collections.map do |c| + { id: id(c), aircraft_id: id(a), name: c.name, description: c.description, visibility: c.visibility, starred: c.starred, components: [] } + end, + documents: a.documents.map do |d| + { id: id(d), aircraft_id: id(a), collection_id: id(d.document_collection_id), doc_type: d.doc_type, name: d.name, + description: d.description, visibility: d.visibility, components: [] } + end, + document_images: a.documents.includes(pages_attachments: :blob).flat_map do |d| + d.pages.map { |page| { id: id(page), document_id: id(d), notes: "", image: file(page, "health/documents") } } + end, + logbook_entries: a.logbook_entries.map { logbook_hash(_1) }, + squawks: a.squawks.includes(:reported_by, attachments_attachments: :blob).map { squawk_hash(_1) }, + inspection_types: inspection_types.map { inspection_type_hash(_1, component_ids) }, + inspection_records: a.inspection_records.map do |r| + { id: id(r), date: r.date&.iso8601, aircraft_hours: dec(r.aircraft_hours), inspection_type_id: id(r.inspection_type_id), + logbook_entry_id: id(r.logbook_entry_id), aircraft_id: id(a), documents: [], component: [ id(r.component_id) ].compact, + notes: r.notes } + end, + ads: ads.map { ad_hash(_1, component_ids) }, + ad_compliances: a.ad_compliances.map do |c| + { id: id(c), ad_id: id(c.airworthiness_directive_id), date_complied: c.date_complied&.iso8601, compliance_notes: c.compliance_notes, + permanent: c.permanent, next_due_at_time: dec(c.next_due_at_time), aircraft_hours_at_compliance: dec(c.aircraft_hours_at_compliance), + logbook_entry_id: id(c.logbook_entry_id), inspection_record_id: id(c.inspection_record_id), aircraft_id: id(a), + component_id: id(c.component_id), not_applicable: c.not_applicable } + end, + consumable_records: a.consumable_records.map do |r| + { id: id(r), record_type: r.record_type, aircraft_id: id(a), date: r.date&.iso8601, quantity_added: dec(r.quantity_added), + level_after: dec(r.level_after), consumable_type: r.consumable_type, flight_hours: dec(r.flight_hours), notes: r.notes, + excluded_from_averages: r.excluded_from_averages, flight_log_id: id(r.flight_log_id) } + end, + major_records: a.major_records.map do |r| + { id: id(r), aircraft_id: id(a), record_type: r.record_type, title: r.title, description: r.description, + date_performed: r.date_performed&.iso8601, performed_by: r.performed_by, component_id: id(r.component_id), + form_337_document_id: id(r.form_337_document_id), stc_number: r.stc_number, stc_holder: r.stc_holder, + stc_document_id: id(r.stc_document_id), logbook_entry_id: id(r.logbook_entry_id), aircraft_hours: dec(r.aircraft_hours), + notes: r.notes, has_ica: r.has_ica, ica_notes: r.ica_notes } + end, + notes: a.notes.includes(:user).map do |n| + { id: id(n), aircraft_id: id(a), text: n.text, public: n.public, added_by_display: n.user&.display_name, + added_timestamp: n.created_at&.iso8601, edited_at: n.edited_at&.iso8601 } + end, + oil_analysis_reports: a.oil_analyses.includes(report_attachment: :blob).map { oil_hash(_1) }, + flight_logs: a.flight_logs.includes(:user, track_log_attachment: :blob, engine_data_attachment: :blob).map { flight_hash(_1) }, + features: [ { feature: "airworthiness_enforcement", enabled: a.airworthiness_enforcement } ] + } + end + + private + + def id(record_or_id) + value = record_or_id.respond_to?(:id) ? record_or_id.id : record_or_id + value&.to_s + end + + def dec(value) = value&.to_s + + # Queue an attachment for the archive and return its archive path. Each file + # gets its own folder so original filenames survive. A file missing from + # storage is left out (and counted) rather than failing the export. + def file(attachable, folder) + attachment = attachable.is_a?(ActiveStorage::Attachment) ? attachable : attachable&.attachment + return nil unless attachment + unless attachment.blob.service.exist?(attachment.blob.key) + @missing += 1 + return nil + end + name = attachment.filename.sanitized.presence || "file" + path = "attachments/#{folder}/#{attachment.blob.key}/#{name}" + @files[path] = attachment + path + end + + def aircraft_hash(a) + { id: id(a), tail_number: a.tail_number, make: a.make, model: a.model, serial_number: a.serial_number, + description: a.description, purchased: a.purchased&.iso8601, status: a.status, tach_time: dec(a.tach_time), + tach_time_offset: dec(a.tach_time_offset), hobbs_time: dec(a.hobbs_time), hobbs_time_offset: dec(a.hobbs_time_offset), + picture: file(a.picture, "aircraft_pictures"), icao_type: a.icao_type } + end + + def component_hash(c) + { id: id(c), aircraft_id: id(c.aircraft_id), parent_component_id: id(c.parent_component_id), component_type_id: id(c.component_type_id), + manufacturer: c.manufacturer, model: c.model, serial_number: c.serial_number, install_location: c.install_location, + notes: c.notes, status: c.status, date_in_service: c.date_in_service&.iso8601, hours_in_service: dec(c.hours_in_service), + hours_since_overhaul: dec(c.hours_since_overhaul), overhaul_date: c.overhaul_date&.iso8601, tbo_hours: c.tbo_hours, + tbo_days: c.tbo_days, inspection_hours: c.inspection_hours, inspection_days: c.inspection_days, + replacement_hours: c.replacement_hours, replacement_days: c.replacement_days, tbo_critical: c.tbo_critical, + on_condition: c.on_condition, inspection_critical: c.inspection_critical, replacement_critical: c.replacement_critical } + end + + def logbook_hash(e) + { id: id(e), aircraft_id: id(e.aircraft_id), log_type: e.log_type, date: e.date&.iso8601, text: e.text, + signoff_person: e.signoff_person, signoff_location: e.signoff_location, log_image_id: id(e.document_id), + aircraft_hours_at_entry: dec(e.aircraft_hours_at_entry), component_hours: {}, entry_type: e.entry_type, + page_number: e.page_number, components: [], related_documents: [] } + end + + def squawk_hash(s) + paths = s.attachments.map { file(_1, "health/squawks") } + { id: id(s), aircraft_id: id(s.aircraft_id), component_id: id(s.component_id), priority: s.priority, + issue_reported: s.issue_reported, attachment: paths.first, created_at: s.created_at&.iso8601, + reported_by_display: s.reported_by&.display_name, resolved: s.resolved, notes: s.notes, logbook_entries: [], + attachments: paths, resolved_at: s.resolved_at&.iso8601 } + end + + def inspection_type_hash(t, component_ids) + { id: id(t), name: t.name, recurring: t.recurring, required: t.required, recurring_hours: dec(t.recurring_hours), + recurring_days: t.recurring_days, recurring_months: t.recurring_months, + applicable_aircraft: t.aircraft_ids.include?(@aircraft.id) ? [ id(@aircraft) ] : [], + applicable_component: (t.component_ids & component_ids).map { id(_1) } } + end + + def ad_hash(ad, component_ids) + { id: id(ad), name: ad.name, short_description: ad.short_description, required_action: ad.required_action, + compliance_type: ad.compliance_type, trigger_condition: ad.trigger_condition, recurring: ad.recurring, + recurring_hours: dec(ad.recurring_hours), recurring_months: ad.recurring_months, recurring_days: ad.recurring_days, + bulletin_type: ad.bulletin_type, mandatory: ad.mandatory, + document_id: (id(ad.document_id) if ad.document&.aircraft_id == @aircraft.id), on_inspection_type: [], + applicable_aircraft: ad.aircraft_ids.include?(@aircraft.id) ? [ id(@aircraft) ] : [], + applicable_component: (ad.component_ids & component_ids).map { id(_1) }, + effective_date: ad.effective_date&.iso8601, supersedes: ad.supersedes, superseded_by: ad.superseded_by, + docket_number: ad.docket_number, applicable_models: ad.applicable_models, product_type: ad.product_type, + document_url: ad.document_url } + end + + def oil_hash(o) + { id: id(o), aircraft_id: id(o.aircraft_id), component_id: id(o.component_id), sample_date: o.sample_date&.iso8601, + analysis_date: o.analysis_date&.iso8601, lab: o.lab.to_s, lab_number: o.sample_number.to_s, oil_type: o.oil_type.to_s, + oil_hours: dec(o.oil_hours), engine_hours: dec(o.engine_hours), oil_added_quarts: dec(o.oil_added), + elements_ppm: o.elements, oil_properties: o.oil_properties.presence, lab_comments: o.lab_comments.to_s, + status: o.status, notes: o.notes.to_s, excluded_from_averages: o.excluded_from_averages, + filter_hours: dec(o.filter_hours), element_averages: o.element_averages, report: file(o.report, "oil_reports") } + end + + def flight_hash(f) + { id: id(f), aircraft_id: id(f.aircraft_id), date: f.date&.iso8601, tach_time: dec(f.tach_time), tach_out: dec(f.tach_out), + tach_in: dec(f.tach_in), hobbs_time: dec(f.hobbs_time), hobbs_out: dec(f.hobbs_out), hobbs_in: dec(f.hobbs_in), + departure_location: f.departure_location.to_s, destination_location: f.destination_location.to_s, route: f.route.to_s, + oil_added: dec(f.oil_added), oil_added_type: f.oil_added_type.to_s, oil_level_after: dec(f.oil_level_after), + fuel_added: dec(f.fuel_added), fuel_added_type: f.fuel_added_type.to_s, fuel_level_after: dec(f.fuel_level_after), + track_log: file(f.track_log, "track_logs"), notes: f.notes.to_s, created_at: f.created_at&.iso8601, + engine_data: file(f.engine_data, "engine_data"), pilot_display: f.user&.display_name } + end + end +end diff --git a/app/services/aircraft_archive/importer.rb b/app/services/aircraft_archive/importer.rb new file mode 100644 index 0000000..0564df7 --- /dev/null +++ b/app/services/aircraft_archive/importer.rb @@ -0,0 +1,479 @@ +require "zip" + +module AircraftArchive + # Reads a .sam.zip (from this app or the original Simple Aircraft Manager) and + # creates the aircraft with all its records and files. + # + # preview = AircraftArchive::Importer.check(path) # fast safety checks + # AircraftArchive::Importer.new(path, owner:, tail_number:).run + # + # Archive IDs are only used to link records inside the archive; everything + # gets new IDs here. Component types, inspection types and ADs are shared + # across aircraft, so existing ones with the same name are reused. + # + # No single transaction wraps the import (SQLite would block other writers for + # its whole length). If it fails part-way, the new aircraft and anything else + # it created are removed again. + class Importer + Preview = Struct.new(:tail_number, :make, :model, :source, :exported_at, :counts, keyword_init: true) + RECORD_SECTIONS = %w[components document_collections documents document_images logbook_entries squawks inspection_types + inspection_records ads ad_compliances consumable_records major_records notes oil_analysis_reports flight_logs].freeze + + # Safety and format checks that don't import anything. Raises Error. + def self.check(path) + manifest = with_zip(path) { |zip| validate_zip(zip) } + a = manifest["aircraft"] + Preview.new(tail_number: a["tail_number"], make: a["make"], model: a["model"], source: manifest["source_instance"].presence, + exported_at: (Time.zone.parse(manifest["exported_at"].to_s) rescue nil), + counts: RECORD_SECTIONS.to_h { [ _1, Array(manifest[_1]).size ] }) + end + + def self.with_zip(path) + Zip.validate_entry_sizes = true # a lying size header raises instead of inflating forever + Zip::File.open(path) { |zip| yield zip } + rescue Zip::Error => e + raise Error, "Not a valid .sam.zip archive (#{e.message})" + end + + def self.validate_zip(zip) + uncompressed = compressed = 0 + zip.each do |entry| + raise Error, "The archive contains a link (#{entry.name}), which isn't allowed." if entry.symlink? + raise Error, "The archive contains an unsafe path: #{entry.name}" unless safe_name?(entry.name) + uncompressed += entry.size + compressed += entry.compressed_size + raise Error, "The archive is too large once unpacked." if uncompressed > ARCHIVE_MAX_BYTES + end + raise Error, "The archive is compressed suspiciously well (over #{MAX_RATIO}:1)." if compressed.positive? && uncompressed / compressed > MAX_RATIO + + entry = zip.find_entry("manifest.json") or raise Error, "The archive has no manifest.json. Is it a .sam.zip export?" + raise Error, "manifest.json is too large." if entry.size > MANIFEST_MAX_BYTES + manifest = begin + JSON.parse(entry.get_input_stream.read.force_encoding("UTF-8")) + rescue JSON::ParserError => e + raise Error, "manifest.json couldn't be read (#{e.message.truncate(120)})" + end + raise Error, "manifest.json isn't a JSON object." unless manifest.is_a?(Hash) + + version = manifest["schema_version"] + unless version.is_a?(Integer) && version.between?(1, SCHEMA_VERSION) + raise Error, "Unsupported archive version #{version.inspect} (this app reads versions 1–#{SCHEMA_VERSION})." + end + required = TOP_LEVEL_KEYS - OPTIONAL_KEYS - (version < 2 ? %w[flight_logs] : []) + missing = required - manifest.keys + raise Error, "manifest.json is missing #{missing.to_sentence}." if missing.any? + raise Error, "manifest.json has no aircraft tail number." unless manifest["aircraft"].is_a?(Hash) && manifest["aircraft"]["tail_number"].present? + RECORD_SECTIONS.each { |key| raise Error, "manifest.json #{key} must be a list." unless manifest[key].nil? || manifest[key].is_a?(Array) } + manifest + end + + # Relative, no "..", valid UTF-8 (zip entry names arrive as raw bytes). + def self.safe_name?(name) + name = name.to_s.dup.force_encoding(Encoding::UTF_8) + return false if name.blank? || !name.valid_encoding? || name.start_with?("/", "\\") || name.match?(/\A[A-Za-z]:/) + name.unicode_normalize(:nfc).tr("\\", "/").split("/").exclude?("..") + end + + attr_reader :counts, :warnings, :aircraft + + def initialize(path, owner:, tail_number: nil, progress: nil) + @path = path + @owner = owner + @tail_number = tail_number + @progress = progress || ->(_counts) { } + @counts = Hash.new(0) + @warnings = [] + @map = Hash.new { |h, k| h[k] = {} } # section → archive id → new record id + @created_blobs = [] + @created_shared = [] # component types / inspection types / ADs made by this import + end + + def run + self.class.with_zip(@path) do |zip| + @zip = zip + @manifest = self.class.validate_zip(zip) + import_all + end + { aircraft: @aircraft, counts: @counts, warnings: @warnings } + rescue StandardError + clean_up + raise + end + + private + + def m(key) = Array(@manifest[key]).select { _1.is_a?(Hash) } + + def import_all + import_aircraft + import_component_types + import_components + import_documents + import_logbook + import_squawks + import_inspections + import_ads + import_consumables + import_major_records + import_notes + import_oil_analyses + import_flight_logs + @aircraft.roles.find_or_create_by!(user: @owner, role: "owner") if @owner + @aircraft.log_event!("aircraft", "Aircraft imported from a .sam.zip archive", + notes: [ @manifest["source_instance"].presence && "Source: #{@manifest['source_instance']}", summary ].compact.join(". "), + user: @owner) + end + + def summary = @counts.select { _2.positive? }.map { |k, n| "#{n} #{k.tr('_', ' ')}" }.join(", ") + + def tick(section, n = 1) + @counts[section] += n + @progress.call(@counts) if (@counts.values.sum % 25).zero? + end + + # ---- records ---- + + def import_aircraft + a = @manifest["aircraft"] + @aircraft = Aircraft.new( + tail_number: (@tail_number.presence || a["tail_number"]).to_s.strip.upcase, + make: a["make"].to_s, model: a["model"].to_s, serial_number: a["serial_number"].to_s, description: a["description"].to_s, + purchased: date(a["purchased"]), status: Aircraft::STATUSES.key?(a["status"]) ? a["status"] : "AVAILABLE", + tach_time: num(a["tach_time"] || a["flight_time"]) || 0, tach_time_offset: num(a["tach_time_offset"]) || 0, + hobbs_time: num(a["hobbs_time"]) || 0, hobbs_time_offset: num(a["hobbs_time_offset"]) || 0, + icao_type: a["icao_type"].to_s.upcase.presence&.then { _1.match?(/\A[A-Z0-9]{1,4}\z/) ? _1 : nil } + ) + feature = m("features").find { _1["feature"] == "airworthiness_enforcement" } + @aircraft.airworthiness_enforcement = feature["enabled"] if feature && !feature["enabled"].nil? && @aircraft.has_attribute?(:airworthiness_enforcement) + @aircraft.save!(validate: false) + attach(@aircraft.picture, a["picture"], "aircraft picture") unless a["_missing"] + end + + def import_component_types + m("component_types").each do |data| + name = data["name"].to_s.strip + next if name.blank? + type = ComponentType.where("LOWER(name) = ?", name.downcase).first || + ComponentType.create!(name: name, consumable: data["consumable"] == true).tap { @created_shared << _1 } + @map[:component_type][data["id"].to_s] = type.id + end + end + + def import_components + by_id = m("components").index_by { _1["id"].to_s } + ordered = [] + visit = lambda do |data, seen = []| + return if ordered.include?(data) || seen.include?(data) + parent = by_id[data["parent_component_id"].to_s] + visit.(parent, seen + [ data ]) if parent + ordered << data + end + by_id.each_value { visit.(_1) } + + ordered.each do |data| + type_id = @map[:component_type][data["component_type_id"].to_s] + next warn("Component #{data['manufacturer']} #{data['model']}: unknown component type, skipped.") unless type_id + c = @aircraft.components.new( + component_type_id: type_id, parent_component_id: @map[:component][data["parent_component_id"].to_s], + manufacturer: data["manufacturer"].to_s, model: data["model"].to_s, serial_number: data["serial_number"].to_s, + install_location: data["install_location"].to_s, notes: data["notes"].to_s, + status: Component::STATUSES.key?(data["status"]) ? data["status"] : "SPARE", + date_in_service: date(data["date_in_service"]) || Date.new(1900, 1, 1), hours_in_service: num(data["hours_in_service"]) || 0, + hours_since_overhaul: num(data["hours_since_overhaul"]) || 0, overhaul_date: date(data["overhaul_date"]), + **data.slice(*%w[tbo_hours tbo_days inspection_hours inspection_days replacement_hours replacement_days]).symbolize_keys, + **data.slice(*%w[tbo_critical on_condition inspection_critical replacement_critical]).transform_values { _1 == true }.symbolize_keys + ) + c.save!(validate: false) + @map[:component][data["id"].to_s] = c.id + tick("components") + end + end + + def import_documents + m("document_collections").each do |data| + c = @aircraft.document_collections.new(name: data["name"].to_s.presence || "Untitled", description: data["description"].to_s, + visibility: visibility(data["visibility"]) || "private", starred: data["starred"] == true) + c.save!(validate: false) + @map[:collection][data["id"].to_s] = c.id + end + documents = {} + m("documents").each do |data| + d = @aircraft.documents.new(name: data["name"].to_s.presence || "Untitled", description: data["description"].to_s, + doc_type: data["doc_type"].presence, visibility: visibility(data["visibility"]), + document_collection_id: @map[:collection][data["collection_id"].to_s]) + d.doc_type = "OTHER" unless Document::TYPES.key?(d.doc_type) + d.save!(validate: false) + @map[:document][data["id"].to_s] = d.id + documents[d.id] = d + tick("documents") + end + m("document_images").each do |data| + document = documents[@map[:document][data["document_id"].to_s]] or next + next if data["_missing"] + tick("document_pages") if attach(document.pages, data["image"], "document #{document.name}") + end + end + + def import_logbook + linked = 0 + m("logbook_entries").each do |data| + e = @aircraft.logbook_entries.new( + log_type: LogbookEntry::LOG_TYPES.key?(data["log_type"]) ? data["log_type"] : "AC", + entry_type: LogbookEntry::ENTRY_TYPES.key?(data["entry_type"]) ? data["entry_type"] : LogbookEntry::ENTRY_TYPES.keys.first, + date: date(data["date"]) || Date.new(1900, 1, 1), text: data["text"].to_s.presence || "—", + signoff_person: data["signoff_person"], signoff_location: data["signoff_location"], + aircraft_hours_at_entry: num(data["aircraft_hours_at_entry"]), page_number: data["page_number"], + document_id: @map[:document][data["log_image_id"].to_s] + ) + e.save!(validate: false) + @map[:logbook][data["id"].to_s] = e.id + linked += 1 if Array(data["components"]).any? || data["component_hours"].present? + tick("logbook_entries") + end + warn("#{linked} logbook #{linked == 1 ? 'entry was' : 'entries were'} linked to components; those links aren't kept here.") if linked.positive? + end + + def import_squawks + m("squawks").each do |data| + s = @aircraft.squawks.new( + priority: Squawk::PRIORITIES.key?(data["priority"]) ? data["priority"] : 3, issue_reported: data["issue_reported"].to_s.presence || "—", + notes: data["notes"], resolved: data["resolved"] == true, component_id: @map[:component][data["component_id"].to_s], + created_at: time(data["created_at"]) || Time.current + ) + s.resolved_at = time(data["resolved_at"]) || (s.created_at if s.resolved) + s.save!(validate: false) + paths = Array(data["attachments"]).presence || [ data["attachment"] ].compact + paths.each { attach(s.attachments, _1, "squawk “#{s.issue_reported.truncate(40)}”") } unless data["_missing"] + tick("squawks") + end + end + + def import_inspections + m("inspection_types").each do |data| + name = data["name"].to_s.strip + next if name.blank? + type = InspectionType.where("LOWER(name) = ?", name.downcase).first + if type + warn("Inspection type “#{name}” already existed here and was reused.") if type.recurring != (data["recurring"] == true) + else + type = InspectionType.new(name: name, recurring: data["recurring"] == true, required: data["required"] != false, + recurring_hours: num(data["recurring_hours"]), recurring_days: data["recurring_days"], + recurring_months: data["recurring_months"]) + type.save!(validate: false) + @created_shared << type + end + type.aircraft << @aircraft if Array(data["applicable_aircraft"]).any? && !type.aircraft.include?(@aircraft) + components(data["applicable_component"]).each { |c| type.components << c unless type.components.include?(c) } + @map[:inspection_type][data["id"].to_s] = type.id + end + m("inspection_records").each do |data| + type_id = @map[:inspection_type][data["inspection_type_id"].to_s] + next warn("An inspection record from #{data['date']} has no inspection type and was skipped.") unless type_id + r = @aircraft.inspection_records.new( + inspection_type_id: type_id, date: date(data["date"]) || Date.new(1900, 1, 1), aircraft_hours: num(data["aircraft_hours"]), + logbook_entry_id: @map[:logbook][data["logbook_entry_id"].to_s], notes: data["notes"], + component_id: @map[:component][Array(data["component"]).first.to_s] + ) + r.save!(validate: false) + @map[:inspection_record][data["id"].to_s] = r.id + tick("inspection_records") + end + end + + def import_ads + m("ads").each do |data| + name = data["name"].to_s.strip + next if name.blank? + ad = AirworthinessDirective.where("LOWER(name) = ?", name.downcase).first + unless ad + ad = AirworthinessDirective.new( + name: name, short_description: data["short_description"].to_s, required_action: data["required_action"].to_s, + compliance_type: AirworthinessDirective::COMPLIANCE_TYPES.key?(data["compliance_type"]) ? data["compliance_type"] : "standard", + trigger_condition: data["trigger_condition"].to_s, recurring: data["recurring"] == true, + recurring_hours: num(data["recurring_hours"]), recurring_months: data["recurring_months"], recurring_days: data["recurring_days"], + bulletin_type: AirworthinessDirective::BULLETIN_TYPES.key?(data["bulletin_type"]) ? data["bulletin_type"] : "ad", + mandatory: data["mandatory"] != false, document_id: @map[:document][data["document_id"].to_s], + effective_date: date(data["effective_date"]), supersedes: data["supersedes"], superseded_by: data["superseded_by"], + docket_number: data["docket_number"], applicable_models: data["applicable_models"], product_type: data["product_type"], + document_url: data["document_url"] + ) + ad.save!(validate: false) + @created_shared << ad + end + ad.aircraft << @aircraft if Array(data["applicable_aircraft"]).any? && !ad.aircraft.include?(@aircraft) + components(data["applicable_component"]).each { |c| ad.components << c unless ad.components.include?(c) } + @map[:ad][data["id"].to_s] = ad.id + tick("ads") + end + m("ad_compliances").each do |data| + ad_id = @map[:ad][data["ad_id"].to_s] or next + c = @aircraft.ad_compliances.new( + airworthiness_directive_id: ad_id, date_complied: date(data["date_complied"]) || Date.new(1900, 1, 1), + compliance_notes: data["compliance_notes"].to_s, permanent: data["permanent"] == true, + next_due_at_time: num(data["next_due_at_time"]) || 0, aircraft_hours_at_compliance: num(data["aircraft_hours_at_compliance"]), + logbook_entry_id: @map[:logbook][data["logbook_entry_id"].to_s], + inspection_record_id: @map[:inspection_record][data["inspection_record_id"].to_s], + component_id: @map[:component][data["component_id"].to_s], not_applicable: data["not_applicable"] == true + ) + c.save!(validate: false) + tick("ad_compliances") + end + end + + def import_consumables + @consumable_flight_links = {} + m("consumable_records").each do |data| + next unless ConsumableRecord::TYPES.key?(data["record_type"]) + r = @aircraft.consumable_records.new( + record_type: data["record_type"], date: date(data["date"]) || Date.new(1900, 1, 1), quantity_added: num(data["quantity_added"]) || 0, + level_after: num(data["level_after"]), consumable_type: data["consumable_type"].to_s, flight_hours: num(data["flight_hours"]) || 0, + notes: data["notes"].to_s, excluded_from_averages: data["excluded_from_averages"] == true + ) + r.save!(validate: false) + @consumable_flight_links[r.id] = data["flight_log_id"].to_s if data["flight_log_id"] + tick("consumable_records") + end + end + + def import_major_records + m("major_records").each do |data| + r = @aircraft.major_records.new( + record_type: MajorRecord::RECORD_TYPES.key?(data["record_type"]) ? data["record_type"] : "repair", + title: data["title"].to_s.presence || "—", description: data["description"].to_s, date_performed: date(data["date_performed"]), + performed_by: data["performed_by"].to_s, component_id: @map[:component][data["component_id"].to_s], + form_337_document_id: @map[:document][data["form_337_document_id"].to_s], stc_number: data["stc_number"].to_s, + stc_holder: data["stc_holder"].to_s, stc_document_id: @map[:document][data["stc_document_id"].to_s], + logbook_entry_id: @map[:logbook][data["logbook_entry_id"].to_s], aircraft_hours: num(data["aircraft_hours"]), + notes: data["notes"].to_s, has_ica: data["has_ica"] == true, ica_notes: data["ica_notes"] + ) + r.save!(validate: false) + tick("major_records") + end + end + + def import_notes + m("notes").each do |data| + at = time(data["added_timestamp"]) || Time.current + n = @aircraft.notes.new(text: data["text"].to_s.presence || "—", public: data["public"] == true, created_at: at, + edited_at: time(data["edited_at"])) + n.text = "#{n.text}\n\n— #{data['added_by_display']}" if data["added_by_display"].present? + n.save!(validate: false) + tick("notes") + end + end + + def import_oil_analyses + m("oil_analysis_reports").each do |data| + o = @aircraft.oil_analyses.new( + component_id: @map[:component][data["component_id"].to_s], sample_date: date(data["sample_date"]) || Date.new(1900, 1, 1), + analysis_date: date(data["analysis_date"]), lab: data["lab"].presence, sample_number: data["lab_number"].presence, + oil_type: data["oil_type"].presence, oil_hours: num(data["oil_hours"]), engine_hours: num(data["engine_hours"]), + filter_hours: num(data["filter_hours"]), oil_added: num(data["oil_added_quarts"]), + elements: data["elements_ppm"].is_a?(Hash) ? data["elements_ppm"] : {}, + element_averages: data["element_averages"].is_a?(Hash) ? data["element_averages"] : {}, + oil_properties: data["oil_properties"].is_a?(Hash) ? data["oil_properties"] : {}, + lab_comments: data["lab_comments"].presence, notes: data["notes"].presence, + status: OilAnalysis::STATUSES.key?(data["status"]) ? data["status"] : nil, + excluded_from_averages: data["excluded_from_averages"] == true + ) + o.save!(validate: false) + attach(o.report, data["report"], "oil analysis #{o.sample_date}") if data["report"] + tick("oil_analyses") + end + end + + def import_flight_logs + m("flight_logs").each do |data| + f = @aircraft.flight_logs.new( + date: date(data["date"]) || Date.new(1900, 1, 1), tach_time: num(data["tach_time"]) || 0, tach_out: num(data["tach_out"]), + tach_in: num(data["tach_in"]), hobbs_time: num(data["hobbs_time"]), hobbs_out: num(data["hobbs_out"]), + hobbs_in: num(data["hobbs_in"]), departure_location: data["departure_location"].to_s, + destination_location: data["destination_location"].to_s, route: data["route"].to_s, oil_added: num(data["oil_added"]), + oil_added_type: data["oil_added_type"].to_s, oil_level_after: num(data["oil_level_after"]), fuel_added: num(data["fuel_added"]), + fuel_added_type: data["fuel_added_type"].to_s, fuel_level_after: num(data["fuel_level_after"]), notes: data["notes"].to_s, + created_at: time(data["created_at"]) || Time.current + ) + f.save!(validate: false) + @map[:flight_log][data["id"].to_s] = f.id + attach(f.track_log, data["track_log"], "flight #{f.date} track log") if data["track_log"] && !data["_missing"] + attach(f.engine_data, data["engine_data"], "flight #{f.date} engine data") if data["engine_data"] + tick("flight_logs") + end + (@consumable_flight_links || {}).each do |record_id, archive_flight_id| + flight_id = @map[:flight_log][archive_flight_id] or next + ConsumableRecord.where(id: record_id).update_all(flight_log_id: flight_id) + end + end + + # ---- files ---- + + # Copy one archive file into storage and attach it. Problems with a single + # file become warnings; the record is kept without it. + def attach(target, archive_path, what) + return false if archive_path.blank? + blob = upload(archive_path.to_s) + target.attach(blob) + true + rescue Error => e + warn("#{what.upcase_first}: #{e.message}") + false + end + + def upload(archive_path) + raise Error, "unsafe file path #{archive_path}" unless self.class.safe_name?(archive_path) + entry = @zip.find_entry(archive_path) or raise Error, "file #{File.basename(archive_path)} isn't in the archive" + raise Error, "#{File.basename(archive_path)} is too large" if entry.size > FILE_MAX_BYTES + ext = AircraftArchive.extension(archive_path) + raise Error, "#{File.basename(archive_path)} isn't an allowed file type" unless EXTENSIONS.include?(ext) + + Tempfile.create([ "sam-import", ".#{ext}" ], binmode: true) do |tmp| + entry.get_input_stream { |input| IO.copy_stream(input, tmp) } + tmp.rewind + head = tmp.read(16).to_s + raise Error, "#{File.basename(archive_path)} doesn't contain what its name says (.#{ext})" unless AircraftArchive.content_matches?(head, ext) + tmp.rewind + ActiveStorage::Blob.create_and_upload!(io: tmp, filename: File.basename(archive_path)).tap { @created_blobs << _1 } + end + end + + # ---- helpers ---- + + def warn(message) + @warnings << message + nil + end + + def components(ids) = Array(ids).filter_map { @map[:component][_1.to_s] }.then { Component.where(id: _1) } + + def visibility(value) = DocumentCollection::VISIBILITIES.key?(value) ? value : nil + + def date(value) + Date.iso8601(value.to_s[0, 10]) if value.present? + rescue Date::Error + nil + end + + def time(value) + Time.zone.parse(value.to_s) if value.present? + rescue ArgumentError + nil + end + + def num(value) + return nil if value.blank? + BigDecimal(value.to_s) + rescue ArgumentError + nil + end + + # Undo a failed import: the aircraft (and its records, via dependent + # destroys), shared records only this import created, and uploaded files. + def clean_up + @aircraft&.destroy if @aircraft&.persisted? + @created_shared.reverse_each { |record| record.destroy if record.class.exists?(record.id) } + @created_blobs.each { |blob| blob.purge if ActiveStorage::Blob.exists?(blob.id) } + rescue StandardError => e + Rails.logger.error("[aircraft import] clean-up failed: #{e.class}: #{e.message}") + end + end +end diff --git a/app/services/logbook_transcriber.rb b/app/services/logbook_transcriber.rb new file mode 100644 index 0000000..418e8c2 --- /dev/null +++ b/app/services/logbook_transcriber.rb @@ -0,0 +1,91 @@ +require "anthropic" + +# Reads scanned logbook pages with Claude and returns the entries on them. +# One call handles one batch: several page images, or one PDF. +# +# LogbookTranscriber.new.transcribe([ { data: bytes, media_type: "image/jpeg" }, … ], carried: [...]) +# +# `carried` are entries already read from page 0 of this batch (the last page of +# the previous one), so an entry continuing onto the next page is completed +# rather than duplicated. +class LogbookTranscriber + class Error < StandardError; end + + MODEL = ENV.fetch("LOGBOOK_AI_MODEL", "claude-opus-5-5") + EFFORT = ENV.fetch("LOGBOOK_AI_EFFORT", "high") # handwriting rewards a careful read + MAX_TOKENS = 64_000 + PROMPT = File.read(File.join(__dir__, "logbook_transcriber/prompt.txt")).freeze + SCHEMA = JSON.parse(File.read(File.join(__dir__, "logbook_transcriber/schema.json"))).freeze + FALLBACK_BETA = "server-side-fallback-2026-07-01" + IMAGE_TYPES = %w[image/jpeg image/png image/gif image/webp].freeze + MAX_IMAGE_BYTES = 5.megabytes # the API's per-image limit + MAX_PDF_BYTES = 30.megabytes # request limit is 32 MB + + Result = Struct.new(:entries, :non_logbook_pages, :unparseable_pages, :model, keyword_init: true) + + class << self + attr_writer :client # tests swap in a fake + + def client = @client ||= Anthropic::Client.new + end + + def initialize(client: self.class.client, model: MODEL, effort: EFFORT) + @client = client + @model = model + @effort = effort + end + + def transcribe(pages, carried: []) + stream = @client.beta.messages.stream( + model: @model, + max_tokens: MAX_TOKENS, + system_: PROMPT, + output_config: { effort: @effort.to_sym, format_: { type: :json_schema, schema: SCHEMA } }, + # If a safety classifier declines, the API retries on its default substitute model. + fallbacks: :default, + betas: [ FALLBACK_BETA ], + messages: [ { role: "user", content: content(pages, carried) } ] + ) + message = stream.accumulated_message + case message.stop_reason + when :refusal then raise Error, "Claude declined to read these pages" + when :max_tokens then raise Error, "the transcription was too long to finish; try fewer pages per file" + end + + text = message.content.select { _1.type == :text }.map(&:text).join + data = JSON.parse(text) + Result.new(entries: Array(data["entries"]), non_logbook_pages: Array(data["non_logbook_pages"]), + unparseable_pages: Array(data["unparseable_pages"]), model: message.model.to_s) + rescue JSON::ParserError + raise Error, "Claude's answer wasn't valid JSON" + rescue Anthropic::Errors::RateLimitError + raise Error, "the Anthropic API is rate limiting requests; try again in a few minutes" + rescue Anthropic::Errors::AuthenticationError, Anthropic::Errors::PermissionDeniedError + raise Error, "the Anthropic API key was rejected" + rescue Anthropic::Errors::APIStatusError => e + raise Error, "the Anthropic API returned an error (#{e.status})" + rescue Anthropic::Errors::APIConnectionError + raise Error, "couldn't reach the Anthropic API" + end + + private + + def content(pages, carried) + blocks = pages.each_with_index.flat_map do |page, i| + source = { type: "base64", media_type: page[:media_type], data: Base64.strict_encode64(page[:data]) } + if page[:media_type] == "application/pdf" + [ { type: "text", text: "This PDF holds the logbook pages; page indices count its pages from 0." }, + { type: "document", source: source } ] + else + [ { type: "text", text: "Page #{i}:" }, { type: "image", source: source } ] + end + end + if carried.any? + context = carried.map do |e| + "- #{e['date'] || 'undated'} (#{e['log_type']}, signed #{e['signoff_person'] || 'unsigned'}): #{e['text'].to_s.truncate(400)}" + end + blocks << { type: "text", text: "Entries already read from page 0 (it was the last page of the previous batch):\n#{context.join("\n")}" } + end + blocks << { type: "text", text: "Extract every logbook entry from these pages." } + end +end diff --git a/app/services/logbook_transcriber/prompt.txt b/app/services/logbook_transcriber/prompt.txt new file mode 100644 index 0000000..11ba8de --- /dev/null +++ b/app/services/logbook_transcriber/prompt.txt @@ -0,0 +1,36 @@ +You are an expert aviation maintenance logbook analyst with deep knowledge of FAA regulations, +aircraft maintenance records, and standard logbook formats. + +You will get numbered pages of an aircraft logbook (page 0 is the first page in this request). +Extract every logbook entry visible. + +Rules: +- A single entry may span two consecutive pages (look for "continued" or entries that cut off mid-sentence). +- Several separate entries may appear on one page (separated by dates, rules, or spacing). +- Some pages are not logbook entries: FAA Form 337 (major repair/alteration), yellow tags (8130-3), + weight and balance records, equipment lists, AD lists, or other administrative forms. List them in + non_logbook_pages instead of extracting entries from them. +- Dates: convert MM/DD/YYYY, MM/DD/YY, or written dates to YYYY-MM-DD. If the year is ambiguous, use the + surrounding entries. Use null only if the date is truly unreadable. +- Signoffs: include the mechanic's name and certificate number (e.g. "A&P 123456", IA number) or the repair + station number. + +Entry types: +- MAINTENANCE: repairs, part replacements, AD compliance, alterations, STCs, 337s referenced +- INSPECTION: annual, 100-hour, phase checks, pre-buy, progressive, conditional inspections +- FLIGHT: test flights, ferry flights, return-to-service flights +- HOURS_UPDATE: an hours entry with no specific work described +- OTHER: administrative entries, certifications + +Fields: +- log_type: AC (airframe), ENG (engine), PROP (propeller), or OTHER +- text: the entry text verbatim, or a close paraphrase where wording is unclear; write [?] for illegible words +- aircraft_hours: the tach/total time the entry records (a number), or null if none is written +- signoff_person / signoff_location: a string, or null +- page_start / page_end: page indices within this request +- confidence: high (clearly legible), medium (some guesswork), low (mostly illegible) +- unparseable_pages: pages too illegible to extract anything from + +Overlapping pages: when the request says entries were already read from page 0, return each of those +entries exactly once, completed with any text that continues onto later pages, and do not return any +other entry that lies wholly on page 0. diff --git a/app/services/logbook_transcriber/schema.json b/app/services/logbook_transcriber/schema.json new file mode 100644 index 0000000..80098f0 --- /dev/null +++ b/app/services/logbook_transcriber/schema.json @@ -0,0 +1,30 @@ +{ + "type": "object", + "properties": { + "entries": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { "type": ["string", "null"], "description": "YYYY-MM-DD, or null if unreadable" }, + "log_type": { "type": "string", "enum": ["AC", "ENG", "PROP", "OTHER"] }, + "entry_type": { "type": "string", "enum": ["MAINTENANCE", "INSPECTION", "FLIGHT", "HOURS_UPDATE", "OTHER"] }, + "text": { "type": "string" }, + "aircraft_hours": { "type": ["number", "null"] }, + "signoff_person": { "type": ["string", "null"] }, + "signoff_location": { "type": ["string", "null"] }, + "page_start": { "type": "integer" }, + "page_end": { "type": "integer" }, + "confidence": { "type": "string", "enum": ["high", "medium", "low"] } + }, + "required": ["date", "log_type", "entry_type", "text", "aircraft_hours", "signoff_person", "signoff_location", + "page_start", "page_end", "confidence"], + "additionalProperties": false + } + }, + "non_logbook_pages": { "type": "array", "items": { "type": "integer" } }, + "unparseable_pages": { "type": "array", "items": { "type": "integer" } } + }, + "required": ["entries", "non_logbook_pages", "unparseable_pages"], + "additionalProperties": false +} diff --git a/app/services/oil_report_parser.rb b/app/services/oil_report_parser.rb index e727bda..56d96ff 100644 --- a/app/services/oil_report_parser.rb +++ b/app/services/oil_report_parser.rb @@ -1,207 +1,43 @@ require "pdf-reader" -# Reads an AvLab (Aviation Laboratories) oil analysis report PDF. A report -# holds the current sample plus up to two previous ones, each with its dates, -# hours, lab verdict, comments and a ppm table (value, lab average beneath). +# Reads an oil analysis lab report PDF into sample records. Supported labs: +# - Aviation Laboratories (AvLab): the current sample plus up to two previous +# ones, each with its own verdict, comments and lab averages. +# - Blackstone Laboratories: a grid of up to ~5 samples (newest first) with 20 +# elements, oil properties, and the lab's universal averages. # -# The PDF's text layer is messy: it hides the cell color codes ("#00CC00") as -# larger text on top of the results. So this works from positioned characters, -# drops the color-code layer by font size, rebuilds words and lines, and maps -# each result to its element column by x position. +# Every sample comes back as a hash of OilAnalysis attributes plus :current. class OilReportParser class Error < StandardError; end - LAB = "AvLab".freeze - COLOR_CODE_FONT_SIZE = 8.5 # the hidden color codes are 9pt; real text is 7–8pt - COLUMNS = { - "Iron" => "iron", "Copper" => "copper", "Nickel" => "nickel", "Chromium" => "chromium", - "Silver" => "silver", "Magnesium" => "magnesium", "Aluminum" => "aluminum", "Lead" => "lead", - "Silicon" => "silicon", "Titanium" => "titanium", "Tin" => "tin", "Moly." => "molybdenum" - }.freeze - LABELS = { - "Sample Date:" => :sample_date, "Analysis Date:" => :analysis_date, "Sample Number:" => :sample_number, - "TSN/TSO:" => :engine_hours, "Oil Hours:" => :oil_hours, "Filter Hours:" => :filter_hours, - "Oil Added:" => :oil_added - }.freeze - LABEL_MAX_X = 108 # left-hand labels end before here… - VALUE_X = 105..160 # …and their values sit in this column - - Char = Struct.new(:text, :x, :endx, :y, :page) - Word = Struct.new(:text, :x, :endx, :y, :page) do - def center = (x + endx) / 2.0 - end - Line = Struct.new(:y, :page, :words) do - def text = words.map(&:text).join(" ") - end - Report = Struct.new(:lab, :tail_number, :engine_serial, :engine_model, :samples, keyword_init: true) + Report = Struct.new(:lab, :tail_number, :engine_serial, :engine_model, :oil_type, :samples, keyword_init: true) def self.from_pdf(io) reader = PDF::Reader.new(io) - chars = reader.pages.each_with_index.flat_map do |page, index| - receiver = PDF::Reader::PageTextReceiver.new - page.walk(receiver) - # Keep overlapping glyphs: a real "0" under a hidden one would otherwise be dropped. - receiver.runs(merge: false, skip_overlapping: false).filter_map do |run| - next if run.font_size >= COLOR_CODE_FONT_SIZE || run.text.strip.empty? - Char.new(run.text, run.x, run.endx, run.y, index) - end + receivers = reader.pages.map { |page| PDF::Reader::PageTextReceiver.new.tap { page.walk(_1) } } + text = receivers.flat_map { |r| r.runs(merge: true, skip_overlapping: false).map(&:text) }.join(" ") + + if text.match?(/blackstone-labs\.com|BLACKSTONE LABORATORIES/i) + Blackstone.new(receivers.first.runs(merge: true, skip_overlapping: false)).parse + elsif text.match?(/OIL ANALYSIS RESULTS IN PARTS PER MILLION|AVIATION LABORATORIES|avlab\.com/i) + Avlab.from_receivers(receivers) + else + raise Error, "Not a recognized lab report (AvLab and Blackstone PDFs are supported)" end - new(chars).parse rescue PDF::Reader::MalformedPDFError, PDF::Reader::UnsupportedFeatureError => e raise Error, "Couldn't read the PDF (#{e.message})" end - def initialize(chars) - @lines = build_lines(chars) + # "1,903" → 1903, "<0.5" → 0.5, "5 qts" → 5, ".3" → 0.3; nil for "-", "N/A"… + def self.number(text) + digits = text.to_s.delete(",").strip.sub(/\A[<>]\s*/, "")[/\A\d*\.?\d+/] or return nil + BigDecimal(digits.sub(/\A\./, "0.")) end - def parse - unless @lines.any? { _1.text.include?("OIL ANALYSIS RESULTS IN PARTS PER MILLION") } - raise Error, "Not an AvLab oil analysis report" - end - samples = sample_blocks.map { parse_sample(_1) }.select { _1[:sample_date] } - raise Error, "No samples found in the report" if samples.empty? - - Report.new(lab: LAB, tail_number: after("Tail No.:"), engine_serial: after("Engine S/N:"), - engine_model: after("Engine Model:", words: 3), samples: samples) - end - - private - - # Characters → lines (same baseline ±1pt) → words (glyphs closer than a space). - def build_lines(chars) - lines = [] - chars.sort_by { [ _1.page, -_1.y ] }.each do |c| - line = lines.reverse_each.find { _1.page == c.page && (_1.y - c.y).abs <= 1.0 } - line ? line.words << c : lines << Line.new(c.y, c.page, [ c ]) - end - lines.each do |line| - line.words = line.words.sort_by(&:x).each_with_object([]) do |c, words| - last = words.last - if last && c.x - last.endx <= 1.6 - last.text += c.text - last.endx = c.endx - else - words << Word.new(c.text.dup, c.x, c.endx, c.y, c.page) - end - end - end - end - - def header_lines - first = @lines.find { _1.words.first&.text.in?(%w[CURRENT PREVIOUS]) } - first ? @lines.take_while { _1 != first } : @lines - end - - # The word(s) following a label on the same line, e.g. "Tail No.:" → "N12345". - def after(label, words: 1) - parts = label.split - header_lines.each do |line| - texts = line.words.map(&:text) - texts.each_cons(parts.size).with_index do |window, i| - next unless window == parts - found = texts[i + parts.size, words]&.take_while { !_1.end_with?(":") } - return found.join(" ").presence if found&.any? - end - end - nil - end - - # Lines from each "CURRENT SAMPLE" / "PREVIOUS SAMPLE n" heading to the next. - def sample_blocks - starts = @lines.each_index.select { |i| @lines[i].words.first&.text.in?(%w[CURRENT PREVIOUS]) && @lines[i].words[1]&.text == "SAMPLE" } - starts.each_with_index.map { |s, n| @lines[s...(starts[n + 1] || @lines.size)] } - end - - def parse_sample(block) - heading = block.first - verdict = heading.words.select { _1.x > 140 && _1.x < 470 }.map(&:text).join(" ") - fields = labeled_values(block) - comments = block.find { _1.words.first&.text == "Comments:" }&.words&.drop(1)&.map(&:text)&.join(" ") - - { - current: heading.words.first.text == "CURRENT", - sample_date: date(fields[:sample_date]), - analysis_date: date(fields[:analysis_date]), - sample_number: fields[:sample_number], - engine_hours: number(fields[:engine_hours]), - oil_hours: number(fields[:oil_hours]), - filter_hours: number(fields[:filter_hours]), - oil_added: number(fields[:oil_added]), - status: status(verdict), - lab_comments: [ (verdict unless status(verdict) == "normal"), comments ].compact_blank.join("\n").presence - }.merge(results(block)) - end - - # Left-hand "Label:" lines and the values beside them. A value isn't always on - # the label's baseline, so each value goes to the nearest label above or below. - def labeled_values(block) - labels = block.filter_map do |line| - text = line.words.select { _1.x < LABEL_MAX_X - 20 || (_1.x < LABEL_MAX_X && _1.text.end_with?(":")) }.map(&:text).join(" ") - [ LABELS[text], line.y ] if LABELS[text] - end - values = block.drop(1).flat_map { |line| line.words.select { VALUE_X.cover?(_1.x) } } # skip the heading - values.each_with_object({}) do |word, found| - key, = labels.min_by { (_2 - word.y).abs } - found[key] ||= word.text if key - end - end - - # The ppm row under the element headings, and the (average) row under that. - def results(block) - header = block.find { |line| COLUMNS.keys.count { line.words.map(&:text).include?(_1) } >= 6 } or return {} - columns = header.words.filter_map { |w| [ COLUMNS[w.text], w.center ] if COLUMNS[w.text] } - below = block.select { _1.y < header.y && _1.y > header.y - 25 } - - below.each_with_object({}) do |line, out| - # Results start right of the label column ("Cylinder Type: steel" shares these lines). - merge_less_than(line.words.select { _1.x >= VALUE_X.end }).each do |word| - element, = columns.min_by { (_2 - word.center).abs } - if word.text.start_with?("(") - out[:"#{element}_avg"] = ppm(word.text.delete("()")) - else - out[element.to_sym] = ppm(word.text) - end - end - end - end - - # "<" and "0.1" are drawn as separate words. - def merge_less_than(words) - words.each_with_object([]) do |w, out| - if out.last&.text == "<" - out.last.text = "< #{w.text}" - out.last.endx = w.endx - else - out << w.dup - end - end - end - - # "N/A" → nil; "< 0.1" (below detection) → 0; ".3" → 0.3 - def ppm(text) - return nil if text.blank? || text == "N/A" - return 0 if text.start_with?("<") - number(text) - end - - def number(text) - BigDecimal(text.to_s.strip.sub(/\A\./, "0.")) - rescue ArgumentError, TypeError - nil - end - - def date(text) + # M/D/YYYY + def self.date(text) Date.strptime(text, "%m/%d/%Y") if text rescue Date::Error nil end - - def status(verdict) - case verdict.upcase - when /ABNORMAL|HIGH|CRITICAL|SEVERE|PROBLEM/ then "high" - when /APPEARS NORMAL/ then "normal" - else "elevated" - end - end end diff --git a/app/services/oil_report_parser/avlab.rb b/app/services/oil_report_parser/avlab.rb new file mode 100644 index 0000000..216a695 --- /dev/null +++ b/app/services/oil_report_parser/avlab.rb @@ -0,0 +1,208 @@ +class OilReportParser + # Aviation Laboratories (AvLab) report. Each sample block has its dates, + # hours, the lab's verdict, comments and a ppm table (value, lab average beneath). + # + # The PDF's text layer is messy: it hides the cell color codes ("#00CC00") as + # larger text on top of the results. So this works from positioned characters, + # drops the color-code layer by font size, rebuilds words and lines, and maps + # each result to its element column by x position. + class Avlab + LAB = "AvLab".freeze + COLOR_CODE_FONT_SIZE = 8.5 # the hidden color codes are 9pt; real text is 7–8pt + WORD_GAP = 1.0 # glyphs closer than this are one word (spaces run 1.3pt+) + COLUMNS = { + "Iron" => "iron", "Copper" => "copper", "Nickel" => "nickel", "Chromium" => "chromium", + "Silver" => "silver", "Magnesium" => "magnesium", "Aluminum" => "aluminum", "Lead" => "lead", + "Silicon" => "silicon", "Titanium" => "titanium", "Tin" => "tin", "Moly." => "molybdenum" + }.freeze + LABELS = { + "Sample Date:" => :sample_date, "Analysis Date:" => :analysis_date, "Sample Number:" => :sample_number, + "TSN/TSO:" => :engine_hours, "Oil Hours:" => :oil_hours, "Filter Hours:" => :filter_hours, + "Oil Added:" => :oil_added + }.freeze + LABEL_MAX_X = 108 # left-hand labels end before here… + VALUE_X = 105..160 # …and their values sit in this column + + Char = Struct.new(:text, :x, :endx, :y, :page) + Word = Struct.new(:text, :x, :endx, :y, :page) do + def center = (x + endx) / 2.0 + end + Line = Struct.new(:y, :page, :words) do + def text = words.map(&:text).join(" ") + end + + def self.from_receivers(receivers) + chars = receivers.each_with_index.flat_map do |receiver, index| + # Keep overlapping glyphs: a real "0" under a hidden one would otherwise be dropped. + receiver.runs(merge: false, skip_overlapping: false).filter_map do |run| + next if run.font_size >= COLOR_CODE_FONT_SIZE || run.text.strip.empty? + Char.new(run.text, run.x, run.endx, run.y, index) + end + end + new(chars).parse + end + + def initialize(chars) + @lines = build_lines(chars) + end + + def parse + unless @lines.any? { _1.text.include?("OIL ANALYSIS RESULTS IN PARTS PER MILLION") } + raise Error, "Not an AvLab oil analysis report" + end + samples = sample_blocks.map { parse_sample(_1) }.select { _1[:sample_date] } + raise Error, "No samples found in the report" if samples.empty? + + Report.new(lab: LAB, tail_number: after("Tail No.:"), engine_serial: after("Engine S/N:"), + engine_model: after("Engine Model:", words: 3), samples: samples) + end + + private + + # Characters → lines (same baseline ±1pt) → words (glyphs closer than a space). + def build_lines(chars) + lines = [] + chars.sort_by { [ _1.page, -_1.y ] }.each do |c| + line = lines.reverse_each.find { _1.page == c.page && (_1.y - c.y).abs <= 1.0 } + line ? line.words << c : lines << Line.new(c.y, c.page, [ c ]) + end + lines.each do |line| + line.words = line.words.sort_by(&:x).each_with_object([]) do |c, words| + last = words.last + if last && c.x - last.endx <= WORD_GAP + last.text += c.text + last.endx = c.endx + else + words << Word.new(c.text.dup, c.x, c.endx, c.y, c.page) + end + end + end + end + + def header_lines + first = @lines.find { _1.words.first&.text.in?(%w[CURRENT PREVIOUS]) } + first ? @lines.take_while { _1 != first } : @lines + end + + # The word(s) following a label on the same line, e.g. "Tail No.:" → "N12345". + def after(label, words: 1) + parts = label.split + header_lines.each do |line| + texts = line.words.map(&:text) + texts.each_cons(parts.size).with_index do |window, i| + next unless window == parts + found = texts[i + parts.size, words]&.take_while { !_1.end_with?(":") } + return found.join(" ").presence if found&.any? + end + end + nil + end + + # Lines from each "CURRENT SAMPLE" / "PREVIOUS SAMPLE n" heading to the next. + def sample_blocks + starts = @lines.each_index.select { |i| @lines[i].words.first&.text.in?(%w[CURRENT PREVIOUS]) && @lines[i].words[1]&.text == "SAMPLE" } + starts.each_with_index.map { |s, n| @lines[s...(starts[n + 1] || @lines.size)] } + end + + def parse_sample(block) + heading = block.first + verdict = heading.words.select { _1.x > 140 && _1.x < 470 }.map(&:text).join(" ") + fields = labeled_values(block) + comments = comments(block) + elements, averages = results(block) + + { + current: heading.words.first.text == "CURRENT", + sample_date: OilReportParser.date(fields[:sample_date]), + analysis_date: OilReportParser.date(fields[:analysis_date]), + sample_number: fields[:sample_number], + engine_hours: OilReportParser.number(fields[:engine_hours]), + oil_hours: OilReportParser.number(fields[:oil_hours]), + filter_hours: OilReportParser.number(fields[:filter_hours]), + oil_added: OilReportParser.number(fields[:oil_added]), + status: status(verdict), + lab_comments: [ (verdict unless status(verdict) == "normal"), comments ].compact_blank.join("\n").presence, + elements: elements, + element_averages: averages + } + end + + # Left-hand "Label:" lines and the values beside them. A value isn't always on + # the label's baseline, so each value goes to the nearest label above or below. + # Every label counts, including ones we don't keep (e.g. "Filter Wt. (mgs):"), + # so their values don't drift onto a neighbouring label. + def labeled_values(block) + labels = block.filter_map do |line| + text = line.words.select { _1.x < LABEL_MAX_X - 20 || (_1.x < LABEL_MAX_X && _1.text.end_with?(":")) }.map(&:text).join(" ") + [ LABELS[text], line.y ] if text.end_with?(":") + end + values = block.drop(1).flat_map { |line| line.words.select { VALUE_X.cover?(_1.x) } } # skip the heading + values.each_with_object({}) do |word, found| + key, = labels.min_by { (_2 - word.y).abs } + found[key] ||= word.text if key + end + end + + # "Comments: …" and the lines that continue it at the same left margin. + def comments(block) + start = block.index { _1.words.first&.text == "Comments:" } or return nil + first = block[start] + margin = first.words.first.x + lines = [ first.words.drop(1) ] + block[(start + 1)..].each_with_index do |line, i| + previous = block[start + i] + break unless (line.words.first.x - margin).abs <= 3 && previous.y - line.y <= 14 && line.page == previous.page + lines << line.words + end + lines.flatten.map(&:text).join(" ").presence + end + + # The ppm row under the element headings, and the (average) row under that. + def results(block) + elements, averages = {}, {} + header = block.find { |line| COLUMNS.keys.count { line.words.map(&:text).include?(_1) } >= 6 } or return [ elements, averages ] + columns = header.words.filter_map { |w| [ COLUMNS[w.text], w.center ] if COLUMNS[w.text] } + below = block.select { _1.y < header.y && _1.y > header.y - 25 } + + below.each do |line| + # Results start right of the label column ("Cylinder Type: steel" shares these lines). + merge_less_than(line.words.select { _1.x >= VALUE_X.end }).each do |word| + element, = columns.min_by { (_2 - word.center).abs } + if word.text.start_with?("(") + averages[element] = ppm(word.text.delete("()")) + else + elements[element] = ppm(word.text) + end + end + end + [ elements.compact, averages.compact ] + end + + # "<" and "0.1" are drawn as separate words. + def merge_less_than(words) + words.each_with_object([]) do |w, out| + if out.last&.text == "<" + out.last.text = "< #{w.text}" + out.last.endx = w.endx + else + out << w.dup + end + end + end + + # "N/A" → nil; "< 0.1" (below detection) → 0; ".3" → 0.3 + def ppm(text) + return nil if text.blank? || text == "N/A" + return 0 if text.start_with?("<") + OilReportParser.number(text) + end + + def status(verdict) + case verdict.upcase + when /ABNORMAL|HIGH|CRITICAL|SEVERE|PROBLEM/ then "action_required" + when /APPEARS NORMAL/ then "normal" + else "monitor" + end + end + end +end diff --git a/app/services/oil_report_parser/blackstone.rb b/app/services/oil_report_parser/blackstone.rb new file mode 100644 index 0000000..e5a65e7 --- /dev/null +++ b/app/services/oil_report_parser/blackstone.rb @@ -0,0 +1,114 @@ +class OilReportParser + # Blackstone Laboratories report: one page with a header (lab number, unit ID, + # oil type), the lab's comments, then a grid with one column per sample (newest + # on the left, dated across the top) and one row per element / oil property. + # Two extra columns hold averages: this unit's own (skipped) and the lab's + # universal averages for the engine type (kept as the comparison average). + # + # Numbers are right-aligned under their sample dates, so values are matched to + # a sample by their right edge. + class Blackstone + LAB = "Blackstone".freeze + ELEMENTS = %w[aluminum chromium iron copper lead tin molybdenum nickel manganese silver titanium + potassium boron silicon sodium calcium magnesium phosphorus zinc barium].freeze + PROPERTIES = { + /\ASUS Viscosity/ => "viscosity_sus_210f", /\AcSt Viscosity/ => "viscosity_cst_100c", + /\AFlashpoint/ => "flashpoint_f", /\AFuel %/ => "fuel_percent", /\AAntifreeze %/ => "antifreeze_percent", + /\AWater %/ => "water_percent", /\AInsolubles %/ => "insolubles_percent", + /\ATBN\z/ => "total_base_number", /\ATAN\z/ => "total_acid_number" + }.freeze + LABEL_MAX_X = 130 # row labels sit in the left margin + ROW_SLOP = 3.5 # a row's values can sit a couple of points off its label's baseline + COLUMN_SLOP = 8 # how far a value's right edge may be from its date's + DATE = %r{\A\d{1,2}/\d{1,2}/\d{4}\z} + # A US registration in the "unit ID" field; anything else (owners often put an + # engine name there) isn't treated as a tail number. + TAIL = /\AN[1-9][0-9A-Z]{0,4}\z/i + + Run = Struct.new(:text, :x, :endx, :y) + + def initialize(runs) + # The vertical side captions ("UNIT INFORMATION"…) are single letters at x≈43. + @runs = runs.filter_map { Run.new(_1.text.strip, _1.x, _1.endx, _1.y) if _1.x >= 48 && _1.text.strip.present? } + end + + def parse + date_label = label("Sample Date") or raise Error, "No sample dates found in the Blackstone report" + dates = row(date_label).select { DATE.match?(_1.text) }.sort_by(&:x) + raise Error, "No sample dates found in the Blackstone report" if dates.empty? + @columns = dates.map(&:endx) + @universal_x = (@runs.find { _1.text == "UNIVERSAL" }&.x || 530) - 10 + + oil_hours, engine_hours, oil_added = [ "MI/HR on Oil", "MI/HR on Unit", "Make Up Oil Added" ].map { values(label(_1)) } + elements, universal = ELEMENTS.each_with_object([ {}, {} ]) do |el, (by_el, avg)| + by_el[el] = values(label(el.upcase)) + avg[el] = universal_value(label(el.upcase)) + end + properties = PROPERTIES.to_h { |pattern, key| [ key, values(label(pattern)) ] } + universal = universal.transform_values { OilReportParser.number(_1) }.compact + lab_number = after("LAB NUMBER:") + + samples = dates.each_with_index.map do |date, i| + { + current: i.zero?, + sample_date: OilReportParser.date(date.text), + sample_number: (lab_number if i.zero?), + oil_hours: OilReportParser.number(oil_hours[i]), + engine_hours: OilReportParser.number(engine_hours[i]), + oil_added: OilReportParser.number(oil_added[i]), + status: nil, # Blackstone gives comments, not a rating + lab_comments: (comments if i.zero?), + elements: elements.transform_values { OilReportParser.number(_1[i]) }.compact, + element_averages: universal, + oil_properties: properties.transform_values { OilReportParser.number(_1[i]) }.compact + } + end.select { _1[:sample_date] } + + unit = after("UNIT ID:") + Report.new(lab: LAB, tail_number: (unit if TAIL.match?(unit.to_s)), engine_serial: nil, + engine_model: after("MAKE/MODEL:"), oil_type: after("OIL TYPE & GRADE:"), samples: samples) + end + + private + + # The left-margin run that labels a row. + def label(match) + @runs.find { _1.x < LABEL_MAX_X && (match.is_a?(Regexp) ? match.match?(_1.text) : _1.text == match) } + end + + # Runs on the same row as a label, to its right. + def row(label_run) + return [] unless label_run + @runs.select { (_1.y - label_run.y).abs <= ROW_SLOP && _1.x > label_run.endx } + end + + # The row's value text for each sample column (nil where blank). + def values(label_run) + found = Array.new(@columns.size) + row(label_run).each do |run| + next if run.x >= @universal_x + i = @columns.each_index.min_by { (@columns[_1] - run.endx).abs } + found[i] ||= run.text if (@columns[i] - run.endx).abs <= COLUMN_SLOP + end + found + end + + def universal_value(label_run) = row(label_run).find { _1.x >= @universal_x }&.text + + # The value printed after a header label, e.g. "UNIT ID:" → "N12345". + def after(text) + label_run = @runs.find { _1.text == text } or return nil + @runs.select { (_1.y - label_run.y).abs <= 2 && _1.x > label_run.endx }.min_by(&:x)&.text + end + + # The comment paragraph between the contact block and the results grid. + # It starts with who it's addressed to ("OWNER:"), which is dropped. + def comments + contact = @runs.select { _1.text.match?(/\A(PHONE|FAX|ALT PHONE|EMAIL):/) } + top = contact.map(&:y).min or return nil + grid = label("MI/HR on Oil")&.y || 0 + lines = @runs.select { _1.y < top - 2 && _1.y > grid + 6 }.sort_by { [ -_1.y, _1.x ] } + lines.map(&:text).join(" ").sub(/\A[A-Z]+:\s*/, "").squeeze(" ").presence + end + end +end diff --git a/app/views/aircraft/index.html.erb b/app/views/aircraft/index.html.erb index f911652..2569111 100644 --- a/app/views/aircraft/index.html.erb +++ b/app/views/aircraft/index.html.erb @@ -1,7 +1,10 @@ <% content_for :title, "Fleet" %>
+ Upload a .sam.zip archive exported from Simple Aircraft Manager (this app or the original). + Everything in it is added as a new aircraft that you own: components, logbook, squawks, inspections, ADs, + documents, oil and fuel, oil analyses, flights and their files. +
+ <% if @preview %> +Choose the file again along with the new tail number.
<% end %> +Leave blank to use the archive's. Needed when that tail number is already here.
+<%= @import.filename %><%= " · from #{@import.source}" if @import.source.present? %>
+ <% if @import.in_progress? %> ++ <%= @import.status == "queued" ? "Waiting to start…" : "Importing…" %>
+ <% if @import.counts.any? %>So far: <%= @import.summary %>.
<% end %> +This page updates by itself. Large archives with many documents can take a few minutes.
+ <% elsif @import.done? %> +Imported.
+<%= @import.summary.presence&.upcase_first || "No records" %>.
+ <% if @import.aircraft %><%= link_to "Open #{@import.aircraft.tail_number} →", aircraft_path(@import.aircraft), class: "btn-primary" %>
<% end %> + <% else %> +The import failed. Nothing was added.
+<%= @import.error %>
+<%= link_to "Try another file", new_aircraft_import_path, class: "btn-secondary" %>
+ <% end %> + <% if @import.warnings.any? %> +<%= pluralize(@import.warnings.size, "note") %>
+Everything recorded for <%= @aircraft.tail_number %>, with its files, in one .sam.zip archive. Import it from the Fleet page here or into another Simple Aircraft Manager (including the original).
+
+ The pages are saved as a document, in file-name order (name them page-001, page-002…).
+ <% if ai %>Claude can also read them and add each entry to the logbook for you.<% else %>
+ To have entries transcribed automatically, set an ANTHROPIC_API_KEY for the app.<% end %>
+
JPEG, PNG, WebP or PDF. For transcription, images up to 5 MB each and PDFs up to 30 MB.
+<%= pluralize(@import.entries_created, "entry") %> so far. This page updates by itself; a long logbook can take several minutes.
++ <%= @import.transcribe ? "Done: #{pluralize(@import.entries_created, 'logbook entry')} added." : "Pages saved." %> +
+ <% if @import.transcribe && @import.entries_created.positive? %> +Check the new entries against the scans: handwriting isn't always read perfectly.
+ <% end %> + <% else %> +The import stopped: <%= @import.error %>
+ <% if @import.entries_created.positive? %><%= pluralize(@import.entries_created, "entry") %> were added before it stopped.
<% end %> + <% end %> + +<%= pluralize(@import.warnings.size, "note") %>
+