Maintenance commands: doctor, records check, restore, sign out all, more

- sam:doctor: health report (secret key, address, HTTPS, SMTP and SSO
  reachable, Google Drive, migrations, disk, jobs, backup/check/off-site/
  procedures/audit recency); exits 1 on a failure
- records:check: backup now + history and signature check; exits 1 on a problem
- backups:list / backups:restore[file]: checks the copy (nightly or Drive
  .gz), refuses while the server runs, moves the current DB aside, asks
  (CONFIRM=yes to skip)
- users:sign_out_all, sam:records_package[tail], email:test[address],
  offsite:backup, offsite:restore_test
- Admin guide: maintenance commands table, restore steps use backups:restore

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rickyandClaude Opus 5.5 committed 2026-11-07 16:42:52 -05:00
1 parent 20fe485f10
commit 1412470cd5
11 files changed
+434 -11

No files matched your search

+4 -4
View File
@@ -7,14 +7,14 @@ class OffsiteBackupJob < ApplicationJob
def perform(kind = "backup")
return unless OffsiteConnection.connected?
run = kind == "restore_test" ? OffsiteBackup.restore_test! : backup
run = kind == "restore_test" ? OffsiteBackup.restore_test! : self.class.backup
return if run.nil? || run.ok || !MailSettings.configured?
User.active.where(admin: true).find_each { AdminMailer.offsite_alert(_1, run).deliver_now }
end
private
def backup
# Copies the latest verified backup (making one if there's none from the
# last day). Returns the OffsiteRun. Also used by bin/rails offsite:backup.
def self.backup
latest = BackupRun.where(ok: true).order(:created_at, :id).last
latest = DatabaseBackup.run! unless latest && latest.created_at > 1.day.ago && latest.path.present? && File.exist?(latest.path)
return OffsiteRun.create!(kind: "backup", ok: false, message: "No verified database backup to copy: #{latest.message}") unless latest.ok
+2 -1
View File
@@ -10,8 +10,9 @@ class AdminMailer < ApplicationMailer
mail subject: run.kind == "restore_test" ? "Off-site restore test failed" : "Off-site backup failed", to: user.email_address
end
def test_email(user)
def test_email(user, via: "from Admin → About")
@user = user
@via = via
mail subject: "Test email from Simple Aircraft Manager", to: user.email_address
end
end
+8
View File
@@ -65,6 +65,14 @@ class AccountRecovery
[ user, password ]
end
# Ends every session of every user (suspected compromise): everyone signs
# in again, through the lockout and two-factor checks.
def self.sign_out_all!
count = Session.delete_all
Rails.logger.warn("[account recovery] all #{count} sessions ended from the command line")
count
end
def self.generate_password = SecureRandom.alphanumeric(20).scan(/.{5}/).join("-")
def self.log(user, what) = Rails.logger.warn("[account recovery] #{user.email_address}: #{what} from the command line")
+79
View File
@@ -0,0 +1,79 @@
require "zlib"
# Puts a database backup in place of the live database (bin/rails
# backups:restore), with the app stopped: a nightly copy from storage/backups,
# or a copy downloaded from Google Drive (.sqlite3.gz). The copy is checked
# first, and the current database is moved aside, never deleted.
class DatabaseRestore
class Error < StandardError; end
attr_reader :source, :target
def self.newest_backup = Dir[DatabaseBackup::DIR.join("sam-*.sqlite3")].max_by { File.mtime(_1) }
def self.live_database = File.expand_path(ActiveRecord::Base.connection_db_config.database.to_s, Rails.root)
def initialize(source: nil, target: self.class.live_database, pidfile: Rails.root.join("tmp/pids/server.pid"))
@pidfile = Pathname(pidfile)
@source = source.presence || self.class.newest_backup or raise Error, "No backup found in #{DatabaseBackup::DIR}; give a file: backups:restore[path]"
@target = File.expand_path(target.to_s, Rails.root)
raise Error, "#{@source} doesn't exist" unless File.file?(@source)
end
# Opens the copy (unzipping a .gz to a temporary file) and checks it.
# Returns a description; raises if it isn't a usable database.
def check
with_plain_copy { |path| describe(path) }
end
# Swaps the copy in. Returns the folder the previous database was moved to.
def restore!
raise Error, "The app seems to be running (#{running_reason}). Stop it first; with Docker: docker compose stop web, then docker compose run --rm web bin/rails backups:restore" if running_reason
with_plain_copy do |path|
describe(path)
ActiveRecord::Base.connection_handler.clear_all_connections! if @target == self.class.live_database
aside = File.join(File.dirname(@target), "pre-restore-#{Time.current.utc.strftime('%Y%m%d-%H%M%S')}")
FileUtils.mkdir_p(aside)
[ "", "-wal", "-shm" ].each { |ext| FileUtils.mv("#{@target}#{ext}", aside) if File.exist?("#{@target}#{ext}") }
FileUtils.cp(path, @target)
Rails.logger.warn("[restore] #{@target} replaced by #{@source}; previous database in #{aside}")
aside
end
end
private
def with_plain_copy
return yield(@source) unless @source.end_with?(".gz")
Dir.mktmpdir("sam-restore") do |dir|
path = File.join(dir, "restore.sqlite3")
Zlib::GzipReader.open(@source) { |gz| File.open(path, "wb") { IO.copy_stream(gz, _1) } }
yield path
end
rescue Zlib::GzipFile::Error => e
raise Error, "#{@source} isn't a valid .gz file (#{e.message})"
end
def describe(path)
db = SQLite3::Database.new(path, readonly: true)
result = db.get_first_value("PRAGMA integrity_check")
raise Error, "integrity check failed: #{result}" unless result == "ok"
counts = %w[aircraft logbook_entries entry_signatures users].to_h { [ _1, db.get_first_value("SELECT COUNT(*) FROM #{_1}") ] }
version = db.get_first_value("SELECT MAX(version) FROM schema_migrations")
"#{File.basename(@source)}: SQLite integrity check ok; #{counts['aircraft']} aircraft, #{counts['logbook_entries']} logbook entries, " \
"#{counts['entry_signatures']} signatures, #{counts['users']} users; schema #{version}"
rescue SQLite3::Exception => e
raise Error, "#{File.basename(@source)} isn't a usable database (#{e.message})"
ensure
db&.close
end
# A live server process (Puma's pid file) means the app is running.
def running_reason
return unless @pidfile.exist?
pid = @pidfile.read.to_i
Process.kill(0, pid) && "server process #{pid}" if pid.positive?
rescue Errno::ESRCH, Errno::EPERM
nil
end
end
+117
View File
@@ -0,0 +1,117 @@
require "net/http"
require "socket"
require "open3"
# A health report for the installation (bin/rails sam:doctor): settings,
# connections to email, single sign-on and Google Drive, migrations, disk
# space, background jobs, and how recent the backups, checks and audit are.
class SystemDoctor
Check = Data.define(:status, :name, :detail) # status: :ok, :warn, :fail
def checks
[ secret_key, host, https, email, sso, password_login, migrations, disk, jobs, backup, integrity, offsite, procedures, audit ].flatten.compact
end
def self.report
doctor = new
list = doctor.checks
lines = list.map { |c| "#{{ ok: ' ok ', warn: ' WARN', fail: ' FAIL' }[c.status]} #{c.name.ljust(22)} #{c.detail}" }
[ "Simple Aircraft Manager #{SystemInfo.new.revision} (#{Rails.env})", *lines ].join("\n").then { [ _1, list ] }
end
private
def ok(name, detail) = Check.new(:ok, name, detail)
def warning(name, detail) = Check.new(:warn, name, detail)
def failed(name, detail) = Check.new(:fail, name, detail)
def production? = Rails.env.production?
def secret_key
return ok("Secret key", "development/test key") unless production?
ENV["SECRET_KEY_BASE"].present? || ENV["RAILS_MASTER_KEY"].present? ? ok("Secret key", "set (keep a copy off the server)") : warning("Secret key", "SECRET_KEY_BASE isn't set in the environment")
end
def host
url = Sso.app_url
production? && ENV["APP_HOST"].blank? && ENV["APP_URL"].blank? ? warning("Address", "APP_HOST/APP_URL not set: email links point at #{url}") : ok("Address", url)
end
def https = Rails.application.config.force_ssl ? ok("HTTPS", "enforced (FORCE_SSL)") : warning("HTTPS", "not enforced by the app (FORCE_SSL=false): only behind a TLS proxy or on a private network")
def email
return warning("Email", "not configured (set SMTP_ADDRESS): no password resets, sign-in links or alerts") unless MailSettings.configured?
s = ActionMailer::Base.smtp_settings
return ok("Email", "#{ActionMailer::Base.delivery_method} delivery") unless ActionMailer::Base.delivery_method == :smtp
Socket.tcp(s[:address], s[:port], connect_timeout: 5).close
ok("Email", "SMTP #{s[:address]}:#{s[:port]} reachable (send one with bin/rails \"email:test[you@example.com]\")")
rescue StandardError => e
failed("Email", "can't reach SMTP #{s[:address]}:#{s[:port]} (#{e.class.name.demodulize})")
end
def sso
return unless Sso.enabled?
uri = URI("#{ENV['OIDC_ISSUER'].to_s.chomp('/')}/.well-known/openid-configuration")
res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 5, read_timeout: 5) { _1.get(uri.request_uri) }
res.is_a?(Net::HTTPSuccess) ? ok("Single sign-on", "#{Sso.label} issuer reachable") : failed("Single sign-on", "#{uri} answered #{res.code}")
rescue StandardError => e
failed("Single sign-on", "can't reach #{Sso.label} (#{e.class.name.demodulize}: #{e.message.truncate(80)})")
end
def password_login
return ok("Password sign-in", "on") if Sso.password_login?
ok("Password sign-in", "off (single sign-on only; invited mechanics can't create accounts)")
end
def migrations
pending = ActiveRecord::Base.connection_pool.migration_context.needs_migration?
pending ? failed("Database", "migrations pending: run bin/rails db:migrate (or restart the container)") : ok("Database", "schema up to date")
end
def disk
out = Open3.capture2("df", "-Pk", Rails.root.join("storage").to_s).first.lines.last.to_s.split
return warning("Disk space", "couldn't check") if out.size < 5
free_gb = out[3].to_f / 1024 / 1024
used = out[4].to_i
detail = "#{free_gb.round(1)} GB free on the storage volume (#{used}% used)"
free_gb < 1 || used > 95 ? failed("Disk space", detail) : (free_gb < 5 || used > 85 ? warning("Disk space", detail) : ok("Disk space", detail))
end
def jobs
return unless defined?(SolidQueue::Process)
count = SolidQueue::Process.where(last_heartbeat_at: 5.minutes.ago..).count
count.positive? ? ok("Background jobs", "#{count} running (nightly backup, reminders, AD search)") : warning("Background jobs", "none seen in the last 5 minutes: is the app running with SOLID_QUEUE_IN_PUMA?")
rescue StandardError => e
warning("Background jobs", "couldn't check (#{e.class.name.demodulize})")
end
def backup = age_check("Last backup", BackupRun.order(:created_at).last) { "#{_1.ok ? 'ok' : 'FAILED'}: #{_1.message.to_s.truncate(70)}" }
def integrity = age_check("Records check", IntegrityCheck.order(:created_at).last) { _1.ok ? "no problems" : "#{Array(_1.problems).size} problem(s)" }
def offsite
return warning("Off-site backup", "Google Drive not connected (Admin → Off-site Backup), or describe another off-site copy in the procedures") unless OffsiteConnection.connected?
OffsiteConnection.current.client.account_email
run = OffsiteRun.latest_backup
run ? age_check("Off-site backup", run) { "#{_1.ok ? 'ok' : 'FAILED'}: Google Drive #{OffsiteConnection.current.account_email}" } : warning("Off-site backup", "Google Drive connected; no copy yet")
rescue GoogleDriveClient::Error => e
failed("Off-site backup", "Google Drive: #{e.message.truncate(90)}")
end
def procedures
r = ProceduresRevision.current
r ? ok("Procedures", "revision #{r.number}, adopted #{r.adopted_at.to_date}") : warning("Procedures", "not adopted (Admin → Recordkeeping Procedures)")
end
def audit
return warning("Records audit", "never done (Admin → Records Audits)") unless AuditRun.latest
due = AuditRun.next_due
detail = "next due #{due} (last #{AuditRun.latest.performed_at.to_date})"
due < Date.current ? warning("Records audit", "overdue: #{detail}") : ok("Records audit", detail)
end
def age_check(name, record)
return warning(name, "never run") unless record
detail = "#{record.created_at.to_date}, #{yield(record)}"
return failed(name, detail) unless record.ok
record.created_at < 2.days.ago ? warning(name, "#{detail} (more than 2 days ago)") : ok(name, detail)
end
end
+1 -1
View File
@@ -1,2 +1,2 @@
<p>This is a test email from Simple Aircraft Manager, sent by <%= @user.email_address %> from Admin → About.</p>
<p>This is a test email from Simple Aircraft Manager, sent by <%= @user.email_address %> <%= @via %>.</p>
<p>Email is working: password resets and sign-in links will arrive.</p>
+1 -1
View File
@@ -1,3 +1,3 @@
This is a test email from Simple Aircraft Manager, sent by <%= @user.email_address %> from Admin → About.
This is a test email from Simple Aircraft Manager, sent by <%= @user.email_address %> <%= @via %>.
Email is working: password resets and sign-in links will arrive.