mirror of
https://github.com/rickbarrette/simple-aircraft-manager-rails.git
synced 2026-11-17 23:50:36 -05:00
Maintenance commands: doctor, records check, restore, sign out all, more
- sam:doctor: health report (secret key, address, HTTPS, SMTP and SSO reachable, Google Drive, migrations, disk, jobs, backup/check/off-site/ procedures/audit recency); exits 1 on a failure - records:check: backup now + history and signature check; exits 1 on a problem - backups:list / backups:restore[file]: checks the copy (nightly or Drive .gz), refuses while the server runs, moves the current DB aside, asks (CONFIRM=yes to skip) - users:sign_out_all, sam:records_package[tail], email:test[address], offsite:backup, offsite:restore_test - Admin guide: maintenance commands table, restore steps use backups:restore Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
20fe485f10
commit
1412470cd5
11 files changed
+434
-11
No files matched your search
@@ -101,6 +101,8 @@ rsync -aH --exclude '*.sqlite3*' storage/ backup/ # uploaded files (-H keeps th
|
||||
|
||||
Alternatively, stop the container and copy the volume.
|
||||
|
||||
To restore, stop the app and run `bin/rails backups:restore` (the newest nightly copy, or a file such as one downloaded from Google Drive); it checks the copy first and keeps the current database aside. Then `bin/rails records:check` verifies the history and signatures. `bin/rails sam:doctor` gives a health report of the whole installation. All commands are in the [administration guide](docs/guide/administration.md#maintenance-commands).
|
||||
|
||||
The app also makes a verified copy of the main database every night in `storage/backups` (Admin → About shows the last one), so a backup of the storage volume always contains a consistent database snapshot.
|
||||
|
||||
For an off-site copy without extra tools, **Admin → Off-site Backup** connects a Google Drive account: each night the verified database copy and any new uploaded files are copied there, and a restore test checks them. Setup and restoring are in the [administration guide](docs/guide/administration.md#off-site-backup-to-google-drive). Keep `SECRET_KEY_BASE` somewhere safe off the server: stored secrets (API keys, the Drive connection) and electronic signature seals depend on it.
|
||||
|
||||
@@ -7,14 +7,14 @@ class OffsiteBackupJob < ApplicationJob
|
||||
|
||||
def perform(kind = "backup")
|
||||
return unless OffsiteConnection.connected?
|
||||
run = kind == "restore_test" ? OffsiteBackup.restore_test! : backup
|
||||
run = kind == "restore_test" ? OffsiteBackup.restore_test! : self.class.backup
|
||||
return if run.nil? || run.ok || !MailSettings.configured?
|
||||
User.active.where(admin: true).find_each { AdminMailer.offsite_alert(_1, run).deliver_now }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def backup
|
||||
# Copies the latest verified backup (making one if there's none from the
|
||||
# last day). Returns the OffsiteRun. Also used by bin/rails offsite:backup.
|
||||
def self.backup
|
||||
latest = BackupRun.where(ok: true).order(:created_at, :id).last
|
||||
latest = DatabaseBackup.run! unless latest && latest.created_at > 1.day.ago && latest.path.present? && File.exist?(latest.path)
|
||||
return OffsiteRun.create!(kind: "backup", ok: false, message: "No verified database backup to copy: #{latest.message}") unless latest.ok
|
||||
|
||||
@@ -10,8 +10,9 @@ class AdminMailer < ApplicationMailer
|
||||
mail subject: run.kind == "restore_test" ? "Off-site restore test failed" : "Off-site backup failed", to: user.email_address
|
||||
end
|
||||
|
||||
def test_email(user)
|
||||
def test_email(user, via: "from Admin → About")
|
||||
@user = user
|
||||
@via = via
|
||||
mail subject: "Test email from Simple Aircraft Manager", to: user.email_address
|
||||
end
|
||||
end
|
||||
@@ -65,6 +65,14 @@ class AccountRecovery
|
||||
[ user, password ]
|
||||
end
|
||||
|
||||
# Ends every session of every user (suspected compromise): everyone signs
|
||||
# in again, through the lockout and two-factor checks.
|
||||
def self.sign_out_all!
|
||||
count = Session.delete_all
|
||||
Rails.logger.warn("[account recovery] all #{count} sessions ended from the command line")
|
||||
count
|
||||
end
|
||||
|
||||
def self.generate_password = SecureRandom.alphanumeric(20).scan(/.{5}/).join("-")
|
||||
|
||||
def self.log(user, what) = Rails.logger.warn("[account recovery] #{user.email_address}: #{what} from the command line")
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
require "zlib"
|
||||
|
||||
# Puts a database backup in place of the live database (bin/rails
|
||||
# backups:restore), with the app stopped: a nightly copy from storage/backups,
|
||||
# or a copy downloaded from Google Drive (.sqlite3.gz). The copy is checked
|
||||
# first, and the current database is moved aside, never deleted.
|
||||
class DatabaseRestore
|
||||
class Error < StandardError; end
|
||||
|
||||
attr_reader :source, :target
|
||||
|
||||
def self.newest_backup = Dir[DatabaseBackup::DIR.join("sam-*.sqlite3")].max_by { File.mtime(_1) }
|
||||
|
||||
def self.live_database = File.expand_path(ActiveRecord::Base.connection_db_config.database.to_s, Rails.root)
|
||||
|
||||
def initialize(source: nil, target: self.class.live_database, pidfile: Rails.root.join("tmp/pids/server.pid"))
|
||||
@pidfile = Pathname(pidfile)
|
||||
@source = source.presence || self.class.newest_backup or raise Error, "No backup found in #{DatabaseBackup::DIR}; give a file: backups:restore[path]"
|
||||
@target = File.expand_path(target.to_s, Rails.root)
|
||||
raise Error, "#{@source} doesn't exist" unless File.file?(@source)
|
||||
end
|
||||
|
||||
# Opens the copy (unzipping a .gz to a temporary file) and checks it.
|
||||
# Returns a description; raises if it isn't a usable database.
|
||||
def check
|
||||
with_plain_copy { |path| describe(path) }
|
||||
end
|
||||
|
||||
# Swaps the copy in. Returns the folder the previous database was moved to.
|
||||
def restore!
|
||||
raise Error, "The app seems to be running (#{running_reason}). Stop it first; with Docker: docker compose stop web, then docker compose run --rm web bin/rails backups:restore" if running_reason
|
||||
with_plain_copy do |path|
|
||||
describe(path)
|
||||
ActiveRecord::Base.connection_handler.clear_all_connections! if @target == self.class.live_database
|
||||
aside = File.join(File.dirname(@target), "pre-restore-#{Time.current.utc.strftime('%Y%m%d-%H%M%S')}")
|
||||
FileUtils.mkdir_p(aside)
|
||||
[ "", "-wal", "-shm" ].each { |ext| FileUtils.mv("#{@target}#{ext}", aside) if File.exist?("#{@target}#{ext}") }
|
||||
FileUtils.cp(path, @target)
|
||||
Rails.logger.warn("[restore] #{@target} replaced by #{@source}; previous database in #{aside}")
|
||||
aside
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def with_plain_copy
|
||||
return yield(@source) unless @source.end_with?(".gz")
|
||||
Dir.mktmpdir("sam-restore") do |dir|
|
||||
path = File.join(dir, "restore.sqlite3")
|
||||
Zlib::GzipReader.open(@source) { |gz| File.open(path, "wb") { IO.copy_stream(gz, _1) } }
|
||||
yield path
|
||||
end
|
||||
rescue Zlib::GzipFile::Error => e
|
||||
raise Error, "#{@source} isn't a valid .gz file (#{e.message})"
|
||||
end
|
||||
|
||||
def describe(path)
|
||||
db = SQLite3::Database.new(path, readonly: true)
|
||||
result = db.get_first_value("PRAGMA integrity_check")
|
||||
raise Error, "integrity check failed: #{result}" unless result == "ok"
|
||||
counts = %w[aircraft logbook_entries entry_signatures users].to_h { [ _1, db.get_first_value("SELECT COUNT(*) FROM #{_1}") ] }
|
||||
version = db.get_first_value("SELECT MAX(version) FROM schema_migrations")
|
||||
"#{File.basename(@source)}: SQLite integrity check ok; #{counts['aircraft']} aircraft, #{counts['logbook_entries']} logbook entries, " \
|
||||
"#{counts['entry_signatures']} signatures, #{counts['users']} users; schema #{version}"
|
||||
rescue SQLite3::Exception => e
|
||||
raise Error, "#{File.basename(@source)} isn't a usable database (#{e.message})"
|
||||
ensure
|
||||
db&.close
|
||||
end
|
||||
|
||||
# A live server process (Puma's pid file) means the app is running.
|
||||
def running_reason
|
||||
return unless @pidfile.exist?
|
||||
pid = @pidfile.read.to_i
|
||||
Process.kill(0, pid) && "server process #{pid}" if pid.positive?
|
||||
rescue Errno::ESRCH, Errno::EPERM
|
||||
nil
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,117 @@
|
||||
require "net/http"
|
||||
require "socket"
|
||||
require "open3"
|
||||
|
||||
# A health report for the installation (bin/rails sam:doctor): settings,
|
||||
# connections to email, single sign-on and Google Drive, migrations, disk
|
||||
# space, background jobs, and how recent the backups, checks and audit are.
|
||||
class SystemDoctor
|
||||
Check = Data.define(:status, :name, :detail) # status: :ok, :warn, :fail
|
||||
|
||||
def checks
|
||||
[ secret_key, host, https, email, sso, password_login, migrations, disk, jobs, backup, integrity, offsite, procedures, audit ].flatten.compact
|
||||
end
|
||||
|
||||
def self.report
|
||||
doctor = new
|
||||
list = doctor.checks
|
||||
lines = list.map { |c| "#{{ ok: ' ok ', warn: ' WARN', fail: ' FAIL' }[c.status]} #{c.name.ljust(22)} #{c.detail}" }
|
||||
[ "Simple Aircraft Manager #{SystemInfo.new.revision} (#{Rails.env})", *lines ].join("\n").then { [ _1, list ] }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def ok(name, detail) = Check.new(:ok, name, detail)
|
||||
def warning(name, detail) = Check.new(:warn, name, detail)
|
||||
def failed(name, detail) = Check.new(:fail, name, detail)
|
||||
def production? = Rails.env.production?
|
||||
|
||||
def secret_key
|
||||
return ok("Secret key", "development/test key") unless production?
|
||||
ENV["SECRET_KEY_BASE"].present? || ENV["RAILS_MASTER_KEY"].present? ? ok("Secret key", "set (keep a copy off the server)") : warning("Secret key", "SECRET_KEY_BASE isn't set in the environment")
|
||||
end
|
||||
|
||||
def host
|
||||
url = Sso.app_url
|
||||
production? && ENV["APP_HOST"].blank? && ENV["APP_URL"].blank? ? warning("Address", "APP_HOST/APP_URL not set: email links point at #{url}") : ok("Address", url)
|
||||
end
|
||||
|
||||
def https = Rails.application.config.force_ssl ? ok("HTTPS", "enforced (FORCE_SSL)") : warning("HTTPS", "not enforced by the app (FORCE_SSL=false): only behind a TLS proxy or on a private network")
|
||||
|
||||
def email
|
||||
return warning("Email", "not configured (set SMTP_ADDRESS): no password resets, sign-in links or alerts") unless MailSettings.configured?
|
||||
s = ActionMailer::Base.smtp_settings
|
||||
return ok("Email", "#{ActionMailer::Base.delivery_method} delivery") unless ActionMailer::Base.delivery_method == :smtp
|
||||
Socket.tcp(s[:address], s[:port], connect_timeout: 5).close
|
||||
ok("Email", "SMTP #{s[:address]}:#{s[:port]} reachable (send one with bin/rails \"email:test[you@example.com]\")")
|
||||
rescue StandardError => e
|
||||
failed("Email", "can't reach SMTP #{s[:address]}:#{s[:port]} (#{e.class.name.demodulize})")
|
||||
end
|
||||
|
||||
def sso
|
||||
return unless Sso.enabled?
|
||||
uri = URI("#{ENV['OIDC_ISSUER'].to_s.chomp('/')}/.well-known/openid-configuration")
|
||||
res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 5, read_timeout: 5) { _1.get(uri.request_uri) }
|
||||
res.is_a?(Net::HTTPSuccess) ? ok("Single sign-on", "#{Sso.label} issuer reachable") : failed("Single sign-on", "#{uri} answered #{res.code}")
|
||||
rescue StandardError => e
|
||||
failed("Single sign-on", "can't reach #{Sso.label} (#{e.class.name.demodulize}: #{e.message.truncate(80)})")
|
||||
end
|
||||
|
||||
def password_login
|
||||
return ok("Password sign-in", "on") if Sso.password_login?
|
||||
ok("Password sign-in", "off (single sign-on only; invited mechanics can't create accounts)")
|
||||
end
|
||||
|
||||
def migrations
|
||||
pending = ActiveRecord::Base.connection_pool.migration_context.needs_migration?
|
||||
pending ? failed("Database", "migrations pending: run bin/rails db:migrate (or restart the container)") : ok("Database", "schema up to date")
|
||||
end
|
||||
|
||||
def disk
|
||||
out = Open3.capture2("df", "-Pk", Rails.root.join("storage").to_s).first.lines.last.to_s.split
|
||||
return warning("Disk space", "couldn't check") if out.size < 5
|
||||
free_gb = out[3].to_f / 1024 / 1024
|
||||
used = out[4].to_i
|
||||
detail = "#{free_gb.round(1)} GB free on the storage volume (#{used}% used)"
|
||||
free_gb < 1 || used > 95 ? failed("Disk space", detail) : (free_gb < 5 || used > 85 ? warning("Disk space", detail) : ok("Disk space", detail))
|
||||
end
|
||||
|
||||
def jobs
|
||||
return unless defined?(SolidQueue::Process)
|
||||
count = SolidQueue::Process.where(last_heartbeat_at: 5.minutes.ago..).count
|
||||
count.positive? ? ok("Background jobs", "#{count} running (nightly backup, reminders, AD search)") : warning("Background jobs", "none seen in the last 5 minutes: is the app running with SOLID_QUEUE_IN_PUMA?")
|
||||
rescue StandardError => e
|
||||
warning("Background jobs", "couldn't check (#{e.class.name.demodulize})")
|
||||
end
|
||||
|
||||
def backup = age_check("Last backup", BackupRun.order(:created_at).last) { "#{_1.ok ? 'ok' : 'FAILED'}: #{_1.message.to_s.truncate(70)}" }
|
||||
def integrity = age_check("Records check", IntegrityCheck.order(:created_at).last) { _1.ok ? "no problems" : "#{Array(_1.problems).size} problem(s)" }
|
||||
|
||||
def offsite
|
||||
return warning("Off-site backup", "Google Drive not connected (Admin → Off-site Backup), or describe another off-site copy in the procedures") unless OffsiteConnection.connected?
|
||||
OffsiteConnection.current.client.account_email
|
||||
run = OffsiteRun.latest_backup
|
||||
run ? age_check("Off-site backup", run) { "#{_1.ok ? 'ok' : 'FAILED'}: Google Drive #{OffsiteConnection.current.account_email}" } : warning("Off-site backup", "Google Drive connected; no copy yet")
|
||||
rescue GoogleDriveClient::Error => e
|
||||
failed("Off-site backup", "Google Drive: #{e.message.truncate(90)}")
|
||||
end
|
||||
|
||||
def procedures
|
||||
r = ProceduresRevision.current
|
||||
r ? ok("Procedures", "revision #{r.number}, adopted #{r.adopted_at.to_date}") : warning("Procedures", "not adopted (Admin → Recordkeeping Procedures)")
|
||||
end
|
||||
|
||||
def audit
|
||||
return warning("Records audit", "never done (Admin → Records Audits)") unless AuditRun.latest
|
||||
due = AuditRun.next_due
|
||||
detail = "next due #{due} (last #{AuditRun.latest.performed_at.to_date})"
|
||||
due < Date.current ? warning("Records audit", "overdue: #{detail}") : ok("Records audit", detail)
|
||||
end
|
||||
|
||||
def age_check(name, record)
|
||||
return warning(name, "never run") unless record
|
||||
detail = "#{record.created_at.to_date}, #{yield(record)}"
|
||||
return failed(name, detail) unless record.ok
|
||||
record.created_at < 2.days.ago ? warning(name, "#{detail} (more than 2 days ago)") : ok(name, detail)
|
||||
end
|
||||
end
|
||||
@@ -1,2 +1,2 @@
|
||||
<p>This is a test email from Simple Aircraft Manager, sent by <%= @user.email_address %> from Admin → About.</p>
|
||||
<p>This is a test email from Simple Aircraft Manager, sent by <%= @user.email_address %> <%= @via %>.</p>
|
||||
<p>Email is working: password resets and sign-in links will arrive.</p>
|
||||
@@ -1,3 +1,3 @@
|
||||
This is a test email from Simple Aircraft Manager, sent by <%= @user.email_address %> from Admin → About.
|
||||
This is a test email from Simple Aircraft Manager, sent by <%= @user.email_address %> <%= @via %>.
|
||||
|
||||
Email is working: password resets and sign-in links will arrive.
|
||||
@@ -88,10 +88,28 @@ The first backup starts straight away. It copies every uploaded file, so it can
|
||||
If the server is lost:
|
||||
|
||||
1. Set up the app again (see the main README) with the **same `SECRET_KEY_BASE`**, but don't start it yet.
|
||||
2. From Google Drive, download the newest file in **Simple Aircraft Manager backups → database**, unzip it (`gunzip sam-….sqlite3.gz`), and put it in place of `storage/production.sqlite3` (delete any `production.sqlite3-wal` and `-shm` files next to it).
|
||||
3. Start the app and sign in. The records are back, but uploaded files are still missing.
|
||||
4. Restore the files: `bin/rails offsite:restore_files` downloads them from Drive (the connection is in the restored database). If that doesn't work (for example the `SECRET_KEY_BASE` is lost), download the **files** folder from Drive, unzip it on the server and run `bin/rails "offsite:restore_files[/path/to/files]"`.
|
||||
5. Check the About page (run **Back up and check now**) and open a few entries and documents.
|
||||
2. From Google Drive, download the newest file in **Simple Aircraft Manager backups → database** (`sam-….sqlite3.gz`) and copy it onto the server.
|
||||
3. Put it in place: `bin/rails "backups:restore[/path/to/sam-….sqlite3.gz]"`. With Docker, run it in a one-off container while the app is stopped: `docker compose run --rm web bin/rails "backups:restore[/rails/storage/sam-….sqlite3.gz]"` (copy the file into the storage volume first). It checks the copy, keeps any existing database aside, and asks before replacing.
|
||||
4. Start the app and sign in. The records are back, but uploaded files are still missing.
|
||||
5. Restore the files: `bin/rails offsite:restore_files` downloads them from Drive (the connection is in the restored database). If that doesn't work (for example the `SECRET_KEY_BASE` is lost), download the **files** folder from Drive, unzip it on the server and run `bin/rails "offsite:restore_files[/path/to/files]"`.
|
||||
6. Check everything: `bin/rails records:check` (history and signatures intact) and `bin/rails sam:doctor`, then open a few entries and documents.
|
||||
|
||||
The same `backups:restore` (with no file) puts back the newest nightly copy from `storage/backups`, for example after a mistake that can't be undone in the app; `bin/rails backups:list` shows the copies.
|
||||
|
||||
## Maintenance commands
|
||||
|
||||
Run on the server like the account recovery commands (`docker compose exec web` in front, or the container's Console in Portainer).
|
||||
|
||||
| Command | What it does |
|
||||
|---|---|
|
||||
| `bin/rails sam:doctor` | Health report: secret key, address, HTTPS, email (SMTP reachable), single sign-on (issuer reachable), Google Drive, migrations, disk space, background jobs, and how recent the backup, records check, off-site copy, procedures and audit are. Exits non-zero if something fails |
|
||||
| `bin/rails records:check` | Backs up the database now and checks the record history and every signature; prints any problems and exits non-zero |
|
||||
| `bin/rails backups:list` | Lists the nightly database copies |
|
||||
| `bin/rails "backups:restore[file]"` | Replaces the database with a backup (newest nightly copy if no file; a Google Drive `.sqlite3.gz` works too). The app must be stopped: `docker compose stop web`, then `docker compose run --rm web bin/rails backups:restore`. The current database is moved to `storage/pre-restore-…`, never deleted. `CONFIRM=yes` skips the question |
|
||||
| `bin/rails users:sign_out_all` | Signs every user out of every device, if you think an account or session was compromised |
|
||||
| `bin/rails "sam:records_package[N12345]"` | Writes the aircraft's records package to a zip, for when the web side is down |
|
||||
| `bin/rails "email:test[you@example.com]"` | Sends a test email and shows the mail settings |
|
||||
| `bin/rails offsite:backup` / `offsite:restore_test` | Runs the Google Drive copy or a restore test now and prints the result |
|
||||
|
||||
## Recordkeeping procedures
|
||||
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# Maintenance and recovery commands. In Docker: docker compose exec web bin/rails <task>
|
||||
# (backups:restore needs the app stopped: docker compose stop web, then
|
||||
# docker compose run --rm web bin/rails backups:restore).
|
||||
|
||||
namespace :sam do
|
||||
desc "Health report: settings, email, single sign-on, Google Drive, migrations, disk, jobs, backups, audit"
|
||||
task doctor: :environment do
|
||||
text, checks = SystemDoctor.report
|
||||
puts text
|
||||
exit 1 if checks.any? { _1.status == :fail }
|
||||
end
|
||||
|
||||
desc "Write an aircraft's records package (entries with signatures, history, procedures, data) to a zip"
|
||||
task :records_package, %i[tail_number path] => :environment do |_, args|
|
||||
aircraft = Aircraft.find_by("UPPER(tail_number) = ?", args[:tail_number].to_s.upcase) or abort "No aircraft #{args[:tail_number].inspect}"
|
||||
package = RecordsPackage.new(aircraft, source: "command line")
|
||||
path = args[:path].presence || package.filename
|
||||
package.write_to(path)
|
||||
puts "Wrote #{path} (#{ActiveSupport::NumberHelper.number_to_human_size(File.size(path))})"
|
||||
end
|
||||
end
|
||||
|
||||
namespace :records do
|
||||
desc "Back up the database now and check the records (history chains and every signature); exits 1 on a problem"
|
||||
task check: :environment do
|
||||
backup = DatabaseBackup.run!
|
||||
puts "Backup: #{backup.ok ? 'ok' : 'FAILED'}: #{backup.message}"
|
||||
check = RecordsIntegrity.check!
|
||||
puts "Records check: #{check.ok ? 'ok' : 'PROBLEMS'}: #{check.versions_checked} history entries and #{check.signatures_checked} signatures checked"
|
||||
Array(check.problems).first(50).each { puts " - #{_1}" }
|
||||
exit 1 unless backup.ok && check.ok
|
||||
end
|
||||
end
|
||||
|
||||
namespace :backups do
|
||||
desc "List the nightly database copies in storage/backups"
|
||||
task list: :environment do
|
||||
files = Dir[DatabaseBackup::DIR.join("sam-*.sqlite3")].sort_by { File.mtime(_1) }.reverse
|
||||
abort "No backups in #{DatabaseBackup::DIR}" if files.empty?
|
||||
files.each { puts "#{_1} #{ActiveSupport::NumberHelper.number_to_human_size(File.size(_1))} #{File.mtime(_1).utc.iso8601}" }
|
||||
end
|
||||
|
||||
desc "Replace the database with a backup (newest nightly copy, or a file incl. a Google Drive .sqlite3.gz); app must be stopped. CONFIRM=yes to skip the prompt"
|
||||
task :restore, [ :file ] => :environment do |_, args|
|
||||
restore = DatabaseRestore.new(source: args[:file])
|
||||
puts "Restoring from: #{restore.check}"
|
||||
puts "Into: #{restore.target}"
|
||||
unless ENV["CONFIRM"] == "yes"
|
||||
abort "Run again with CONFIRM=yes to go ahead." unless $stdin.tty?
|
||||
print "Replace the current database with this backup? The current one is kept aside. [y/N] "
|
||||
abort "Cancelled." unless $stdin.gets.to_s.strip.downcase.start_with?("y")
|
||||
end
|
||||
aside = restore.restore!
|
||||
puts "Restored. The previous database is in #{aside}."
|
||||
puts "Next: start the app (it applies any newer migrations), restore uploaded files if needed (bin/rails offsite:restore_files),"
|
||||
puts "then run bin/rails records:check and bin/rails sam:doctor."
|
||||
rescue DatabaseRestore::Error => e
|
||||
abort "Error: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
namespace :users do
|
||||
desc "Sign every user out of every device (suspected compromise)"
|
||||
task sign_out_all: :environment do
|
||||
puts "Ended #{AccountRecovery.sign_out_all!} sessions. Everyone signs in again."
|
||||
end
|
||||
end
|
||||
|
||||
namespace :email do
|
||||
desc "Send a test email to an address, to check the SMTP settings"
|
||||
task :test, [ :address ] => :environment do |_, args|
|
||||
address = args[:address].to_s.strip
|
||||
abort "Give an address: bin/rails \"email:test[you@example.com]\"" if address.blank?
|
||||
MailSettings.summary.each { |k, v| puts "#{k}: #{v}" }
|
||||
user = User.find_by(email_address: address.downcase) || User.new(email_address: address, name: address)
|
||||
AdminMailer.test_email(user, via: "from the server's command line").deliver_now
|
||||
puts "Sent to #{address}."
|
||||
rescue StandardError => e
|
||||
abort "Sending failed: #{e.class.name.demodulize}: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
namespace :offsite do
|
||||
desc "Copy the latest backup and new files to Google Drive now"
|
||||
task backup: :environment do
|
||||
abort "Google Drive isn't connected (Admin → Off-site Backup)." unless OffsiteConnection.connected?
|
||||
run = OffsiteBackupJob.backup
|
||||
puts "#{run.ok ? 'OK' : 'FAILED'}: #{run.message}"
|
||||
exit 1 unless run.ok
|
||||
end
|
||||
|
||||
desc "Download the newest database copy and a few files from Google Drive and check them"
|
||||
task restore_test: :environment do
|
||||
abort "Google Drive isn't connected (Admin → Off-site Backup)." unless OffsiteConnection.connected?
|
||||
run = OffsiteBackup.restore_test!
|
||||
puts "#{run.ok ? 'OK' : 'FAILED'}: #{run.message}"
|
||||
exit 1 unless run.ok
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,99 @@
|
||||
require "test_helper"
|
||||
require "rake"
|
||||
|
||||
# Maintenance and recovery commands: restore, records check, doctor,
|
||||
# sign everyone out, records package, test email, off-site.
|
||||
class MaintenanceCommandsTest < ActionDispatch::IntegrationTest
|
||||
include ActionMailer::TestHelper
|
||||
|
||||
setup do
|
||||
@dir = Dir.mktmpdir("sam-maint")
|
||||
Rails.application.load_tasks if Rake::Task.tasks.none? { _1.name == "sam:doctor" }
|
||||
end
|
||||
|
||||
teardown { FileUtils.rm_rf(@dir) }
|
||||
|
||||
def rake(name, *args)
|
||||
Rake::Task[name].reenable
|
||||
out, = capture_io { Rake::Task[name].invoke(*args) }
|
||||
out
|
||||
end
|
||||
|
||||
def sqlite(path, entries: 1)
|
||||
db = SQLite3::Database.new(path)
|
||||
%w[aircraft logbook_entries entry_signatures users].each { db.execute("CREATE TABLE #{_1} (id integer)") }
|
||||
db.execute("CREATE TABLE schema_migrations (version varchar)")
|
||||
db.execute("INSERT INTO schema_migrations VALUES ('20261108050000')")
|
||||
entries.times { db.execute("INSERT INTO logbook_entries VALUES (1)") }
|
||||
db.close
|
||||
path
|
||||
end
|
||||
|
||||
test "restoring a backup checks it, keeps the current database aside, and refuses while the app runs" do
|
||||
source = sqlite(File.join(@dir, "sam-20261101-033000.sqlite3"), entries: 3)
|
||||
target = sqlite(File.join(@dir, "production.sqlite3"))
|
||||
File.write("#{target}-wal", "x")
|
||||
restore = DatabaseRestore.new(source: source, target: target, pidfile: File.join(@dir, "none.pid"))
|
||||
assert_includes restore.check, "3 logbook entries"
|
||||
aside = restore.restore!
|
||||
assert File.exist?(File.join(aside, "production.sqlite3"))
|
||||
assert File.exist?(File.join(aside, "production.sqlite3-wal"))
|
||||
refute File.exist?("#{target}-wal")
|
||||
assert_equal 3, SQLite3::Database.new(target).get_first_value("SELECT COUNT(*) FROM logbook_entries")
|
||||
|
||||
gz = File.join(@dir, "sam-20261102.sqlite3.gz")
|
||||
Zlib::GzipWriter.open(gz) { |w| w.write(File.binread(source)) }
|
||||
assert_includes DatabaseRestore.new(source: gz, target: target).check, "3 logbook entries"
|
||||
|
||||
File.write(File.join(@dir, "junk.sqlite3"), "not a database")
|
||||
assert_raises(DatabaseRestore::Error) { DatabaseRestore.new(source: File.join(@dir, "junk.sqlite3"), target: target).check }
|
||||
|
||||
pidfile = File.join(@dir, "server.pid")
|
||||
File.write(pidfile, Process.pid.to_s)
|
||||
error = assert_raises(DatabaseRestore::Error) { DatabaseRestore.new(source: source, target: target, pidfile: pidfile).restore! }
|
||||
assert_includes error.message, "running"
|
||||
end
|
||||
|
||||
test "records:check reports the backup and the records check" do
|
||||
backup = BackupRun.create!(ok: true, message: "Verified")
|
||||
DatabaseBackup.singleton_class.alias_method :original_run!, :run!
|
||||
DatabaseBackup.define_singleton_method(:run!) { backup }
|
||||
out = rake("records:check")
|
||||
assert_includes out, "Backup: ok: Verified"
|
||||
assert_match(/Records check: ok/, out)
|
||||
ensure
|
||||
DatabaseBackup.singleton_class.alias_method :run!, :original_run!
|
||||
end
|
||||
|
||||
test "sam:doctor reports each area without failing on a healthy test setup" do
|
||||
stub_request(:any, /.*/).to_return(status: 200, body: "{}")
|
||||
_, checks = SystemDoctor.report
|
||||
names = checks.map(&:name)
|
||||
assert_includes names, "Database"
|
||||
assert_includes names, "Last backup"
|
||||
assert_equal :ok, checks.find { _1.name == "Database" }.status
|
||||
assert_equal :warn, checks.find { _1.name == "Last backup" }.status # never run
|
||||
assert_includes rake("sam:doctor"), "Records audit"
|
||||
end
|
||||
|
||||
test "users:sign_out_all ends every session" do
|
||||
user = create_user("jane@example.com")
|
||||
sign_in user
|
||||
assert_includes rake("users:sign_out_all"), "Ended 1 sessions"
|
||||
get root_path
|
||||
assert_redirected_to new_session_path
|
||||
end
|
||||
|
||||
test "sam:records_package writes the zip; email:test sends; off-site commands need Drive" do
|
||||
owner = create_user("owner@example.com")
|
||||
create_aircraft(owner: owner, tail_number: "N12345")
|
||||
path = File.join(@dir, "package.zip")
|
||||
assert_includes rake("sam:records_package", "n12345", path), "Wrote #{path}"
|
||||
assert Zip::File.open(path) { |z| z.find_entry("README.txt") }
|
||||
|
||||
assert_emails(1) { rake("email:test", "someone@example.com") }
|
||||
assert_includes ActionMailer::Base.deliveries.last.text_part.body.to_s, "from the server's command line"
|
||||
|
||||
assert_raises(SystemExit) { capture_io { Rake::Task["offsite:backup"].reenable; Rake::Task["offsite:backup"].invoke } }
|
||||
end
|
||||
end
|
||||
Reference in new issue
Block a user