mirror of
https://github.com/rickbarrette/simple-aircraft-manager-rails.git
synced 2026-11-17 23:50:36 -05:00
Off-site backup to Google Drive
- Admin → Off-site Backup: connect a Google account with your own OAuth client (drive.file scope; refresh token and secret encrypted) - Nightly after the verified database backup: upload the gzipped snapshot (newest N kept, Settings) and every uploaded file not yet copied - Restore test downloads the newest copy, opens and checks it, and checks sampled files against their checksums; failures emailed to admins - About, procedures document, policy completeness and audits use it - offsite:restore_files rake task; restore steps in the admin guide Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fe3fba1736
commit
7694719582
31 files changed
+853
-20
No files matched your search
@@ -0,0 +1,150 @@
|
||||
require "net/http"
|
||||
|
||||
# Minimal Google Drive v3 client for the off-site backup: OAuth (offline
|
||||
# access, drive.file scope), folders, resumable uploads, listing,
|
||||
# downloading and deleting the files this app created.
|
||||
class GoogleDriveClient
|
||||
class Error < StandardError; end
|
||||
class AuthError < Error; end # revoked or wrong credentials: reconnect
|
||||
|
||||
AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth".freeze
|
||||
TOKEN_URL = "https://oauth2.googleapis.com/token".freeze
|
||||
REVOKE_URL = "https://oauth2.googleapis.com/revoke".freeze
|
||||
API = "https://www.googleapis.com/drive/v3".freeze
|
||||
UPLOAD = "https://www.googleapis.com/upload/drive/v3/files".freeze
|
||||
SCOPE = "https://www.googleapis.com/auth/drive.file".freeze
|
||||
FOLDER = "application/vnd.google-apps.folder".freeze
|
||||
|
||||
def self.authorize_url(client_id:, redirect_uri:, state:)
|
||||
"#{AUTH_URL}?#{URI.encode_www_form(client_id: client_id, redirect_uri: redirect_uri, response_type: 'code', scope: SCOPE,
|
||||
access_type: 'offline', prompt: 'consent', include_granted_scopes: 'true', state: state)}"
|
||||
end
|
||||
|
||||
# The refresh token from the code Google sent back.
|
||||
def self.exchange_code(client_id:, client_secret:, code:, redirect_uri:)
|
||||
token = token_request(client_id: client_id, client_secret: client_secret, code: code, redirect_uri: redirect_uri, grant_type: "authorization_code")
|
||||
token["refresh_token"].presence or raise AuthError, "Google didn't return a refresh token; remove the app's access in your Google account and connect again"
|
||||
end
|
||||
|
||||
def self.revoke(token)
|
||||
res = Net::HTTP.post_form(URI(REVOKE_URL), token: token)
|
||||
raise Error, "revoke failed (#{res.code})" unless res.is_a?(Net::HTTPSuccess)
|
||||
rescue SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout => e
|
||||
raise Error, e.message
|
||||
end
|
||||
|
||||
def self.token_request(**form)
|
||||
res = Net::HTTP.post_form(URI(TOKEN_URL), form)
|
||||
body = JSON.parse(res.body) rescue {}
|
||||
return body if res.is_a?(Net::HTTPSuccess)
|
||||
raise AuthError, "Google refused the credentials (#{body['error_description'] || body['error'] || res.code})" if res.code.to_i.in?([ 400, 401 ])
|
||||
raise Error, "Google token request failed (#{res.code})"
|
||||
rescue SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout => e
|
||||
raise Error, "couldn't reach Google: #{e.message}"
|
||||
end
|
||||
|
||||
def initialize(client_id:, client_secret:, refresh_token:)
|
||||
@client_id = client_id
|
||||
@client_secret = client_secret
|
||||
@refresh_token = refresh_token
|
||||
end
|
||||
|
||||
def account_email = request(:get, "#{API}/about", query: { fields: "user(emailAddress)" }).dig("user", "emailAddress")
|
||||
|
||||
def folder?(id)
|
||||
file = request(:get, "#{API}/files/#{CGI.escape(id)}", query: { fields: "id,mimeType,trashed" })
|
||||
file["mimeType"] == FOLDER && !file["trashed"]
|
||||
rescue Error => e
|
||||
raise if e.is_a?(AuthError)
|
||||
false
|
||||
end
|
||||
|
||||
def create_folder(name, parent: nil)
|
||||
request(:post, "#{API}/files", query: { fields: "id" }, json: { name: name, mimeType: FOLDER, parents: Array(parent) }).fetch("id")
|
||||
end
|
||||
|
||||
# Uploads an IO of known size (resumable session, one PUT); returns the file id.
|
||||
def upload(io, name:, parent:, size:, content_type: "application/octet-stream", properties: {})
|
||||
session = request(:post, UPLOAD, query: { uploadType: "resumable", fields: "id" },
|
||||
json: { name: name, parents: [ parent ], appProperties: properties },
|
||||
headers: { "X-Upload-Content-Type" => content_type, "X-Upload-Content-Length" => size.to_s }, raw: true)
|
||||
location = session["Location"] or raise Error, "Google didn't start the upload"
|
||||
put = Net::HTTP::Put.new(URI(location))
|
||||
put["Content-Length"] = size.to_s
|
||||
put["Content-Type"] = content_type
|
||||
put.body_stream = io
|
||||
JSON.parse(perform(URI(location), put, timeout: 3600).body).fetch("id")
|
||||
end
|
||||
|
||||
# [{ "id", "name", "size", "createdTime", "appProperties" }] in a folder.
|
||||
def list(parent)
|
||||
files = []
|
||||
page = nil
|
||||
loop do
|
||||
q = { q: "'#{parent.gsub("'", "")}' in parents and trashed = false", pageSize: 1000,
|
||||
fields: "nextPageToken,files(id,name,size,createdTime,appProperties)", orderBy: "name" }
|
||||
q[:pageToken] = page if page
|
||||
body = request(:get, "#{API}/files", query: q)
|
||||
files.concat(body.fetch("files", []))
|
||||
page = body["nextPageToken"] or break
|
||||
end
|
||||
files
|
||||
end
|
||||
|
||||
# Streams a file's content into the IO.
|
||||
def download(id, to:)
|
||||
uri = URI("#{API}/files/#{CGI.escape(id)}?alt=media")
|
||||
get = Net::HTTP::Get.new(uri)
|
||||
get["Authorization"] = "Bearer #{access_token}"
|
||||
Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 15, read_timeout: 600) do |http|
|
||||
http.request(get) do |res|
|
||||
raise_for(res) unless res.is_a?(Net::HTTPSuccess)
|
||||
res.read_body { to.write(_1) }
|
||||
end
|
||||
end
|
||||
to.rewind if to.respond_to?(:rewind)
|
||||
to
|
||||
rescue SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout => e
|
||||
raise Error, "couldn't reach Google Drive: #{e.message}"
|
||||
end
|
||||
|
||||
def delete(id) = request(:delete, "#{API}/files/#{CGI.escape(id)}")
|
||||
|
||||
private
|
||||
|
||||
def access_token
|
||||
return @access_token if @access_token && @expires_at > Time.current
|
||||
token = self.class.token_request(client_id: @client_id, client_secret: @client_secret, refresh_token: @refresh_token, grant_type: "refresh_token")
|
||||
@expires_at = Time.current + token.fetch("expires_in", 3600).to_i - 60
|
||||
@access_token = token.fetch("access_token")
|
||||
end
|
||||
|
||||
def request(method, url, query: nil, json: nil, headers: {}, raw: false)
|
||||
uri = URI(url)
|
||||
uri.query = URI.encode_www_form(query) if query
|
||||
req = Net::HTTP.const_get(method.to_s.capitalize).new(uri)
|
||||
req["Authorization"] = "Bearer #{access_token}"
|
||||
headers.each { req[_1] = _2 }
|
||||
if json
|
||||
req["Content-Type"] = "application/json; charset=UTF-8"
|
||||
req.body = json.to_json
|
||||
end
|
||||
res = perform(uri, req)
|
||||
return res if raw
|
||||
res.body.present? ? JSON.parse(res.body) : {}
|
||||
end
|
||||
|
||||
def perform(uri, req, timeout: 60)
|
||||
res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 15, read_timeout: timeout, write_timeout: timeout) { _1.request(req) }
|
||||
raise_for(res) unless res.is_a?(Net::HTTPSuccess)
|
||||
res
|
||||
rescue SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout => e
|
||||
raise Error, "couldn't reach Google Drive: #{e.message}"
|
||||
end
|
||||
|
||||
def raise_for(res)
|
||||
message = (JSON.parse(res.body.to_s).dig("error", "message") rescue nil) || res.message
|
||||
raise AuthError, "Google Drive refused access (#{message}); connect again" if res.code.to_i == 401
|
||||
raise Error, "Google Drive: #{message} (#{res.code})"
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,112 @@
|
||||
require "zlib"
|
||||
|
||||
# Off-site copy to Google Drive (AC 120-78B 3.2.7), after the nightly
|
||||
# database backup: the verified database snapshot (gzipped; the newest few
|
||||
# kept) and every uploaded file not yet copied (never deleted off-site,
|
||||
# so a file removed by mistake can still be recovered).
|
||||
#
|
||||
# database/sam-20261108-033000.sqlite3.gz
|
||||
# files/<blob key> (appProperties: filename, checksum)
|
||||
class OffsiteBackup
|
||||
SAMPLE_FILES = 3
|
||||
|
||||
def self.run!(database_path) = new.run!(database_path)
|
||||
def self.restore_test! = new.restore_test!
|
||||
|
||||
def initialize(connection = OffsiteConnection.current)
|
||||
@connection = connection
|
||||
end
|
||||
|
||||
def run!(database_path)
|
||||
raise "Google Drive isn't connected" unless @connection.connected?
|
||||
drive = @connection.client
|
||||
@connection.ensure_folders!(drive)
|
||||
bytes = upload_database(drive, database_path)
|
||||
pruned = prune(drive)
|
||||
uploaded, failed = upload_files(drive)
|
||||
message = "Database copy uploaded (#{ActiveSupport::NumberHelper.number_to_human_size(bytes)}); #{uploaded.size} new #{'file'.pluralize(uploaded.size)} copied"
|
||||
message += "; #{failed.size} couldn't be copied: #{failed.first(3).join('; ')}" if failed.any?
|
||||
OffsiteRun.create!(kind: "backup", ok: failed.empty?, bytes: bytes + uploaded.sum, files_uploaded: uploaded.size, message: message,
|
||||
details: { "account" => @connection.account_email, "pruned" => pruned, "failed" => failed.first(20) })
|
||||
rescue StandardError => e
|
||||
Rails.logger.error("[offsite] #{e.class}: #{e.message}")
|
||||
OffsiteRun.create!(kind: "backup", ok: false, message: "#{e.class.name.demodulize}: #{e.message}".truncate(500))
|
||||
end
|
||||
|
||||
# Downloads the newest database copy from Drive, opens it and checks it,
|
||||
# and downloads a few random files and checks them against their checksums.
|
||||
def restore_test!
|
||||
raise "Google Drive isn't connected" unless @connection.connected?
|
||||
drive = @connection.client
|
||||
latest = drive.list(@connection.database_folder_id).select { _1["name"].end_with?(".sqlite3.gz") }.max_by { _1["name"] } or raise "no database copy in Google Drive yet"
|
||||
db_result = Dir.mktmpdir("sam-restore") { |dir| check_database(drive, latest, dir) }
|
||||
samples = OffsiteFile.includes(:blob).order(Arel.sql("RANDOM()")).limit(SAMPLE_FILES).to_a
|
||||
bad = samples.reject { check_file(drive, _1) }
|
||||
message = "#{latest['name']}: #{db_result}. #{samples.size - bad.size} of #{samples.size} sampled files match their checksums"
|
||||
OffsiteRun.create!(kind: "restore_test", ok: bad.empty?, message: message,
|
||||
details: { "database" => latest["name"], "files_checked" => samples.map { _1.blob.filename.to_s }, "mismatched" => bad.map { _1.blob.filename.to_s } })
|
||||
rescue StandardError => e
|
||||
Rails.logger.error("[offsite restore test] #{e.class}: #{e.message}")
|
||||
OffsiteRun.create!(kind: "restore_test", ok: false, message: "#{e.class.name.demodulize}: #{e.message}".truncate(500))
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def upload_database(drive, path)
|
||||
Tempfile.create([ "sam-offsite", ".gz" ], binmode: true) do |gz|
|
||||
Zlib::GzipWriter.wrap(File.open(gz.path, "wb")) { |w| File.open(path, "rb") { IO.copy_stream(_1, w) } }
|
||||
size = File.size(gz.path)
|
||||
File.open(gz.path, "rb") do |io|
|
||||
drive.upload(io, name: "#{File.basename(path)}.gz", parent: @connection.database_folder_id, size: size, content_type: "application/gzip")
|
||||
end
|
||||
size
|
||||
end
|
||||
end
|
||||
|
||||
def prune(drive)
|
||||
keep = Setting.integer("offsite_backups_kept")
|
||||
copies = drive.list(@connection.database_folder_id).select { _1["name"].start_with?("sam-") }.sort_by { _1["name"] }.reverse
|
||||
copies.drop(keep).each { drive.delete(_1["id"]) }.map { _1["name"] }
|
||||
end
|
||||
|
||||
# Returns [sizes uploaded, failure messages].
|
||||
def upload_files(drive)
|
||||
uploaded = []
|
||||
failed = []
|
||||
ActiveStorage::Blob.where.not(id: OffsiteFile.select(:blob_id)).find_each do |blob|
|
||||
blob.open do |file|
|
||||
id = drive.upload(file, name: blob.key, parent: @connection.files_folder_id, size: blob.byte_size,
|
||||
content_type: blob.content_type.presence || "application/octet-stream",
|
||||
properties: { filename: blob.filename.to_s.first(100), checksum: blob.checksum.to_s })
|
||||
OffsiteFile.create!(blob: blob, drive_file_id: id, bytes: blob.byte_size)
|
||||
uploaded << blob.byte_size
|
||||
end
|
||||
rescue GoogleDriveClient::AuthError
|
||||
raise
|
||||
rescue StandardError => e
|
||||
failed << "#{blob.filename}: #{e.message.truncate(120)}"
|
||||
end
|
||||
[ uploaded, failed ]
|
||||
end
|
||||
|
||||
def check_database(drive, file, dir)
|
||||
gz = File.join(dir, "db.sqlite3.gz")
|
||||
File.open(gz, "wb") { drive.download(file["id"], to: _1) }
|
||||
path = File.join(dir, "db.sqlite3")
|
||||
Zlib::GzipReader.open(gz) { |r| File.open(path, "wb") { IO.copy_stream(r, _1) } }
|
||||
db = SQLite3::Database.new(path, readonly: true)
|
||||
result = db.get_first_value("PRAGMA integrity_check")
|
||||
raise "integrity check of the downloaded copy failed: #{result}" unless result == "ok"
|
||||
entries, signatures = %w[logbook_entries entry_signatures].map { db.get_first_value("SELECT COUNT(*) FROM #{_1}") }
|
||||
"opened, SQLite integrity check ok, #{entries} logbook entries and #{signatures} signatures (#{LogbookEntry.count} and #{EntrySignature.count} now)"
|
||||
ensure
|
||||
db&.close
|
||||
end
|
||||
|
||||
def check_file(drive, offsite)
|
||||
io = drive.download(offsite.drive_file_id, to: StringIO.new(+"", "wb"))
|
||||
Digest::MD5.base64digest(io.string) == offsite.blob.checksum
|
||||
rescue GoogleDriveClient::Error
|
||||
false
|
||||
end
|
||||
end
|
||||
@@ -28,6 +28,7 @@ class RecordsAudit
|
||||
Item.new("backup", "A verified database backup was made in the last 2 days", true,
|
||||
backup.present? && backup.ok && backup.created_at > 2.days.ago,
|
||||
backup ? "#{backup.created_at.to_date}: #{backup.message}" : "Never run"),
|
||||
*offsite_items,
|
||||
Item.new("procedures", "The procedures document is adopted and its details filled in", true,
|
||||
revision.present? && @policy.complete?,
|
||||
revision ? "Revision #{revision.number}, adopted #{revision.adopted_at.to_date}" : "Not adopted yet"),
|
||||
@@ -62,8 +63,18 @@ class RecordsAudit
|
||||
AuditRun.create!(performed_by: user, performed_at: Time.current, passed: results.all? { _1["passed"] }, results: results, notes: notes.to_s.strip.first(5000))
|
||||
end
|
||||
|
||||
# The latest automatic restore test (shown next to the manual restore item).
|
||||
def restore_test = OffsiteConnection.connected? ? OffsiteRun.latest_restore_test : nil
|
||||
|
||||
private
|
||||
|
||||
def offsite_items
|
||||
return [] unless OffsiteConnection.connected?
|
||||
run = OffsiteRun.latest_backup
|
||||
[ Item.new("offsite", "The backup was copied off-site (Google Drive) in the last 2 days", true, run.present? && run.ok && run.recent?,
|
||||
run ? "#{run.created_at.to_date}: #{run.message}" : "Never run") ]
|
||||
end
|
||||
|
||||
def since_label = @since ? @since.to_date.to_s : "the start"
|
||||
def signatures_since = EntrySignature.where(signed_at: (@since || Time.at(0))..)
|
||||
def versions_since = AppRevision.where(first_seen_at: (@since || Time.at(0))..).order(:first_seen_at)
|
||||
|
||||
Reference in new issue
Block a user