diff --git a/app/controllers/direct_uploads_controller.rb b/app/controllers/direct_uploads_controller.rb new file mode 100644 index 0000000..4585809 --- /dev/null +++ b/app/controllers/direct_uploads_controller.rb @@ -0,0 +1,25 @@ +# Direct uploads (the browser sends each file straight to storage, then submits +# only its signed id). Active Storage's own endpoint lets anyone create blobs; +# this one requires a signed-in user and applies the upload rules up front. +class DirectUploadsController < ActiveStorage::DirectUploadsController + include Authentication + before_action :check_upload, only: :create + + private + + def blob_args + @blob_args ||= super.tap do |args| + args[:content_type] = Marcel::MimeType.for(name: args[:filename].to_s) if args[:content_type].blank? + end + end + + def check_upload + error = + if !UploadValidation::ALLOWED_CONTENT_TYPES.include?(blob_args[:content_type]) + "#{blob_args[:filename]} isn't an allowed file type" + elsif blob_args[:byte_size].to_i > UploadValidation::MAX_SIZE + "#{blob_args[:filename]} is over #{UploadValidation::MAX_SIZE / 1.megabyte} MB" + end + render json: { error: error }, status: :unprocessable_entity if error + end +end diff --git a/app/controllers/documents_controller.rb b/app/controllers/documents_controller.rb index 29d4f86..d943a77 100644 --- a/app/controllers/documents_controller.rb +++ b/app/controllers/documents_controller.rb @@ -26,6 +26,52 @@ class DocumentsController < ApplicationController end end + def bulk_new + @bulk = bulk_defaults.merge(document_collection_id: params[:collection_id]) + end + + # Each file becomes a document named after it; files given the same name are + # combined, in the order sent, as the pages of one document. Files arrive as + # direct-upload signed ids (or as plain uploads when JavaScript is off). + def bulk_create + @bulk = bulk_defaults.merge(params.slice(:doc_type, :visibility, :document_collection_id, :new_collection).permit!.to_h.symbolize_keys) + files = Array(params[:files]).compact_blank + return bulk_error("Choose some files to upload.") if files.empty? + return bulk_error("Upload at most #{MAX_BULK_FILES} files at a time.") if files.size > MAX_BULK_FILES + + names = Array(params[:names]) + rejected = [] + groups = {} + files.each_with_index do |file, i| + upload = file.is_a?(String) ? unattached_blob(file) : file + next rejected << "file #{i + 1} (upload expired, try again)" unless upload + name = names[i].to_s.squish.presence || File.basename(upload_filename(upload), ".*").tr("_", " ").squish.presence || "Untitled" + (groups[name.first(255)] ||= []) << upload + end + + collection_id, error = bulk_collection + return bulk_error(error) if error + created = [] + groups.each do |name, pages| + doc = @aircraft.documents.new(name: name, doc_type: @bulk[:doc_type], visibility: @bulk[:visibility].presence, + document_collection_id: collection_id, pages: pages) + if doc.save + created << doc + else + rejected << "#{name} (#{doc.errors.full_messages.to_sentence})" + pages.each { _1.purge_later if _1.is_a?(ActiveStorage::Blob) } + end + end + if created.empty? + @aircraft.document_collections.find_by(id: collection_id)&.destroy if @bulk[:new_collection].present? + return bulk_error("Nothing was uploaded: #{rejected.to_sentence}.") + end + + log_event("document", "#{helpers.pluralize(created.size, 'document')} uploaded", notes: created.map(&:name).join(", ").truncate(1000)) + flash[:alert] = "Not uploaded: #{rejected.to_sentence}." if rejected.any? + redirect_to aircraft_documents_path(@aircraft), notice: "#{helpers.pluralize(created.size, 'document')} uploaded.", status: :see_other + end + def edit; end def update @@ -50,6 +96,33 @@ class DocumentsController < ApplicationController private + MAX_BULK_FILES = 500 + + def bulk_defaults = { doc_type: "OTHER", visibility: nil, document_collection_id: nil, new_collection: nil } + + def bulk_error(message) + @bulk_error = message + render :bulk_new, status: :unprocessable_entity + end + + # A blob from a direct upload that hasn't been attached anywhere yet, so a + # signed id lifted from an existing file's URL can't be re-attached here. + def unattached_blob(signed_id) + blob = ActiveStorage::Blob.find_signed(signed_id) + blob if blob && !blob.attachments.exists? + end + + def upload_filename(upload) + (upload.respond_to?(:original_filename) ? upload.original_filename : upload.filename).to_s + end + + # [collection id or nil, error]: the chosen collection, or a new one. + def bulk_collection + return [ scoped_id(@aircraft.document_collections, @bulk[:document_collection_id]), nil ] if @bulk[:new_collection].blank? + col = @aircraft.document_collections.create(name: @bulk[:new_collection].squish, visibility: @bulk[:visibility].presence || "private") + col.persisted? ? [ col.id, nil ] : [ nil, "Couldn't create the collection: #{col.errors.full_messages.to_sentence}." ] + end + def set_document = @document = @aircraft.documents.find(params[:id]) def document_params diff --git a/app/javascript/controllers/bulk_upload_controller.js b/app/javascript/controllers/bulk_upload_controller.js new file mode 100644 index 0000000..8e3d322 --- /dev/null +++ b/app/javascript/controllers/bulk_upload_controller.js @@ -0,0 +1,207 @@ +import { Controller } from "@hotwired/stimulus" +import { DirectUpload } from "@rails/activestorage" + +// Bulk document upload. Picked files (or a whole folder) are listed with an +// editable document name each; files sharing a name become one document. On +// submit every file goes straight to storage (a few at a time, with progress), +// then the form is submitted with just the signed ids and names. +// +// Without JavaScript the file field posts the files itself and the server +// names each document after its file. +export default class extends Controller { + static targets = [ "input", "folder", "folderField", "panel", "list", "summary", "group", "groupField", "submit", "status" ] + static values = { url: String, maxBytes: Number } + + connect() { + this.items = [] + this.ready = false + this.uploading = false + this.inputTarget.removeAttribute("name") // the signed ids are sent instead + if ("webkitdirectory" in this.folderTarget) this.folderFieldTarget.hidden = false + this.beforeUnload = (event) => { if (this.uploading) event.preventDefault() } + window.addEventListener("beforeunload", this.beforeUnload) + } + + disconnect() { + window.removeEventListener("beforeunload", this.beforeUnload) + } + + add(event) { + for (const file of event.target.files) { + const item = { file, path: file.webkitRelativePath || file.name, state: "pending", progress: 0 } + if (file.size > this.maxBytesValue) Object.assign(item, { state: "error", error: `over ${Math.round(this.maxBytesValue / 1048576)} MB` }) + this.items.push(item) + } + event.target.value = "" + this.items.sort((a, b) => a.path.localeCompare(b.path, undefined, { numeric: true, sensitivity: "base" })) + this.items.forEach((item) => { if (!item.custom) item.name = this.defaultName(item) }) + this.groupFieldTarget.hidden = !this.items.some((item) => item.path.includes("/")) + this.render() + } + + regroup() { + this.items.forEach((item) => { item.custom = false; item.name = this.defaultName(item) }) + this.render() + } + + clear() { + if (this.uploading) return + this.items = [] + this.render() + } + + defaultName(item) { + const parts = item.path.split("/") + if (this.groupTarget.checked && parts.length > 1) return parts[parts.length - 2] + return parts[parts.length - 1].replace(/\.[^.]+$/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim() || "Untitled" + } + + render() { + this.panelTarget.hidden = this.items.length === 0 + this.listTarget.replaceChildren(...this.items.map((item, index) => this.row(item, index))) + this.updateSummary() + } + + row(item, index) { + const li = document.createElement("li") + li.className = "flex items-start gap-3 px-3 py-2" + + const body = document.createElement("div") + body.className = "min-w-0 flex-1" + const name = document.createElement("input") + name.type = "text" + name.className = "input" + name.value = item.name + name.setAttribute("aria-label", `Document name for ${item.path}`) + name.disabled = this.uploading + name.addEventListener("input", () => { item.name = name.value; item.custom = true; this.updateSummary() }) + + const meta = document.createElement("p") + meta.className = "mt-1 truncate text-xs text-slate-500" + const bar = document.createElement("div") + bar.className = "mt-1 h-1 overflow-hidden rounded bg-slate-100" + bar.hidden = item.state === "pending" || item.state === "error" + const fill = document.createElement("div") + fill.className = "h-full bg-sky-600 transition-[width]" + bar.append(fill) + body.append(name, meta, bar) + + const remove = document.createElement("button") + remove.type = "button" + remove.className = "btn-secondary btn-sm" + remove.textContent = "✕" + remove.setAttribute("aria-label", `Remove ${item.path}`) + remove.disabled = this.uploading + remove.addEventListener("click", () => { this.items.splice(index, 1); this.render() }) + + li.append(body, remove) + item.update = () => { + const status = { pending: "", uploading: ` · ${item.progress}%`, done: " · uploaded", error: ` · ${item.error}` }[item.state] + meta.textContent = `${item.path} · ${this.size(item.file.size)}${status}` + meta.classList.toggle("text-red-600", item.state === "error") + meta.classList.toggle("text-slate-500", item.state !== "error") + bar.hidden = item.state === "pending" || item.state === "error" + fill.style.width = `${item.state === "done" ? 100 : item.progress}%` + } + item.update() + return li + } + + updateSummary() { + const files = this.items.length + const docs = new Set(this.items.map((item) => item.name.trim() || "Untitled")).size + this.summaryTarget.textContent = `${files} ${files === 1 ? "file" : "files"} → ${docs} ${docs === 1 ? "document" : "documents"}` + this.submitTarget.textContent = files ? `Upload ${docs} ${docs === 1 ? "document" : "documents"}` : "Upload" + } + + size(bytes) { + if (bytes < 1024) return `${bytes} B` + if (bytes < 1048576) return `${Math.round(bytes / 1024)} KB` + return `${(bytes / 1048576).toFixed(1)} MB` + } + + async submit(event) { + if (this.ready) return + event.preventDefault() + if (this.uploading) return + if (this.items.length === 0) return this.say("Choose some files to upload.") + if (this.items.some((item) => item.file.size > this.maxBytesValue)) { + return this.say("Remove the files marked in red first.") + } + + this.items.forEach((item) => { if (item.state === "error") item.state = "pending" }) + this.setUploading(true) + const queue = this.items.filter((item) => item.state === "pending") + const total = queue.length + let finished = 0 + this.say(`Uploading 0 of ${total}…`) + const worker = async () => { + for (let item = queue.shift(); item; item = queue.shift()) { + await this.upload(item) + this.say(`Uploading ${++finished} of ${total}…`) + } + } + await Promise.all([ worker(), worker(), worker() ]) + + const failed = this.items.filter((item) => item.state === "error").length + if (failed) { + this.setUploading(false) + return this.say(`${failed} ${failed === 1 ? "file" : "files"} didn't upload. Press Upload to retry, or remove ${failed === 1 ? "it" : "them"}.`) + } + + this.element.querySelectorAll("input[data-bulk-upload-hidden]").forEach((el) => el.remove()) + for (const item of this.items) { + this.hidden("files[]", item.signedId) + this.hidden("names[]", item.name) + } + this.uploading = false + this.ready = true + this.say("Saving…") + // When nothing was left to upload this all ran inside the click's submit + // event, during which the browser ignores requestSubmit, so wait a tick. + setTimeout(() => this.element.requestSubmit()) + } + + upload(item) { + item.state = "uploading" + item.progress = 0 + item.update() + return new Promise((resolve) => { + const delegate = { + directUploadWillStoreFileWithXHR: (xhr) => { + xhr.upload.addEventListener("progress", (e) => { + if (e.lengthComputable) { item.progress = Math.round((e.loaded / e.total) * 100); item.update() } + }) + } + } + new DirectUpload(item.file, this.urlValue, delegate).create((error, blob) => { + if (error) { + Object.assign(item, { state: "error", error: /Status: 422/.test(error) ? "not an allowed file type" : "upload failed" }) + } else { + Object.assign(item, { state: "done", signedId: blob.signed_id }) + } + item.update() + resolve() + }) + }) + } + + hidden(name, value) { + const input = document.createElement("input") + input.type = "hidden" + input.name = name + input.value = value + input.dataset.bulkUploadHidden = "" + this.element.append(input) + } + + setUploading(on) { + this.uploading = on + this.submitTarget.disabled = on + this.render() + } + + say(message) { + this.statusTarget.textContent = message + } +} diff --git a/app/views/documents/_form.html.erb b/app/views/documents/_form.html.erb index bab13d7..71cb709 100644 --- a/app/views/documents/_form.html.erb +++ b/app/views/documents/_form.html.erb @@ -19,6 +19,9 @@ <% end %> <% end %> + <% unless document.persisted? %> +

Lots of files? <%= link_to "Bulk upload", new_bulk_aircraft_documents_path(aircraft, collection_id: document.document_collection_id), class: "link" %> makes a document from each one.

+ <% end %>
<%= f.submit "Save Document" %><%= link_to "Cancel", aircraft_documents_path(aircraft), class: "btn-secondary" %>
<% end %> <% if document.persisted? %>
<%= delete_button "Delete document", aircraft_document_path(aircraft, document) %>
<% end %> diff --git a/app/views/documents/bulk_new.html.erb b/app/views/documents/bulk_new.html.erb new file mode 100644 index 0000000..c59161a --- /dev/null +++ b/app/views/documents/bulk_new.html.erb @@ -0,0 +1,68 @@ +<% content_for :title, "#{@aircraft.tail_number} Bulk Upload" %> +<%= render "shared/aircraft_header", aircraft: @aircraft %> +<% file_classes = "block w-full text-sm text-slate-600 file:mr-3 file:rounded-md file:border-0 file:bg-slate-100 file:px-3 file:py-1.5 file:text-sm file:font-medium hover:file:bg-slate-200" %> +<%= form_with url: bulk_aircraft_documents_path(@aircraft), multipart: true, class: "card card-body max-w-3xl space-y-4", + data: { controller: "bulk-upload", bulk_upload_url_value: rails_direct_uploads_path, + bulk_upload_max_bytes_value: UploadValidation::MAX_SIZE, action: "submit->bulk-upload#submit" } do %> +
+

Bulk upload

+

Each file becomes its own document, named after the file. Give files the same name to combine them, in order, as the pages of one document.

+
+ <% if @bulk_error %> + + <% end %> + +
+
+ <%= label_tag :doc_type, "Type", class: "label" %> + <%= select_tag :doc_type, options_for_select(Document::TYPES.invert, @bulk[:doc_type]), class: "input" %> +
+
+ <%= label_tag :visibility, "Share-link visibility", class: "label" %> + <%= select_tag :visibility, options_for_select([ [ "Inherit from collection", "" ] ] + DocumentCollection::VISIBILITIES.invert.to_a, @bulk[:visibility]), class: "input" %> +
+
+ <%= label_tag :document_collection_id, "Collection", class: "label" %> + <%= select_tag :document_collection_id, + options_for_select([ [ "— None —", "" ] ] + @aircraft.document_collections.ordered.map { [ _1.name, _1.id ] }, @bulk[:document_collection_id]), class: "input" %> +
+
+ <%= label_tag :new_collection, "…or a new collection", class: "label" %> + <%= text_field_tag :new_collection, @bulk[:new_collection], class: "input", placeholder: "e.g. 2025 Annual" %> +
+
+ +
+
+ <%= label_tag :files, "Files", class: "label" %> + <%= file_field_tag "files[]", id: "files", multiple: true, class: file_classes, + data: { bulk_upload_target: "input", action: "change->bulk-upload#add" } %> +
+ +
+

Images, PDFs or text, up to <%= UploadValidation::MAX_SIZE / 1.megabyte %> MB each and <%= DocumentsController::MAX_BULK_FILES %> at a time.

+ + + +
+ + <%= link_to "Cancel", aircraft_documents_path(@aircraft), class: "btn-secondary" %> +

+
+<% end %> diff --git a/app/views/documents/index.html.erb b/app/views/documents/index.html.erb index 9d9c5bd..82d495c 100644 --- a/app/views/documents/index.html.erb +++ b/app/views/documents/index.html.erb @@ -1,8 +1,9 @@ <% content_for :title, "#{@aircraft.tail_number} Documents" %> <%= render "shared/aircraft_header", aircraft: @aircraft %> -
+
<% if owner? %> <%= link_to "+ Collection", new_aircraft_document_collection_path(@aircraft), class: "btn-secondary" %> + <%= link_to "Bulk Upload", new_bulk_aircraft_documents_path(@aircraft), class: "btn-secondary" %> <%= link_to "+ Upload Document", new_aircraft_document_path(@aircraft), class: "btn-primary" %> <% end %>
diff --git a/config/application.rb b/config/application.rb index f679f01..cfcf926 100644 --- a/config/application.rb +++ b/config/application.rb @@ -14,7 +14,7 @@ module SimpleAircraftManager # Please, add to the `ignore` list any other `lib` subdirectories that do # not contain `.rb` files, or that should not be reloaded or eager loaded. # Common ones are `templates`, `generators`, or `middleware`, for example. - config.autoload_lib(ignore: %w[assets tasks]) + config.autoload_lib(ignore: %w[active_storage assets tasks]) # Configuration for the application, engines, and railties goes here. # diff --git a/config/importmap.rb b/config/importmap.rb index 70cf5fa..8da46ce 100644 --- a/config/importmap.rb +++ b/config/importmap.rb @@ -8,3 +8,4 @@ pin_all_from "app/javascript/controllers", under: "controllers" pin "chartkick", to: "chartkick.js" pin "Chart.bundle", to: "Chart.bundle.js" pin "next_due" +pin "@rails/activestorage", to: "activestorage.esm.js" diff --git a/config/recurring.yml b/config/recurring.yml index b4207f9..3fb1828 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -13,3 +13,9 @@ production: clear_solid_queue_finished_jobs: command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" schedule: every hour at minute 12 + purge_abandoned_uploads: + command: "ActiveStorage::Blob.unattached.where(created_at: ..2.days.ago).find_each(&:purge_later)" + schedule: every day at 4am + dedupe_storage: + command: "ActiveStorage::Blob.service.try(:sweep)" + schedule: every day at 4:30am diff --git a/config/routes.rb b/config/routes.rb index e436319..140a22c 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,8 @@ Rails.application.routes.draw do + # Signed-in users only, with the upload rules checked before any bytes are sent + # (overrides Active Storage's open endpoint at the same path). + post "rails/active_storage/direct_uploads", to: "direct_uploads#create" + resource :session resources :passwords, param: :token resource :setup, only: %i[new create], controller: "setup" @@ -45,7 +49,12 @@ Rails.application.routes.draw do end resources :logbook_imports, path: "logbook/import", only: %i[new create show] resources :logbook_entries - resources :documents + resources :documents do + collection do + get :bulk, action: :bulk_new, as: :new_bulk + post :bulk, action: :bulk_create + end + end resources :document_collections, except: %i[index show] resources :inspections, only: :index resources :inspection_types, except: %i[index show] diff --git a/config/storage.yml b/config/storage.yml index 4942ab6..190873d 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -1,9 +1,9 @@ test: - service: Disk + service: DedupDisk root: <%= Rails.root.join("tmp/storage") %> local: - service: Disk + service: DedupDisk root: <%= Rails.root.join("storage") %> # Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) diff --git a/lib/active_storage/service/dedup_disk_service.rb b/lib/active_storage/service/dedup_disk_service.rb new file mode 100644 index 0000000..55d5ae1 --- /dev/null +++ b/lib/active_storage/service/dedup_disk_service.rb @@ -0,0 +1,107 @@ +require "active_storage/service/disk_service" + +module ActiveStorage + # Disk storage that keeps one copy of each distinct file. A stored file's + # bytes live once under .objects/ab/cd/, and every blob (and variant) + # with those bytes is a hard link to it, so uploading the same scan twice + # costs no extra space. Blobs, filenames and URLs are unchanged. + # + # Linked files must never be written in place, so an upload over an existing + # key first unlinks it. If the filesystem can't hard-link (some network + # shares), files are simply kept as plain copies. + # + # #sweep links in files stored before this service (or that failed to link) + # and removes objects no blob uses any more; it runs nightly. + class Service::DedupDiskService < Service::DiskService + OBJECTS_DIR = ".objects".freeze + + def upload(key, io, checksum: nil, **) + release(path_for(key)) + super + adopt(path_for(key)) + end + + def compose(source_keys, destination_key, **) + release(path_for(destination_key)) + super + adopt(path_for(destination_key)) + end + + def delete(key) + instrument :delete, key: key do + release(path_for(key)) + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + prefix_path = path_for(prefix) + prefix_path += "/" if prefix.end_with?("/") + Dir.glob("#{escape_glob_metacharacters(prefix_path)}*").each do |path| + files = File.directory?(path) ? Dir.glob("#{escape_glob_metacharacters(path)}/**/*") : [ path ] + files.each { release(_1) if File.file?(_1) } + FileUtils.rm_rf(path) + end + end + end + + # Returns { linked:, removed:, saved_bytes: }. + def sweep + linked = Dir.glob(File.join(escape_glob_metacharacters(root), "??", "??", "**", "*")).count do |path| + File.file?(path) && !File.symlink?(path) && adopt(path) == :shared + end + removed = 0 + saved = 0 + Dir.glob(File.join(escape_glob_metacharacters(objects_root), "**", "*")).each do |object| + next unless File.file?(object) + stat = File.stat(object) + if stat.nlink == 1 + File.delete(object) + removed += 1 + else + saved += stat.size * (stat.nlink - 2) + end + end + { linked: linked, removed: removed, saved_bytes: saved } + end + + private + + def objects_root = File.join(root, OBJECTS_DIR) + + def object_path(digest) = File.join(objects_root, digest[0..1], digest[2..3], digest) + + # Points a freshly written file at the stored copy of its bytes (or makes it + # the stored copy). Returns :stored, :shared, or nil if left alone. + def adopt(path) + return unless File.stat(path).nlink == 1 + object = object_path(Digest::SHA256.file(path).hexdigest) + FileUtils.mkdir_p(File.dirname(object)) + begin + File.link(path, object) + :stored + rescue Errno::EEXIST + return unless File.size(object) == File.size(path) + temp = "#{path}.#{SecureRandom.hex(6)}.link" + File.link(object, temp) + File.rename(temp, path) # atomic swap: readers see one file or the other + :shared + end + rescue SystemCallError => e + Rails.logger.warn("[dedup storage] kept a plain copy of #{path}: #{e.class}: #{e.message}") + nil + end + + # Unlinks a blob's file, and its stored object if nothing else uses it. + def release(path) + stat = File.stat(path) + if stat.nlink == 2 + object = object_path(Digest::SHA256.file(path).hexdigest) + File.delete(object) if File.exist?(object) && File.stat(object).ino == stat.ino + end + File.delete(path) + rescue Errno::ENOENT + # already gone + end + end +end diff --git a/lib/tasks/storage.rake b/lib/tasks/storage.rake new file mode 100644 index 0000000..980e3e3 --- /dev/null +++ b/lib/tasks/storage.rake @@ -0,0 +1,10 @@ +namespace :storage do + desc "Store each distinct uploaded file once (hard links) and drop unused copies" + task dedupe: :environment do + service = ActiveStorage::Blob.service + abort "The #{service.class.name.demodulize} service doesn't deduplicate." unless service.respond_to?(:sweep) + result = service.sweep + puts "Linked #{result[:linked]} duplicate files, removed #{result[:removed]} unused copies; " \ + "#{ActiveSupport::NumberHelper.number_to_human_size(result[:saved_bytes])} saved by sharing." + end +end diff --git a/test/integration/document_bulk_upload_test.rb b/test/integration/document_bulk_upload_test.rb new file mode 100644 index 0000000..60e1490 --- /dev/null +++ b/test/integration/document_bulk_upload_test.rb @@ -0,0 +1,133 @@ +require "test_helper" + +class DocumentBulkUploadTest < ActionDispatch::IntegrationTest + setup do + @owner = create_user("owner@example.com") + @pilot = create_user("pilot@example.com") + @a = create_aircraft(owner: @owner, pilot: @pilot) + end + + def upload(name, body = "page", type = "image/jpeg") + Rack::Test::UploadedFile.new(StringIO.new(body), type, original_filename: name) + end + + def direct_blob(name, type = "image/jpeg") + ActiveStorage::Blob.create_and_upload!(io: StringIO.new("bytes"), filename: name, content_type: type) + end + + test "owner sees the bulk page; pilots and outsiders can't use it" do + sign_in @owner + get aircraft_documents_path(@a) + assert_select "a[href='#{new_bulk_aircraft_documents_path(@a)}']", text: "Bulk Upload" + get new_bulk_aircraft_documents_path(@a) + assert_response :success + assert_select "form[data-controller='bulk-upload'][data-bulk-upload-url-value='/rails/active_storage/direct_uploads']" + assert_select "input[type=file][name='files[]'][multiple]" + assert_select "input[type=file][webkitdirectory]" + + sign_in @pilot + post bulk_aircraft_documents_path(@a), params: { files: [ upload("a.jpg") ] } + assert_redirected_to aircraft_path(@a) + assert_equal 0, Document.count + + sign_in create_user + get new_bulk_aircraft_documents_path(@a) + assert_response :not_found + end + + test "plain uploads (no JavaScript) become one document per file, named after the file" do + sign_in @owner + col = @a.document_collections.create!(name: "Annual", visibility: "private") + post bulk_aircraft_documents_path(@a), params: { + files: [ upload("weight_and_balance.pdf", "%PDF", "application/pdf"), upload("Form 337.jpg") ], + doc_type: "ALTER", visibility: "maintenance", document_collection_id: col.id + } + assert_redirected_to aircraft_documents_path(@a) + assert_equal "2 documents uploaded.", flash[:notice] + docs = @a.documents.order(:name) + assert_equal [ "Form 337", "weight and balance" ], docs.map(&:name) + assert docs.all? { _1.doc_type == "ALTER" && _1.visibility == "maintenance" && _1.document_collection == col && _1.pages.size == 1 } + assert_equal "2 documents uploaded", @a.events.last.event_name + end + + test "direct uploads: same name combines pages in order; a new collection is made" do + sign_in @owner + blobs = %w[p1.jpg p2.jpg receipt.jpg].map { direct_blob(_1) } + post bulk_aircraft_documents_path(@a), params: { + files: blobs.map(&:signed_id), names: [ "Engine log 1", "Engine log 1", "Prop overhaul receipt" ], + new_collection: "Logbooks", doc_type: "LOG" + } + assert_redirected_to aircraft_documents_path(@a) + col = @a.document_collections.find_by!(name: "Logbooks") + assert_equal "private", col.visibility + log = col.documents.find_by!(name: "Engine log 1") + assert_equal %w[p1.jpg p2.jpg], log.pages.map { _1.filename.to_s } + assert_nil log.visibility + assert_equal 1, col.documents.find_by!(name: "Prop overhaul receipt").pages.size + end + + test "already-attached blobs and bad files are refused; partial success is reported" do + sign_in @owner + other = @a.documents.create!(name: "Existing", doc_type: "OTHER", pages: [ upload("x.jpg") ]) + good = direct_blob("good.jpg") + bad = direct_blob("tool.exe", "application/x-msdownload") + post bulk_aircraft_documents_path(@a), params: { files: [ other.pages.first.blob.signed_id, good.signed_id, bad.signed_id ] } + assert_redirected_to aircraft_documents_path(@a) + assert_equal "1 document uploaded.", flash[:notice] + assert_match(/file 1 \(upload expired/, flash[:alert]) + assert_match(/tool \(Pages tool.exe is not an allowed file type\)/, flash[:alert]) + assert_equal [ "Existing", "good" ], @a.documents.order(:name).map(&:name) + assert_equal 1, other.reload.pages.size + end + + test "nothing valid re-renders the form and drops the new collection" do + sign_in @owner + post bulk_aircraft_documents_path(@a), params: { new_collection: "Junk" } + assert_response :unprocessable_entity + assert_includes response.body, "Choose some files to upload." + + post bulk_aircraft_documents_path(@a), params: { files: [ upload("a.exe", "MZ", "application/x-msdownload") ], new_collection: "Junk" } + assert_response :unprocessable_entity + assert_includes response.body, "Nothing was uploaded" + assert_select "input[name=new_collection][value=Junk]" + assert_equal 0, @a.document_collections.count + end + + test "direct-upload endpoint needs a session and checks type and size first" do + blob = ->(filename, type, size = 10) { { blob: { filename: filename, byte_size: size, checksum: "Q2hlY2tzdW0=", content_type: type } } } + post "/rails/active_storage/direct_uploads", params: blob.("a.jpg", "image/jpeg"), as: :json + assert_redirected_to new_session_path + assert_equal 0, ActiveStorage::Blob.count + + sign_in @owner + post "/rails/active_storage/direct_uploads", params: blob.("a.exe", "application/x-msdownload"), as: :json + assert_response :unprocessable_entity + assert_match "isn't an allowed file type", response.parsed_body["error"] + post "/rails/active_storage/direct_uploads", params: blob.("big.pdf", "application/pdf", 51.megabytes), as: :json + assert_response :unprocessable_entity + + post "/rails/active_storage/direct_uploads", params: blob.("scan.png", ""), as: :json + assert_response :success + assert_equal "image/png", ActiveStorage::Blob.last.content_type + assert response.parsed_body["signed_id"].present? + end + + test "the same file direct-uploaded twice is stored once on disk" do + sign_in @owner + body = "%PDF-1.4 identical scan" + ids = 2.times.map do |i| + post "/rails/active_storage/direct_uploads", as: :json, params: { blob: { + filename: "scan#{i}.pdf", byte_size: body.bytesize, checksum: OpenSSL::Digest::MD5.base64digest(body), content_type: "application/pdf" } } + signed_id, upload = response.parsed_body.values_at("signed_id", "direct_upload") + put URI(upload["url"]).request_uri, params: body, headers: upload["headers"] + assert_response :no_content + signed_id + end + post bulk_aircraft_documents_path(@a), params: { files: ids } + assert_redirected_to aircraft_documents_path(@a) + paths = @a.documents.order(:name).map { ActiveStorage::Blob.service.path_for(_1.pages.first.key) } + assert_equal 2, paths.size + assert_equal File.stat(paths[0]).ino, File.stat(paths[1]).ino + assert_equal %w[scan0 scan1], @a.documents.order(:name).map(&:name) + end +end diff --git a/test/lib/dedup_disk_service_test.rb b/test/lib/dedup_disk_service_test.rb new file mode 100644 index 0000000..0ddd8ff --- /dev/null +++ b/test/lib/dedup_disk_service_test.rb @@ -0,0 +1,87 @@ +require "test_helper" +require "active_storage/service/dedup_disk_service" + +class DedupDiskServiceTest < ActiveSupport::TestCase + include ActiveJob::TestHelper + + setup do + @root = Dir.mktmpdir + @service = ActiveStorage::Service::DedupDiskService.new(root: @root) + end + + teardown { FileUtils.rm_rf(@root) } + + def put(key, body) = @service.upload(key, StringIO.new(body), checksum: OpenSSL::Digest::MD5.base64digest(body)) + def path(key) = @service.path_for(key) + def objects = Dir.glob(File.join(@root, ".objects/**/*")).select { File.file?(_1) } + + test "identical files share one stored copy; different files don't" do + put "aaaa1111", "same scan" + put "bbbb2222", "same scan" + put "cccc3333", "other scan" + assert_equal File.stat(path("aaaa1111")).ino, File.stat(path("bbbb2222")).ino + refute_equal File.stat(path("aaaa1111")).ino, File.stat(path("cccc3333")).ino + assert_equal 2, objects.size + assert_equal "same scan", @service.download("bbbb2222") + end + + test "deleting one copy keeps the other; deleting the last frees the object" do + put "aaaa1111", "same scan" + put "bbbb2222", "same scan" + @service.delete("aaaa1111") + assert_equal "same scan", @service.download("bbbb2222") + assert_equal 1, objects.size + @service.delete("bbbb2222") + assert_empty objects + refute @service.exist?("bbbb2222") + end + + test "re-uploading a shared key never changes the other blob" do + put "aaaa1111", "same scan" + put "bbbb2222", "same scan" + put "aaaa1111", "replaced" + assert_equal "replaced", @service.download("aaaa1111") + assert_equal "same scan", @service.download("bbbb2222") + end + + test "a checksum mismatch stores nothing" do + assert_raises(ActiveStorage::IntegrityError) { @service.upload("aaaa1111", StringIO.new("bytes"), checksum: "bm9wZQ==") } + refute @service.exist?("aaaa1111") + assert_empty objects + end + + test "variants under a prefix are released too" do + put "variants/aaaa1111/thumb", "same scan" + put "bbbb2222", "same scan" + @service.delete_prefixed("variants/aaaa1111/") + refute @service.exist?("variants/aaaa1111/thumb") + assert_equal "same scan", @service.download("bbbb2222") + assert_equal 1, objects.size + end + + test "sweep links files stored before dedup and clears unused objects" do + plain = ActiveStorage::Service::DiskService.new(root: @root) + plain.upload("aaaa1111", StringIO.new("old scan")) + plain.upload("bbbb2222", StringIO.new("old scan")) + put "cccc3333", "gone soon" + File.delete(path("cccc3333")) # e.g. a crash between unlinks + + assert_equal({ linked: 1, removed: 1, saved_bytes: 8 }, @service.sweep) + assert_equal File.stat(path("aaaa1111")).ino, File.stat(path("bbbb2222")).ino + assert_equal({ linked: 0, removed: 0, saved_bytes: 8 }, @service.sweep) + end + + test "attachments through Active Storage share storage but keep their own filenames" do + a = create_aircraft + one = a.documents.create!(name: "Receipt", doc_type: "OTHER", pages: [ { io: StringIO.new("%PDF same"), filename: "receipt.pdf", content_type: "application/pdf" } ]) + two = a.documents.create!(name: "Invoice", doc_type: "OTHER", pages: [ { io: StringIO.new("%PDF same"), filename: "invoice.pdf", content_type: "application/pdf" } ]) + service = ActiveStorage::Blob.service + assert_kind_of ActiveStorage::Service::DedupDiskService, service + keys = [ one, two ].map { _1.pages.first.blob.key } + assert_equal File.stat(service.path_for(keys[0])).ino, File.stat(service.path_for(keys[1])).ino + assert_equal %w[receipt.pdf invoice.pdf], [ one, two ].map { _1.pages.first.filename.to_s } + perform_enqueued_jobs { one.destroy! } + assert_equal 1, ActiveStorage::Blob.count + assert_equal "%PDF same", two.pages.first.download + end +end