Bulk document upload and deduplicated file storage

- Bulk Upload page: pick many files or a folder; each file becomes a
  document named after it, and files sharing a name become one multi-page
  document. Files go up as direct uploads with per-file progress.
- Direct uploads require a signed-in user and check type/size up front;
  only fresh, unattached blobs can be attached; abandoned uploads are
  purged nightly.
- DedupDisk storage service: each distinct file is stored once under its
  SHA-256 and blobs hard-link to it; nightly sweep (and storage:dedupe)
  links pre-existing duplicates and drops unused objects.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rickyandClaude Opus 5.5 committed 2026-11-07 08:20:22 -05:00
1 parent 230a569b77
commit 2bfcb845ca
15 files changed
+735 -5

No files matched your search

@@ -0,0 +1,25 @@
# Direct uploads (the browser sends each file straight to storage, then submits
# only its signed id). Active Storage's own endpoint lets anyone create blobs;
# this one requires a signed-in user and applies the upload rules up front.
class DirectUploadsController < ActiveStorage::DirectUploadsController
include Authentication
before_action :check_upload, only: :create
private
def blob_args
@blob_args ||= super.tap do |args|
args[:content_type] = Marcel::MimeType.for(name: args[:filename].to_s) if args[:content_type].blank?
end
end
def check_upload
error =
if !UploadValidation::ALLOWED_CONTENT_TYPES.include?(blob_args[:content_type])
"#{blob_args[:filename]} isn't an allowed file type"
elsif blob_args[:byte_size].to_i > UploadValidation::MAX_SIZE
"#{blob_args[:filename]} is over #{UploadValidation::MAX_SIZE / 1.megabyte} MB"
end
render json: { error: error }, status: :unprocessable_entity if error
end
end
+73
View File
@@ -26,6 +26,52 @@ class DocumentsController < ApplicationController
end
end
def bulk_new
@bulk = bulk_defaults.merge(document_collection_id: params[:collection_id])
end
# Each file becomes a document named after it; files given the same name are
# combined, in the order sent, as the pages of one document. Files arrive as
# direct-upload signed ids (or as plain uploads when JavaScript is off).
def bulk_create
@bulk = bulk_defaults.merge(params.slice(:doc_type, :visibility, :document_collection_id, :new_collection).permit!.to_h.symbolize_keys)
files = Array(params[:files]).compact_blank
return bulk_error("Choose some files to upload.") if files.empty?
return bulk_error("Upload at most #{MAX_BULK_FILES} files at a time.") if files.size > MAX_BULK_FILES
names = Array(params[:names])
rejected = []
groups = {}
files.each_with_index do |file, i|
upload = file.is_a?(String) ? unattached_blob(file) : file
next rejected << "file #{i + 1} (upload expired, try again)" unless upload
name = names[i].to_s.squish.presence || File.basename(upload_filename(upload), ".*").tr("_", " ").squish.presence || "Untitled"
(groups[name.first(255)] ||= []) << upload
end
collection_id, error = bulk_collection
return bulk_error(error) if error
created = []
groups.each do |name, pages|
doc = @aircraft.documents.new(name: name, doc_type: @bulk[:doc_type], visibility: @bulk[:visibility].presence,
document_collection_id: collection_id, pages: pages)
if doc.save
created << doc
else
rejected << "#{name} (#{doc.errors.full_messages.to_sentence})"
pages.each { _1.purge_later if _1.is_a?(ActiveStorage::Blob) }
end
end
if created.empty?
@aircraft.document_collections.find_by(id: collection_id)&.destroy if @bulk[:new_collection].present?
return bulk_error("Nothing was uploaded: #{rejected.to_sentence}.")
end
log_event("document", "#{helpers.pluralize(created.size, 'document')} uploaded", notes: created.map(&:name).join(", ").truncate(1000))
flash[:alert] = "Not uploaded: #{rejected.to_sentence}." if rejected.any?
redirect_to aircraft_documents_path(@aircraft), notice: "#{helpers.pluralize(created.size, 'document')} uploaded.", status: :see_other
end
def edit; end
def update
@@ -50,6 +96,33 @@ class DocumentsController < ApplicationController
private
MAX_BULK_FILES = 500
def bulk_defaults = { doc_type: "OTHER", visibility: nil, document_collection_id: nil, new_collection: nil }
def bulk_error(message)
@bulk_error = message
render :bulk_new, status: :unprocessable_entity
end
# A blob from a direct upload that hasn't been attached anywhere yet, so a
# signed id lifted from an existing file's URL can't be re-attached here.
def unattached_blob(signed_id)
blob = ActiveStorage::Blob.find_signed(signed_id)
blob if blob && !blob.attachments.exists?
end
def upload_filename(upload)
(upload.respond_to?(:original_filename) ? upload.original_filename : upload.filename).to_s
end
# [collection id or nil, error]: the chosen collection, or a new one.
def bulk_collection
return [ scoped_id(@aircraft.document_collections, @bulk[:document_collection_id]), nil ] if @bulk[:new_collection].blank?
col = @aircraft.document_collections.create(name: @bulk[:new_collection].squish, visibility: @bulk[:visibility].presence || "private")
col.persisted? ? [ col.id, nil ] : [ nil, "Couldn't create the collection: #{col.errors.full_messages.to_sentence}." ]
end
def set_document = @document = @aircraft.documents.find(params[:id])
def document_params
@@ -0,0 +1,207 @@
import { Controller } from "@hotwired/stimulus"
import { DirectUpload } from "@rails/activestorage"
// Bulk document upload. Picked files (or a whole folder) are listed with an
// editable document name each; files sharing a name become one document. On
// submit every file goes straight to storage (a few at a time, with progress),
// then the form is submitted with just the signed ids and names.
//
// Without JavaScript the file field posts the files itself and the server
// names each document after its file.
export default class extends Controller {
static targets = [ "input", "folder", "folderField", "panel", "list", "summary", "group", "groupField", "submit", "status" ]
static values = { url: String, maxBytes: Number }
connect() {
this.items = []
this.ready = false
this.uploading = false
this.inputTarget.removeAttribute("name") // the signed ids are sent instead
if ("webkitdirectory" in this.folderTarget) this.folderFieldTarget.hidden = false
this.beforeUnload = (event) => { if (this.uploading) event.preventDefault() }
window.addEventListener("beforeunload", this.beforeUnload)
}
disconnect() {
window.removeEventListener("beforeunload", this.beforeUnload)
}
add(event) {
for (const file of event.target.files) {
const item = { file, path: file.webkitRelativePath || file.name, state: "pending", progress: 0 }
if (file.size > this.maxBytesValue) Object.assign(item, { state: "error", error: `over ${Math.round(this.maxBytesValue / 1048576)} MB` })
this.items.push(item)
}
event.target.value = ""
this.items.sort((a, b) => a.path.localeCompare(b.path, undefined, { numeric: true, sensitivity: "base" }))
this.items.forEach((item) => { if (!item.custom) item.name = this.defaultName(item) })
this.groupFieldTarget.hidden = !this.items.some((item) => item.path.includes("/"))
this.render()
}
regroup() {
this.items.forEach((item) => { item.custom = false; item.name = this.defaultName(item) })
this.render()
}
clear() {
if (this.uploading) return
this.items = []
this.render()
}
defaultName(item) {
const parts = item.path.split("/")
if (this.groupTarget.checked && parts.length > 1) return parts[parts.length - 2]
return parts[parts.length - 1].replace(/\.[^.]+$/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim() || "Untitled"
}
render() {
this.panelTarget.hidden = this.items.length === 0
this.listTarget.replaceChildren(...this.items.map((item, index) => this.row(item, index)))
this.updateSummary()
}
row(item, index) {
const li = document.createElement("li")
li.className = "flex items-start gap-3 px-3 py-2"
const body = document.createElement("div")
body.className = "min-w-0 flex-1"
const name = document.createElement("input")
name.type = "text"
name.className = "input"
name.value = item.name
name.setAttribute("aria-label", `Document name for ${item.path}`)
name.disabled = this.uploading
name.addEventListener("input", () => { item.name = name.value; item.custom = true; this.updateSummary() })
const meta = document.createElement("p")
meta.className = "mt-1 truncate text-xs text-slate-500"
const bar = document.createElement("div")
bar.className = "mt-1 h-1 overflow-hidden rounded bg-slate-100"
bar.hidden = item.state === "pending" || item.state === "error"
const fill = document.createElement("div")
fill.className = "h-full bg-sky-600 transition-[width]"
bar.append(fill)
body.append(name, meta, bar)
const remove = document.createElement("button")
remove.type = "button"
remove.className = "btn-secondary btn-sm"
remove.textContent = "✕"
remove.setAttribute("aria-label", `Remove ${item.path}`)
remove.disabled = this.uploading
remove.addEventListener("click", () => { this.items.splice(index, 1); this.render() })
li.append(body, remove)
item.update = () => {
const status = { pending: "", uploading: ` · ${item.progress}%`, done: " · uploaded", error: ` · ${item.error}` }[item.state]
meta.textContent = `${item.path} · ${this.size(item.file.size)}${status}`
meta.classList.toggle("text-red-600", item.state === "error")
meta.classList.toggle("text-slate-500", item.state !== "error")
bar.hidden = item.state === "pending" || item.state === "error"
fill.style.width = `${item.state === "done" ? 100 : item.progress}%`
}
item.update()
return li
}
updateSummary() {
const files = this.items.length
const docs = new Set(this.items.map((item) => item.name.trim() || "Untitled")).size
this.summaryTarget.textContent = `${files} ${files === 1 ? "file" : "files"} → ${docs} ${docs === 1 ? "document" : "documents"}`
this.submitTarget.textContent = files ? `Upload ${docs} ${docs === 1 ? "document" : "documents"}` : "Upload"
}
size(bytes) {
if (bytes < 1024) return `${bytes} B`
if (bytes < 1048576) return `${Math.round(bytes / 1024)} KB`
return `${(bytes / 1048576).toFixed(1)} MB`
}
async submit(event) {
if (this.ready) return
event.preventDefault()
if (this.uploading) return
if (this.items.length === 0) return this.say("Choose some files to upload.")
if (this.items.some((item) => item.file.size > this.maxBytesValue)) {
return this.say("Remove the files marked in red first.")
}
this.items.forEach((item) => { if (item.state === "error") item.state = "pending" })
this.setUploading(true)
const queue = this.items.filter((item) => item.state === "pending")
const total = queue.length
let finished = 0
this.say(`Uploading 0 of ${total}…`)
const worker = async () => {
for (let item = queue.shift(); item; item = queue.shift()) {
await this.upload(item)
this.say(`Uploading ${++finished} of ${total}…`)
}
}
await Promise.all([ worker(), worker(), worker() ])
const failed = this.items.filter((item) => item.state === "error").length
if (failed) {
this.setUploading(false)
return this.say(`${failed} ${failed === 1 ? "file" : "files"} didn't upload. Press Upload to retry, or remove ${failed === 1 ? "it" : "them"}.`)
}
this.element.querySelectorAll("input[data-bulk-upload-hidden]").forEach((el) => el.remove())
for (const item of this.items) {
this.hidden("files[]", item.signedId)
this.hidden("names[]", item.name)
}
this.uploading = false
this.ready = true
this.say("Saving…")
// When nothing was left to upload this all ran inside the click's submit
// event, during which the browser ignores requestSubmit, so wait a tick.
setTimeout(() => this.element.requestSubmit())
}
upload(item) {
item.state = "uploading"
item.progress = 0
item.update()
return new Promise((resolve) => {
const delegate = {
directUploadWillStoreFileWithXHR: (xhr) => {
xhr.upload.addEventListener("progress", (e) => {
if (e.lengthComputable) { item.progress = Math.round((e.loaded / e.total) * 100); item.update() }
})
}
}
new DirectUpload(item.file, this.urlValue, delegate).create((error, blob) => {
if (error) {
Object.assign(item, { state: "error", error: /Status: 422/.test(error) ? "not an allowed file type" : "upload failed" })
} else {
Object.assign(item, { state: "done", signedId: blob.signed_id })
}
item.update()
resolve()
})
})
}
hidden(name, value) {
const input = document.createElement("input")
input.type = "hidden"
input.name = name
input.value = value
input.dataset.bulkUploadHidden = ""
this.element.append(input)
}
setUploading(on) {
this.uploading = on
this.submitTarget.disabled = on
this.render()
}
say(message) {
this.statusTarget.textContent = message
}
}
+3
View File
@@ -19,6 +19,9 @@
<% end %>
</div>
<% end %>
<% unless document.persisted? %>
<p class="text-sm text-slate-500">Lots of files? <%= link_to "Bulk upload", new_bulk_aircraft_documents_path(aircraft, collection_id: document.document_collection_id), class: "link" %> makes a document from each one.</p>
<% end %>
<div class="flex gap-2"><%= f.submit "Save Document" %><%= link_to "Cancel", aircraft_documents_path(aircraft), class: "btn-secondary" %></div>
<% end %>
<% if document.persisted? %><div class="mt-4"><%= delete_button "Delete document", aircraft_document_path(aircraft, document) %></div><% end %>
+68
View File
@@ -0,0 +1,68 @@
<% content_for :title, "#{@aircraft.tail_number} Bulk Upload" %>
<%= render "shared/aircraft_header", aircraft: @aircraft %>
<% file_classes = "block w-full text-sm text-slate-600 file:mr-3 file:rounded-md file:border-0 file:bg-slate-100 file:px-3 file:py-1.5 file:text-sm file:font-medium hover:file:bg-slate-200" %>
<%= form_with url: bulk_aircraft_documents_path(@aircraft), multipart: true, class: "card card-body max-w-3xl space-y-4",
data: { controller: "bulk-upload", bulk_upload_url_value: rails_direct_uploads_path,
bulk_upload_max_bytes_value: UploadValidation::MAX_SIZE, action: "submit->bulk-upload#submit" } do %>
<div>
<h2 class="card-title">Bulk upload</h2>
<p class="text-sm text-slate-500">Each file becomes its own document, named after the file. Give files the same name to combine them, in order, as the pages of one document.</p>
</div>
<% if @bulk_error %>
<div class="rounded-md bg-red-50 p-3 text-sm text-red-700 ring-1 ring-red-200" role="alert"><%= @bulk_error %></div>
<% end %>
<div class="grid gap-4 sm:grid-cols-2">
<div>
<%= label_tag :doc_type, "Type", class: "label" %>
<%= select_tag :doc_type, options_for_select(Document::TYPES.invert, @bulk[:doc_type]), class: "input" %>
</div>
<div>
<%= label_tag :visibility, "Share-link visibility", class: "label" %>
<%= select_tag :visibility, options_for_select([ [ "Inherit from collection", "" ] ] + DocumentCollection::VISIBILITIES.invert.to_a, @bulk[:visibility]), class: "input" %>
</div>
<div>
<%= label_tag :document_collection_id, "Collection", class: "label" %>
<%= select_tag :document_collection_id,
options_for_select([ [ "— None —", "" ] ] + @aircraft.document_collections.ordered.map { [ _1.name, _1.id ] }, @bulk[:document_collection_id]), class: "input" %>
</div>
<div>
<%= label_tag :new_collection, "…or a new collection", class: "label" %>
<%= text_field_tag :new_collection, @bulk[:new_collection], class: "input", placeholder: "e.g. 2025 Annual" %>
</div>
</div>
<div class="grid gap-4 sm:grid-cols-2">
<div>
<%= label_tag :files, "Files", class: "label" %>
<%= file_field_tag "files[]", id: "files", multiple: true, class: file_classes,
data: { bulk_upload_target: "input", action: "change->bulk-upload#add" } %>
</div>
<div data-bulk-upload-target="folderField" hidden>
<%= label_tag :folder, "…or a whole folder", class: "label" %>
<input type="file" id="folder" webkitdirectory multiple class="<%= file_classes %>"
data-bulk-upload-target="folder" data-action="change->bulk-upload#add">
</div>
</div>
<p class="hint !mt-1">Images, PDFs or text, up to <%= UploadValidation::MAX_SIZE / 1.megabyte %> MB each and <%= DocumentsController::MAX_BULK_FILES %> at a time.</p>
<div data-bulk-upload-target="panel" hidden class="space-y-2">
<div class="flex flex-wrap items-center justify-between gap-2">
<p class="text-sm font-medium text-slate-700" data-bulk-upload-target="summary"></p>
<div class="flex flex-wrap items-center gap-3">
<label class="flex items-center gap-2 text-sm text-slate-700" data-bulk-upload-target="groupField" hidden>
<input type="checkbox" class="h-4 w-4 rounded border-slate-300" data-bulk-upload-target="group" data-action="bulk-upload#regroup">
One document per folder
</label>
<button type="button" class="btn-secondary btn-sm" data-action="bulk-upload#clear">Clear</button>
</div>
</div>
<ul class="divide-y divide-slate-100 rounded-md ring-1 ring-slate-200" data-bulk-upload-target="list"></ul>
</div>
<div class="flex flex-wrap items-center gap-2">
<button type="submit" class="btn-primary" data-bulk-upload-target="submit">Upload</button>
<%= link_to "Cancel", aircraft_documents_path(@aircraft), class: "btn-secondary" %>
<p class="text-sm text-slate-500" data-bulk-upload-target="status" role="status"></p>
</div>
<% end %>
+2 -1
View File
@@ -1,8 +1,9 @@
<% content_for :title, "#{@aircraft.tail_number} Documents" %>
<%= render "shared/aircraft_header", aircraft: @aircraft %>
<div class="mb-4 flex justify-end gap-2">
<div class="mb-4 flex flex-wrap justify-end gap-2">
<% if owner? %>
<%= link_to "+ Collection", new_aircraft_document_collection_path(@aircraft), class: "btn-secondary" %>
<%= link_to "Bulk Upload", new_bulk_aircraft_documents_path(@aircraft), class: "btn-secondary" %>
<%= link_to "+ Upload Document", new_aircraft_document_path(@aircraft), class: "btn-primary" %>
<% end %>
</div>
+1 -1
View File
@@ -14,7 +14,7 @@ module SimpleAircraftManager
# Please, add to the `ignore` list any other `lib` subdirectories that do
# not contain `.rb` files, or that should not be reloaded or eager loaded.
# Common ones are `templates`, `generators`, or `middleware`, for example.
config.autoload_lib(ignore: %w[assets tasks])
config.autoload_lib(ignore: %w[active_storage assets tasks])
# Configuration for the application, engines, and railties goes here.
#
+1
View File
@@ -8,3 +8,4 @@ pin_all_from "app/javascript/controllers", under: "controllers"
pin "chartkick", to: "chartkick.js"
pin "Chart.bundle", to: "Chart.bundle.js"
pin "next_due"
pin "@rails/activestorage", to: "activestorage.esm.js"
+6
View File
@@ -13,3 +13,9 @@ production:
clear_solid_queue_finished_jobs:
command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
schedule: every hour at minute 12
purge_abandoned_uploads:
command: "ActiveStorage::Blob.unattached.where(created_at: ..2.days.ago).find_each(&:purge_later)"
schedule: every day at 4am
dedupe_storage:
command: "ActiveStorage::Blob.service.try(:sweep)"
schedule: every day at 4:30am
+10 -1
View File
@@ -1,4 +1,8 @@
Rails.application.routes.draw do
# Signed-in users only, with the upload rules checked before any bytes are sent
# (overrides Active Storage's open endpoint at the same path).
post "rails/active_storage/direct_uploads", to: "direct_uploads#create"
resource :session
resources :passwords, param: :token
resource :setup, only: %i[new create], controller: "setup"
@@ -45,7 +49,12 @@ Rails.application.routes.draw do
end
resources :logbook_imports, path: "logbook/import", only: %i[new create show]
resources :logbook_entries
resources :documents
resources :documents do
collection do
get :bulk, action: :bulk_new, as: :new_bulk
post :bulk, action: :bulk_create
end
end
resources :document_collections, except: %i[index show]
resources :inspections, only: :index
resources :inspection_types, except: %i[index show]
+2 -2
View File
@@ -1,9 +1,9 @@
test:
service: Disk
service: DedupDisk
root: <%= Rails.root.join("tmp/storage") %>
local:
service: Disk
service: DedupDisk
root: <%= Rails.root.join("storage") %>
# Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
@@ -0,0 +1,107 @@
require "active_storage/service/disk_service"
module ActiveStorage
# Disk storage that keeps one copy of each distinct file. A stored file's
# bytes live once under .objects/ab/cd/<sha256>, and every blob (and variant)
# with those bytes is a hard link to it, so uploading the same scan twice
# costs no extra space. Blobs, filenames and URLs are unchanged.
#
# Linked files must never be written in place, so an upload over an existing
# key first unlinks it. If the filesystem can't hard-link (some network
# shares), files are simply kept as plain copies.
#
# #sweep links in files stored before this service (or that failed to link)
# and removes objects no blob uses any more; it runs nightly.
class Service::DedupDiskService < Service::DiskService
OBJECTS_DIR = ".objects".freeze
def upload(key, io, checksum: nil, **)
release(path_for(key))
super
adopt(path_for(key))
end
def compose(source_keys, destination_key, **)
release(path_for(destination_key))
super
adopt(path_for(destination_key))
end
def delete(key)
instrument :delete, key: key do
release(path_for(key))
end
end
def delete_prefixed(prefix)
instrument :delete_prefixed, prefix: prefix do
prefix_path = path_for(prefix)
prefix_path += "/" if prefix.end_with?("/")
Dir.glob("#{escape_glob_metacharacters(prefix_path)}*").each do |path|
files = File.directory?(path) ? Dir.glob("#{escape_glob_metacharacters(path)}/**/*") : [ path ]
files.each { release(_1) if File.file?(_1) }
FileUtils.rm_rf(path)
end
end
end
# Returns { linked:, removed:, saved_bytes: }.
def sweep
linked = Dir.glob(File.join(escape_glob_metacharacters(root), "??", "??", "**", "*")).count do |path|
File.file?(path) && !File.symlink?(path) && adopt(path) == :shared
end
removed = 0
saved = 0
Dir.glob(File.join(escape_glob_metacharacters(objects_root), "**", "*")).each do |object|
next unless File.file?(object)
stat = File.stat(object)
if stat.nlink == 1
File.delete(object)
removed += 1
else
saved += stat.size * (stat.nlink - 2)
end
end
{ linked: linked, removed: removed, saved_bytes: saved }
end
private
def objects_root = File.join(root, OBJECTS_DIR)
def object_path(digest) = File.join(objects_root, digest[0..1], digest[2..3], digest)
# Points a freshly written file at the stored copy of its bytes (or makes it
# the stored copy). Returns :stored, :shared, or nil if left alone.
def adopt(path)
return unless File.stat(path).nlink == 1
object = object_path(Digest::SHA256.file(path).hexdigest)
FileUtils.mkdir_p(File.dirname(object))
begin
File.link(path, object)
:stored
rescue Errno::EEXIST
return unless File.size(object) == File.size(path)
temp = "#{path}.#{SecureRandom.hex(6)}.link"
File.link(object, temp)
File.rename(temp, path) # atomic swap: readers see one file or the other
:shared
end
rescue SystemCallError => e
Rails.logger.warn("[dedup storage] kept a plain copy of #{path}: #{e.class}: #{e.message}")
nil
end
# Unlinks a blob's file, and its stored object if nothing else uses it.
def release(path)
stat = File.stat(path)
if stat.nlink == 2
object = object_path(Digest::SHA256.file(path).hexdigest)
File.delete(object) if File.exist?(object) && File.stat(object).ino == stat.ino
end
File.delete(path)
rescue Errno::ENOENT
# already gone
end
end
end
+10
View File
@@ -0,0 +1,10 @@
namespace :storage do
desc "Store each distinct uploaded file once (hard links) and drop unused copies"
task dedupe: :environment do
service = ActiveStorage::Blob.service
abort "The #{service.class.name.demodulize} service doesn't deduplicate." unless service.respond_to?(:sweep)
result = service.sweep
puts "Linked #{result[:linked]} duplicate files, removed #{result[:removed]} unused copies; " \
"#{ActiveSupport::NumberHelper.number_to_human_size(result[:saved_bytes])} saved by sharing."
end
end
@@ -0,0 +1,133 @@
require "test_helper"
class DocumentBulkUploadTest < ActionDispatch::IntegrationTest
setup do
@owner = create_user("owner@example.com")
@pilot = create_user("pilot@example.com")
@a = create_aircraft(owner: @owner, pilot: @pilot)
end
def upload(name, body = "page", type = "image/jpeg")
Rack::Test::UploadedFile.new(StringIO.new(body), type, original_filename: name)
end
def direct_blob(name, type = "image/jpeg")
ActiveStorage::Blob.create_and_upload!(io: StringIO.new("bytes"), filename: name, content_type: type)
end
test "owner sees the bulk page; pilots and outsiders can't use it" do
sign_in @owner
get aircraft_documents_path(@a)
assert_select "a[href='#{new_bulk_aircraft_documents_path(@a)}']", text: "Bulk Upload"
get new_bulk_aircraft_documents_path(@a)
assert_response :success
assert_select "form[data-controller='bulk-upload'][data-bulk-upload-url-value='/rails/active_storage/direct_uploads']"
assert_select "input[type=file][name='files[]'][multiple]"
assert_select "input[type=file][webkitdirectory]"
sign_in @pilot
post bulk_aircraft_documents_path(@a), params: { files: [ upload("a.jpg") ] }
assert_redirected_to aircraft_path(@a)
assert_equal 0, Document.count
sign_in create_user
get new_bulk_aircraft_documents_path(@a)
assert_response :not_found
end
test "plain uploads (no JavaScript) become one document per file, named after the file" do
sign_in @owner
col = @a.document_collections.create!(name: "Annual", visibility: "private")
post bulk_aircraft_documents_path(@a), params: {
files: [ upload("weight_and_balance.pdf", "%PDF", "application/pdf"), upload("Form 337.jpg") ],
doc_type: "ALTER", visibility: "maintenance", document_collection_id: col.id
}
assert_redirected_to aircraft_documents_path(@a)
assert_equal "2 documents uploaded.", flash[:notice]
docs = @a.documents.order(:name)
assert_equal [ "Form 337", "weight and balance" ], docs.map(&:name)
assert docs.all? { _1.doc_type == "ALTER" && _1.visibility == "maintenance" && _1.document_collection == col && _1.pages.size == 1 }
assert_equal "2 documents uploaded", @a.events.last.event_name
end
test "direct uploads: same name combines pages in order; a new collection is made" do
sign_in @owner
blobs = %w[p1.jpg p2.jpg receipt.jpg].map { direct_blob(_1) }
post bulk_aircraft_documents_path(@a), params: {
files: blobs.map(&:signed_id), names: [ "Engine log 1", "Engine log 1", "Prop overhaul receipt" ],
new_collection: "Logbooks", doc_type: "LOG"
}
assert_redirected_to aircraft_documents_path(@a)
col = @a.document_collections.find_by!(name: "Logbooks")
assert_equal "private", col.visibility
log = col.documents.find_by!(name: "Engine log 1")
assert_equal %w[p1.jpg p2.jpg], log.pages.map { _1.filename.to_s }
assert_nil log.visibility
assert_equal 1, col.documents.find_by!(name: "Prop overhaul receipt").pages.size
end
test "already-attached blobs and bad files are refused; partial success is reported" do
sign_in @owner
other = @a.documents.create!(name: "Existing", doc_type: "OTHER", pages: [ upload("x.jpg") ])
good = direct_blob("good.jpg")
bad = direct_blob("tool.exe", "application/x-msdownload")
post bulk_aircraft_documents_path(@a), params: { files: [ other.pages.first.blob.signed_id, good.signed_id, bad.signed_id ] }
assert_redirected_to aircraft_documents_path(@a)
assert_equal "1 document uploaded.", flash[:notice]
assert_match(/file 1 \(upload expired/, flash[:alert])
assert_match(/tool \(Pages tool.exe is not an allowed file type\)/, flash[:alert])
assert_equal [ "Existing", "good" ], @a.documents.order(:name).map(&:name)
assert_equal 1, other.reload.pages.size
end
test "nothing valid re-renders the form and drops the new collection" do
sign_in @owner
post bulk_aircraft_documents_path(@a), params: { new_collection: "Junk" }
assert_response :unprocessable_entity
assert_includes response.body, "Choose some files to upload."
post bulk_aircraft_documents_path(@a), params: { files: [ upload("a.exe", "MZ", "application/x-msdownload") ], new_collection: "Junk" }
assert_response :unprocessable_entity
assert_includes response.body, "Nothing was uploaded"
assert_select "input[name=new_collection][value=Junk]"
assert_equal 0, @a.document_collections.count
end
test "direct-upload endpoint needs a session and checks type and size first" do
blob = ->(filename, type, size = 10) { { blob: { filename: filename, byte_size: size, checksum: "Q2hlY2tzdW0=", content_type: type } } }
post "/rails/active_storage/direct_uploads", params: blob.("a.jpg", "image/jpeg"), as: :json
assert_redirected_to new_session_path
assert_equal 0, ActiveStorage::Blob.count
sign_in @owner
post "/rails/active_storage/direct_uploads", params: blob.("a.exe", "application/x-msdownload"), as: :json
assert_response :unprocessable_entity
assert_match "isn't an allowed file type", response.parsed_body["error"]
post "/rails/active_storage/direct_uploads", params: blob.("big.pdf", "application/pdf", 51.megabytes), as: :json
assert_response :unprocessable_entity
post "/rails/active_storage/direct_uploads", params: blob.("scan.png", ""), as: :json
assert_response :success
assert_equal "image/png", ActiveStorage::Blob.last.content_type
assert response.parsed_body["signed_id"].present?
end
test "the same file direct-uploaded twice is stored once on disk" do
sign_in @owner
body = "%PDF-1.4 identical scan"
ids = 2.times.map do |i|
post "/rails/active_storage/direct_uploads", as: :json, params: { blob: {
filename: "scan#{i}.pdf", byte_size: body.bytesize, checksum: OpenSSL::Digest::MD5.base64digest(body), content_type: "application/pdf" } }
signed_id, upload = response.parsed_body.values_at("signed_id", "direct_upload")
put URI(upload["url"]).request_uri, params: body, headers: upload["headers"]
assert_response :no_content
signed_id
end
post bulk_aircraft_documents_path(@a), params: { files: ids }
assert_redirected_to aircraft_documents_path(@a)
paths = @a.documents.order(:name).map { ActiveStorage::Blob.service.path_for(_1.pages.first.key) }
assert_equal 2, paths.size
assert_equal File.stat(paths[0]).ino, File.stat(paths[1]).ino
assert_equal %w[scan0 scan1], @a.documents.order(:name).map(&:name)
end
end
+87
View File
@@ -0,0 +1,87 @@
require "test_helper"
require "active_storage/service/dedup_disk_service"
class DedupDiskServiceTest < ActiveSupport::TestCase
include ActiveJob::TestHelper
setup do
@root = Dir.mktmpdir
@service = ActiveStorage::Service::DedupDiskService.new(root: @root)
end
teardown { FileUtils.rm_rf(@root) }
def put(key, body) = @service.upload(key, StringIO.new(body), checksum: OpenSSL::Digest::MD5.base64digest(body))
def path(key) = @service.path_for(key)
def objects = Dir.glob(File.join(@root, ".objects/**/*")).select { File.file?(_1) }
test "identical files share one stored copy; different files don't" do
put "aaaa1111", "same scan"
put "bbbb2222", "same scan"
put "cccc3333", "other scan"
assert_equal File.stat(path("aaaa1111")).ino, File.stat(path("bbbb2222")).ino
refute_equal File.stat(path("aaaa1111")).ino, File.stat(path("cccc3333")).ino
assert_equal 2, objects.size
assert_equal "same scan", @service.download("bbbb2222")
end
test "deleting one copy keeps the other; deleting the last frees the object" do
put "aaaa1111", "same scan"
put "bbbb2222", "same scan"
@service.delete("aaaa1111")
assert_equal "same scan", @service.download("bbbb2222")
assert_equal 1, objects.size
@service.delete("bbbb2222")
assert_empty objects
refute @service.exist?("bbbb2222")
end
test "re-uploading a shared key never changes the other blob" do
put "aaaa1111", "same scan"
put "bbbb2222", "same scan"
put "aaaa1111", "replaced"
assert_equal "replaced", @service.download("aaaa1111")
assert_equal "same scan", @service.download("bbbb2222")
end
test "a checksum mismatch stores nothing" do
assert_raises(ActiveStorage::IntegrityError) { @service.upload("aaaa1111", StringIO.new("bytes"), checksum: "bm9wZQ==") }
refute @service.exist?("aaaa1111")
assert_empty objects
end
test "variants under a prefix are released too" do
put "variants/aaaa1111/thumb", "same scan"
put "bbbb2222", "same scan"
@service.delete_prefixed("variants/aaaa1111/")
refute @service.exist?("variants/aaaa1111/thumb")
assert_equal "same scan", @service.download("bbbb2222")
assert_equal 1, objects.size
end
test "sweep links files stored before dedup and clears unused objects" do
plain = ActiveStorage::Service::DiskService.new(root: @root)
plain.upload("aaaa1111", StringIO.new("old scan"))
plain.upload("bbbb2222", StringIO.new("old scan"))
put "cccc3333", "gone soon"
File.delete(path("cccc3333")) # e.g. a crash between unlinks
assert_equal({ linked: 1, removed: 1, saved_bytes: 8 }, @service.sweep)
assert_equal File.stat(path("aaaa1111")).ino, File.stat(path("bbbb2222")).ino
assert_equal({ linked: 0, removed: 0, saved_bytes: 8 }, @service.sweep)
end
test "attachments through Active Storage share storage but keep their own filenames" do
a = create_aircraft
one = a.documents.create!(name: "Receipt", doc_type: "OTHER", pages: [ { io: StringIO.new("%PDF same"), filename: "receipt.pdf", content_type: "application/pdf" } ])
two = a.documents.create!(name: "Invoice", doc_type: "OTHER", pages: [ { io: StringIO.new("%PDF same"), filename: "invoice.pdf", content_type: "application/pdf" } ])
service = ActiveStorage::Blob.service
assert_kind_of ActiveStorage::Service::DedupDiskService, service
keys = [ one, two ].map { _1.pages.first.blob.key }
assert_equal File.stat(service.path_for(keys[0])).ino, File.stat(service.path_for(keys[1])).ino
assert_equal %w[receipt.pdf invoice.pdf], [ one, two ].map { _1.pages.first.filename.to_s }
perform_enqueued_jobs { one.destroy! }
assert_equal 1, ActiveStorage::Blob.count
assert_equal "%PDF same", two.pages.first.download
end
end