Bulk document upload and deduplicated file storage

- Bulk Upload page: pick many files or a folder; each file becomes a
  document named after it, and files sharing a name become one multi-page
  document. Files go up as direct uploads with per-file progress.
- Direct uploads require a signed-in user and check type/size up front;
  only fresh, unattached blobs can be attached; abandoned uploads are
  purged nightly.
- DedupDisk storage service: each distinct file is stored once under its
  SHA-256 and blobs hard-link to it; nightly sweep (and storage:dedupe)
  links pre-existing duplicates and drops unused objects.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rickyandClaude Opus 5.5 committed 2026-11-07 08:20:22 -05:00
1 parent 230a569b77
commit 2bfcb845ca
15 files changed
+735 -5

No files matched your search

@@ -0,0 +1,133 @@
require "test_helper"
class DocumentBulkUploadTest < ActionDispatch::IntegrationTest
setup do
@owner = create_user("owner@example.com")
@pilot = create_user("pilot@example.com")
@a = create_aircraft(owner: @owner, pilot: @pilot)
end
def upload(name, body = "page", type = "image/jpeg")
Rack::Test::UploadedFile.new(StringIO.new(body), type, original_filename: name)
end
def direct_blob(name, type = "image/jpeg")
ActiveStorage::Blob.create_and_upload!(io: StringIO.new("bytes"), filename: name, content_type: type)
end
test "owner sees the bulk page; pilots and outsiders can't use it" do
sign_in @owner
get aircraft_documents_path(@a)
assert_select "a[href='#{new_bulk_aircraft_documents_path(@a)}']", text: "Bulk Upload"
get new_bulk_aircraft_documents_path(@a)
assert_response :success
assert_select "form[data-controller='bulk-upload'][data-bulk-upload-url-value='/rails/active_storage/direct_uploads']"
assert_select "input[type=file][name='files[]'][multiple]"
assert_select "input[type=file][webkitdirectory]"
sign_in @pilot
post bulk_aircraft_documents_path(@a), params: { files: [ upload("a.jpg") ] }
assert_redirected_to aircraft_path(@a)
assert_equal 0, Document.count
sign_in create_user
get new_bulk_aircraft_documents_path(@a)
assert_response :not_found
end
test "plain uploads (no JavaScript) become one document per file, named after the file" do
sign_in @owner
col = @a.document_collections.create!(name: "Annual", visibility: "private")
post bulk_aircraft_documents_path(@a), params: {
files: [ upload("weight_and_balance.pdf", "%PDF", "application/pdf"), upload("Form 337.jpg") ],
doc_type: "ALTER", visibility: "maintenance", document_collection_id: col.id
}
assert_redirected_to aircraft_documents_path(@a)
assert_equal "2 documents uploaded.", flash[:notice]
docs = @a.documents.order(:name)
assert_equal [ "Form 337", "weight and balance" ], docs.map(&:name)
assert docs.all? { _1.doc_type == "ALTER" && _1.visibility == "maintenance" && _1.document_collection == col && _1.pages.size == 1 }
assert_equal "2 documents uploaded", @a.events.last.event_name
end
test "direct uploads: same name combines pages in order; a new collection is made" do
sign_in @owner
blobs = %w[p1.jpg p2.jpg receipt.jpg].map { direct_blob(_1) }
post bulk_aircraft_documents_path(@a), params: {
files: blobs.map(&:signed_id), names: [ "Engine log 1", "Engine log 1", "Prop overhaul receipt" ],
new_collection: "Logbooks", doc_type: "LOG"
}
assert_redirected_to aircraft_documents_path(@a)
col = @a.document_collections.find_by!(name: "Logbooks")
assert_equal "private", col.visibility
log = col.documents.find_by!(name: "Engine log 1")
assert_equal %w[p1.jpg p2.jpg], log.pages.map { _1.filename.to_s }
assert_nil log.visibility
assert_equal 1, col.documents.find_by!(name: "Prop overhaul receipt").pages.size
end
test "already-attached blobs and bad files are refused; partial success is reported" do
sign_in @owner
other = @a.documents.create!(name: "Existing", doc_type: "OTHER", pages: [ upload("x.jpg") ])
good = direct_blob("good.jpg")
bad = direct_blob("tool.exe", "application/x-msdownload")
post bulk_aircraft_documents_path(@a), params: { files: [ other.pages.first.blob.signed_id, good.signed_id, bad.signed_id ] }
assert_redirected_to aircraft_documents_path(@a)
assert_equal "1 document uploaded.", flash[:notice]
assert_match(/file 1 \(upload expired/, flash[:alert])
assert_match(/tool \(Pages tool.exe is not an allowed file type\)/, flash[:alert])
assert_equal [ "Existing", "good" ], @a.documents.order(:name).map(&:name)
assert_equal 1, other.reload.pages.size
end
test "nothing valid re-renders the form and drops the new collection" do
sign_in @owner
post bulk_aircraft_documents_path(@a), params: { new_collection: "Junk" }
assert_response :unprocessable_entity
assert_includes response.body, "Choose some files to upload."
post bulk_aircraft_documents_path(@a), params: { files: [ upload("a.exe", "MZ", "application/x-msdownload") ], new_collection: "Junk" }
assert_response :unprocessable_entity
assert_includes response.body, "Nothing was uploaded"
assert_select "input[name=new_collection][value=Junk]"
assert_equal 0, @a.document_collections.count
end
test "direct-upload endpoint needs a session and checks type and size first" do
blob = ->(filename, type, size = 10) { { blob: { filename: filename, byte_size: size, checksum: "Q2hlY2tzdW0=", content_type: type } } }
post "/rails/active_storage/direct_uploads", params: blob.("a.jpg", "image/jpeg"), as: :json
assert_redirected_to new_session_path
assert_equal 0, ActiveStorage::Blob.count
sign_in @owner
post "/rails/active_storage/direct_uploads", params: blob.("a.exe", "application/x-msdownload"), as: :json
assert_response :unprocessable_entity
assert_match "isn't an allowed file type", response.parsed_body["error"]
post "/rails/active_storage/direct_uploads", params: blob.("big.pdf", "application/pdf", 51.megabytes), as: :json
assert_response :unprocessable_entity
post "/rails/active_storage/direct_uploads", params: blob.("scan.png", ""), as: :json
assert_response :success
assert_equal "image/png", ActiveStorage::Blob.last.content_type
assert response.parsed_body["signed_id"].present?
end
test "the same file direct-uploaded twice is stored once on disk" do
sign_in @owner
body = "%PDF-1.4 identical scan"
ids = 2.times.map do |i|
post "/rails/active_storage/direct_uploads", as: :json, params: { blob: {
filename: "scan#{i}.pdf", byte_size: body.bytesize, checksum: OpenSSL::Digest::MD5.base64digest(body), content_type: "application/pdf" } }
signed_id, upload = response.parsed_body.values_at("signed_id", "direct_upload")
put URI(upload["url"]).request_uri, params: body, headers: upload["headers"]
assert_response :no_content
signed_id
end
post bulk_aircraft_documents_path(@a), params: { files: ids }
assert_redirected_to aircraft_documents_path(@a)
paths = @a.documents.order(:name).map { ActiveStorage::Blob.service.path_for(_1.pages.first.key) }
assert_equal 2, paths.size
assert_equal File.stat(paths[0]).ino, File.stat(paths[1]).ino
assert_equal %w[scan0 scan1], @a.documents.order(:name).map(&:name)
end
end
+87
View File
@@ -0,0 +1,87 @@
require "test_helper"
require "active_storage/service/dedup_disk_service"
class DedupDiskServiceTest < ActiveSupport::TestCase
include ActiveJob::TestHelper
setup do
@root = Dir.mktmpdir
@service = ActiveStorage::Service::DedupDiskService.new(root: @root)
end
teardown { FileUtils.rm_rf(@root) }
def put(key, body) = @service.upload(key, StringIO.new(body), checksum: OpenSSL::Digest::MD5.base64digest(body))
def path(key) = @service.path_for(key)
def objects = Dir.glob(File.join(@root, ".objects/**/*")).select { File.file?(_1) }
test "identical files share one stored copy; different files don't" do
put "aaaa1111", "same scan"
put "bbbb2222", "same scan"
put "cccc3333", "other scan"
assert_equal File.stat(path("aaaa1111")).ino, File.stat(path("bbbb2222")).ino
refute_equal File.stat(path("aaaa1111")).ino, File.stat(path("cccc3333")).ino
assert_equal 2, objects.size
assert_equal "same scan", @service.download("bbbb2222")
end
test "deleting one copy keeps the other; deleting the last frees the object" do
put "aaaa1111", "same scan"
put "bbbb2222", "same scan"
@service.delete("aaaa1111")
assert_equal "same scan", @service.download("bbbb2222")
assert_equal 1, objects.size
@service.delete("bbbb2222")
assert_empty objects
refute @service.exist?("bbbb2222")
end
test "re-uploading a shared key never changes the other blob" do
put "aaaa1111", "same scan"
put "bbbb2222", "same scan"
put "aaaa1111", "replaced"
assert_equal "replaced", @service.download("aaaa1111")
assert_equal "same scan", @service.download("bbbb2222")
end
test "a checksum mismatch stores nothing" do
assert_raises(ActiveStorage::IntegrityError) { @service.upload("aaaa1111", StringIO.new("bytes"), checksum: "bm9wZQ==") }
refute @service.exist?("aaaa1111")
assert_empty objects
end
test "variants under a prefix are released too" do
put "variants/aaaa1111/thumb", "same scan"
put "bbbb2222", "same scan"
@service.delete_prefixed("variants/aaaa1111/")
refute @service.exist?("variants/aaaa1111/thumb")
assert_equal "same scan", @service.download("bbbb2222")
assert_equal 1, objects.size
end
test "sweep links files stored before dedup and clears unused objects" do
plain = ActiveStorage::Service::DiskService.new(root: @root)
plain.upload("aaaa1111", StringIO.new("old scan"))
plain.upload("bbbb2222", StringIO.new("old scan"))
put "cccc3333", "gone soon"
File.delete(path("cccc3333")) # e.g. a crash between unlinks
assert_equal({ linked: 1, removed: 1, saved_bytes: 8 }, @service.sweep)
assert_equal File.stat(path("aaaa1111")).ino, File.stat(path("bbbb2222")).ino
assert_equal({ linked: 0, removed: 0, saved_bytes: 8 }, @service.sweep)
end
test "attachments through Active Storage share storage but keep their own filenames" do
a = create_aircraft
one = a.documents.create!(name: "Receipt", doc_type: "OTHER", pages: [ { io: StringIO.new("%PDF same"), filename: "receipt.pdf", content_type: "application/pdf" } ])
two = a.documents.create!(name: "Invoice", doc_type: "OTHER", pages: [ { io: StringIO.new("%PDF same"), filename: "invoice.pdf", content_type: "application/pdf" } ])
service = ActiveStorage::Blob.service
assert_kind_of ActiveStorage::Service::DedupDiskService, service
keys = [ one, two ].map { _1.pages.first.blob.key }
assert_equal File.stat(service.path_for(keys[0])).ino, File.stat(service.path_for(keys[1])).ino
assert_equal %w[receipt.pdf invoice.pdf], [ one, two ].map { _1.pages.first.filename.to_s }
perform_enqueued_jobs { one.destroy! }
assert_equal 1, ActiveStorage::Blob.count
assert_equal "%PDF same", two.pages.first.download
end
end