Bulk document upload and deduplicated file storage

- Bulk Upload page: pick many files or a folder; each file becomes a
  document named after it, and files sharing a name become one multi-page
  document. Files go up as direct uploads with per-file progress.
- Direct uploads require a signed-in user and check type/size up front;
  only fresh, unattached blobs can be attached; abandoned uploads are
  purged nightly.
- DedupDisk storage service: each distinct file is stored once under its
  SHA-256 and blobs hard-link to it; nightly sweep (and storage:dedupe)
  links pre-existing duplicates and drops unused objects.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rickyandClaude Opus 5.5 committed 2026-11-07 08:20:22 -05:00
1 parent 230a569b77
commit 2bfcb845ca
15 files changed
+735 -5

No files matched your search

@@ -0,0 +1,25 @@
# Direct uploads (the browser sends each file straight to storage, then submits
# only its signed id). Active Storage's own endpoint lets anyone create blobs;
# this one requires a signed-in user and applies the upload rules up front.
class DirectUploadsController < ActiveStorage::DirectUploadsController
include Authentication
before_action :check_upload, only: :create
private
def blob_args
@blob_args ||= super.tap do |args|
args[:content_type] = Marcel::MimeType.for(name: args[:filename].to_s) if args[:content_type].blank?
end
end
def check_upload
error =
if !UploadValidation::ALLOWED_CONTENT_TYPES.include?(blob_args[:content_type])
"#{blob_args[:filename]} isn't an allowed file type"
elsif blob_args[:byte_size].to_i > UploadValidation::MAX_SIZE
"#{blob_args[:filename]} is over #{UploadValidation::MAX_SIZE / 1.megabyte} MB"
end
render json: { error: error }, status: :unprocessable_entity if error
end
end
+73
View File
@@ -26,6 +26,52 @@ class DocumentsController < ApplicationController
end
end
def bulk_new
@bulk = bulk_defaults.merge(document_collection_id: params[:collection_id])
end
# Each file becomes a document named after it; files given the same name are
# combined, in the order sent, as the pages of one document. Files arrive as
# direct-upload signed ids (or as plain uploads when JavaScript is off).
def bulk_create
@bulk = bulk_defaults.merge(params.slice(:doc_type, :visibility, :document_collection_id, :new_collection).permit!.to_h.symbolize_keys)
files = Array(params[:files]).compact_blank
return bulk_error("Choose some files to upload.") if files.empty?
return bulk_error("Upload at most #{MAX_BULK_FILES} files at a time.") if files.size > MAX_BULK_FILES
names = Array(params[:names])
rejected = []
groups = {}
files.each_with_index do |file, i|
upload = file.is_a?(String) ? unattached_blob(file) : file
next rejected << "file #{i + 1} (upload expired, try again)" unless upload
name = names[i].to_s.squish.presence || File.basename(upload_filename(upload), ".*").tr("_", " ").squish.presence || "Untitled"
(groups[name.first(255)] ||= []) << upload
end
collection_id, error = bulk_collection
return bulk_error(error) if error
created = []
groups.each do |name, pages|
doc = @aircraft.documents.new(name: name, doc_type: @bulk[:doc_type], visibility: @bulk[:visibility].presence,
document_collection_id: collection_id, pages: pages)
if doc.save
created << doc
else
rejected << "#{name} (#{doc.errors.full_messages.to_sentence})"
pages.each { _1.purge_later if _1.is_a?(ActiveStorage::Blob) }
end
end
if created.empty?
@aircraft.document_collections.find_by(id: collection_id)&.destroy if @bulk[:new_collection].present?
return bulk_error("Nothing was uploaded: #{rejected.to_sentence}.")
end
log_event("document", "#{helpers.pluralize(created.size, 'document')} uploaded", notes: created.map(&:name).join(", ").truncate(1000))
flash[:alert] = "Not uploaded: #{rejected.to_sentence}." if rejected.any?
redirect_to aircraft_documents_path(@aircraft), notice: "#{helpers.pluralize(created.size, 'document')} uploaded.", status: :see_other
end
def edit; end
def update
@@ -50,6 +96,33 @@ class DocumentsController < ApplicationController
private
MAX_BULK_FILES = 500
def bulk_defaults = { doc_type: "OTHER", visibility: nil, document_collection_id: nil, new_collection: nil }
def bulk_error(message)
@bulk_error = message
render :bulk_new, status: :unprocessable_entity
end
# A blob from a direct upload that hasn't been attached anywhere yet, so a
# signed id lifted from an existing file's URL can't be re-attached here.
def unattached_blob(signed_id)
blob = ActiveStorage::Blob.find_signed(signed_id)
blob if blob && !blob.attachments.exists?
end
def upload_filename(upload)
(upload.respond_to?(:original_filename) ? upload.original_filename : upload.filename).to_s
end
# [collection id or nil, error]: the chosen collection, or a new one.
def bulk_collection
return [ scoped_id(@aircraft.document_collections, @bulk[:document_collection_id]), nil ] if @bulk[:new_collection].blank?
col = @aircraft.document_collections.create(name: @bulk[:new_collection].squish, visibility: @bulk[:visibility].presence || "private")
col.persisted? ? [ col.id, nil ] : [ nil, "Couldn't create the collection: #{col.errors.full_messages.to_sentence}." ]
end
def set_document = @document = @aircraft.documents.find(params[:id])
def document_params
@@ -0,0 +1,207 @@
import { Controller } from "@hotwired/stimulus"
import { DirectUpload } from "@rails/activestorage"
// Bulk document upload. Picked files (or a whole folder) are listed with an
// editable document name each; files sharing a name become one document. On
// submit every file goes straight to storage (a few at a time, with progress),
// then the form is submitted with just the signed ids and names.
//
// Without JavaScript the file field posts the files itself and the server
// names each document after its file.
export default class extends Controller {
static targets = [ "input", "folder", "folderField", "panel", "list", "summary", "group", "groupField", "submit", "status" ]
static values = { url: String, maxBytes: Number }
connect() {
this.items = []
this.ready = false
this.uploading = false
this.inputTarget.removeAttribute("name") // the signed ids are sent instead
if ("webkitdirectory" in this.folderTarget) this.folderFieldTarget.hidden = false
this.beforeUnload = (event) => { if (this.uploading) event.preventDefault() }
window.addEventListener("beforeunload", this.beforeUnload)
}
disconnect() {
window.removeEventListener("beforeunload", this.beforeUnload)
}
add(event) {
for (const file of event.target.files) {
const item = { file, path: file.webkitRelativePath || file.name, state: "pending", progress: 0 }
if (file.size > this.maxBytesValue) Object.assign(item, { state: "error", error: `over ${Math.round(this.maxBytesValue / 1048576)} MB` })
this.items.push(item)
}
event.target.value = ""
this.items.sort((a, b) => a.path.localeCompare(b.path, undefined, { numeric: true, sensitivity: "base" }))
this.items.forEach((item) => { if (!item.custom) item.name = this.defaultName(item) })
this.groupFieldTarget.hidden = !this.items.some((item) => item.path.includes("/"))
this.render()
}
regroup() {
this.items.forEach((item) => { item.custom = false; item.name = this.defaultName(item) })
this.render()
}
clear() {
if (this.uploading) return
this.items = []
this.render()
}
defaultName(item) {
const parts = item.path.split("/")
if (this.groupTarget.checked && parts.length > 1) return parts[parts.length - 2]
return parts[parts.length - 1].replace(/\.[^.]+$/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim() || "Untitled"
}
render() {
this.panelTarget.hidden = this.items.length === 0
this.listTarget.replaceChildren(...this.items.map((item, index) => this.row(item, index)))
this.updateSummary()
}
row(item, index) {
const li = document.createElement("li")
li.className = "flex items-start gap-3 px-3 py-2"
const body = document.createElement("div")
body.className = "min-w-0 flex-1"
const name = document.createElement("input")
name.type = "text"
name.className = "input"
name.value = item.name
name.setAttribute("aria-label", `Document name for ${item.path}`)
name.disabled = this.uploading
name.addEventListener("input", () => { item.name = name.value; item.custom = true; this.updateSummary() })
const meta = document.createElement("p")
meta.className = "mt-1 truncate text-xs text-slate-500"
const bar = document.createElement("div")
bar.className = "mt-1 h-1 overflow-hidden rounded bg-slate-100"
bar.hidden = item.state === "pending" || item.state === "error"
const fill = document.createElement("div")
fill.className = "h-full bg-sky-600 transition-[width]"
bar.append(fill)
body.append(name, meta, bar)
const remove = document.createElement("button")
remove.type = "button"
remove.className = "btn-secondary btn-sm"
remove.textContent = "✕"
remove.setAttribute("aria-label", `Remove ${item.path}`)
remove.disabled = this.uploading
remove.addEventListener("click", () => { this.items.splice(index, 1); this.render() })
li.append(body, remove)
item.update = () => {
const status = { pending: "", uploading: ` · ${item.progress}%`, done: " · uploaded", error: ` · ${item.error}` }[item.state]
meta.textContent = `${item.path} · ${this.size(item.file.size)}${status}`
meta.classList.toggle("text-red-600", item.state === "error")
meta.classList.toggle("text-slate-500", item.state !== "error")
bar.hidden = item.state === "pending" || item.state === "error"
fill.style.width = `${item.state === "done" ? 100 : item.progress}%`
}
item.update()
return li
}
updateSummary() {
const files = this.items.length
const docs = new Set(this.items.map((item) => item.name.trim() || "Untitled")).size
this.summaryTarget.textContent = `${files} ${files === 1 ? "file" : "files"} → ${docs} ${docs === 1 ? "document" : "documents"}`
this.submitTarget.textContent = files ? `Upload ${docs} ${docs === 1 ? "document" : "documents"}` : "Upload"
}
size(bytes) {
if (bytes < 1024) return `${bytes} B`
if (bytes < 1048576) return `${Math.round(bytes / 1024)} KB`
return `${(bytes / 1048576).toFixed(1)} MB`
}
async submit(event) {
if (this.ready) return
event.preventDefault()
if (this.uploading) return
if (this.items.length === 0) return this.say("Choose some files to upload.")
if (this.items.some((item) => item.file.size > this.maxBytesValue)) {
return this.say("Remove the files marked in red first.")
}
this.items.forEach((item) => { if (item.state === "error") item.state = "pending" })
this.setUploading(true)
const queue = this.items.filter((item) => item.state === "pending")
const total = queue.length
let finished = 0
this.say(`Uploading 0 of ${total}…`)
const worker = async () => {
for (let item = queue.shift(); item; item = queue.shift()) {
await this.upload(item)
this.say(`Uploading ${++finished} of ${total}…`)
}
}
await Promise.all([ worker(), worker(), worker() ])
const failed = this.items.filter((item) => item.state === "error").length
if (failed) {
this.setUploading(false)
return this.say(`${failed} ${failed === 1 ? "file" : "files"} didn't upload. Press Upload to retry, or remove ${failed === 1 ? "it" : "them"}.`)
}
this.element.querySelectorAll("input[data-bulk-upload-hidden]").forEach((el) => el.remove())
for (const item of this.items) {
this.hidden("files[]", item.signedId)
this.hidden("names[]", item.name)
}
this.uploading = false
this.ready = true
this.say("Saving…")
// When nothing was left to upload this all ran inside the click's submit
// event, during which the browser ignores requestSubmit, so wait a tick.
setTimeout(() => this.element.requestSubmit())
}
upload(item) {
item.state = "uploading"
item.progress = 0
item.update()
return new Promise((resolve) => {
const delegate = {
directUploadWillStoreFileWithXHR: (xhr) => {
xhr.upload.addEventListener("progress", (e) => {
if (e.lengthComputable) { item.progress = Math.round((e.loaded / e.total) * 100); item.update() }
})
}
}
new DirectUpload(item.file, this.urlValue, delegate).create((error, blob) => {
if (error) {
Object.assign(item, { state: "error", error: /Status: 422/.test(error) ? "not an allowed file type" : "upload failed" })
} else {
Object.assign(item, { state: "done", signedId: blob.signed_id })
}
item.update()
resolve()
})
})
}
hidden(name, value) {
const input = document.createElement("input")
input.type = "hidden"
input.name = name
input.value = value
input.dataset.bulkUploadHidden = ""
this.element.append(input)
}
setUploading(on) {
this.uploading = on
this.submitTarget.disabled = on
this.render()
}
say(message) {
this.statusTarget.textContent = message
}
}
+3
View File
@@ -19,6 +19,9 @@
<% end %>
</div>
<% end %>
<% unless document.persisted? %>
<p class="text-sm text-slate-500">Lots of files? <%= link_to "Bulk upload", new_bulk_aircraft_documents_path(aircraft, collection_id: document.document_collection_id), class: "link" %> makes a document from each one.</p>
<% end %>
<div class="flex gap-2"><%= f.submit "Save Document" %><%= link_to "Cancel", aircraft_documents_path(aircraft), class: "btn-secondary" %></div>
<% end %>
<% if document.persisted? %><div class="mt-4"><%= delete_button "Delete document", aircraft_document_path(aircraft, document) %></div><% end %>
+68
View File
@@ -0,0 +1,68 @@
<% content_for :title, "#{@aircraft.tail_number} Bulk Upload" %>
<%= render "shared/aircraft_header", aircraft: @aircraft %>
<% file_classes = "block w-full text-sm text-slate-600 file:mr-3 file:rounded-md file:border-0 file:bg-slate-100 file:px-3 file:py-1.5 file:text-sm file:font-medium hover:file:bg-slate-200" %>
<%= form_with url: bulk_aircraft_documents_path(@aircraft), multipart: true, class: "card card-body max-w-3xl space-y-4",
data: { controller: "bulk-upload", bulk_upload_url_value: rails_direct_uploads_path,
bulk_upload_max_bytes_value: UploadValidation::MAX_SIZE, action: "submit->bulk-upload#submit" } do %>
<div>
<h2 class="card-title">Bulk upload</h2>
<p class="text-sm text-slate-500">Each file becomes its own document, named after the file. Give files the same name to combine them, in order, as the pages of one document.</p>
</div>
<% if @bulk_error %>
<div class="rounded-md bg-red-50 p-3 text-sm text-red-700 ring-1 ring-red-200" role="alert"><%= @bulk_error %></div>
<% end %>
<div class="grid gap-4 sm:grid-cols-2">
<div>
<%= label_tag :doc_type, "Type", class: "label" %>
<%= select_tag :doc_type, options_for_select(Document::TYPES.invert, @bulk[:doc_type]), class: "input" %>
</div>
<div>
<%= label_tag :visibility, "Share-link visibility", class: "label" %>
<%= select_tag :visibility, options_for_select([ [ "Inherit from collection", "" ] ] + DocumentCollection::VISIBILITIES.invert.to_a, @bulk[:visibility]), class: "input" %>
</div>
<div>
<%= label_tag :document_collection_id, "Collection", class: "label" %>
<%= select_tag :document_collection_id,
options_for_select([ [ "— None —", "" ] ] + @aircraft.document_collections.ordered.map { [ _1.name, _1.id ] }, @bulk[:document_collection_id]), class: "input" %>
</div>
<div>
<%= label_tag :new_collection, "…or a new collection", class: "label" %>
<%= text_field_tag :new_collection, @bulk[:new_collection], class: "input", placeholder: "e.g. 2025 Annual" %>
</div>
</div>
<div class="grid gap-4 sm:grid-cols-2">
<div>
<%= label_tag :files, "Files", class: "label" %>
<%= file_field_tag "files[]", id: "files", multiple: true, class: file_classes,
data: { bulk_upload_target: "input", action: "change->bulk-upload#add" } %>
</div>
<div data-bulk-upload-target="folderField" hidden>
<%= label_tag :folder, "…or a whole folder", class: "label" %>
<input type="file" id="folder" webkitdirectory multiple class="<%= file_classes %>"
data-bulk-upload-target="folder" data-action="change->bulk-upload#add">
</div>
</div>
<p class="hint !mt-1">Images, PDFs or text, up to <%= UploadValidation::MAX_SIZE / 1.megabyte %> MB each and <%= DocumentsController::MAX_BULK_FILES %> at a time.</p>
<div data-bulk-upload-target="panel" hidden class="space-y-2">
<div class="flex flex-wrap items-center justify-between gap-2">
<p class="text-sm font-medium text-slate-700" data-bulk-upload-target="summary"></p>
<div class="flex flex-wrap items-center gap-3">
<label class="flex items-center gap-2 text-sm text-slate-700" data-bulk-upload-target="groupField" hidden>
<input type="checkbox" class="h-4 w-4 rounded border-slate-300" data-bulk-upload-target="group" data-action="bulk-upload#regroup">
One document per folder
</label>
<button type="button" class="btn-secondary btn-sm" data-action="bulk-upload#clear">Clear</button>
</div>
</div>
<ul class="divide-y divide-slate-100 rounded-md ring-1 ring-slate-200" data-bulk-upload-target="list"></ul>
</div>
<div class="flex flex-wrap items-center gap-2">
<button type="submit" class="btn-primary" data-bulk-upload-target="submit">Upload</button>
<%= link_to "Cancel", aircraft_documents_path(@aircraft), class: "btn-secondary" %>
<p class="text-sm text-slate-500" data-bulk-upload-target="status" role="status"></p>
</div>
<% end %>
+2 -1
View File
@@ -1,8 +1,9 @@
<% content_for :title, "#{@aircraft.tail_number} Documents" %>
<%= render "shared/aircraft_header", aircraft: @aircraft %>
<div class="mb-4 flex justify-end gap-2">
<div class="mb-4 flex flex-wrap justify-end gap-2">
<% if owner? %>
<%= link_to "+ Collection", new_aircraft_document_collection_path(@aircraft), class: "btn-secondary" %>
<%= link_to "Bulk Upload", new_bulk_aircraft_documents_path(@aircraft), class: "btn-secondary" %>
<%= link_to "+ Upload Document", new_aircraft_document_path(@aircraft), class: "btn-primary" %>
<% end %>
</div>