mirror of
https://github.com/rickbarrette/simple-aircraft-manager-rails.git
synced 2026-11-17 15:50:36 -05:00
Bulk document upload and deduplicated file storage
- Bulk Upload page: pick many files or a folder; each file becomes a document named after it, and files sharing a name become one multi-page document. Files go up as direct uploads with per-file progress. - Direct uploads require a signed-in user and check type/size up front; only fresh, unattached blobs can be attached; abandoned uploads are purged nightly. - DedupDisk storage service: each distinct file is stored once under its SHA-256 and blobs hard-link to it; nightly sweep (and storage:dedupe) links pre-existing duplicates and drops unused objects. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
230a569b77
commit
2bfcb845ca
15 files changed
+735
-5
No files matched your search
@@ -0,0 +1,25 @@
|
||||
# Direct uploads (the browser sends each file straight to storage, then submits
|
||||
# only its signed id). Active Storage's own endpoint lets anyone create blobs;
|
||||
# this one requires a signed-in user and applies the upload rules up front.
|
||||
class DirectUploadsController < ActiveStorage::DirectUploadsController
|
||||
include Authentication
|
||||
before_action :check_upload, only: :create
|
||||
|
||||
private
|
||||
|
||||
def blob_args
|
||||
@blob_args ||= super.tap do |args|
|
||||
args[:content_type] = Marcel::MimeType.for(name: args[:filename].to_s) if args[:content_type].blank?
|
||||
end
|
||||
end
|
||||
|
||||
def check_upload
|
||||
error =
|
||||
if !UploadValidation::ALLOWED_CONTENT_TYPES.include?(blob_args[:content_type])
|
||||
"#{blob_args[:filename]} isn't an allowed file type"
|
||||
elsif blob_args[:byte_size].to_i > UploadValidation::MAX_SIZE
|
||||
"#{blob_args[:filename]} is over #{UploadValidation::MAX_SIZE / 1.megabyte} MB"
|
||||
end
|
||||
render json: { error: error }, status: :unprocessable_entity if error
|
||||
end
|
||||
end
|
||||
@@ -26,6 +26,52 @@ class DocumentsController < ApplicationController
|
||||
end
|
||||
end
|
||||
|
||||
def bulk_new
|
||||
@bulk = bulk_defaults.merge(document_collection_id: params[:collection_id])
|
||||
end
|
||||
|
||||
# Each file becomes a document named after it; files given the same name are
|
||||
# combined, in the order sent, as the pages of one document. Files arrive as
|
||||
# direct-upload signed ids (or as plain uploads when JavaScript is off).
|
||||
def bulk_create
|
||||
@bulk = bulk_defaults.merge(params.slice(:doc_type, :visibility, :document_collection_id, :new_collection).permit!.to_h.symbolize_keys)
|
||||
files = Array(params[:files]).compact_blank
|
||||
return bulk_error("Choose some files to upload.") if files.empty?
|
||||
return bulk_error("Upload at most #{MAX_BULK_FILES} files at a time.") if files.size > MAX_BULK_FILES
|
||||
|
||||
names = Array(params[:names])
|
||||
rejected = []
|
||||
groups = {}
|
||||
files.each_with_index do |file, i|
|
||||
upload = file.is_a?(String) ? unattached_blob(file) : file
|
||||
next rejected << "file #{i + 1} (upload expired, try again)" unless upload
|
||||
name = names[i].to_s.squish.presence || File.basename(upload_filename(upload), ".*").tr("_", " ").squish.presence || "Untitled"
|
||||
(groups[name.first(255)] ||= []) << upload
|
||||
end
|
||||
|
||||
collection_id, error = bulk_collection
|
||||
return bulk_error(error) if error
|
||||
created = []
|
||||
groups.each do |name, pages|
|
||||
doc = @aircraft.documents.new(name: name, doc_type: @bulk[:doc_type], visibility: @bulk[:visibility].presence,
|
||||
document_collection_id: collection_id, pages: pages)
|
||||
if doc.save
|
||||
created << doc
|
||||
else
|
||||
rejected << "#{name} (#{doc.errors.full_messages.to_sentence})"
|
||||
pages.each { _1.purge_later if _1.is_a?(ActiveStorage::Blob) }
|
||||
end
|
||||
end
|
||||
if created.empty?
|
||||
@aircraft.document_collections.find_by(id: collection_id)&.destroy if @bulk[:new_collection].present?
|
||||
return bulk_error("Nothing was uploaded: #{rejected.to_sentence}.")
|
||||
end
|
||||
|
||||
log_event("document", "#{helpers.pluralize(created.size, 'document')} uploaded", notes: created.map(&:name).join(", ").truncate(1000))
|
||||
flash[:alert] = "Not uploaded: #{rejected.to_sentence}." if rejected.any?
|
||||
redirect_to aircraft_documents_path(@aircraft), notice: "#{helpers.pluralize(created.size, 'document')} uploaded.", status: :see_other
|
||||
end
|
||||
|
||||
def edit; end
|
||||
|
||||
def update
|
||||
@@ -50,6 +96,33 @@ class DocumentsController < ApplicationController
|
||||
|
||||
private
|
||||
|
||||
MAX_BULK_FILES = 500
|
||||
|
||||
def bulk_defaults = { doc_type: "OTHER", visibility: nil, document_collection_id: nil, new_collection: nil }
|
||||
|
||||
def bulk_error(message)
|
||||
@bulk_error = message
|
||||
render :bulk_new, status: :unprocessable_entity
|
||||
end
|
||||
|
||||
# A blob from a direct upload that hasn't been attached anywhere yet, so a
|
||||
# signed id lifted from an existing file's URL can't be re-attached here.
|
||||
def unattached_blob(signed_id)
|
||||
blob = ActiveStorage::Blob.find_signed(signed_id)
|
||||
blob if blob && !blob.attachments.exists?
|
||||
end
|
||||
|
||||
def upload_filename(upload)
|
||||
(upload.respond_to?(:original_filename) ? upload.original_filename : upload.filename).to_s
|
||||
end
|
||||
|
||||
# [collection id or nil, error]: the chosen collection, or a new one.
|
||||
def bulk_collection
|
||||
return [ scoped_id(@aircraft.document_collections, @bulk[:document_collection_id]), nil ] if @bulk[:new_collection].blank?
|
||||
col = @aircraft.document_collections.create(name: @bulk[:new_collection].squish, visibility: @bulk[:visibility].presence || "private")
|
||||
col.persisted? ? [ col.id, nil ] : [ nil, "Couldn't create the collection: #{col.errors.full_messages.to_sentence}." ]
|
||||
end
|
||||
|
||||
def set_document = @document = @aircraft.documents.find(params[:id])
|
||||
|
||||
def document_params
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import { Controller } from "@hotwired/stimulus"
|
||||
import { DirectUpload } from "@rails/activestorage"
|
||||
|
||||
// Bulk document upload. Picked files (or a whole folder) are listed with an
|
||||
// editable document name each; files sharing a name become one document. On
|
||||
// submit every file goes straight to storage (a few at a time, with progress),
|
||||
// then the form is submitted with just the signed ids and names.
|
||||
//
|
||||
// Without JavaScript the file field posts the files itself and the server
|
||||
// names each document after its file.
|
||||
export default class extends Controller {
|
||||
static targets = [ "input", "folder", "folderField", "panel", "list", "summary", "group", "groupField", "submit", "status" ]
|
||||
static values = { url: String, maxBytes: Number }
|
||||
|
||||
connect() {
|
||||
this.items = []
|
||||
this.ready = false
|
||||
this.uploading = false
|
||||
this.inputTarget.removeAttribute("name") // the signed ids are sent instead
|
||||
if ("webkitdirectory" in this.folderTarget) this.folderFieldTarget.hidden = false
|
||||
this.beforeUnload = (event) => { if (this.uploading) event.preventDefault() }
|
||||
window.addEventListener("beforeunload", this.beforeUnload)
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
window.removeEventListener("beforeunload", this.beforeUnload)
|
||||
}
|
||||
|
||||
add(event) {
|
||||
for (const file of event.target.files) {
|
||||
const item = { file, path: file.webkitRelativePath || file.name, state: "pending", progress: 0 }
|
||||
if (file.size > this.maxBytesValue) Object.assign(item, { state: "error", error: `over ${Math.round(this.maxBytesValue / 1048576)} MB` })
|
||||
this.items.push(item)
|
||||
}
|
||||
event.target.value = ""
|
||||
this.items.sort((a, b) => a.path.localeCompare(b.path, undefined, { numeric: true, sensitivity: "base" }))
|
||||
this.items.forEach((item) => { if (!item.custom) item.name = this.defaultName(item) })
|
||||
this.groupFieldTarget.hidden = !this.items.some((item) => item.path.includes("/"))
|
||||
this.render()
|
||||
}
|
||||
|
||||
regroup() {
|
||||
this.items.forEach((item) => { item.custom = false; item.name = this.defaultName(item) })
|
||||
this.render()
|
||||
}
|
||||
|
||||
clear() {
|
||||
if (this.uploading) return
|
||||
this.items = []
|
||||
this.render()
|
||||
}
|
||||
|
||||
defaultName(item) {
|
||||
const parts = item.path.split("/")
|
||||
if (this.groupTarget.checked && parts.length > 1) return parts[parts.length - 2]
|
||||
return parts[parts.length - 1].replace(/\.[^.]+$/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim() || "Untitled"
|
||||
}
|
||||
|
||||
render() {
|
||||
this.panelTarget.hidden = this.items.length === 0
|
||||
this.listTarget.replaceChildren(...this.items.map((item, index) => this.row(item, index)))
|
||||
this.updateSummary()
|
||||
}
|
||||
|
||||
row(item, index) {
|
||||
const li = document.createElement("li")
|
||||
li.className = "flex items-start gap-3 px-3 py-2"
|
||||
|
||||
const body = document.createElement("div")
|
||||
body.className = "min-w-0 flex-1"
|
||||
const name = document.createElement("input")
|
||||
name.type = "text"
|
||||
name.className = "input"
|
||||
name.value = item.name
|
||||
name.setAttribute("aria-label", `Document name for ${item.path}`)
|
||||
name.disabled = this.uploading
|
||||
name.addEventListener("input", () => { item.name = name.value; item.custom = true; this.updateSummary() })
|
||||
|
||||
const meta = document.createElement("p")
|
||||
meta.className = "mt-1 truncate text-xs text-slate-500"
|
||||
const bar = document.createElement("div")
|
||||
bar.className = "mt-1 h-1 overflow-hidden rounded bg-slate-100"
|
||||
bar.hidden = item.state === "pending" || item.state === "error"
|
||||
const fill = document.createElement("div")
|
||||
fill.className = "h-full bg-sky-600 transition-[width]"
|
||||
bar.append(fill)
|
||||
body.append(name, meta, bar)
|
||||
|
||||
const remove = document.createElement("button")
|
||||
remove.type = "button"
|
||||
remove.className = "btn-secondary btn-sm"
|
||||
remove.textContent = "✕"
|
||||
remove.setAttribute("aria-label", `Remove ${item.path}`)
|
||||
remove.disabled = this.uploading
|
||||
remove.addEventListener("click", () => { this.items.splice(index, 1); this.render() })
|
||||
|
||||
li.append(body, remove)
|
||||
item.update = () => {
|
||||
const status = { pending: "", uploading: ` · ${item.progress}%`, done: " · uploaded", error: ` · ${item.error}` }[item.state]
|
||||
meta.textContent = `${item.path} · ${this.size(item.file.size)}${status}`
|
||||
meta.classList.toggle("text-red-600", item.state === "error")
|
||||
meta.classList.toggle("text-slate-500", item.state !== "error")
|
||||
bar.hidden = item.state === "pending" || item.state === "error"
|
||||
fill.style.width = `${item.state === "done" ? 100 : item.progress}%`
|
||||
}
|
||||
item.update()
|
||||
return li
|
||||
}
|
||||
|
||||
updateSummary() {
|
||||
const files = this.items.length
|
||||
const docs = new Set(this.items.map((item) => item.name.trim() || "Untitled")).size
|
||||
this.summaryTarget.textContent = `${files} ${files === 1 ? "file" : "files"} → ${docs} ${docs === 1 ? "document" : "documents"}`
|
||||
this.submitTarget.textContent = files ? `Upload ${docs} ${docs === 1 ? "document" : "documents"}` : "Upload"
|
||||
}
|
||||
|
||||
size(bytes) {
|
||||
if (bytes < 1024) return `${bytes} B`
|
||||
if (bytes < 1048576) return `${Math.round(bytes / 1024)} KB`
|
||||
return `${(bytes / 1048576).toFixed(1)} MB`
|
||||
}
|
||||
|
||||
async submit(event) {
|
||||
if (this.ready) return
|
||||
event.preventDefault()
|
||||
if (this.uploading) return
|
||||
if (this.items.length === 0) return this.say("Choose some files to upload.")
|
||||
if (this.items.some((item) => item.file.size > this.maxBytesValue)) {
|
||||
return this.say("Remove the files marked in red first.")
|
||||
}
|
||||
|
||||
this.items.forEach((item) => { if (item.state === "error") item.state = "pending" })
|
||||
this.setUploading(true)
|
||||
const queue = this.items.filter((item) => item.state === "pending")
|
||||
const total = queue.length
|
||||
let finished = 0
|
||||
this.say(`Uploading 0 of ${total}…`)
|
||||
const worker = async () => {
|
||||
for (let item = queue.shift(); item; item = queue.shift()) {
|
||||
await this.upload(item)
|
||||
this.say(`Uploading ${++finished} of ${total}…`)
|
||||
}
|
||||
}
|
||||
await Promise.all([ worker(), worker(), worker() ])
|
||||
|
||||
const failed = this.items.filter((item) => item.state === "error").length
|
||||
if (failed) {
|
||||
this.setUploading(false)
|
||||
return this.say(`${failed} ${failed === 1 ? "file" : "files"} didn't upload. Press Upload to retry, or remove ${failed === 1 ? "it" : "them"}.`)
|
||||
}
|
||||
|
||||
this.element.querySelectorAll("input[data-bulk-upload-hidden]").forEach((el) => el.remove())
|
||||
for (const item of this.items) {
|
||||
this.hidden("files[]", item.signedId)
|
||||
this.hidden("names[]", item.name)
|
||||
}
|
||||
this.uploading = false
|
||||
this.ready = true
|
||||
this.say("Saving…")
|
||||
// When nothing was left to upload this all ran inside the click's submit
|
||||
// event, during which the browser ignores requestSubmit, so wait a tick.
|
||||
setTimeout(() => this.element.requestSubmit())
|
||||
}
|
||||
|
||||
upload(item) {
|
||||
item.state = "uploading"
|
||||
item.progress = 0
|
||||
item.update()
|
||||
return new Promise((resolve) => {
|
||||
const delegate = {
|
||||
directUploadWillStoreFileWithXHR: (xhr) => {
|
||||
xhr.upload.addEventListener("progress", (e) => {
|
||||
if (e.lengthComputable) { item.progress = Math.round((e.loaded / e.total) * 100); item.update() }
|
||||
})
|
||||
}
|
||||
}
|
||||
new DirectUpload(item.file, this.urlValue, delegate).create((error, blob) => {
|
||||
if (error) {
|
||||
Object.assign(item, { state: "error", error: /Status: 422/.test(error) ? "not an allowed file type" : "upload failed" })
|
||||
} else {
|
||||
Object.assign(item, { state: "done", signedId: blob.signed_id })
|
||||
}
|
||||
item.update()
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
hidden(name, value) {
|
||||
const input = document.createElement("input")
|
||||
input.type = "hidden"
|
||||
input.name = name
|
||||
input.value = value
|
||||
input.dataset.bulkUploadHidden = ""
|
||||
this.element.append(input)
|
||||
}
|
||||
|
||||
setUploading(on) {
|
||||
this.uploading = on
|
||||
this.submitTarget.disabled = on
|
||||
this.render()
|
||||
}
|
||||
|
||||
say(message) {
|
||||
this.statusTarget.textContent = message
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,9 @@
|
||||
<% end %>
|
||||
</div>
|
||||
<% end %>
|
||||
<% unless document.persisted? %>
|
||||
<p class="text-sm text-slate-500">Lots of files? <%= link_to "Bulk upload", new_bulk_aircraft_documents_path(aircraft, collection_id: document.document_collection_id), class: "link" %> makes a document from each one.</p>
|
||||
<% end %>
|
||||
<div class="flex gap-2"><%= f.submit "Save Document" %><%= link_to "Cancel", aircraft_documents_path(aircraft), class: "btn-secondary" %></div>
|
||||
<% end %>
|
||||
<% if document.persisted? %><div class="mt-4"><%= delete_button "Delete document", aircraft_document_path(aircraft, document) %></div><% end %>
|
||||
@@ -0,0 +1,68 @@
|
||||
<% content_for :title, "#{@aircraft.tail_number} Bulk Upload" %>
|
||||
<%= render "shared/aircraft_header", aircraft: @aircraft %>
|
||||
<% file_classes = "block w-full text-sm text-slate-600 file:mr-3 file:rounded-md file:border-0 file:bg-slate-100 file:px-3 file:py-1.5 file:text-sm file:font-medium hover:file:bg-slate-200" %>
|
||||
<%= form_with url: bulk_aircraft_documents_path(@aircraft), multipart: true, class: "card card-body max-w-3xl space-y-4",
|
||||
data: { controller: "bulk-upload", bulk_upload_url_value: rails_direct_uploads_path,
|
||||
bulk_upload_max_bytes_value: UploadValidation::MAX_SIZE, action: "submit->bulk-upload#submit" } do %>
|
||||
<div>
|
||||
<h2 class="card-title">Bulk upload</h2>
|
||||
<p class="text-sm text-slate-500">Each file becomes its own document, named after the file. Give files the same name to combine them, in order, as the pages of one document.</p>
|
||||
</div>
|
||||
<% if @bulk_error %>
|
||||
<div class="rounded-md bg-red-50 p-3 text-sm text-red-700 ring-1 ring-red-200" role="alert"><%= @bulk_error %></div>
|
||||
<% end %>
|
||||
|
||||
<div class="grid gap-4 sm:grid-cols-2">
|
||||
<div>
|
||||
<%= label_tag :doc_type, "Type", class: "label" %>
|
||||
<%= select_tag :doc_type, options_for_select(Document::TYPES.invert, @bulk[:doc_type]), class: "input" %>
|
||||
</div>
|
||||
<div>
|
||||
<%= label_tag :visibility, "Share-link visibility", class: "label" %>
|
||||
<%= select_tag :visibility, options_for_select([ [ "Inherit from collection", "" ] ] + DocumentCollection::VISIBILITIES.invert.to_a, @bulk[:visibility]), class: "input" %>
|
||||
</div>
|
||||
<div>
|
||||
<%= label_tag :document_collection_id, "Collection", class: "label" %>
|
||||
<%= select_tag :document_collection_id,
|
||||
options_for_select([ [ "— None —", "" ] ] + @aircraft.document_collections.ordered.map { [ _1.name, _1.id ] }, @bulk[:document_collection_id]), class: "input" %>
|
||||
</div>
|
||||
<div>
|
||||
<%= label_tag :new_collection, "…or a new collection", class: "label" %>
|
||||
<%= text_field_tag :new_collection, @bulk[:new_collection], class: "input", placeholder: "e.g. 2025 Annual" %>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid gap-4 sm:grid-cols-2">
|
||||
<div>
|
||||
<%= label_tag :files, "Files", class: "label" %>
|
||||
<%= file_field_tag "files[]", id: "files", multiple: true, class: file_classes,
|
||||
data: { bulk_upload_target: "input", action: "change->bulk-upload#add" } %>
|
||||
</div>
|
||||
<div data-bulk-upload-target="folderField" hidden>
|
||||
<%= label_tag :folder, "…or a whole folder", class: "label" %>
|
||||
<input type="file" id="folder" webkitdirectory multiple class="<%= file_classes %>"
|
||||
data-bulk-upload-target="folder" data-action="change->bulk-upload#add">
|
||||
</div>
|
||||
</div>
|
||||
<p class="hint !mt-1">Images, PDFs or text, up to <%= UploadValidation::MAX_SIZE / 1.megabyte %> MB each and <%= DocumentsController::MAX_BULK_FILES %> at a time.</p>
|
||||
|
||||
<div data-bulk-upload-target="panel" hidden class="space-y-2">
|
||||
<div class="flex flex-wrap items-center justify-between gap-2">
|
||||
<p class="text-sm font-medium text-slate-700" data-bulk-upload-target="summary"></p>
|
||||
<div class="flex flex-wrap items-center gap-3">
|
||||
<label class="flex items-center gap-2 text-sm text-slate-700" data-bulk-upload-target="groupField" hidden>
|
||||
<input type="checkbox" class="h-4 w-4 rounded border-slate-300" data-bulk-upload-target="group" data-action="bulk-upload#regroup">
|
||||
One document per folder
|
||||
</label>
|
||||
<button type="button" class="btn-secondary btn-sm" data-action="bulk-upload#clear">Clear</button>
|
||||
</div>
|
||||
</div>
|
||||
<ul class="divide-y divide-slate-100 rounded-md ring-1 ring-slate-200" data-bulk-upload-target="list"></ul>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<button type="submit" class="btn-primary" data-bulk-upload-target="submit">Upload</button>
|
||||
<%= link_to "Cancel", aircraft_documents_path(@aircraft), class: "btn-secondary" %>
|
||||
<p class="text-sm text-slate-500" data-bulk-upload-target="status" role="status"></p>
|
||||
</div>
|
||||
<% end %>
|
||||
@@ -1,8 +1,9 @@
|
||||
<% content_for :title, "#{@aircraft.tail_number} Documents" %>
|
||||
<%= render "shared/aircraft_header", aircraft: @aircraft %>
|
||||
<div class="mb-4 flex justify-end gap-2">
|
||||
<div class="mb-4 flex flex-wrap justify-end gap-2">
|
||||
<% if owner? %>
|
||||
<%= link_to "+ Collection", new_aircraft_document_collection_path(@aircraft), class: "btn-secondary" %>
|
||||
<%= link_to "Bulk Upload", new_bulk_aircraft_documents_path(@aircraft), class: "btn-secondary" %>
|
||||
<%= link_to "+ Upload Document", new_aircraft_document_path(@aircraft), class: "btn-primary" %>
|
||||
<% end %>
|
||||
</div>
|
||||
|
||||
Reference in new issue
Block a user