mirror of
https://github.com/rickbarrette/simple-aircraft-manager-rails.git
synced 2026-11-17 15:50:36 -05:00
Bulk document upload and deduplicated file storage
- Bulk Upload page: pick many files or a folder; each file becomes a document named after it, and files sharing a name become one multi-page document. Files go up as direct uploads with per-file progress. - Direct uploads require a signed-in user and check type/size up front; only fresh, unattached blobs can be attached; abandoned uploads are purged nightly. - DedupDisk storage service: each distinct file is stored once under its SHA-256 and blobs hard-link to it; nightly sweep (and storage:dedupe) links pre-existing duplicates and drops unused objects. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
230a569b77
commit
2bfcb845ca
15 files changed
+735
-5
No files matched your search
@@ -0,0 +1,25 @@
|
|||||||
|
# Direct uploads (the browser sends each file straight to storage, then submits
|
||||||
|
# only its signed id). Active Storage's own endpoint lets anyone create blobs;
|
||||||
|
# this one requires a signed-in user and applies the upload rules up front.
|
||||||
|
class DirectUploadsController < ActiveStorage::DirectUploadsController
|
||||||
|
include Authentication
|
||||||
|
before_action :check_upload, only: :create
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def blob_args
|
||||||
|
@blob_args ||= super.tap do |args|
|
||||||
|
args[:content_type] = Marcel::MimeType.for(name: args[:filename].to_s) if args[:content_type].blank?
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def check_upload
|
||||||
|
error =
|
||||||
|
if !UploadValidation::ALLOWED_CONTENT_TYPES.include?(blob_args[:content_type])
|
||||||
|
"#{blob_args[:filename]} isn't an allowed file type"
|
||||||
|
elsif blob_args[:byte_size].to_i > UploadValidation::MAX_SIZE
|
||||||
|
"#{blob_args[:filename]} is over #{UploadValidation::MAX_SIZE / 1.megabyte} MB"
|
||||||
|
end
|
||||||
|
render json: { error: error }, status: :unprocessable_entity if error
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -26,6 +26,52 @@ class DocumentsController < ApplicationController
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def bulk_new
|
||||||
|
@bulk = bulk_defaults.merge(document_collection_id: params[:collection_id])
|
||||||
|
end
|
||||||
|
|
||||||
|
# Each file becomes a document named after it; files given the same name are
|
||||||
|
# combined, in the order sent, as the pages of one document. Files arrive as
|
||||||
|
# direct-upload signed ids (or as plain uploads when JavaScript is off).
|
||||||
|
def bulk_create
|
||||||
|
@bulk = bulk_defaults.merge(params.slice(:doc_type, :visibility, :document_collection_id, :new_collection).permit!.to_h.symbolize_keys)
|
||||||
|
files = Array(params[:files]).compact_blank
|
||||||
|
return bulk_error("Choose some files to upload.") if files.empty?
|
||||||
|
return bulk_error("Upload at most #{MAX_BULK_FILES} files at a time.") if files.size > MAX_BULK_FILES
|
||||||
|
|
||||||
|
names = Array(params[:names])
|
||||||
|
rejected = []
|
||||||
|
groups = {}
|
||||||
|
files.each_with_index do |file, i|
|
||||||
|
upload = file.is_a?(String) ? unattached_blob(file) : file
|
||||||
|
next rejected << "file #{i + 1} (upload expired, try again)" unless upload
|
||||||
|
name = names[i].to_s.squish.presence || File.basename(upload_filename(upload), ".*").tr("_", " ").squish.presence || "Untitled"
|
||||||
|
(groups[name.first(255)] ||= []) << upload
|
||||||
|
end
|
||||||
|
|
||||||
|
collection_id, error = bulk_collection
|
||||||
|
return bulk_error(error) if error
|
||||||
|
created = []
|
||||||
|
groups.each do |name, pages|
|
||||||
|
doc = @aircraft.documents.new(name: name, doc_type: @bulk[:doc_type], visibility: @bulk[:visibility].presence,
|
||||||
|
document_collection_id: collection_id, pages: pages)
|
||||||
|
if doc.save
|
||||||
|
created << doc
|
||||||
|
else
|
||||||
|
rejected << "#{name} (#{doc.errors.full_messages.to_sentence})"
|
||||||
|
pages.each { _1.purge_later if _1.is_a?(ActiveStorage::Blob) }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
if created.empty?
|
||||||
|
@aircraft.document_collections.find_by(id: collection_id)&.destroy if @bulk[:new_collection].present?
|
||||||
|
return bulk_error("Nothing was uploaded: #{rejected.to_sentence}.")
|
||||||
|
end
|
||||||
|
|
||||||
|
log_event("document", "#{helpers.pluralize(created.size, 'document')} uploaded", notes: created.map(&:name).join(", ").truncate(1000))
|
||||||
|
flash[:alert] = "Not uploaded: #{rejected.to_sentence}." if rejected.any?
|
||||||
|
redirect_to aircraft_documents_path(@aircraft), notice: "#{helpers.pluralize(created.size, 'document')} uploaded.", status: :see_other
|
||||||
|
end
|
||||||
|
|
||||||
def edit; end
|
def edit; end
|
||||||
|
|
||||||
def update
|
def update
|
||||||
@@ -50,6 +96,33 @@ class DocumentsController < ApplicationController
|
|||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
|
MAX_BULK_FILES = 500
|
||||||
|
|
||||||
|
def bulk_defaults = { doc_type: "OTHER", visibility: nil, document_collection_id: nil, new_collection: nil }
|
||||||
|
|
||||||
|
def bulk_error(message)
|
||||||
|
@bulk_error = message
|
||||||
|
render :bulk_new, status: :unprocessable_entity
|
||||||
|
end
|
||||||
|
|
||||||
|
# A blob from a direct upload that hasn't been attached anywhere yet, so a
|
||||||
|
# signed id lifted from an existing file's URL can't be re-attached here.
|
||||||
|
def unattached_blob(signed_id)
|
||||||
|
blob = ActiveStorage::Blob.find_signed(signed_id)
|
||||||
|
blob if blob && !blob.attachments.exists?
|
||||||
|
end
|
||||||
|
|
||||||
|
def upload_filename(upload)
|
||||||
|
(upload.respond_to?(:original_filename) ? upload.original_filename : upload.filename).to_s
|
||||||
|
end
|
||||||
|
|
||||||
|
# [collection id or nil, error]: the chosen collection, or a new one.
|
||||||
|
def bulk_collection
|
||||||
|
return [ scoped_id(@aircraft.document_collections, @bulk[:document_collection_id]), nil ] if @bulk[:new_collection].blank?
|
||||||
|
col = @aircraft.document_collections.create(name: @bulk[:new_collection].squish, visibility: @bulk[:visibility].presence || "private")
|
||||||
|
col.persisted? ? [ col.id, nil ] : [ nil, "Couldn't create the collection: #{col.errors.full_messages.to_sentence}." ]
|
||||||
|
end
|
||||||
|
|
||||||
def set_document = @document = @aircraft.documents.find(params[:id])
|
def set_document = @document = @aircraft.documents.find(params[:id])
|
||||||
|
|
||||||
def document_params
|
def document_params
|
||||||
|
|||||||
@@ -0,0 +1,207 @@
|
|||||||
|
import { Controller } from "@hotwired/stimulus"
|
||||||
|
import { DirectUpload } from "@rails/activestorage"
|
||||||
|
|
||||||
|
// Bulk document upload. Picked files (or a whole folder) are listed with an
|
||||||
|
// editable document name each; files sharing a name become one document. On
|
||||||
|
// submit every file goes straight to storage (a few at a time, with progress),
|
||||||
|
// then the form is submitted with just the signed ids and names.
|
||||||
|
//
|
||||||
|
// Without JavaScript the file field posts the files itself and the server
|
||||||
|
// names each document after its file.
|
||||||
|
export default class extends Controller {
|
||||||
|
static targets = [ "input", "folder", "folderField", "panel", "list", "summary", "group", "groupField", "submit", "status" ]
|
||||||
|
static values = { url: String, maxBytes: Number }
|
||||||
|
|
||||||
|
connect() {
|
||||||
|
this.items = []
|
||||||
|
this.ready = false
|
||||||
|
this.uploading = false
|
||||||
|
this.inputTarget.removeAttribute("name") // the signed ids are sent instead
|
||||||
|
if ("webkitdirectory" in this.folderTarget) this.folderFieldTarget.hidden = false
|
||||||
|
this.beforeUnload = (event) => { if (this.uploading) event.preventDefault() }
|
||||||
|
window.addEventListener("beforeunload", this.beforeUnload)
|
||||||
|
}
|
||||||
|
|
||||||
|
disconnect() {
|
||||||
|
window.removeEventListener("beforeunload", this.beforeUnload)
|
||||||
|
}
|
||||||
|
|
||||||
|
add(event) {
|
||||||
|
for (const file of event.target.files) {
|
||||||
|
const item = { file, path: file.webkitRelativePath || file.name, state: "pending", progress: 0 }
|
||||||
|
if (file.size > this.maxBytesValue) Object.assign(item, { state: "error", error: `over ${Math.round(this.maxBytesValue / 1048576)} MB` })
|
||||||
|
this.items.push(item)
|
||||||
|
}
|
||||||
|
event.target.value = ""
|
||||||
|
this.items.sort((a, b) => a.path.localeCompare(b.path, undefined, { numeric: true, sensitivity: "base" }))
|
||||||
|
this.items.forEach((item) => { if (!item.custom) item.name = this.defaultName(item) })
|
||||||
|
this.groupFieldTarget.hidden = !this.items.some((item) => item.path.includes("/"))
|
||||||
|
this.render()
|
||||||
|
}
|
||||||
|
|
||||||
|
regroup() {
|
||||||
|
this.items.forEach((item) => { item.custom = false; item.name = this.defaultName(item) })
|
||||||
|
this.render()
|
||||||
|
}
|
||||||
|
|
||||||
|
clear() {
|
||||||
|
if (this.uploading) return
|
||||||
|
this.items = []
|
||||||
|
this.render()
|
||||||
|
}
|
||||||
|
|
||||||
|
defaultName(item) {
|
||||||
|
const parts = item.path.split("/")
|
||||||
|
if (this.groupTarget.checked && parts.length > 1) return parts[parts.length - 2]
|
||||||
|
return parts[parts.length - 1].replace(/\.[^.]+$/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim() || "Untitled"
|
||||||
|
}
|
||||||
|
|
||||||
|
render() {
|
||||||
|
this.panelTarget.hidden = this.items.length === 0
|
||||||
|
this.listTarget.replaceChildren(...this.items.map((item, index) => this.row(item, index)))
|
||||||
|
this.updateSummary()
|
||||||
|
}
|
||||||
|
|
||||||
|
row(item, index) {
|
||||||
|
const li = document.createElement("li")
|
||||||
|
li.className = "flex items-start gap-3 px-3 py-2"
|
||||||
|
|
||||||
|
const body = document.createElement("div")
|
||||||
|
body.className = "min-w-0 flex-1"
|
||||||
|
const name = document.createElement("input")
|
||||||
|
name.type = "text"
|
||||||
|
name.className = "input"
|
||||||
|
name.value = item.name
|
||||||
|
name.setAttribute("aria-label", `Document name for ${item.path}`)
|
||||||
|
name.disabled = this.uploading
|
||||||
|
name.addEventListener("input", () => { item.name = name.value; item.custom = true; this.updateSummary() })
|
||||||
|
|
||||||
|
const meta = document.createElement("p")
|
||||||
|
meta.className = "mt-1 truncate text-xs text-slate-500"
|
||||||
|
const bar = document.createElement("div")
|
||||||
|
bar.className = "mt-1 h-1 overflow-hidden rounded bg-slate-100"
|
||||||
|
bar.hidden = item.state === "pending" || item.state === "error"
|
||||||
|
const fill = document.createElement("div")
|
||||||
|
fill.className = "h-full bg-sky-600 transition-[width]"
|
||||||
|
bar.append(fill)
|
||||||
|
body.append(name, meta, bar)
|
||||||
|
|
||||||
|
const remove = document.createElement("button")
|
||||||
|
remove.type = "button"
|
||||||
|
remove.className = "btn-secondary btn-sm"
|
||||||
|
remove.textContent = "✕"
|
||||||
|
remove.setAttribute("aria-label", `Remove ${item.path}`)
|
||||||
|
remove.disabled = this.uploading
|
||||||
|
remove.addEventListener("click", () => { this.items.splice(index, 1); this.render() })
|
||||||
|
|
||||||
|
li.append(body, remove)
|
||||||
|
item.update = () => {
|
||||||
|
const status = { pending: "", uploading: ` · ${item.progress}%`, done: " · uploaded", error: ` · ${item.error}` }[item.state]
|
||||||
|
meta.textContent = `${item.path} · ${this.size(item.file.size)}${status}`
|
||||||
|
meta.classList.toggle("text-red-600", item.state === "error")
|
||||||
|
meta.classList.toggle("text-slate-500", item.state !== "error")
|
||||||
|
bar.hidden = item.state === "pending" || item.state === "error"
|
||||||
|
fill.style.width = `${item.state === "done" ? 100 : item.progress}%`
|
||||||
|
}
|
||||||
|
item.update()
|
||||||
|
return li
|
||||||
|
}
|
||||||
|
|
||||||
|
updateSummary() {
|
||||||
|
const files = this.items.length
|
||||||
|
const docs = new Set(this.items.map((item) => item.name.trim() || "Untitled")).size
|
||||||
|
this.summaryTarget.textContent = `${files} ${files === 1 ? "file" : "files"} → ${docs} ${docs === 1 ? "document" : "documents"}`
|
||||||
|
this.submitTarget.textContent = files ? `Upload ${docs} ${docs === 1 ? "document" : "documents"}` : "Upload"
|
||||||
|
}
|
||||||
|
|
||||||
|
size(bytes) {
|
||||||
|
if (bytes < 1024) return `${bytes} B`
|
||||||
|
if (bytes < 1048576) return `${Math.round(bytes / 1024)} KB`
|
||||||
|
return `${(bytes / 1048576).toFixed(1)} MB`
|
||||||
|
}
|
||||||
|
|
||||||
|
async submit(event) {
|
||||||
|
if (this.ready) return
|
||||||
|
event.preventDefault()
|
||||||
|
if (this.uploading) return
|
||||||
|
if (this.items.length === 0) return this.say("Choose some files to upload.")
|
||||||
|
if (this.items.some((item) => item.file.size > this.maxBytesValue)) {
|
||||||
|
return this.say("Remove the files marked in red first.")
|
||||||
|
}
|
||||||
|
|
||||||
|
this.items.forEach((item) => { if (item.state === "error") item.state = "pending" })
|
||||||
|
this.setUploading(true)
|
||||||
|
const queue = this.items.filter((item) => item.state === "pending")
|
||||||
|
const total = queue.length
|
||||||
|
let finished = 0
|
||||||
|
this.say(`Uploading 0 of ${total}…`)
|
||||||
|
const worker = async () => {
|
||||||
|
for (let item = queue.shift(); item; item = queue.shift()) {
|
||||||
|
await this.upload(item)
|
||||||
|
this.say(`Uploading ${++finished} of ${total}…`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await Promise.all([ worker(), worker(), worker() ])
|
||||||
|
|
||||||
|
const failed = this.items.filter((item) => item.state === "error").length
|
||||||
|
if (failed) {
|
||||||
|
this.setUploading(false)
|
||||||
|
return this.say(`${failed} ${failed === 1 ? "file" : "files"} didn't upload. Press Upload to retry, or remove ${failed === 1 ? "it" : "them"}.`)
|
||||||
|
}
|
||||||
|
|
||||||
|
this.element.querySelectorAll("input[data-bulk-upload-hidden]").forEach((el) => el.remove())
|
||||||
|
for (const item of this.items) {
|
||||||
|
this.hidden("files[]", item.signedId)
|
||||||
|
this.hidden("names[]", item.name)
|
||||||
|
}
|
||||||
|
this.uploading = false
|
||||||
|
this.ready = true
|
||||||
|
this.say("Saving…")
|
||||||
|
// When nothing was left to upload this all ran inside the click's submit
|
||||||
|
// event, during which the browser ignores requestSubmit, so wait a tick.
|
||||||
|
setTimeout(() => this.element.requestSubmit())
|
||||||
|
}
|
||||||
|
|
||||||
|
upload(item) {
|
||||||
|
item.state = "uploading"
|
||||||
|
item.progress = 0
|
||||||
|
item.update()
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const delegate = {
|
||||||
|
directUploadWillStoreFileWithXHR: (xhr) => {
|
||||||
|
xhr.upload.addEventListener("progress", (e) => {
|
||||||
|
if (e.lengthComputable) { item.progress = Math.round((e.loaded / e.total) * 100); item.update() }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
new DirectUpload(item.file, this.urlValue, delegate).create((error, blob) => {
|
||||||
|
if (error) {
|
||||||
|
Object.assign(item, { state: "error", error: /Status: 422/.test(error) ? "not an allowed file type" : "upload failed" })
|
||||||
|
} else {
|
||||||
|
Object.assign(item, { state: "done", signedId: blob.signed_id })
|
||||||
|
}
|
||||||
|
item.update()
|
||||||
|
resolve()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
hidden(name, value) {
|
||||||
|
const input = document.createElement("input")
|
||||||
|
input.type = "hidden"
|
||||||
|
input.name = name
|
||||||
|
input.value = value
|
||||||
|
input.dataset.bulkUploadHidden = ""
|
||||||
|
this.element.append(input)
|
||||||
|
}
|
||||||
|
|
||||||
|
setUploading(on) {
|
||||||
|
this.uploading = on
|
||||||
|
this.submitTarget.disabled = on
|
||||||
|
this.render()
|
||||||
|
}
|
||||||
|
|
||||||
|
say(message) {
|
||||||
|
this.statusTarget.textContent = message
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,6 +19,9 @@
|
|||||||
<% end %>
|
<% end %>
|
||||||
</div>
|
</div>
|
||||||
<% end %>
|
<% end %>
|
||||||
|
<% unless document.persisted? %>
|
||||||
|
<p class="text-sm text-slate-500">Lots of files? <%= link_to "Bulk upload", new_bulk_aircraft_documents_path(aircraft, collection_id: document.document_collection_id), class: "link" %> makes a document from each one.</p>
|
||||||
|
<% end %>
|
||||||
<div class="flex gap-2"><%= f.submit "Save Document" %><%= link_to "Cancel", aircraft_documents_path(aircraft), class: "btn-secondary" %></div>
|
<div class="flex gap-2"><%= f.submit "Save Document" %><%= link_to "Cancel", aircraft_documents_path(aircraft), class: "btn-secondary" %></div>
|
||||||
<% end %>
|
<% end %>
|
||||||
<% if document.persisted? %><div class="mt-4"><%= delete_button "Delete document", aircraft_document_path(aircraft, document) %></div><% end %>
|
<% if document.persisted? %><div class="mt-4"><%= delete_button "Delete document", aircraft_document_path(aircraft, document) %></div><% end %>
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
<% content_for :title, "#{@aircraft.tail_number} Bulk Upload" %>
|
||||||
|
<%= render "shared/aircraft_header", aircraft: @aircraft %>
|
||||||
|
<% file_classes = "block w-full text-sm text-slate-600 file:mr-3 file:rounded-md file:border-0 file:bg-slate-100 file:px-3 file:py-1.5 file:text-sm file:font-medium hover:file:bg-slate-200" %>
|
||||||
|
<%= form_with url: bulk_aircraft_documents_path(@aircraft), multipart: true, class: "card card-body max-w-3xl space-y-4",
|
||||||
|
data: { controller: "bulk-upload", bulk_upload_url_value: rails_direct_uploads_path,
|
||||||
|
bulk_upload_max_bytes_value: UploadValidation::MAX_SIZE, action: "submit->bulk-upload#submit" } do %>
|
||||||
|
<div>
|
||||||
|
<h2 class="card-title">Bulk upload</h2>
|
||||||
|
<p class="text-sm text-slate-500">Each file becomes its own document, named after the file. Give files the same name to combine them, in order, as the pages of one document.</p>
|
||||||
|
</div>
|
||||||
|
<% if @bulk_error %>
|
||||||
|
<div class="rounded-md bg-red-50 p-3 text-sm text-red-700 ring-1 ring-red-200" role="alert"><%= @bulk_error %></div>
|
||||||
|
<% end %>
|
||||||
|
|
||||||
|
<div class="grid gap-4 sm:grid-cols-2">
|
||||||
|
<div>
|
||||||
|
<%= label_tag :doc_type, "Type", class: "label" %>
|
||||||
|
<%= select_tag :doc_type, options_for_select(Document::TYPES.invert, @bulk[:doc_type]), class: "input" %>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<%= label_tag :visibility, "Share-link visibility", class: "label" %>
|
||||||
|
<%= select_tag :visibility, options_for_select([ [ "Inherit from collection", "" ] ] + DocumentCollection::VISIBILITIES.invert.to_a, @bulk[:visibility]), class: "input" %>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<%= label_tag :document_collection_id, "Collection", class: "label" %>
|
||||||
|
<%= select_tag :document_collection_id,
|
||||||
|
options_for_select([ [ "— None —", "" ] ] + @aircraft.document_collections.ordered.map { [ _1.name, _1.id ] }, @bulk[:document_collection_id]), class: "input" %>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<%= label_tag :new_collection, "…or a new collection", class: "label" %>
|
||||||
|
<%= text_field_tag :new_collection, @bulk[:new_collection], class: "input", placeholder: "e.g. 2025 Annual" %>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="grid gap-4 sm:grid-cols-2">
|
||||||
|
<div>
|
||||||
|
<%= label_tag :files, "Files", class: "label" %>
|
||||||
|
<%= file_field_tag "files[]", id: "files", multiple: true, class: file_classes,
|
||||||
|
data: { bulk_upload_target: "input", action: "change->bulk-upload#add" } %>
|
||||||
|
</div>
|
||||||
|
<div data-bulk-upload-target="folderField" hidden>
|
||||||
|
<%= label_tag :folder, "…or a whole folder", class: "label" %>
|
||||||
|
<input type="file" id="folder" webkitdirectory multiple class="<%= file_classes %>"
|
||||||
|
data-bulk-upload-target="folder" data-action="change->bulk-upload#add">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p class="hint !mt-1">Images, PDFs or text, up to <%= UploadValidation::MAX_SIZE / 1.megabyte %> MB each and <%= DocumentsController::MAX_BULK_FILES %> at a time.</p>
|
||||||
|
|
||||||
|
<div data-bulk-upload-target="panel" hidden class="space-y-2">
|
||||||
|
<div class="flex flex-wrap items-center justify-between gap-2">
|
||||||
|
<p class="text-sm font-medium text-slate-700" data-bulk-upload-target="summary"></p>
|
||||||
|
<div class="flex flex-wrap items-center gap-3">
|
||||||
|
<label class="flex items-center gap-2 text-sm text-slate-700" data-bulk-upload-target="groupField" hidden>
|
||||||
|
<input type="checkbox" class="h-4 w-4 rounded border-slate-300" data-bulk-upload-target="group" data-action="bulk-upload#regroup">
|
||||||
|
One document per folder
|
||||||
|
</label>
|
||||||
|
<button type="button" class="btn-secondary btn-sm" data-action="bulk-upload#clear">Clear</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<ul class="divide-y divide-slate-100 rounded-md ring-1 ring-slate-200" data-bulk-upload-target="list"></ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-wrap items-center gap-2">
|
||||||
|
<button type="submit" class="btn-primary" data-bulk-upload-target="submit">Upload</button>
|
||||||
|
<%= link_to "Cancel", aircraft_documents_path(@aircraft), class: "btn-secondary" %>
|
||||||
|
<p class="text-sm text-slate-500" data-bulk-upload-target="status" role="status"></p>
|
||||||
|
</div>
|
||||||
|
<% end %>
|
||||||
@@ -1,8 +1,9 @@
|
|||||||
<% content_for :title, "#{@aircraft.tail_number} Documents" %>
|
<% content_for :title, "#{@aircraft.tail_number} Documents" %>
|
||||||
<%= render "shared/aircraft_header", aircraft: @aircraft %>
|
<%= render "shared/aircraft_header", aircraft: @aircraft %>
|
||||||
<div class="mb-4 flex justify-end gap-2">
|
<div class="mb-4 flex flex-wrap justify-end gap-2">
|
||||||
<% if owner? %>
|
<% if owner? %>
|
||||||
<%= link_to "+ Collection", new_aircraft_document_collection_path(@aircraft), class: "btn-secondary" %>
|
<%= link_to "+ Collection", new_aircraft_document_collection_path(@aircraft), class: "btn-secondary" %>
|
||||||
|
<%= link_to "Bulk Upload", new_bulk_aircraft_documents_path(@aircraft), class: "btn-secondary" %>
|
||||||
<%= link_to "+ Upload Document", new_aircraft_document_path(@aircraft), class: "btn-primary" %>
|
<%= link_to "+ Upload Document", new_aircraft_document_path(@aircraft), class: "btn-primary" %>
|
||||||
<% end %>
|
<% end %>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ module SimpleAircraftManager
|
|||||||
# Please, add to the `ignore` list any other `lib` subdirectories that do
|
# Please, add to the `ignore` list any other `lib` subdirectories that do
|
||||||
# not contain `.rb` files, or that should not be reloaded or eager loaded.
|
# not contain `.rb` files, or that should not be reloaded or eager loaded.
|
||||||
# Common ones are `templates`, `generators`, or `middleware`, for example.
|
# Common ones are `templates`, `generators`, or `middleware`, for example.
|
||||||
config.autoload_lib(ignore: %w[assets tasks])
|
config.autoload_lib(ignore: %w[active_storage assets tasks])
|
||||||
|
|
||||||
# Configuration for the application, engines, and railties goes here.
|
# Configuration for the application, engines, and railties goes here.
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -8,3 +8,4 @@ pin_all_from "app/javascript/controllers", under: "controllers"
|
|||||||
pin "chartkick", to: "chartkick.js"
|
pin "chartkick", to: "chartkick.js"
|
||||||
pin "Chart.bundle", to: "Chart.bundle.js"
|
pin "Chart.bundle", to: "Chart.bundle.js"
|
||||||
pin "next_due"
|
pin "next_due"
|
||||||
|
pin "@rails/activestorage", to: "activestorage.esm.js"
|
||||||
@@ -13,3 +13,9 @@ production:
|
|||||||
clear_solid_queue_finished_jobs:
|
clear_solid_queue_finished_jobs:
|
||||||
command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
|
command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
|
||||||
schedule: every hour at minute 12
|
schedule: every hour at minute 12
|
||||||
|
purge_abandoned_uploads:
|
||||||
|
command: "ActiveStorage::Blob.unattached.where(created_at: ..2.days.ago).find_each(&:purge_later)"
|
||||||
|
schedule: every day at 4am
|
||||||
|
dedupe_storage:
|
||||||
|
command: "ActiveStorage::Blob.service.try(:sweep)"
|
||||||
|
schedule: every day at 4:30am
|
||||||
+10
-1
@@ -1,4 +1,8 @@
|
|||||||
Rails.application.routes.draw do
|
Rails.application.routes.draw do
|
||||||
|
# Signed-in users only, with the upload rules checked before any bytes are sent
|
||||||
|
# (overrides Active Storage's open endpoint at the same path).
|
||||||
|
post "rails/active_storage/direct_uploads", to: "direct_uploads#create"
|
||||||
|
|
||||||
resource :session
|
resource :session
|
||||||
resources :passwords, param: :token
|
resources :passwords, param: :token
|
||||||
resource :setup, only: %i[new create], controller: "setup"
|
resource :setup, only: %i[new create], controller: "setup"
|
||||||
@@ -45,7 +49,12 @@ Rails.application.routes.draw do
|
|||||||
end
|
end
|
||||||
resources :logbook_imports, path: "logbook/import", only: %i[new create show]
|
resources :logbook_imports, path: "logbook/import", only: %i[new create show]
|
||||||
resources :logbook_entries
|
resources :logbook_entries
|
||||||
resources :documents
|
resources :documents do
|
||||||
|
collection do
|
||||||
|
get :bulk, action: :bulk_new, as: :new_bulk
|
||||||
|
post :bulk, action: :bulk_create
|
||||||
|
end
|
||||||
|
end
|
||||||
resources :document_collections, except: %i[index show]
|
resources :document_collections, except: %i[index show]
|
||||||
resources :inspections, only: :index
|
resources :inspections, only: :index
|
||||||
resources :inspection_types, except: %i[index show]
|
resources :inspection_types, except: %i[index show]
|
||||||
|
|||||||
+2
-2
@@ -1,9 +1,9 @@
|
|||||||
test:
|
test:
|
||||||
service: Disk
|
service: DedupDisk
|
||||||
root: <%= Rails.root.join("tmp/storage") %>
|
root: <%= Rails.root.join("tmp/storage") %>
|
||||||
|
|
||||||
local:
|
local:
|
||||||
service: Disk
|
service: DedupDisk
|
||||||
root: <%= Rails.root.join("storage") %>
|
root: <%= Rails.root.join("storage") %>
|
||||||
|
|
||||||
# Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
|
# Use bin/rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
require "active_storage/service/disk_service"
|
||||||
|
|
||||||
|
module ActiveStorage
|
||||||
|
# Disk storage that keeps one copy of each distinct file. A stored file's
|
||||||
|
# bytes live once under .objects/ab/cd/<sha256>, and every blob (and variant)
|
||||||
|
# with those bytes is a hard link to it, so uploading the same scan twice
|
||||||
|
# costs no extra space. Blobs, filenames and URLs are unchanged.
|
||||||
|
#
|
||||||
|
# Linked files must never be written in place, so an upload over an existing
|
||||||
|
# key first unlinks it. If the filesystem can't hard-link (some network
|
||||||
|
# shares), files are simply kept as plain copies.
|
||||||
|
#
|
||||||
|
# #sweep links in files stored before this service (or that failed to link)
|
||||||
|
# and removes objects no blob uses any more; it runs nightly.
|
||||||
|
class Service::DedupDiskService < Service::DiskService
|
||||||
|
OBJECTS_DIR = ".objects".freeze
|
||||||
|
|
||||||
|
def upload(key, io, checksum: nil, **)
|
||||||
|
release(path_for(key))
|
||||||
|
super
|
||||||
|
adopt(path_for(key))
|
||||||
|
end
|
||||||
|
|
||||||
|
def compose(source_keys, destination_key, **)
|
||||||
|
release(path_for(destination_key))
|
||||||
|
super
|
||||||
|
adopt(path_for(destination_key))
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete(key)
|
||||||
|
instrument :delete, key: key do
|
||||||
|
release(path_for(key))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_prefixed(prefix)
|
||||||
|
instrument :delete_prefixed, prefix: prefix do
|
||||||
|
prefix_path = path_for(prefix)
|
||||||
|
prefix_path += "/" if prefix.end_with?("/")
|
||||||
|
Dir.glob("#{escape_glob_metacharacters(prefix_path)}*").each do |path|
|
||||||
|
files = File.directory?(path) ? Dir.glob("#{escape_glob_metacharacters(path)}/**/*") : [ path ]
|
||||||
|
files.each { release(_1) if File.file?(_1) }
|
||||||
|
FileUtils.rm_rf(path)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Returns { linked:, removed:, saved_bytes: }.
|
||||||
|
def sweep
|
||||||
|
linked = Dir.glob(File.join(escape_glob_metacharacters(root), "??", "??", "**", "*")).count do |path|
|
||||||
|
File.file?(path) && !File.symlink?(path) && adopt(path) == :shared
|
||||||
|
end
|
||||||
|
removed = 0
|
||||||
|
saved = 0
|
||||||
|
Dir.glob(File.join(escape_glob_metacharacters(objects_root), "**", "*")).each do |object|
|
||||||
|
next unless File.file?(object)
|
||||||
|
stat = File.stat(object)
|
||||||
|
if stat.nlink == 1
|
||||||
|
File.delete(object)
|
||||||
|
removed += 1
|
||||||
|
else
|
||||||
|
saved += stat.size * (stat.nlink - 2)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
{ linked: linked, removed: removed, saved_bytes: saved }
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def objects_root = File.join(root, OBJECTS_DIR)
|
||||||
|
|
||||||
|
def object_path(digest) = File.join(objects_root, digest[0..1], digest[2..3], digest)
|
||||||
|
|
||||||
|
# Points a freshly written file at the stored copy of its bytes (or makes it
|
||||||
|
# the stored copy). Returns :stored, :shared, or nil if left alone.
|
||||||
|
def adopt(path)
|
||||||
|
return unless File.stat(path).nlink == 1
|
||||||
|
object = object_path(Digest::SHA256.file(path).hexdigest)
|
||||||
|
FileUtils.mkdir_p(File.dirname(object))
|
||||||
|
begin
|
||||||
|
File.link(path, object)
|
||||||
|
:stored
|
||||||
|
rescue Errno::EEXIST
|
||||||
|
return unless File.size(object) == File.size(path)
|
||||||
|
temp = "#{path}.#{SecureRandom.hex(6)}.link"
|
||||||
|
File.link(object, temp)
|
||||||
|
File.rename(temp, path) # atomic swap: readers see one file or the other
|
||||||
|
:shared
|
||||||
|
end
|
||||||
|
rescue SystemCallError => e
|
||||||
|
Rails.logger.warn("[dedup storage] kept a plain copy of #{path}: #{e.class}: #{e.message}")
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
|
||||||
|
# Unlinks a blob's file, and its stored object if nothing else uses it.
|
||||||
|
def release(path)
|
||||||
|
stat = File.stat(path)
|
||||||
|
if stat.nlink == 2
|
||||||
|
object = object_path(Digest::SHA256.file(path).hexdigest)
|
||||||
|
File.delete(object) if File.exist?(object) && File.stat(object).ino == stat.ino
|
||||||
|
end
|
||||||
|
File.delete(path)
|
||||||
|
rescue Errno::ENOENT
|
||||||
|
# already gone
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
namespace :storage do
|
||||||
|
desc "Store each distinct uploaded file once (hard links) and drop unused copies"
|
||||||
|
task dedupe: :environment do
|
||||||
|
service = ActiveStorage::Blob.service
|
||||||
|
abort "The #{service.class.name.demodulize} service doesn't deduplicate." unless service.respond_to?(:sweep)
|
||||||
|
result = service.sweep
|
||||||
|
puts "Linked #{result[:linked]} duplicate files, removed #{result[:removed]} unused copies; " \
|
||||||
|
"#{ActiveSupport::NumberHelper.number_to_human_size(result[:saved_bytes])} saved by sharing."
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
require "test_helper"
|
||||||
|
|
||||||
|
class DocumentBulkUploadTest < ActionDispatch::IntegrationTest
|
||||||
|
setup do
|
||||||
|
@owner = create_user("owner@example.com")
|
||||||
|
@pilot = create_user("pilot@example.com")
|
||||||
|
@a = create_aircraft(owner: @owner, pilot: @pilot)
|
||||||
|
end
|
||||||
|
|
||||||
|
def upload(name, body = "page", type = "image/jpeg")
|
||||||
|
Rack::Test::UploadedFile.new(StringIO.new(body), type, original_filename: name)
|
||||||
|
end
|
||||||
|
|
||||||
|
def direct_blob(name, type = "image/jpeg")
|
||||||
|
ActiveStorage::Blob.create_and_upload!(io: StringIO.new("bytes"), filename: name, content_type: type)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "owner sees the bulk page; pilots and outsiders can't use it" do
|
||||||
|
sign_in @owner
|
||||||
|
get aircraft_documents_path(@a)
|
||||||
|
assert_select "a[href='#{new_bulk_aircraft_documents_path(@a)}']", text: "Bulk Upload"
|
||||||
|
get new_bulk_aircraft_documents_path(@a)
|
||||||
|
assert_response :success
|
||||||
|
assert_select "form[data-controller='bulk-upload'][data-bulk-upload-url-value='/rails/active_storage/direct_uploads']"
|
||||||
|
assert_select "input[type=file][name='files[]'][multiple]"
|
||||||
|
assert_select "input[type=file][webkitdirectory]"
|
||||||
|
|
||||||
|
sign_in @pilot
|
||||||
|
post bulk_aircraft_documents_path(@a), params: { files: [ upload("a.jpg") ] }
|
||||||
|
assert_redirected_to aircraft_path(@a)
|
||||||
|
assert_equal 0, Document.count
|
||||||
|
|
||||||
|
sign_in create_user
|
||||||
|
get new_bulk_aircraft_documents_path(@a)
|
||||||
|
assert_response :not_found
|
||||||
|
end
|
||||||
|
|
||||||
|
test "plain uploads (no JavaScript) become one document per file, named after the file" do
|
||||||
|
sign_in @owner
|
||||||
|
col = @a.document_collections.create!(name: "Annual", visibility: "private")
|
||||||
|
post bulk_aircraft_documents_path(@a), params: {
|
||||||
|
files: [ upload("weight_and_balance.pdf", "%PDF", "application/pdf"), upload("Form 337.jpg") ],
|
||||||
|
doc_type: "ALTER", visibility: "maintenance", document_collection_id: col.id
|
||||||
|
}
|
||||||
|
assert_redirected_to aircraft_documents_path(@a)
|
||||||
|
assert_equal "2 documents uploaded.", flash[:notice]
|
||||||
|
docs = @a.documents.order(:name)
|
||||||
|
assert_equal [ "Form 337", "weight and balance" ], docs.map(&:name)
|
||||||
|
assert docs.all? { _1.doc_type == "ALTER" && _1.visibility == "maintenance" && _1.document_collection == col && _1.pages.size == 1 }
|
||||||
|
assert_equal "2 documents uploaded", @a.events.last.event_name
|
||||||
|
end
|
||||||
|
|
||||||
|
test "direct uploads: same name combines pages in order; a new collection is made" do
|
||||||
|
sign_in @owner
|
||||||
|
blobs = %w[p1.jpg p2.jpg receipt.jpg].map { direct_blob(_1) }
|
||||||
|
post bulk_aircraft_documents_path(@a), params: {
|
||||||
|
files: blobs.map(&:signed_id), names: [ "Engine log 1", "Engine log 1", "Prop overhaul receipt" ],
|
||||||
|
new_collection: "Logbooks", doc_type: "LOG"
|
||||||
|
}
|
||||||
|
assert_redirected_to aircraft_documents_path(@a)
|
||||||
|
col = @a.document_collections.find_by!(name: "Logbooks")
|
||||||
|
assert_equal "private", col.visibility
|
||||||
|
log = col.documents.find_by!(name: "Engine log 1")
|
||||||
|
assert_equal %w[p1.jpg p2.jpg], log.pages.map { _1.filename.to_s }
|
||||||
|
assert_nil log.visibility
|
||||||
|
assert_equal 1, col.documents.find_by!(name: "Prop overhaul receipt").pages.size
|
||||||
|
end
|
||||||
|
|
||||||
|
test "already-attached blobs and bad files are refused; partial success is reported" do
|
||||||
|
sign_in @owner
|
||||||
|
other = @a.documents.create!(name: "Existing", doc_type: "OTHER", pages: [ upload("x.jpg") ])
|
||||||
|
good = direct_blob("good.jpg")
|
||||||
|
bad = direct_blob("tool.exe", "application/x-msdownload")
|
||||||
|
post bulk_aircraft_documents_path(@a), params: { files: [ other.pages.first.blob.signed_id, good.signed_id, bad.signed_id ] }
|
||||||
|
assert_redirected_to aircraft_documents_path(@a)
|
||||||
|
assert_equal "1 document uploaded.", flash[:notice]
|
||||||
|
assert_match(/file 1 \(upload expired/, flash[:alert])
|
||||||
|
assert_match(/tool \(Pages tool.exe is not an allowed file type\)/, flash[:alert])
|
||||||
|
assert_equal [ "Existing", "good" ], @a.documents.order(:name).map(&:name)
|
||||||
|
assert_equal 1, other.reload.pages.size
|
||||||
|
end
|
||||||
|
|
||||||
|
test "nothing valid re-renders the form and drops the new collection" do
|
||||||
|
sign_in @owner
|
||||||
|
post bulk_aircraft_documents_path(@a), params: { new_collection: "Junk" }
|
||||||
|
assert_response :unprocessable_entity
|
||||||
|
assert_includes response.body, "Choose some files to upload."
|
||||||
|
|
||||||
|
post bulk_aircraft_documents_path(@a), params: { files: [ upload("a.exe", "MZ", "application/x-msdownload") ], new_collection: "Junk" }
|
||||||
|
assert_response :unprocessable_entity
|
||||||
|
assert_includes response.body, "Nothing was uploaded"
|
||||||
|
assert_select "input[name=new_collection][value=Junk]"
|
||||||
|
assert_equal 0, @a.document_collections.count
|
||||||
|
end
|
||||||
|
|
||||||
|
test "direct-upload endpoint needs a session and checks type and size first" do
|
||||||
|
blob = ->(filename, type, size = 10) { { blob: { filename: filename, byte_size: size, checksum: "Q2hlY2tzdW0=", content_type: type } } }
|
||||||
|
post "/rails/active_storage/direct_uploads", params: blob.("a.jpg", "image/jpeg"), as: :json
|
||||||
|
assert_redirected_to new_session_path
|
||||||
|
assert_equal 0, ActiveStorage::Blob.count
|
||||||
|
|
||||||
|
sign_in @owner
|
||||||
|
post "/rails/active_storage/direct_uploads", params: blob.("a.exe", "application/x-msdownload"), as: :json
|
||||||
|
assert_response :unprocessable_entity
|
||||||
|
assert_match "isn't an allowed file type", response.parsed_body["error"]
|
||||||
|
post "/rails/active_storage/direct_uploads", params: blob.("big.pdf", "application/pdf", 51.megabytes), as: :json
|
||||||
|
assert_response :unprocessable_entity
|
||||||
|
|
||||||
|
post "/rails/active_storage/direct_uploads", params: blob.("scan.png", ""), as: :json
|
||||||
|
assert_response :success
|
||||||
|
assert_equal "image/png", ActiveStorage::Blob.last.content_type
|
||||||
|
assert response.parsed_body["signed_id"].present?
|
||||||
|
end
|
||||||
|
|
||||||
|
test "the same file direct-uploaded twice is stored once on disk" do
|
||||||
|
sign_in @owner
|
||||||
|
body = "%PDF-1.4 identical scan"
|
||||||
|
ids = 2.times.map do |i|
|
||||||
|
post "/rails/active_storage/direct_uploads", as: :json, params: { blob: {
|
||||||
|
filename: "scan#{i}.pdf", byte_size: body.bytesize, checksum: OpenSSL::Digest::MD5.base64digest(body), content_type: "application/pdf" } }
|
||||||
|
signed_id, upload = response.parsed_body.values_at("signed_id", "direct_upload")
|
||||||
|
put URI(upload["url"]).request_uri, params: body, headers: upload["headers"]
|
||||||
|
assert_response :no_content
|
||||||
|
signed_id
|
||||||
|
end
|
||||||
|
post bulk_aircraft_documents_path(@a), params: { files: ids }
|
||||||
|
assert_redirected_to aircraft_documents_path(@a)
|
||||||
|
paths = @a.documents.order(:name).map { ActiveStorage::Blob.service.path_for(_1.pages.first.key) }
|
||||||
|
assert_equal 2, paths.size
|
||||||
|
assert_equal File.stat(paths[0]).ino, File.stat(paths[1]).ino
|
||||||
|
assert_equal %w[scan0 scan1], @a.documents.order(:name).map(&:name)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
require "test_helper"
|
||||||
|
require "active_storage/service/dedup_disk_service"
|
||||||
|
|
||||||
|
class DedupDiskServiceTest < ActiveSupport::TestCase
|
||||||
|
include ActiveJob::TestHelper
|
||||||
|
|
||||||
|
setup do
|
||||||
|
@root = Dir.mktmpdir
|
||||||
|
@service = ActiveStorage::Service::DedupDiskService.new(root: @root)
|
||||||
|
end
|
||||||
|
|
||||||
|
teardown { FileUtils.rm_rf(@root) }
|
||||||
|
|
||||||
|
def put(key, body) = @service.upload(key, StringIO.new(body), checksum: OpenSSL::Digest::MD5.base64digest(body))
|
||||||
|
def path(key) = @service.path_for(key)
|
||||||
|
def objects = Dir.glob(File.join(@root, ".objects/**/*")).select { File.file?(_1) }
|
||||||
|
|
||||||
|
test "identical files share one stored copy; different files don't" do
|
||||||
|
put "aaaa1111", "same scan"
|
||||||
|
put "bbbb2222", "same scan"
|
||||||
|
put "cccc3333", "other scan"
|
||||||
|
assert_equal File.stat(path("aaaa1111")).ino, File.stat(path("bbbb2222")).ino
|
||||||
|
refute_equal File.stat(path("aaaa1111")).ino, File.stat(path("cccc3333")).ino
|
||||||
|
assert_equal 2, objects.size
|
||||||
|
assert_equal "same scan", @service.download("bbbb2222")
|
||||||
|
end
|
||||||
|
|
||||||
|
test "deleting one copy keeps the other; deleting the last frees the object" do
|
||||||
|
put "aaaa1111", "same scan"
|
||||||
|
put "bbbb2222", "same scan"
|
||||||
|
@service.delete("aaaa1111")
|
||||||
|
assert_equal "same scan", @service.download("bbbb2222")
|
||||||
|
assert_equal 1, objects.size
|
||||||
|
@service.delete("bbbb2222")
|
||||||
|
assert_empty objects
|
||||||
|
refute @service.exist?("bbbb2222")
|
||||||
|
end
|
||||||
|
|
||||||
|
test "re-uploading a shared key never changes the other blob" do
|
||||||
|
put "aaaa1111", "same scan"
|
||||||
|
put "bbbb2222", "same scan"
|
||||||
|
put "aaaa1111", "replaced"
|
||||||
|
assert_equal "replaced", @service.download("aaaa1111")
|
||||||
|
assert_equal "same scan", @service.download("bbbb2222")
|
||||||
|
end
|
||||||
|
|
||||||
|
test "a checksum mismatch stores nothing" do
|
||||||
|
assert_raises(ActiveStorage::IntegrityError) { @service.upload("aaaa1111", StringIO.new("bytes"), checksum: "bm9wZQ==") }
|
||||||
|
refute @service.exist?("aaaa1111")
|
||||||
|
assert_empty objects
|
||||||
|
end
|
||||||
|
|
||||||
|
test "variants under a prefix are released too" do
|
||||||
|
put "variants/aaaa1111/thumb", "same scan"
|
||||||
|
put "bbbb2222", "same scan"
|
||||||
|
@service.delete_prefixed("variants/aaaa1111/")
|
||||||
|
refute @service.exist?("variants/aaaa1111/thumb")
|
||||||
|
assert_equal "same scan", @service.download("bbbb2222")
|
||||||
|
assert_equal 1, objects.size
|
||||||
|
end
|
||||||
|
|
||||||
|
test "sweep links files stored before dedup and clears unused objects" do
|
||||||
|
plain = ActiveStorage::Service::DiskService.new(root: @root)
|
||||||
|
plain.upload("aaaa1111", StringIO.new("old scan"))
|
||||||
|
plain.upload("bbbb2222", StringIO.new("old scan"))
|
||||||
|
put "cccc3333", "gone soon"
|
||||||
|
File.delete(path("cccc3333")) # e.g. a crash between unlinks
|
||||||
|
|
||||||
|
assert_equal({ linked: 1, removed: 1, saved_bytes: 8 }, @service.sweep)
|
||||||
|
assert_equal File.stat(path("aaaa1111")).ino, File.stat(path("bbbb2222")).ino
|
||||||
|
assert_equal({ linked: 0, removed: 0, saved_bytes: 8 }, @service.sweep)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "attachments through Active Storage share storage but keep their own filenames" do
|
||||||
|
a = create_aircraft
|
||||||
|
one = a.documents.create!(name: "Receipt", doc_type: "OTHER", pages: [ { io: StringIO.new("%PDF same"), filename: "receipt.pdf", content_type: "application/pdf" } ])
|
||||||
|
two = a.documents.create!(name: "Invoice", doc_type: "OTHER", pages: [ { io: StringIO.new("%PDF same"), filename: "invoice.pdf", content_type: "application/pdf" } ])
|
||||||
|
service = ActiveStorage::Blob.service
|
||||||
|
assert_kind_of ActiveStorage::Service::DedupDiskService, service
|
||||||
|
keys = [ one, two ].map { _1.pages.first.blob.key }
|
||||||
|
assert_equal File.stat(service.path_for(keys[0])).ino, File.stat(service.path_for(keys[1])).ino
|
||||||
|
assert_equal %w[receipt.pdf invoice.pdf], [ one, two ].map { _1.pages.first.filename.to_s }
|
||||||
|
perform_enqueued_jobs { one.destroy! }
|
||||||
|
assert_equal 1, ActiveStorage::Blob.count
|
||||||
|
assert_equal "%PDF same", two.pages.first.download
|
||||||
|
end
|
||||||
|
end
|
||||||
Reference in new issue
Block a user