mirror of
https://github.com/rickbarrette/simple-aircraft-manager-rails.git
synced 2026-11-17 15:50:36 -05:00
Bulk document upload and deduplicated file storage
- Bulk Upload page: pick many files or a folder; each file becomes a document named after it, and files sharing a name become one multi-page document. Files go up as direct uploads with per-file progress. - Direct uploads require a signed-in user and check type/size up front; only fresh, unattached blobs can be attached; abandoned uploads are purged nightly. - DedupDisk storage service: each distinct file is stored once under its SHA-256 and blobs hard-link to it; nightly sweep (and storage:dedupe) links pre-existing duplicates and drops unused objects. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
230a569b77
commit
2bfcb845ca
15 files changed
+735
-5
No files matched your search
@@ -0,0 +1,107 @@
|
||||
require "active_storage/service/disk_service"
|
||||
|
||||
module ActiveStorage
|
||||
# Disk storage that keeps one copy of each distinct file. A stored file's
|
||||
# bytes live once under .objects/ab/cd/<sha256>, and every blob (and variant)
|
||||
# with those bytes is a hard link to it, so uploading the same scan twice
|
||||
# costs no extra space. Blobs, filenames and URLs are unchanged.
|
||||
#
|
||||
# Linked files must never be written in place, so an upload over an existing
|
||||
# key first unlinks it. If the filesystem can't hard-link (some network
|
||||
# shares), files are simply kept as plain copies.
|
||||
#
|
||||
# #sweep links in files stored before this service (or that failed to link)
|
||||
# and removes objects no blob uses any more; it runs nightly.
|
||||
class Service::DedupDiskService < Service::DiskService
|
||||
OBJECTS_DIR = ".objects".freeze
|
||||
|
||||
def upload(key, io, checksum: nil, **)
|
||||
release(path_for(key))
|
||||
super
|
||||
adopt(path_for(key))
|
||||
end
|
||||
|
||||
def compose(source_keys, destination_key, **)
|
||||
release(path_for(destination_key))
|
||||
super
|
||||
adopt(path_for(destination_key))
|
||||
end
|
||||
|
||||
def delete(key)
|
||||
instrument :delete, key: key do
|
||||
release(path_for(key))
|
||||
end
|
||||
end
|
||||
|
||||
def delete_prefixed(prefix)
|
||||
instrument :delete_prefixed, prefix: prefix do
|
||||
prefix_path = path_for(prefix)
|
||||
prefix_path += "/" if prefix.end_with?("/")
|
||||
Dir.glob("#{escape_glob_metacharacters(prefix_path)}*").each do |path|
|
||||
files = File.directory?(path) ? Dir.glob("#{escape_glob_metacharacters(path)}/**/*") : [ path ]
|
||||
files.each { release(_1) if File.file?(_1) }
|
||||
FileUtils.rm_rf(path)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Returns { linked:, removed:, saved_bytes: }.
|
||||
def sweep
|
||||
linked = Dir.glob(File.join(escape_glob_metacharacters(root), "??", "??", "**", "*")).count do |path|
|
||||
File.file?(path) && !File.symlink?(path) && adopt(path) == :shared
|
||||
end
|
||||
removed = 0
|
||||
saved = 0
|
||||
Dir.glob(File.join(escape_glob_metacharacters(objects_root), "**", "*")).each do |object|
|
||||
next unless File.file?(object)
|
||||
stat = File.stat(object)
|
||||
if stat.nlink == 1
|
||||
File.delete(object)
|
||||
removed += 1
|
||||
else
|
||||
saved += stat.size * (stat.nlink - 2)
|
||||
end
|
||||
end
|
||||
{ linked: linked, removed: removed, saved_bytes: saved }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def objects_root = File.join(root, OBJECTS_DIR)
|
||||
|
||||
def object_path(digest) = File.join(objects_root, digest[0..1], digest[2..3], digest)
|
||||
|
||||
# Points a freshly written file at the stored copy of its bytes (or makes it
|
||||
# the stored copy). Returns :stored, :shared, or nil if left alone.
|
||||
def adopt(path)
|
||||
return unless File.stat(path).nlink == 1
|
||||
object = object_path(Digest::SHA256.file(path).hexdigest)
|
||||
FileUtils.mkdir_p(File.dirname(object))
|
||||
begin
|
||||
File.link(path, object)
|
||||
:stored
|
||||
rescue Errno::EEXIST
|
||||
return unless File.size(object) == File.size(path)
|
||||
temp = "#{path}.#{SecureRandom.hex(6)}.link"
|
||||
File.link(object, temp)
|
||||
File.rename(temp, path) # atomic swap: readers see one file or the other
|
||||
:shared
|
||||
end
|
||||
rescue SystemCallError => e
|
||||
Rails.logger.warn("[dedup storage] kept a plain copy of #{path}: #{e.class}: #{e.message}")
|
||||
nil
|
||||
end
|
||||
|
||||
# Unlinks a blob's file, and its stored object if nothing else uses it.
|
||||
def release(path)
|
||||
stat = File.stat(path)
|
||||
if stat.nlink == 2
|
||||
object = object_path(Digest::SHA256.file(path).hexdigest)
|
||||
File.delete(object) if File.exist?(object) && File.stat(object).ino == stat.ino
|
||||
end
|
||||
File.delete(path)
|
||||
rescue Errno::ENOENT
|
||||
# already gone
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in new issue
Block a user