Bulk document upload and deduplicated file storage

- Bulk Upload page: pick many files or a folder; each file becomes a
  document named after it, and files sharing a name become one multi-page
  document. Files go up as direct uploads with per-file progress.
- Direct uploads require a signed-in user and check type/size up front;
  only fresh, unattached blobs can be attached; abandoned uploads are
  purged nightly.
- DedupDisk storage service: each distinct file is stored once under its
  SHA-256 and blobs hard-link to it; nightly sweep (and storage:dedupe)
  links pre-existing duplicates and drops unused objects.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
rickyandClaude Opus 5.5 committed 2026-11-07 08:20:22 -05:00
1 parent 230a569b77
commit 2bfcb845ca
15 files changed
+735 -5

No files matched your search

@@ -0,0 +1,107 @@
require "active_storage/service/disk_service"
module ActiveStorage
# Disk storage that keeps one copy of each distinct file. A stored file's
# bytes live once under .objects/ab/cd/<sha256>, and every blob (and variant)
# with those bytes is a hard link to it, so uploading the same scan twice
# costs no extra space. Blobs, filenames and URLs are unchanged.
#
# Linked files must never be written in place, so an upload over an existing
# key first unlinks it. If the filesystem can't hard-link (some network
# shares), files are simply kept as plain copies.
#
# #sweep links in files stored before this service (or that failed to link)
# and removes objects no blob uses any more; it runs nightly.
class Service::DedupDiskService < Service::DiskService
OBJECTS_DIR = ".objects".freeze
def upload(key, io, checksum: nil, **)
release(path_for(key))
super
adopt(path_for(key))
end
def compose(source_keys, destination_key, **)
release(path_for(destination_key))
super
adopt(path_for(destination_key))
end
def delete(key)
instrument :delete, key: key do
release(path_for(key))
end
end
def delete_prefixed(prefix)
instrument :delete_prefixed, prefix: prefix do
prefix_path = path_for(prefix)
prefix_path += "/" if prefix.end_with?("/")
Dir.glob("#{escape_glob_metacharacters(prefix_path)}*").each do |path|
files = File.directory?(path) ? Dir.glob("#{escape_glob_metacharacters(path)}/**/*") : [ path ]
files.each { release(_1) if File.file?(_1) }
FileUtils.rm_rf(path)
end
end
end
# Returns { linked:, removed:, saved_bytes: }.
def sweep
linked = Dir.glob(File.join(escape_glob_metacharacters(root), "??", "??", "**", "*")).count do |path|
File.file?(path) && !File.symlink?(path) && adopt(path) == :shared
end
removed = 0
saved = 0
Dir.glob(File.join(escape_glob_metacharacters(objects_root), "**", "*")).each do |object|
next unless File.file?(object)
stat = File.stat(object)
if stat.nlink == 1
File.delete(object)
removed += 1
else
saved += stat.size * (stat.nlink - 2)
end
end
{ linked: linked, removed: removed, saved_bytes: saved }
end
private
def objects_root = File.join(root, OBJECTS_DIR)
def object_path(digest) = File.join(objects_root, digest[0..1], digest[2..3], digest)
# Points a freshly written file at the stored copy of its bytes (or makes it
# the stored copy). Returns :stored, :shared, or nil if left alone.
def adopt(path)
return unless File.stat(path).nlink == 1
object = object_path(Digest::SHA256.file(path).hexdigest)
FileUtils.mkdir_p(File.dirname(object))
begin
File.link(path, object)
:stored
rescue Errno::EEXIST
return unless File.size(object) == File.size(path)
temp = "#{path}.#{SecureRandom.hex(6)}.link"
File.link(object, temp)
File.rename(temp, path) # atomic swap: readers see one file or the other
:shared
end
rescue SystemCallError => e
Rails.logger.warn("[dedup storage] kept a plain copy of #{path}: #{e.class}: #{e.message}")
nil
end
# Unlinks a blob's file, and its stored object if nothing else uses it.
def release(path)
stat = File.stat(path)
if stat.nlink == 2
object = object_path(Digest::SHA256.file(path).hexdigest)
File.delete(object) if File.exist?(object) && File.stat(object).ino == stat.ino
end
File.delete(path)
rescue Errno::ENOENT
# already gone
end
end
end