mirror of
https://github.com/rickbarrette/redmine_qbo.git
synced 2026-11-17 18:20:36 -05:00
- Nicknames now come from the carltonnorthern/nicknames data set (Apache 2.0, ~2,800 pairs, bundled in data/nicknames with its license), plus a few extra groups in EXTRA_NICKNAME_GROUPS. Names match if directly linked, or if both are short for the same longer name, so Bill/William and Bill/Will match but William/Robert don't (the data lists "bill" for both) - Double Metaphone (text gem) matches names that sound alike but are spelled differently: Jon/John, Smyth/Smith, Cathy/Kathy, Steven/Stephen, Jeffrey/Geoffrey - Faster ranking: per-search word-pair scores are cached, whole-name comparison only runs when it can succeed, and prepared search data is kept in process memory until a customer changes (~40 ms per search at 5,000 customers) Adds gem 'text', '~> 1.3' to the Gemfile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BiwYMjhoQvL7XsKKfRz4pM
234 lines
10 KiB
Ruby
234 lines
10 KiB
Ruby
#The MIT License (MIT)
|
||
#
|
||
#Copyright (c) 2016 - 2026 rick barrette
|
||
#
|
||
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||
#
|
||
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||
#
|
||
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||
|
||
require 'csv'
|
||
require 'did_you_mean'
|
||
require 'text'
|
||
|
||
# Finds customers by name and/or phone, tolerant of the ways duplicates sneak in:
|
||
# - nicknames: "Bill Smith" finds "William Smith", "Bob" finds "Robert" (about 2,800 pairs, see data/nicknames)
|
||
# - sound-alikes: "Jon" finds "John", "Smyth" finds "Smith", "Cathy" finds "Kathy" (Double Metaphone)
|
||
# - typos: "Willaim Smtih" finds "William Smith"
|
||
# - word order: "Smith, William" finds "William Smith"
|
||
# - partial: "will smi" finds "William Smith" while typing
|
||
# - phone: any 4+ digits, in any format, matched against both phone numbers
|
||
#
|
||
# Results are ranked best match first. Only id, name and phone numbers are loaded, and the prepared
|
||
# search data is cached until a customer changes, so searching stays fast as the customer list grows.
|
||
class CustomerSearchService
|
||
DEFAULT_LIMIT = 20
|
||
|
||
# Lowest score (0..1) a customer needs to be returned
|
||
MIN_SCORE = 0.72
|
||
|
||
# Bundled nickname data (Apache 2.0, see data/nicknames/README.md)
|
||
NICKNAMES_CSV = File.expand_path('../../data/nicknames/names.csv', __dir__)
|
||
|
||
# Groups missing from the bundled data. Add more here: every name in a group matches every other.
|
||
EXTRA_NICKNAME_GROUPS = [
|
||
%w[william liam],
|
||
%w[alexander xander],
|
||
%w[alexandra lexi],
|
||
%w[theodore theo],
|
||
%w[christopher kit]
|
||
].freeze
|
||
|
||
# Words that don't help tell customers apart
|
||
STOP_WORDS = %w[and the llc inc co corp company].freeze
|
||
|
||
# Returns matching Customer records, best match first.
|
||
def self.call(name: nil, phone: nil, limit: DEFAULT_LIMIT)
|
||
ids = rank(candidates, name: name, phone: phone, limit: limit).map(&:first)
|
||
return Customer.none if ids.empty?
|
||
|
||
Customer.where(id: ids).in_order_of(:id, ids)
|
||
end
|
||
|
||
# Prepared search rows for every customer, cached until any customer changes. Kept in this process's
|
||
# memory as well, so each keystroke doesn't reload it from the Rails cache.
|
||
def self.candidates
|
||
key = ['customer_search_candidates', 2, Customer.count, Customer.maximum(:updated_at).to_i]
|
||
cached_key, cached_rows = @candidates
|
||
return cached_rows if cached_key == key
|
||
|
||
rows = Rails.cache.fetch(key, expires_in: 12.hours) do
|
||
prepare(Customer.pluck(:id, :name, :phone_number, :mobile_phone_number))
|
||
end
|
||
@candidates = [key, rows]
|
||
rows
|
||
end
|
||
|
||
# [[id, name, phone, mobile], ...] -> [[id, name, phone_digits, tokens, sound_codes], ...]
|
||
def self.prepare(rows)
|
||
rows.map do |id, name, phone_number, mobile_number|
|
||
words = tokens(name)
|
||
phones = [phone_number, mobile_number].compact.map { |n| n.to_s.gsub(/\D/, '') }.reject(&:empty?)
|
||
[id, name.to_s, phones, words, words.map { |w| sound_codes(w) }]
|
||
end
|
||
end
|
||
|
||
# Pure ranking over prepared rows (no database), so it can be tested on its own.
|
||
# Returns [[id, score], ...] best first. A query of mostly digits is also treated as a phone number.
|
||
def self.rank(prepared, name: nil, phone: nil, limit: DEFAULT_LIMIT)
|
||
name = name.to_s.strip
|
||
phone_digits = phone.to_s.gsub(/\D/, '')
|
||
|
||
# "401-555-1234" typed into the name box is a phone search
|
||
name_digits = name.gsub(/\D/, '')
|
||
if phone_digits.empty? && name_digits.length >= 4 && name_digits.length >= name.gsub(/\s/, '').length * 0.7
|
||
phone_digits = name_digits
|
||
name = ''
|
||
end
|
||
|
||
query_words = tokens(name)
|
||
return [] if query_words.empty? && phone_digits.length < 4
|
||
|
||
query = query_words.map { |w| [w, sound_codes(w)] }
|
||
memo = {} # word-pair scores for this search; customer names repeat words a lot ("smith", "john")
|
||
|
||
scored = prepared.filter_map do |id, customer_name, phones, words, codes|
|
||
score = [
|
||
phone_digits.length >= 4 ? phone_score(phone_digits, phones) : 0.0,
|
||
query.any? ? name_score(query, words, codes, memo) : 0.0
|
||
].max
|
||
[id, score, customer_name] if score >= MIN_SCORE
|
||
end
|
||
|
||
scored.sort_by { |id, score, customer_name| [-score, customer_name.downcase, id] }
|
||
.first(limit)
|
||
.map { |id, score, _| [id, score.round(3)] }
|
||
end
|
||
|
||
# Lowercase words with accents and punctuation removed: "O'Brien-Smith, José" -> ["obrien", "smith", "jose"]
|
||
def self.tokens(text)
|
||
normalized = text.to_s.unicode_normalize(:nfkd).gsub(/\p{Mn}/, '').downcase
|
||
normalized = normalized.gsub(/['’]/, '').gsub(/[^a-z0-9]+/, ' ')
|
||
words = normalized.split
|
||
meaningful = words - STOP_WORDS
|
||
meaningful.empty? ? words : meaningful
|
||
end
|
||
|
||
# Double Metaphone codes (primary and alternate) for a word; empty for very short or numeric words
|
||
def self.sound_codes(word)
|
||
return [] if word.length < 3 || word.match?(/\d/)
|
||
|
||
Text::Metaphone.double_metaphone(word).compact.reject(&:empty?).uniq
|
||
end
|
||
|
||
def self.phone_score(digits, phones)
|
||
phones.map do |number|
|
||
if number == digits || number.end_with?(digits) && digits.length >= 7 then 1.0
|
||
elsif number.include?(digits) then 0.9
|
||
else 0.0
|
||
end
|
||
end.max || 0.0
|
||
end
|
||
|
||
# Each query word is matched to its best word in the customer's name; the score is the average.
|
||
# A query word with no match at all halves the score, so "bill jones" won't match "william smith".
|
||
def self.name_score(query, words, codes, memo = {})
|
||
return 0.0 if words.empty?
|
||
|
||
best = query.map do |q_word, q_codes|
|
||
scores = (memo[q_word] ||= {})
|
||
words.each_with_index.map do |c_word, i|
|
||
scores.fetch(c_word) { scores[c_word] = word_score(q_word, q_codes, c_word, codes[i]) }
|
||
end.max
|
||
end
|
||
score = best.sum / best.size
|
||
score *= 0.5 if best.any?(&:zero?)
|
||
|
||
return score if score >= 0.9 # already a strong match; skip the whole-name comparison
|
||
|
||
# Also compare the names with spaces removed, for "Mc Donald" vs "McDonald". Only worth it when the
|
||
# strings could reach the 0.92 bar at all: same first letter and nearly the same length.
|
||
query_joined = query.map(&:first).join
|
||
name_joined = words.join
|
||
return score unless query_joined[0] == name_joined[0] && (query_joined.length - name_joined.length).abs <= 2
|
||
|
||
joined = DidYouMean::JaroWinkler.distance(query_joined, name_joined)
|
||
[score, joined >= 0.92 ? joined * 0.9 : 0.0].max
|
||
end
|
||
|
||
def self.word_score(query, query_codes, candidate, candidate_codes)
|
||
return 1.0 if query == candidate
|
||
return 0.95 if nicknames?(query, candidate)
|
||
return 0.9 if query.length >= 2 && candidate.start_with?(query) # still typing: "wil" -> "william"
|
||
return 0.85 if query.length >= 2 && nickname_prefix?(query, candidate) # "bil" -> "william" via "bill"
|
||
|
||
similarity = DidYouMean::JaroWinkler.distance(query, candidate)
|
||
|
||
# Sounds the same ("jon"/"john", "smyth"/"smith"). The similarity check keeps unrelated words that
|
||
# happen to share a code (e.g. "john" and "ann" via the Spanish-J alternate) from matching.
|
||
return 0.88 if similarity >= 0.7 && (query_codes & candidate_codes).any?
|
||
|
||
return 0.8 if query.length >= 4 && candidate.include?(query) # "donald" in "mcdonald"
|
||
return 0.7 if query.length == 1 && candidate.start_with?(query) # an initial
|
||
|
||
# Typos. Short words need to be closer to count.
|
||
threshold = [query.length, candidate.length].min <= 4 ? 0.9 : 0.85
|
||
similarity >= threshold ? similarity * 0.9 : 0.0
|
||
end
|
||
|
||
# Nickname matching treats the data as connections between names (the file lists both directions,
|
||
# e.g. "william -> bill" and "bill -> william"). Two names match if they are connected directly, or both
|
||
# connect to a LONGER name they're short for: "bill" and "will" via "william". Requiring the shared
|
||
# name to be longer stops unrelated full names linking through a short one: "bill" is listed for both
|
||
# William and Robert, but that doesn't make William match Robert.
|
||
def self.nicknames?(a, b)
|
||
related_names(a).include?(b)
|
||
end
|
||
|
||
# A partly typed nickname of the candidate's name, e.g. "bil" for "william" (via "bill")
|
||
def self.nickname_prefix?(query, candidate)
|
||
related_names(candidate).any? { |name| name.start_with?(query) }
|
||
end
|
||
|
||
# All names interchangeable with name, cached per process
|
||
def self.related_names(name)
|
||
cache = (@related_names_cache ||= {})
|
||
cache[name] ||= begin
|
||
graph = nickname_graph
|
||
direct = graph.fetch(name, EMPTY)
|
||
through_full_names = direct.select { |full| full.length > name.length }.flat_map do |full|
|
||
graph.fetch(full, EMPTY).select { |other| other.length < full.length }
|
||
end
|
||
((direct + through_full_names).uniq - [name]).freeze
|
||
end
|
||
end
|
||
|
||
EMPTY = [].freeze
|
||
|
||
# name => [connected names], built once per process from the bundled data plus EXTRA_NICKNAME_GROUPS
|
||
def self.nickname_graph
|
||
@nickname_graph ||= begin
|
||
graph = Hash.new { |h, k| h[k] = [] }
|
||
connect = lambda do |a, b|
|
||
next if a.empty? || b.empty? || a == b
|
||
|
||
graph[a] << b
|
||
graph[b] << a
|
||
end
|
||
|
||
if File.exist?(NICKNAMES_CSV)
|
||
CSV.foreach(NICKNAMES_CSV, headers: true) do |row|
|
||
connect.call(row['name1'].to_s.strip.downcase, row['name2'].to_s.strip.downcase)
|
||
end
|
||
end
|
||
|
||
EXTRA_NICKNAME_GROUPS.each do |group|
|
||
group.combination(2) { |a, b| connect.call(a, b) }
|
||
end
|
||
|
||
graph.transform_values { |names| names.uniq.freeze }.to_h.freeze
|
||
end
|
||
end
|
||
end
|