Files
redmine_qbo/app/services/customer_search_service.rb
T
Claude c4ab97b75e Customer search: bundled nickname data and sound-alike matching
- Nicknames now come from the carltonnorthern/nicknames data set
  (Apache 2.0, ~2,800 pairs, bundled in data/nicknames with its license),
  plus a few extra groups in EXTRA_NICKNAME_GROUPS. Names match if
  directly linked, or if both are short for the same longer name, so
  Bill/William and Bill/Will match but William/Robert don't (the data
  lists "bill" for both)
- Double Metaphone (text gem) matches names that sound alike but are
  spelled differently: Jon/John, Smyth/Smith, Cathy/Kathy,
  Steven/Stephen, Jeffrey/Geoffrey
- Faster ranking: per-search word-pair scores are cached, whole-name
  comparison only runs when it can succeed, and prepared search data is
  kept in process memory until a customer changes (~40 ms per search at
  5,000 customers)

Adds gem 'text', '~> 1.3' to the Gemfile.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BiwYMjhoQvL7XsKKfRz4pM
2026-10-28 22:22:00 -04:00

234 lines
10 KiB
Ruby
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#The MIT License (MIT)
#
#Copyright (c) 2016 - 2026 rick barrette
#
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
require 'csv'
require 'did_you_mean'
require 'text'
# Finds customers by name and/or phone, tolerant of the ways duplicates sneak in:
# - nicknames: "Bill Smith" finds "William Smith", "Bob" finds "Robert" (about 2,800 pairs, see data/nicknames)
# - sound-alikes: "Jon" finds "John", "Smyth" finds "Smith", "Cathy" finds "Kathy" (Double Metaphone)
# - typos: "Willaim Smtih" finds "William Smith"
# - word order: "Smith, William" finds "William Smith"
# - partial: "will smi" finds "William Smith" while typing
# - phone: any 4+ digits, in any format, matched against both phone numbers
#
# Results are ranked best match first. Only id, name and phone numbers are loaded, and the prepared
# search data is cached until a customer changes, so searching stays fast as the customer list grows.
class CustomerSearchService
DEFAULT_LIMIT = 20
# Lowest score (0..1) a customer needs to be returned
MIN_SCORE = 0.72
# Bundled nickname data (Apache 2.0, see data/nicknames/README.md)
NICKNAMES_CSV = File.expand_path('../../data/nicknames/names.csv', __dir__)
# Groups missing from the bundled data. Add more here: every name in a group matches every other.
EXTRA_NICKNAME_GROUPS = [
%w[william liam],
%w[alexander xander],
%w[alexandra lexi],
%w[theodore theo],
%w[christopher kit]
].freeze
# Words that don't help tell customers apart
STOP_WORDS = %w[and the llc inc co corp company].freeze
# Returns matching Customer records, best match first.
def self.call(name: nil, phone: nil, limit: DEFAULT_LIMIT)
ids = rank(candidates, name: name, phone: phone, limit: limit).map(&:first)
return Customer.none if ids.empty?
Customer.where(id: ids).in_order_of(:id, ids)
end
# Prepared search rows for every customer, cached until any customer changes. Kept in this process's
# memory as well, so each keystroke doesn't reload it from the Rails cache.
def self.candidates
key = ['customer_search_candidates', 2, Customer.count, Customer.maximum(:updated_at).to_i]
cached_key, cached_rows = @candidates
return cached_rows if cached_key == key
rows = Rails.cache.fetch(key, expires_in: 12.hours) do
prepare(Customer.pluck(:id, :name, :phone_number, :mobile_phone_number))
end
@candidates = [key, rows]
rows
end
# [[id, name, phone, mobile], ...] -> [[id, name, phone_digits, tokens, sound_codes], ...]
def self.prepare(rows)
rows.map do |id, name, phone_number, mobile_number|
words = tokens(name)
phones = [phone_number, mobile_number].compact.map { |n| n.to_s.gsub(/\D/, '') }.reject(&:empty?)
[id, name.to_s, phones, words, words.map { |w| sound_codes(w) }]
end
end
# Pure ranking over prepared rows (no database), so it can be tested on its own.
# Returns [[id, score], ...] best first. A query of mostly digits is also treated as a phone number.
def self.rank(prepared, name: nil, phone: nil, limit: DEFAULT_LIMIT)
name = name.to_s.strip
phone_digits = phone.to_s.gsub(/\D/, '')
# "401-555-1234" typed into the name box is a phone search
name_digits = name.gsub(/\D/, '')
if phone_digits.empty? && name_digits.length >= 4 && name_digits.length >= name.gsub(/\s/, '').length * 0.7
phone_digits = name_digits
name = ''
end
query_words = tokens(name)
return [] if query_words.empty? && phone_digits.length < 4
query = query_words.map { |w| [w, sound_codes(w)] }
memo = {} # word-pair scores for this search; customer names repeat words a lot ("smith", "john")
scored = prepared.filter_map do |id, customer_name, phones, words, codes|
score = [
phone_digits.length >= 4 ? phone_score(phone_digits, phones) : 0.0,
query.any? ? name_score(query, words, codes, memo) : 0.0
].max
[id, score, customer_name] if score >= MIN_SCORE
end
scored.sort_by { |id, score, customer_name| [-score, customer_name.downcase, id] }
.first(limit)
.map { |id, score, _| [id, score.round(3)] }
end
# Lowercase words with accents and punctuation removed: "O'Brien-Smith, José" -> ["obrien", "smith", "jose"]
def self.tokens(text)
normalized = text.to_s.unicode_normalize(:nfkd).gsub(/\p{Mn}/, '').downcase
normalized = normalized.gsub(/['’]/, '').gsub(/[^a-z0-9]+/, ' ')
words = normalized.split
meaningful = words - STOP_WORDS
meaningful.empty? ? words : meaningful
end
# Double Metaphone codes (primary and alternate) for a word; empty for very short or numeric words
def self.sound_codes(word)
return [] if word.length < 3 || word.match?(/\d/)
Text::Metaphone.double_metaphone(word).compact.reject(&:empty?).uniq
end
def self.phone_score(digits, phones)
phones.map do |number|
if number == digits || number.end_with?(digits) && digits.length >= 7 then 1.0
elsif number.include?(digits) then 0.9
else 0.0
end
end.max || 0.0
end
# Each query word is matched to its best word in the customer's name; the score is the average.
# A query word with no match at all halves the score, so "bill jones" won't match "william smith".
def self.name_score(query, words, codes, memo = {})
return 0.0 if words.empty?
best = query.map do |q_word, q_codes|
scores = (memo[q_word] ||= {})
words.each_with_index.map do |c_word, i|
scores.fetch(c_word) { scores[c_word] = word_score(q_word, q_codes, c_word, codes[i]) }
end.max
end
score = best.sum / best.size
score *= 0.5 if best.any?(&:zero?)
return score if score >= 0.9 # already a strong match; skip the whole-name comparison
# Also compare the names with spaces removed, for "Mc Donald" vs "McDonald". Only worth it when the
# strings could reach the 0.92 bar at all: same first letter and nearly the same length.
query_joined = query.map(&:first).join
name_joined = words.join
return score unless query_joined[0] == name_joined[0] && (query_joined.length - name_joined.length).abs <= 2
joined = DidYouMean::JaroWinkler.distance(query_joined, name_joined)
[score, joined >= 0.92 ? joined * 0.9 : 0.0].max
end
def self.word_score(query, query_codes, candidate, candidate_codes)
return 1.0 if query == candidate
return 0.95 if nicknames?(query, candidate)
return 0.9 if query.length >= 2 && candidate.start_with?(query) # still typing: "wil" -> "william"
return 0.85 if query.length >= 2 && nickname_prefix?(query, candidate) # "bil" -> "william" via "bill"
similarity = DidYouMean::JaroWinkler.distance(query, candidate)
# Sounds the same ("jon"/"john", "smyth"/"smith"). The similarity check keeps unrelated words that
# happen to share a code (e.g. "john" and "ann" via the Spanish-J alternate) from matching.
return 0.88 if similarity >= 0.7 && (query_codes & candidate_codes).any?
return 0.8 if query.length >= 4 && candidate.include?(query) # "donald" in "mcdonald"
return 0.7 if query.length == 1 && candidate.start_with?(query) # an initial
# Typos. Short words need to be closer to count.
threshold = [query.length, candidate.length].min <= 4 ? 0.9 : 0.85
similarity >= threshold ? similarity * 0.9 : 0.0
end
# Nickname matching treats the data as connections between names (the file lists both directions,
# e.g. "william -> bill" and "bill -> william"). Two names match if they are connected directly, or both
# connect to a LONGER name they're short for: "bill" and "will" via "william". Requiring the shared
# name to be longer stops unrelated full names linking through a short one: "bill" is listed for both
# William and Robert, but that doesn't make William match Robert.
def self.nicknames?(a, b)
related_names(a).include?(b)
end
# A partly typed nickname of the candidate's name, e.g. "bil" for "william" (via "bill")
def self.nickname_prefix?(query, candidate)
related_names(candidate).any? { |name| name.start_with?(query) }
end
# All names interchangeable with name, cached per process
def self.related_names(name)
cache = (@related_names_cache ||= {})
cache[name] ||= begin
graph = nickname_graph
direct = graph.fetch(name, EMPTY)
through_full_names = direct.select { |full| full.length > name.length }.flat_map do |full|
graph.fetch(full, EMPTY).select { |other| other.length < full.length }
end
((direct + through_full_names).uniq - [name]).freeze
end
end
EMPTY = [].freeze
# name => [connected names], built once per process from the bundled data plus EXTRA_NICKNAME_GROUPS
def self.nickname_graph
@nickname_graph ||= begin
graph = Hash.new { |h, k| h[k] = [] }
connect = lambda do |a, b|
next if a.empty? || b.empty? || a == b
graph[a] << b
graph[b] << a
end
if File.exist?(NICKNAMES_CSV)
CSV.foreach(NICKNAMES_CSV, headers: true) do |row|
connect.call(row['name1'].to_s.strip.downcase, row['name2'].to_s.strip.downcase)
end
end
EXTRA_NICKNAME_GROUPS.each do |group|
group.combination(2) { |a, b| connect.call(a, b) }
end
graph.transform_values { |names| names.uniq.freeze }.to_h.freeze
end
end
end