From ccc70171b41d7d4a7b44bc76538c84fb6ff94219 Mon Sep 17 00:00:00 2001 From: Rick Barrette Date: Tue, 17 Nov 2026 18:34:21 -0500 Subject: [PATCH] Read paid invoices from Redmine instead of QuickBooks Invoices paid in full in the month come from redmine_qbo's cached invoices (their sales tax and paid-in-full day, redmine_qbo 2026.11.21), so viewing a month no longer reads every paid invoice changed since it began, with their payments and credit memos, from QuickBooks. Sales receipts, refund receipts and credit memos dated in the month are still read from QuickBooks, together, and cached. Read from QuickBooks and Approve first sync the invoices changed since the last sync. Approvers are warned while paid invoices still lack their paid-in-full day (a full sync fills it in). Co-Authored-By: Claude Opus 5.5 --- .../sales_commissions_controller.rb | 1 + app/views/sales_commissions/show.html.erb | 3 + config/locales/en.yml | 3 +- init.rb | 2 +- lib/redmine_qbo_commission/qbo_source.rb | 158 ++++++++---------- test/unit/qbo_source_test.rb | 150 +++++++++-------- 6 files changed, 158 insertions(+), 159 deletions(-) diff --git a/app/controllers/sales_commissions_controller.rb b/app/controllers/sales_commissions_controller.rb index 360704d..d5a0f56 100644 --- a/app/controllers/sales_commissions_controller.rb +++ b/app/controllers/sales_commissions_controller.rb @@ -45,6 +45,7 @@ class SalesCommissionsController < ApplicationController else @report = RedmineQboCommission::MonthReport.new(@month, txns: read_transactions, commission_month: @record) @read_at = RedmineQboCommission::QboSource.read_at(@month) + @missing_paid_on = RedmineQboCommission::QboSource.missing_paid_on? if @can_approve end respond_to do |format| diff --git a/app/views/sales_commissions/show.html.erb b/app/views/sales_commissions/show.html.erb index 23ddb8d..1394cde 100644 --- a/app/views/sales_commissions/show.html.erb +++ b/app/views/sales_commissions/show.html.erb @@ -38,6 +38,9 @@ <% elsif !@month.completed? %>

<%= l(:text_sales_commission_month_in_progress, date: format_date(@month.end_date)) %>

<% end %> +<% if !approved && @can_approve && @missing_paid_on %> +

<%= l(:text_sales_commission_missing_paid_on) %>

+<% end %> <% if !approved && @read_at %>

<%= l(:text_sales_commission_read_at, time: format_time(@read_at)) %>

<% end %> diff --git a/config/locales/en.yml b/config/locales/en.yml index daa030c..921f1ce 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -66,7 +66,8 @@ en: error_sales_commission_qbo: "Couldn't read QuickBooks: %{error}" text_sales_commission_month_in_progress: "This month isn't over until %{date}, so its sales can still change. It can be approved once it's over." - text_sales_commission_read_at: "Read from QuickBooks at %{time}. It's read again every 15 minutes." + text_sales_commission_read_at: "Invoices are Redmine's copy of QuickBooks, kept up to date as they change. Sales receipts, refund receipts and credit memos were read from QuickBooks at %{time} (again every 15 minutes). Read from QuickBooks brings both up to date now." + text_sales_commission_missing_paid_on: "Some paid invoices were synced before Redmine kept the day they were paid in full, so they aren't counted yet. Run a full sync of invoices (Administration » Plugins » QuickBooks » Full sync) once." text_sales_commission_approved_by: "Approved for payroll by %{user} on %{time}." text_sales_commission_threshold_met: "reached" text_sales_commission_threshold_of: "to reach %{amount}" diff --git a/init.rb b/init.rb index 1fed681..16aaad3 100644 --- a/init.rb +++ b/init.rb @@ -22,7 +22,7 @@ Redmine::Plugin.register :redmine_qbo_commission do # The QuickBooks connection lives in redmine_qbo begin - requires_redmine_plugin :redmine_qbo, version_or_higher: '2026.11.0' + requires_redmine_plugin :redmine_qbo, version_or_higher: '2026.11.21' rescue Redmine::PluginNotFound raise 'Please install the redmine_qbo plugin (https://github.com/rickbarrette/redmine_qbo)' end diff --git a/lib/redmine_qbo_commission/qbo_source.rb b/lib/redmine_qbo_commission/qbo_source.rb index 613ce8e..3be1d52 100644 --- a/lib/redmine_qbo_commission/qbo_source.rb +++ b/lib/redmine_qbo_commission/qbo_source.rb @@ -9,44 +9,53 @@ #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. module RedmineQboCommission - # Reads a month's paid sales and deductions from QuickBooks Online (SOP-OPS-013 §4): + # A month's paid sales and deductions (SOP-OPS-013 §4): # - # * Invoices paid in full that month. An invoice is paid in full on the later of its own date and the date of - # the last payment or credit memo applied to it, so a deposit taken months before the work counts in the - # month it was invoiced and the last of it paid. QuickBooks updates an invoice when a payment is applied to - # it, so the invoices with a $0 balance changed since the month began are all that need reading. - # * Sales receipts dated in the month (counter sales paid on the spot), if the settings say so. - # * Refund receipts and credit memos dated in the month, as deductions, if the settings say so. Credit memos - # made only of the excluded products/services (e.g. tech bonus credits) aren't returns and are left out. + # * Invoices paid in full that month, from Redmine's own copy of the invoices (redmine_qbo keeps each invoice's + # sales tax and the day it was paid in full: the later of its date and the date of the last payment or credit + # memo applied to it, so a deposit taken before the work counts in the month the job was invoiced). + # * Sales receipts dated in the month (counter sales paid on the spot), if the settings say so, read from + # QuickBooks. + # * Refund receipts and credit memos dated in the month, as deductions, if the settings say so, read from + # QuickBooks. Credit memos made only of the excluded products/services (e.g. tech bonus credits) aren't + # returns and are left out. # - # Voided transactions (a $0 total) don't count. Amounts are kept without sales tax. + # What's read from QuickBooks is kept for CACHE_TTL; the three reads run at the same time. Voided transactions + # (a $0 total) don't count. Amounts are kept without sales tax. class QboSource CACHE_TTL = 15.minutes BATCH = 1000 - IDS_PER_QUERY = 100 - # The month's transactions, read from QuickBooks at most every CACHE_TTL (refresh: true reads them now) + # The month's transactions. refresh: true first brings Redmine's invoices up to date with QuickBooks (only + # those changed since the last sync), and reads the receipts and credit memos again now. def self.transactions(month, refresh: false) - Rails.cache.delete(cache_key(month)) if refresh - cached = Rails.cache.fetch(cache_key(month), expires_in: CACHE_TTL) do - { 'read_at' => Time.current.iso8601, 'txns' => new(month).fetch.map(&:to_cache) } + if refresh + sync_invoices + Rails.cache.delete(cache_key(month)) end - cached['txns'].map { |hash| Txn.from_h(hash) } + cached = Rails.cache.fetch(cache_key(month), expires_in: CACHE_TTL) do + { 'read_at' => Time.current.iso8601, 'txns' => new(month).receipts.map(&:to_cache) } + end + (new(month).invoices + cached['txns'].map { |hash| Txn.from_h(hash) }).sort_by { |txn| [txn.txn_date, txn.doc_number.to_s] } end - # When the cached transactions were read from QuickBooks, or nil when they aren't cached + # When the receipts and credit memos were read from QuickBooks, or nil when they aren't cached def self.read_at(month) value = Rails.cache.read(cache_key(month)) value && Time.zone.parse(value['read_at'].to_s) end def self.cache_key(month) - "redmine_qbo_commission/txns/#{month.to_param}" + "redmine_qbo_commission/receipts/#{month.to_param}" end - # The day an invoice was paid in full: the later of its date and the dates of what was applied to it - def self.paid_on(txn_date, applied_dates) - [txn_date, *applied_dates.compact].compact.max + # Paid invoices synced before redmine_qbo kept the day they were paid in full (a full invoice sync fills it in) + def self.missing_paid_on? + Invoice.where(balance: 0, paid_on: nil).where('total_amount > 0').exists? + end + + def self.sync_invoices + InvoiceSyncService.new.sync end attr_reader :month @@ -55,81 +64,62 @@ module RedmineQboCommission @month = month end - def fetch - txns = invoices - txns += dated(Quickbooks::Service::SalesReceipt, 'SalesReceipt', 'sales_receipt') if Settings.include_sales_receipts? - txns += dated(Quickbooks::Service::RefundReceipt, 'RefundReceipt', 'refund_receipt') if Settings.deduct_refund_receipts? - txns += credit_memos if Settings.deduct_credit_memos? - txns.sort_by { |txn| [txn.txn_date, txn.doc_number.to_s] } + # Invoices paid in full in the month, from Redmine's invoices + def invoices + Invoice.where(paid_on: month.range).where('total_amount > 0').includes(:customer).map do |invoice| + Txn.new(kind: 'invoice', txn_id: invoice.id.to_s, doc_number: invoice.doc_number.to_s, + customer_name: invoice.customer&.name.to_s, txn_date: invoice.paid_on, + total: invoice.total_amount.to_d, tax: invoice.tax_amount.to_d) + end + end + + # Sales receipts, refund receipts and credit memos dated in the month, read from QuickBooks together + def receipts + reads = [] + reads << ['SalesReceipt', 'sales_receipt'] if Settings.include_sales_receipts? + reads << ['RefundReceipt', 'refund_receipt'] if Settings.deduct_refund_receipts? + reads << ['CreditMemo', 'credit_memo'] if Settings.deduct_credit_memos? + return [] if reads.empty? + + excluded = Settings.excluded_item_ids + services = reads.to_h { |entity, _| [entity, "Quickbooks::Service::#{entity}".constantize] } + qbo = QboConnectionService.current! + # The threads only call QuickBooks; everything else is done here, after they're joined + records = qbo.perform_authenticated_request do |access_token| + threads = reads.map do |entity, _| + service = services[entity].new(company_id: qbo.realm_id, access_token: access_token) + sql = "SELECT * FROM #{entity} WHERE TxnDate >= '#{month.start_date.iso8601}' AND TxnDate <= '#{month.end_date.iso8601}'" + Thread.new { query(service, sql) } + end + reads.map(&:last).zip(threads.map(&:value)) + end + records.flat_map do |kind, list| + list.filter_map do |record| + next unless record.total.to_d.positive? + next if kind == 'credit_memo' && !return?(record, excluded) + + txn(kind, record) + end + end end private - def invoices - since = month.start_date.in_time_zone.iso8601 - remote = query(Quickbooks::Service::Invoice, - "SELECT * FROM Invoice WHERE Balance = '0' AND MetaData.LastUpdatedTime >= '#{since}'") - remote = remote.select { |invoice| invoice.total.to_d.positive? && invoice.txn_date && invoice.txn_date <= month.end_date } - dates = applied_dates(remote) - remote.filter_map do |invoice| - applied = linked(invoice).map { |link| dates[[link.txn_type, link.txn_id.to_s]] } - paid_on = self.class.paid_on(invoice.txn_date, applied) - txn('invoice', invoice, paid_on) if month.range.cover?(paid_on) - end + # A credit memo is a return unless it's made only of the excluded products/services + def return?(memo, excluded) + items = Array(memo.line_items).filter_map { |line| line.sales_item_line_detail&.item_ref&.value&.to_s } + excluded.empty? || items.empty? || !items.all? { |item| excluded.include?(item) } end - def credit_memos - excluded = Settings.excluded_item_ids - dated(Quickbooks::Service::CreditMemo, 'CreditMemo', 'credit_memo') do |memo| - items = Array(memo.line_items).filter_map { |line| line.sales_item_line_detail&.item_ref&.value&.to_s } - excluded.empty? || items.empty? || !items.all? { |item| excluded.include?(item) } - end - end - - # The transactions of a type dated in the month (that the block keeps) - def dated(service_class, entity, kind) - remote = query(service_class, "SELECT * FROM #{entity} WHERE TxnDate >= '#{month.start_date.iso8601}' " \ - "AND TxnDate <= '#{month.end_date.iso8601}'") - remote.filter_map do |record| - next unless record.total.to_d.positive? - next if block_given? && !yield(record) - - txn(kind, record, record.txn_date) - end - end - - # { ["Payment", "123"] => Date } for the payments and credit memos applied to the invoices - def applied_dates(invoices) - ids = Hash.new { |hash, type| hash[type] = [] } - invoices.each { |invoice| linked(invoice).each { |link| ids[link.txn_type] << link.txn_id.to_s } } - { 'Payment' => Quickbooks::Service::Payment, 'CreditMemo' => Quickbooks::Service::CreditMemo } - .each_with_object({}) do |(type, service_class), dates| - ids[type].uniq.each_slice(IDS_PER_QUERY) do |slice| - list = slice.map { |id| "'#{id.gsub(/\D/, '')}'" }.join(',') - query(service_class, "SELECT * FROM #{type} WHERE Id IN (#{list})").each do |record| - dates[[type, record.id.to_s]] = record.txn_date - end - end - end - end - - def linked(invoice) - Array(invoice.linked_transactions).select { |link| %w[Payment CreditMemo].include?(link.txn_type) } - end - - def txn(kind, record, date) + def txn(kind, record) Txn.new(kind: kind, txn_id: record.id.to_s, doc_number: record.doc_number.to_s, - customer_name: record.customer_ref&.name.to_s, txn_date: date, + customer_name: record.customer_ref&.name.to_s, txn_date: record.txn_date, total: record.total.to_d, tax: record.txn_tax_detail&.total_tax.to_d) end - def query(service_class, sql) + def query(service, sql) results = [] - qbo = QboConnectionService.current! - qbo.perform_authenticated_request do |access_token| - service = service_class.new(company_id: qbo.realm_id, access_token: access_token) - service.query_in_batches(sql, per_page: BATCH) { |batch| results.concat(Array(batch)) } - end + service.query_in_batches(sql, per_page: BATCH) { |batch| results.concat(Array(batch)) } results end end diff --git a/test/unit/qbo_source_test.rb b/test/unit/qbo_source_test.rb index ce4fb8b..802f0be 100644 --- a/test/unit/qbo_source_test.rb +++ b/test/unit/qbo_source_test.rb @@ -13,42 +13,66 @@ require File.expand_path('../test_helper', __dir__) class QboSourceTest < ActiveSupport::TestCase include QboCommissionTestHelper + Source = RedmineQboCommission::QboSource + + # Stands in for the QuickBooks connection: runs the block with a token + class FakeQbo + def realm_id + '123' + end + + def perform_authenticated_request + yield OAuth2::AccessToken.new(OAuth2::Client.new('id', 'secret'), 'token') + end + end + + def setup + Invoice.where(id: 9001..9099).delete_all + Customer.where(id: 9500).delete_all + customer = Customer.new(id: 9500, name: 'Jane Customer') + customer.skip_qbo_push = true + customer.save! + QboConnectionService.stubs(:current!).returns(FakeQbo.new) + end + + def local_invoice(id, total, paid_on:, tax: 0, balance: 0) + invoice = Invoice.new(id: id, doc_number: "10#{id}", txn_date: MONTH_START, customer_id: 9500, + total_amount: total, tax_amount: tax, balance: balance, paid_on: paid_on) + invoice.skip_qbo_push = true + invoice.save! + end + # Answers the source's QuickBooks queries from lists of records, by the entity queried class FakeSource < RedmineQboCommission::QboSource - attr_reader :queries - - def initialize(month, records) + def initialize(month, records, queries) super(month) @records = records - @queries = [] + @queries = queries end private - def query(_service_class, sql) + def query(_service, sql) @queries << sql - entity = sql[/FROM (\w+)/, 1] - list = @records.fetch(entity, []) - ids = sql[/Id IN \(([^)]*)\)/, 1] - ids ? list.select { |record| ids.include?("'#{record.id}'") } : list + @records.fetch(sql[/FROM (\w+)/, 1], []) end end - def ref(name) - Quickbooks::Model::BaseReference.new(name) + # Every QboSource made from now on answers from the records (QboSource.transactions makes its own) + def stub_queries(records) + @queries = [] + queries = @queries + Source.define_singleton_method(:new) { |month| FakeSource.allocate.tap { |f| f.send(:initialize, month, records, queries) } } end - def link(type, id) - linked = Quickbooks::Model::LinkedTransaction.new - linked.txn_type = type - linked.txn_id = id.to_s - linked + def teardown + Source.singleton_class.send(:remove_method, :new) if Source.singleton_methods(false).include?(:new) end - def record(klass, id, total, date, tax: nil, links: [], items: []) + def record(klass, id, total, date, tax: nil, items: []) record = klass.new record.id = id.to_s - record.doc_number = "D#{id}" if record.respond_to?(:doc_number=) + record.doc_number = "D#{id}" record.txn_date = date record.total = BigDecimal(total.to_s) record.customer_ref = Quickbooks::Model::BaseReference.new(id.to_s).tap { |r| r.name = "Customer #{id}" } @@ -56,7 +80,6 @@ class QboSourceTest < ActiveSupport::TestCase record.txn_tax_detail = Quickbooks::Model::TransactionTaxDetail.new record.txn_tax_detail.total_tax = BigDecimal(tax.to_s) end - record.linked_transactions = links if record.respond_to?(:linked_transactions=) if items.any? record.line_items = items.map do |item_id| line = Quickbooks::Model::Line.new @@ -68,84 +91,65 @@ class QboSourceTest < ActiveSupport::TestCase record end - def payment(id, date) - record(Quickbooks::Model::Payment, id, 1, date) - end - - def invoice(id, total, date, **options) - record(Quickbooks::Model::Invoice, id, total, date, **options) - end - - def test_paid_on - assert_equal Date.new(2026, 10, 9), RedmineQboCommission::QboSource.paid_on(Date.new(2026, 10, 2), [Date.new(2026, 10, 9), nil]) - # A deposit taken before the invoice: paid in full on the invoice's date - assert_equal Date.new(2026, 10, 2), RedmineQboCommission::QboSource.paid_on(Date.new(2026, 10, 2), [Date.new(2026, 8, 1)]) - end - - def test_invoices_count_in_the_month_paid_in_full - records = { - 'Invoice' => [ - invoice(1, 1070, MONTH_START + 4, tax: 70, links: [link('Payment', 50)]), # paid in October - invoice(2, 500, Date.new(2026, 9, 20), links: [link('Payment', 51)]), # September, paid in October - invoice(3, 800, MONTH_START + 2, links: [link('Payment', 52)]), # paid in November - invoice(4, 300, MONTH_START + 6, links: [link('Payment', 53), link('Estimate', 9)]), # deposit from August - invoice(5, 0, MONTH_START + 6), # voided - invoice(6, 200, Date.new(2026, 11, 3), links: [link('Payment', 50)]), # dated November - invoice(7, 100, Date.new(2026, 9, 3), links: [link('CreditMemo', 70)]) # settled by a September credit memo - ], - 'Payment' => [payment(50, MONTH_START + 9), payment(51, MONTH_START + 1), payment(52, Date.new(2026, 11, 2)), - payment(53, Date.new(2026, 8, 1))], - 'CreditMemo' => [record(Quickbooks::Model::CreditMemo, 70, 100, Date.new(2026, 9, 30))] - } - with_commission_settings('include_sales_receipts' => '0', 'deduct_refund_receipts' => '0', 'deduct_credit_memos' => '0') do - source = FakeSource.new(commission_month, records) - txns = source.fetch - assert_equal %w[2 4 1], txns.map(&:txn_id) - first = txns.find { |txn| txn.txn_id == '1' } - assert_equal MONTH_START + 9, first.txn_date - assert_equal BigDecimal('1000'), first.net - assert_equal 'Customer 1', first.customer_name - assert_equal MONTH_START + 6, txns.find { |txn| txn.txn_id == '4' }.txn_date - assert_match(/Balance = '0' AND MetaData.LastUpdatedTime >= '2026-10-01T00:00:00/, source.queries.first) - end + def test_invoices_paid_in_full_in_the_month_come_from_redmine + local_invoice(9001, 1070, tax: 70, paid_on: MONTH_START + 9) + local_invoice(9002, 500, paid_on: MONTH_START + 1) + local_invoice(9003, 800, paid_on: Date.new(2026, 11, 2)) # paid in November + local_invoice(9004, 300, paid_on: nil, balance: 300) # open + local_invoice(9005, 0, paid_on: MONTH_START + 3) # voided + txns = Source.new(commission_month).invoices.sort_by(&:txn_id) + assert_equal %w[9001 9002], txns.map(&:txn_id) + assert_equal BigDecimal('1000'), txns.first.net + assert_equal MONTH_START + 9, txns.first.txn_date + assert_equal 'Jane Customer', txns.first.customer_name end def test_receipts_and_credit_memos - records = { + stub_queries( 'SalesReceipt' => [record(Quickbooks::Model::SalesReceipt, 1, 107, MONTH_START + 3, tax: 7)], 'RefundReceipt' => [record(Quickbooks::Model::RefundReceipt, 2, 50, MONTH_START + 4)], 'CreditMemo' => [record(Quickbooks::Model::CreditMemo, 3, 40, MONTH_START + 5, items: [12]), record(Quickbooks::Model::CreditMemo, 4, 100, MONTH_START + 5, items: [31]), # tech bonus record(Quickbooks::Model::CreditMemo, 5, 60, MONTH_START + 5, items: [31, 12])] - } + ) with_commission_settings('excluded_item_ids' => ['31']) do - source = FakeSource.new(commission_month, records) - txns = source.fetch + txns = Source.new(commission_month).receipts assert_equal [%w[sales_receipt 1], %w[refund_receipt 2], %w[credit_memo 3], %w[credit_memo 5]], txns.map { |txn| [txn.kind, txn.txn_id] } assert_equal BigDecimal('100'), txns.first.net - assert(source.queries.any? { |sql| sql.include?("TxnDate >= '2026-10-01' AND TxnDate <= '2026-10-31'") }) + assert @queries.all? { |sql| sql.include?("TxnDate >= '2026-10-01' AND TxnDate <= '2026-10-31'") } + assert_equal 3, @queries.size end end def test_settings_leave_out_receipts_and_credit_memos - records = { 'SalesReceipt' => [record(Quickbooks::Model::SalesReceipt, 1, 100, MONTH_START)], - 'RefundReceipt' => [record(Quickbooks::Model::RefundReceipt, 2, 50, MONTH_START)], - 'CreditMemo' => [record(Quickbooks::Model::CreditMemo, 3, 40, MONTH_START)] } + stub_queries({}) with_commission_settings('include_sales_receipts' => '0', 'deduct_refund_receipts' => '0', 'deduct_credit_memos' => '0') do - assert_equal [], FakeSource.new(commission_month, records).fetch + assert_equal [], Source.new(commission_month).receipts + assert_empty @queries end end - def test_transactions_are_cached - source = FakeSource.new(commission_month, { 'SalesReceipt' => [record(Quickbooks::Model::SalesReceipt, 1, 100, MONTH_START)] }) - RedmineQboCommission::QboSource.expects(:new).once.returns(source) + def test_transactions_and_refresh + local_invoice(9001, 200, paid_on: MONTH_START + 2) + stub_queries('RefundReceipt' => [record(Quickbooks::Model::RefundReceipt, 2, 50, MONTH_START + 4)]) + Source.expects(:sync_invoices).once with_memory_cache do - 2.times { assert_equal [BigDecimal('100')], RedmineQboCommission::QboSource.transactions(commission_month).map(&:net) } - assert RedmineQboCommission::QboSource.read_at(commission_month) + 2.times { assert_equal %w[invoice refund_receipt], Source.transactions(commission_month).map(&:kind) } + assert_equal 3, @queries.size # read once, then cached + assert Source.read_at(commission_month) + Source.transactions(commission_month, refresh: true) + assert_equal 6, @queries.size end end + def test_missing_paid_on + local_invoice(9001, 200, paid_on: MONTH_START) + assert_not Source.missing_paid_on? + local_invoice(9002, 200, paid_on: nil) + assert Source.missing_paid_on? + end + private def with_memory_cache