Refactor invoice processing and synchronization; implement InvoiceSyncJob and related services for improved background processing and logging

This commit is contained in:
2026-02-27 22:33:04 -05:00
parent a1cbf9a0a9
commit 03109d5775
9 changed files with 359 additions and 195 deletions

View File

@@ -19,7 +19,7 @@ class InvoiceController < ApplicationController
# Downloads and forwards the invoice pdf
#
def show
logger.info("Processing request for URL: #{request.original_url}")
log "Processing request for URL: #{request.original_url}"
begin
qbo = Qbo.first
qbo.perform_authenticated_request do |access_token|
@@ -27,10 +27,10 @@ class InvoiceController < ApplicationController
# If multiple id's then pull each pdf & combine them
if params[:invoice_ids]
logger.info("Grabbing pdfs for " + params[:invoice_ids].join(', '))
log "Grabbing pdfs for " + params[:invoice_ids].join(', ')
ref = ""
params[:invoice_ids].each do |i|
logger.info("processing " + i)
log "processing " + i
invoice = service.fetch_by_id(i)
ref += " #{invoice.doc_number}"
@pdf << CombinePDF.parse(service.pdf(invoice)) unless @pdf.nil?
@@ -51,4 +51,10 @@ class InvoiceController < ApplicationController
redirect_to :back, flash: { error: I18n.t(:notice_invoice_not_found) }
end
end
private
def log(msg)
Rails.logger.info "[InvoiceController] #{msg}"
end
end

View File

@@ -26,7 +26,7 @@ class QboController < ApplicationController
#
def authenticate
redirect_uri = "#{Setting.protocol}://#{Setting.host_name + qbo_oauth_callback_path}"
logger.info "redirect_uri: #{redirect_uri}"
log "redirect_uri: #{redirect_uri}"
oauth2_client = Qbo.construct_oauth2_client
grant_url = oauth2_client.auth_code.authorize_url(redirect_uri: redirect_uri, response_type: "code", state: SecureRandom.hex(12), scope: "com.intuit.quickbooks.accounting")
redirect_to grant_url
@@ -91,15 +91,15 @@ class QboController < ApplicationController
# Synchronizes the QboCustomer table with QBO
#
def sync
logger.info "Syncing EVERYTHING"
log "Syncing EVERYTHING"
CustomerSyncJob.perform_later(full_sync: true)
EstimateSyncJob.perform_later(full_sync: true)
InvoiceSyncJob.perform_later(full_sync: true)
# Update info in background
Thread.new do
if Qbo.exists?
Invoice.sync
Employee.sync
# Record the last sync time
@@ -113,7 +113,7 @@ class QboController < ApplicationController
# QuickBooks Webhook Callback
def webhook
logger.info "QBO: Webhook received"
log "Webhook received"
signature = request.headers['intuit-signature']
key = Setting.plugin_redmine_qbo['settingsWebhookToken']
@@ -123,7 +123,7 @@ class QboController < ApplicationController
computed = Base64.strict_encode64(OpenSSL::HMAC.digest(digest, key, body))
unless secure_compare(computed, signature)
logger.warn "QBO: Invalid webhook signature"
log "Invalid webhook signature"
head :unauthorized
return
end
@@ -135,8 +135,13 @@ class QboController < ApplicationController
private
# Securely compare two strings to prevent timing attacks. Returns false if either string is blank or if they do not match.
def secure_compare(a, b)
return false if a.blank? || b.blank?
ActiveSupport::SecurityUtils.secure_compare(a, b)
end
def log(msg)
Rails.logger.info "[QboController] #{msg}"
end
end

View File

@@ -0,0 +1,35 @@
#The MIT License (MIT)
#
#Copyright (c) 2016 - 2026 rick barrette
#
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
class InvoiceSyncJob < ApplicationJob
queue_as :default
retry_on StandardError, wait: 5.minutes, attempts: 5
def perform(full_sync: false, id: nil)
qbo = Qbo.first
return unless qbo
log "Starting #{full_sync ? 'full' : 'incremental'} sync for invoice ##{id || 'all'}..."
service = InvoiceSyncService.new(qbo: qbo)
if id.present?
service.sync_by_id(id)
else
service.sync(full_sync: full_sync)
end
end
private
def log(msg)
Rails.logger.info "[InvoiceSyncJob] #{msg}"
end
end

View File

@@ -9,200 +9,26 @@
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
class Invoice < ActiveRecord::Base
has_and_belongs_to_many :issues
belongs_to :customer
validates_presence_of :doc_number, :id, :customer_id, :txn_date
belongs_to :customer
validates :id, presence: true, uniqueness: true
validates :doc_number, :txn_date, presence: true
self.primary_key = :id
# returns a human readable string
# Return the invoice's document number as its string representation
def to_s
return self[:doc_number]
doc_number
end
# sync ALL the invoices
def self.sync
logger.info "Syncing all invoices"
last = Qbo.first.last_sync
query = "SELECT Id, DocNumber FROM Invoice"
query << " WHERE Metadata.LastUpdatedTime >= '#{last.iso8601}' " if last
# TODO actually do something with the above query
# .all() is never called since count is never initialized
qbo = Qbo.first
invoices = qbo.perform_authenticated_request do |access_token|
service = Quickbooks::Service::Invoice.new(company_id: qbo.realm_id, access_token: access_token)
service.all
end
return unless invoices
invoices.each { | invoice |
process_invoice invoice
}
InvoiceSyncJob.perform_later(full_sync: true)
end
#sync by invoice ID
# Sync a single invoice by ID, typically triggered by a webhook notification or manual sync request
def self.sync_by_id(id)
logger.info "Syncing invoice #{id}"
qbo = Qbo.first
qbo.perform_authenticated_request do |access_token|
service = Quickbooks::Service::Invoice.new(company_id: qbo.realm_id, access_token: access_token)
invoice = service.fetch_by_id(id)
process_invoice invoice
end
end
private
# Attach the invoice to the issue
def self.attach_to_issue(issue, invoice)
return if issue.nil?
# skip this issue if the issue customer is not the same as the invoice customer
return if issue.customer_id != invoice.customer_ref.value.to_i
logger.info "Attaching invoice #{invoice.id} to issue #{issue.id}"
invoice = Invoice.find_or_create_by(id: invoice.id)
unless issue.invoices.include?(invoice)
issue.invoices << invoice
issue.save!
end
compare_custom_fields(issue, invoice)
end
# processes the invoice into the database
def self.process_invoice(i)
logger.info "Processing invoice #{i.id}"
# Load the invoice into the database
invoice = Invoice.find_or_create_by(id: i.id)
invoice.doc_number = i.doc_number
invoice.id = i.id
invoice.customer_id = i.customer_ref
invoice.txn_date = i.txn_date
invoice.save!
# Scan the private notes for hashtags and attach to the applicable issues
if not i.private_note.nil?
i.private_note.scan(/#(\w+)/).flatten.each { |issue|
attach_to_issue(Issue.find_by_id(issue.to_i), invoice)
}
end
# Scan the line items for hashtags and attach to the applicable issues
i.line_items.each { |line|
if line.description
line.description.scan(/#(\w+)/).flatten.each { |issue|
attach_to_issue(Issue.find_by_id(issue.to_i), invoice)
}
end
}
end
# compares the custome fields on invoices & issues and updates the invoice as needed
#
# the issue here is when two or more issues share an invoice with the same custom field, but diffrent values
# this condions causes an infinite loop as the webhook is called when an invoice is updated
# TODO maybe add a cf_sync_confict flag to invoices
def self.compare_custom_fields(issue, invoice)
logger.info "Comparing custom fields"
# TODO break if Invoice.find(invoice.id).cf_sync_confict
is_changed = false
logger.debug "Calling :process_invoice_custom_fields hook"
context = Redmine::Hook.call_hook :process_invoice_custom_fields, { issue: issue, invoice: invoice }
# Process updates from the hooks
context.each do |c|
unless c.nil?
logger.debug "Invoice.compare_custom_fields: We have a responce from a hook"
push_updates c[:invoice] if c[:is_changed]
end
end
# Process Issue Custom Values
begin
value = issue.custom_values.find_by(custom_field_id: CustomField.find_by_name(cf.name).id)
# Check to see if the value is blank...
if not value.value.to_s.blank?
# Check to see if the value is diffrent
if not cf.string_value.to_s.eql? value.value.to_s
# update the custom field on the invoice
cf.string_value = value.value.to_s
is_changed = true
end
end
rescue
# Nothing to do here, there is no match
end
push_updates invoice if is_changed
InvoiceSyncJob.perform_later(id: id)
end
# pushes invoice updates
def self.push_updates(invoice)
begin
logger.info "Invoice.push_updates"
qbo = Qbo.first
qbo.perform_authenticated_request do |access_token|
service = Quickbooks::Service::Invoice.new(company_id: qbo.realm_id, access_token: access_token)
service.update invoice
end
rescue
# Do nothing, probaly custome field sync confict on the invoice.
# This is a problem with how it's billed
# TODO Add notes in memo area
# TODO flag Invoice.cf_sync_confict here
logger.error "Failed to update invoice"
end
end
# download the pdf from quickbooks
def pdf
qbo = Qbo.first
qbo.perform_authenticated_request do |access_token|
service = Quickbooks::Service::Invoice.new(company_id: qbo.realm_id, access_token: access_token)
invoice = service.fetch_by_id(id)
return service.pdf(invoice)
end
end
# Magic Method
# Maps Get/Set methods to QBO invoice object
def method_missing(sym, *arguments)
# Check to see if the method exists
if Quickbooks::Model::Invoice.method_defined?(sym)
# download details if required
pull unless @details
method_name = sym.to_s
# Setter
if method_name[-1, 1] == "="
@details.method(method_name).call(arguments[0])
# Getter
else
return @details.method(method_name).call
end
end
end
# pull the details from quickbooks
def pull
begin
raise Exception unless self.id
qbo = Qbo.first
@details = qbo.perform_authenticated_request do |access_token|
service = Quickbooks::Service::Invoice.new(company_id: qbo.realm_id, access_token: access_token)
service.fetch_by_id(self.id)
end
rescue Exception => e
@details = Quickbooks::Model::Invoice.new
end
end
end
end

View File

@@ -0,0 +1,62 @@
#The MIT License (MIT)
#
#Copyright (c) 2016 - 2026 rick barrette
#
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
class InvoiceAttachmentService
def initialize(invoice, remote)
@invoice = invoice
@remote = remote
end
# Attach invoice to issues based on issue IDs found in the invoice's private note and line descriptions
def attach
extract_issue_ids.each do |issue_id|
log "Processing issue ##{issue_id} for invoice ##{@invoice.doc_number}"
issue = Issue.find_by(id: issue_id)
next unless issue
next unless issue.customer&.id == @invoice.customer&.id
unless issue.invoices.exists?(@invoice.id)
issue.invoices << @invoice
issue.save! if issue.changed?
log "Attached invoice ##{@invoice.id} to issue ##{issue.id}"
end
InvoiceCustomFieldSyncService.new(issue, @invoice, @remote).sync
end
end
private
# Extract issue IDs from the invoice's private note and line descriptions
def extract_issue_ids
ids = []
if @remote.private_note.present?
ids += scan(@remote.private_note)
end
Array(@remote.line_items).each do |line|
ids += scan(line.description.to_s)
end
ids.uniq
end
# Scan text for issue IDs in the format #123
def scan(text)
text.scan(/#(\d+)/).flatten.map(&:to_i)
end
def log(msg)
Rails.logger.info "[InvoiceAttachmentService] #{msg}"
end
end

View File

@@ -0,0 +1,69 @@
#The MIT License (MIT)
#
#Copyright (c) 2016 - 2026 rick barrette
#
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
class InvoiceCustomFieldSyncService
def initialize(issue, invoice, remote)
@issue = issue
@invoice = invoice
@remote = remote
end
# Sync custom fields on the issue based on the invoice data, then push changes to QBO if any fields were updated
def sync
return if @invoice.qbo_sync_locked?
log "Syncing custom fields for issue ##{@issue.id} based on invoice ##{@invoice.doc_number}"
changed = false
# Process Invoice Custom Fields via Hooks
Redmine::Hook.call_hook(
:process_invoice_custom_fields,
issue: @issue,
invoice: @remote
).each do |context|
next unless context
changed ||= context[:is_changed]
log "Custom fields updated by hook, marking invoice for push to QBO" if context[:is_changed]
end
# Process Issue Custom Values from any issue custom fields that match the invoice custom fields
begin
value = @issue.custom_values.find_by(custom_field_id: CustomField.find_by_name(cf.name).id)
# Check to see if the value is blank...
if not value.value.to_s.blank?
# Check to see if the value is diffrent
if not cf.string_value.to_s.eql? value.value.to_s
# update the custom field on the invoice
cf.string_value = value.value.to_s
is_changed = true
end
end
rescue
# Nothing to do here, there is no match
end
push_if_changed if changed
end
private
# If any custom fields were changed during the sync process, this method will trigger a push of the invoice data to QuickBooks Online to ensure that the remote data stays in sync with the local changes. It uses the InvoicePushService to handle the actual communication with QBO.
def push_if_changed
InvoicePushService.new(@invoice).push
end
def log(msg)
Rails.logger.info "[InvoiceCustomFieldSyncService] #{msg}"
end
end

View File

@@ -0,0 +1,47 @@
#The MIT License (MIT)
#
#Copyright (c) 2016 - 2026 rick barrette
#
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
class InvoicePushService
def initialize(invoice)
@invoice = invoice
end
# Push invoice changes to QBO if the invoice is linked to any issues with custom field changes that need to be synced
def push
return if @invoice.qbo_sync_locked?
log "Pushing invoice ##{@invoice.id} to QBO due to linked issue custom field changes"
@invoice.update_column(:qbo_sync_locked, true)
qbo = Qbo.first
qbo.perform_authenticated_request do |access_token|
service = Quickbooks::Service::Invoice.new( company_id: qbo.realm_id, access_token: access_token)
remote = service.fetch_by_id(@invoice.id)
# modify remote object here if needed
service.update(remote)
end
rescue => e
Rails.logger.error "[InvoicePushService] #{e.message}"
ensure
@invoice.update_column(:qbo_sync_locked, false)
end
private
def log(msg)
Rails.logger.info "[InvoicePushService] #{msg}"
end
end

View File

@@ -0,0 +1,90 @@
#The MIT License (MIT)
#
#Copyright (c) 2016 - 2026 rick barrette
#
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
class InvoiceSyncService
PAGE_SIZE = 1000
def initialize(qbo:)
@qbo = qbo
end
# Sync all invoices, or only those updated since the last sync
def sync(full_sync: false)
log "Starting #{full_sync ? 'full' : 'incremental'} invoice sync"
@qbo.perform_authenticated_request do |access_token|
service = Quickbooks::Service::Invoice.new(company_id: @qbo.realm_id, access_token: access_token)
page = 1
loop do
collection = fetch_page(service, page, full_sync)
entries = Array(collection&.entries)
break if entries.empty?
entries.each { |remote| persist(remote) }
break if entries.size < PAGE_SIZE
page += 1
end
end
log "Invoice sync complete"
end
# Sync a single invoice by its QBO ID, used for webhook updates
def sync_by_id(id)
@qbo.perform_authenticated_request do |access_token|
service = Quickbooks::Service::Invoice.new(company_id: @qbo.realm_id, access_token: access_token)
remote = service.fetch_by_id(id)
persist(remote)
end
end
private
# Fetch a page of invoices, either all or only those updated since the last sync
def fetch_page(service, page, full_sync)
start_position = (page - 1) * PAGE_SIZE + 1
if full_sync
service.query("SELECT * FROM Invoice STARTPOSITION #{start_position} MAXRESULTS #{PAGE_SIZE}")
else
last_update = Invoice.maximum(:qbo_updated_at) || 1.year.ago
service.query(<<~SQL.squish)
SELECT * FROM Invoice
WHERE MetaData.LastUpdatedTime > '#{last_update.utc.iso8601}'
STARTPOSITION #{start_position}
MAXRESULTS #{PAGE_SIZE}
SQL
end
end
# Create or update a local Invoice record based on the QBO remote data
def persist(remote)
invoice = Invoice.find_or_initialize_by(id: remote.id)
invoice.doc_number = remote.doc_number
invoice.txn_date = remote.txn_date
invoice.due_date = remote.due_date
invoice.total_amount = remote.total
invoice.balance = remote.balance
invoice.qbo_updated_at = remote.meta_data&.last_updated_time
invoice.customer = Customer.find_by(id: remote.customer_ref&.value)
invoice.save!
InvoiceAttachmentService.new(invoice, remote).attach
end
def log(msg)
Rails.logger.info "[InvoiceSyncService] #{msg}"
end
end