From 6da26935f612aee5ae1898307da02b5d40a0ce4f Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Tue, 22 Aug 2023 14:09:18 -0600 Subject: [PATCH 01/10] testing before changing descriptives table --- .../rcpt/update_descriptives_merge_catchup.qy | 34 +- .../modules/rcpt/update_descriptives_new.qy | 925 +++++++++--------- 2 files changed, 485 insertions(+), 474 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy index 0b4ee853..dcbdc295 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy @@ -10,31 +10,41 @@ update_descriptives_merge_catchup "system/query" -- the regular giving. -- Step One: Locate + print 'donghwanHwang' + + --select * from collection a_descriptives_hist; + update identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 set - :a_merged_id = isnull(:dh2:a_merged_id, :dh2:a_hist_id) + :a_merged_id = 999999 + --:a_merged_id = :dh2:a_hist_id + --:a_merged_id = isnull(:dh2:a_merged_id, :dh2:a_hist_id) where (:parameters:fund is null or :parameters:fund = :dh2:a_fund) and (:parameters:donor is null or :parameters:donor = :dh2:p_donor_partner_key) and :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and - :dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason - (:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular - round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup - :dh2:a_count > :dh1:a_count and -- regular has more gifts than catchup - :dh2:a_count >= 2 and -- regular has at least 2 gifts - :dh2:a_first_gift < :dh2:a_last_gift and -- regular gifts are not on the same day - round(:dh2:a_act_average_interval, 0) != 0 and -- regular has an established interval (non-zero) - (:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) and -- gifts are adjacent - datediff(month, condition(:dh1:a_last_gift < :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift), condition(:dh1:a_first_gift > :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) <= 14 and + --:dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason + --(:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular + --round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup + --:dh2:a_count > :dh1:a_count --and -- regular has more gifts than catchup + :dh2:a_count >= 2 --and -- regular has at least 2 gifts + --:dh2:a_first_gift < :dh2:a_last_gift --and -- regular gifts are not on the same day + --round(:dh2:a_act_average_interval, 0) != 0 --and -- regular has an established interval (non-zero) + --(:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) --and -- gifts are adjacent + --datediff(month, condition(:dh1:a_last_gift < :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift), condition(:dh1:a_first_gift > :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) <= 14 and -- checks that the average giving stays similar after the merge would occur - (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and - (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) + --(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and + --(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) ; + --print 'before Merged' + + select * from collection a_descriptives_hist; + print 'FULLY TESTED' -- Step Two: Merge update identity collection a_descriptives_hist dh2, diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index 7ba384c0..c47fa8e0 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -205,16 +205,16 @@ update_descriptives_new "system/query" ; -- Update previous/next interval start/end dates - print 'Updating segment start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); - update - collection a_descriptives_hist dh1 - set - :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift order by :dh2:a_last_gift desc limit 1), - :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift order by :dh2:a_first_gift asc limit 1) - where - (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) - ; + --print 'Updating segment start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- collection a_descriptives_hist dh1 + --set + -- :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift order by :dh2:a_last_gift desc limit 1), + -- :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift order by :dh2:a_first_gift asc limit 1) + --where + -- (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) + --; -- These are the merges that allow the gift histories to come out -- very neat and clean. Some of them are repeated, which is a @@ -222,103 +222,103 @@ update_descriptives_new "system/query" -- and may need to be repeated more for larger data sets. -- Merge in 'catch up' giving - print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy - fund=:parameters:fund, - donor=:parameters:donor - ; - -- Merge in adjacent giving segments - print 'Merge adjacent at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent.qy - fund=:parameters:fund, - donor=:parameters:donor - ; - -- Merge in 'catch up' giving that includes extra gifts - print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy - fund=:parameters:fund, - donor=:parameters:donor - ; - -- Merge in extra giving - print 'Merge extra amounts at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_extra.qy - fund=:parameters:fund, - donor=:parameters:donor - ; - -- Merge in adjacent giving segments - print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy - fund=:parameters:fund, - donor=:parameters:donor - ; - -- Merge in adjacent single-gift segments - print 'Merge adjacent singles at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_singles.qy - fund=:parameters:fund, - donor=:parameters:donor - ; - -- Merge in adjacent giving segments - print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy - fund=:parameters:fund, - donor=:parameters:donor - ; - -- Merge in adjacent giving segments - print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy - fund=:parameters:fund, - donor=:parameters:donor - ; - -- Approximate segments that did not fit in anywhere else - print 'Merge approximates at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_approx.qy - fund=:parameters:fund, - donor=:parameters:donor - ; + -- print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); + -- exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + -- ; + ---- Merge in adjacent giving segments + --print 'Merge adjacent at ' + dateformat(getdate(), 'hh:mm:ss'); + --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + --; + ---- Merge in 'catch up' giving that includes extra gifts + --print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); + --exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + --; + ---- Merge in extra giving + --print 'Merge extra amounts at ' + dateformat(getdate(), 'hh:mm:ss'); + --exec /apps/kardia/modules/rcpt/update_descriptives_merge_extra.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + --; + ---- Merge in adjacent giving segments + --print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + --; + ---- Merge in adjacent single-gift segments + --print 'Merge adjacent singles at ' + dateformat(getdate(), 'hh:mm:ss'); + --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_singles.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + --; + ---- Merge in adjacent giving segments + --print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + --; + ---- Merge in adjacent giving segments + --print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + --; + ---- Approximate segments that did not fit in anywhere else + --print 'Merge approximates at ' + dateformat(getdate(), 'hh:mm:ss'); + --exec /apps/kardia/modules/rcpt/update_descriptives_merge_approx.qy + -- fund=:parameters:fund, + -- donor=:parameters:donor + --; - -- Correct previous/next interval start/end dates - -- Extra gifts are all linked together, and regular gifts are - -- linked together. - print 'Correcting start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); - update - collection a_descriptives_hist dh1 - set - :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and not :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), - :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and not :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) - where - (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - not :dh1:a_is_extra - ; - update - collection a_descriptives_hist dh1 - set - :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), - :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) - where - (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - :dh1:a_is_extra - ; + ---- Correct previous/next interval start/end dates + ---- Extra gifts are all linked together, and regular gifts are + ---- linked together. + --print 'Correcting start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- collection a_descriptives_hist dh1 + --set + -- :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and not :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), + -- :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and not :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) + --where + -- (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and + -- not :dh1:a_is_extra + --; + --update + -- collection a_descriptives_hist dh1 + --set + -- :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), + -- :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) + --where + -- (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and + -- :dh1:a_is_extra + --; - -- Correct NTL Tracking - print 'Fixing next-to-last tracking at ' + dateformat(getdate(), 'hh:mm:ss'); - update - collection a_descriptives_hist dh1 - set - :dh1:a_ntl_gift = (select nth(:t:giftdate, 2) from collection tmp_gift_items t where :t:ledger = :dh1:a_ledger_number and :t:fund = :dh1:a_fund and :t:donor = :dh1:p_donor_partner_key and :t:giftdate <= :dh1:a_last_gift order by :t:giftdate desc) - where - (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - :dh1:a_ntl_gift is null and - :dh1:a_count > 1 - ; + ---- Correct NTL Tracking + --print 'Fixing next-to-last tracking at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- collection a_descriptives_hist dh1 + --set + -- :dh1:a_ntl_gift = (select nth(:t:giftdate, 2) from collection tmp_gift_items t where :t:ledger = :dh1:a_ledger_number and :t:fund = :dh1:a_fund and :t:donor = :dh1:p_donor_partner_key and :t:giftdate <= :dh1:a_last_gift order by :t:giftdate desc) + --where + -- (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and + -- :dh1:a_ntl_gift is null and + -- :dh1:a_count > 1 + --; - -- First history value - print 'Propagating history #1 at ' + dateformat(getdate(), 'hh:mm:ss'); - update + ---- First history value + print 'Propagating history #1 at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives d - set + set :a_hist_1_amount = (select first(:dh:a_amount) from @@ -330,7 +330,7 @@ update_descriptives_new "system/query" order by :dh:a_last_gift desc ), - :a_hist_1_last = (select + :a_hist_1_last = (select first(:dh:a_last_gift) from collection a_descriptives_hist dh @@ -341,406 +341,407 @@ update_descriptives_new "system/query" order by :dh:a_last_gift desc ) - where + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update + ; + update identity collection a_descriptives d, collection a_descriptives_hist dh - set + set :d:a_hist_1_count = :dh:a_count, :d:a_hist_1_first = :dh:a_first_gift, :d:a_hist_1_last = :dh:a_last_gift - where + where :d:a_ledger_number = :dh:a_ledger_number and :d:a_fund = :dh:a_fund and :d:p_donor_partner_key = :dh:p_donor_partner_key and :d:a_hist_1_amount = :dh:a_amount and - :d:a_hist_1_last = :dh:a_last_gift and + :d:a_hist_1_last = :dh:a_last_gift and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ; + select :a_ledger_number, :a_fund, :p_donor_partner_key, :a_hist_1_amount, :a_hist_1_last, :a_hist_1_count, :a_hist_1_first from collection a_descriptives limit 100; + print ' ' + print 'After merging' - -- Second history value - print 'Propagating history #2 at ' + dateformat(getdate(), 'hh:mm:ss'); - update - identity collection a_descriptives d - set - :a_hist_2_amount = (select - first(:dh:a_amount) - from - collection a_descriptives_hist dh - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key and - :dh:a_last_gift < :d:a_hist_1_last - order by - :dh:a_last_gift desc - ), - :a_hist_2_last = (select - first(:dh:a_last_gift) - from - collection a_descriptives_hist dh - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key and - :dh:a_last_gift < :d:a_hist_1_last - order by - :dh:a_last_gift desc - ) - where - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update - identity collection a_descriptives d, - collection a_descriptives_hist dh - set - :d:a_hist_2_count = :dh:a_count, - :d:a_hist_2_first = :dh:a_first_gift - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key and - :d:a_hist_2_amount = :dh:a_amount and - :d:a_hist_2_last = :dh:a_last_gift and - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ---- Second history value + --print 'Propagating history #2 at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- identity collection a_descriptives d + --set + -- :a_hist_2_amount = (select + -- first(:dh:a_amount) + -- from + -- collection a_descriptives_hist dh + -- where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key and + -- :dh:a_last_gift < :d:a_hist_1_last + -- order by + -- :dh:a_last_gift desc + -- ), + -- :a_hist_2_last = (select + -- first(:dh:a_last_gift) + -- from + -- collection a_descriptives_hist dh + -- where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key and + -- :dh:a_last_gift < :d:a_hist_1_last + -- order by + -- :dh:a_last_gift desc + -- ) + --where + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; + --update + -- identity collection a_descriptives d, + -- collection a_descriptives_hist dh + --set + -- :d:a_hist_2_count = :dh:a_count, + -- :d:a_hist_2_first = :dh:a_first_gift + --where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key and + -- :d:a_hist_2_amount = :dh:a_amount and + -- :d:a_hist_2_last = :dh:a_last_gift and + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; - -- Third history value - print 'Propagating history #3 at ' + dateformat(getdate(), 'hh:mm:ss'); - update - identity collection a_descriptives d - set - :a_hist_3_amount = (select - first(:dh:a_amount) - from - collection a_descriptives_hist dh - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key and - :dh:a_last_gift < :d:a_hist_1_last and - :dh:a_last_gift < :d:a_hist_2_last - order by - :dh:a_last_gift desc - ), - :a_hist_3_last = (select - first(:dh:a_last_gift) - from - collection a_descriptives_hist dh - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key and - :dh:a_last_gift < :d:a_hist_1_last and - :dh:a_last_gift < :d:a_hist_2_last - order by - :dh:a_last_gift desc - ) - where - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update - identity collection a_descriptives d, - collection a_descriptives_hist dh - set - :d:a_hist_3_count = :dh:a_count, - :d:a_hist_3_first = :dh:a_first_gift - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key and - :d:a_hist_3_amount = :dh:a_amount and - :d:a_hist_3_last = :dh:a_last_gift and - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ---- Third history value + --print 'Propagating history #3 at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- identity collection a_descriptives d + --set + -- :a_hist_3_amount = (select + -- first(:dh:a_amount) + -- from + -- collection a_descriptives_hist dh + -- where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key and + -- :dh:a_last_gift < :d:a_hist_1_last and + -- :dh:a_last_gift < :d:a_hist_2_last + -- order by + -- :dh:a_last_gift desc + -- ), + -- :a_hist_3_last = (select + -- first(:dh:a_last_gift) + -- from + -- collection a_descriptives_hist dh + -- where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key and + -- :dh:a_last_gift < :d:a_hist_1_last and + -- :dh:a_last_gift < :d:a_hist_2_last + -- order by + -- :dh:a_last_gift desc + -- ) + --where + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; + --update + -- identity collection a_descriptives d, + -- collection a_descriptives_hist dh + --set + -- :d:a_hist_3_count = :dh:a_count, + -- :d:a_hist_3_first = :dh:a_first_gift + --where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key and + -- :d:a_hist_3_amount = :dh:a_amount and + -- :d:a_hist_3_last = :dh:a_last_gift and + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; - -- Overall adjustments to a_act_average_xyz for a_descriptives - -- if the interval is biannual or less, lookback slightly less than a year from the most recent gift - -- the lookahead date is half an interval past the most recent gift - print 'Setting general stats at ' + dateformat(getdate(), 'hh:mm:ss'); - update - identity collection a_descriptives d - set - :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), - :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), - :a_act_average_interval = convert(double, null), - :a_act_average_months = null, - :a_act_average_amount = convert(money, null) - where - :a_act_average_interval < 7 and - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - -- if the interval is greater than bi-annually, check back the last three intervals - update - identity collection a_descriptives d - set - :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), - :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), - :a_act_average_interval = convert(double, null), - :a_act_average_months = null, - :a_act_average_amount = convert(money, null) - where - :a_act_average_interval >= 7 and - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - -- this handles one time gifts (and giving that falls all within half a month) - update - identity collection a_descriptives d - set - :a_act_average_interval = convert(double, null), - :a_act_average_months = null, - :a_act_average_amount = convert(money, null), - :a_act_lookahead_date = condition(getdate() < dateadd(day, 366, :a_last_gift), getdate(), dateadd(day, 366, :a_last_gift)), - :a_act_lookback_date = :a_first_gift - where - :a_act_average_months = 0 and - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update - identity collection a_descriptives d - set - :a_act_average_months = convert(integer, round(datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5)), - :a_act_average_amount = isnull((select condition( - count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, - convert(money, null), - sum(:i:amount) / (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) - ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key), $0), - :a_act_average_interval = (select condition( - count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, - convert(double, null), - (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) / convert(double, count(1)) - ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key) - where - :d:a_act_average_interval is null and - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ---- Overall adjustments to a_act_average_xyz for a_descriptives + ---- if the interval is biannual or less, lookback slightly less than a year from the most recent gift + ---- the lookahead date is half an interval past the most recent gift + --print 'Setting general stats at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- identity collection a_descriptives d + --set + -- :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), + -- :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), + -- :a_act_average_interval = convert(double, null), + -- :a_act_average_months = null, + -- :a_act_average_amount = convert(money, null) + --where + -- :a_act_average_interval < 7 and + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; + ---- if the interval is greater than bi-annually, check back the last three intervals + --update + -- identity collection a_descriptives d + --set + -- :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), + -- :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), + -- :a_act_average_interval = convert(double, null), + -- :a_act_average_months = null, + -- :a_act_average_amount = convert(money, null) + --where + -- :a_act_average_interval >= 7 and + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; + ---- this handles one time gifts (and giving that falls all within half a month) + --update + -- identity collection a_descriptives d + --set + -- :a_act_average_interval = convert(double, null), + -- :a_act_average_months = null, + -- :a_act_average_amount = convert(money, null), + -- :a_act_lookahead_date = condition(getdate() < dateadd(day, 366, :a_last_gift), getdate(), dateadd(day, 366, :a_last_gift)), + -- :a_act_lookback_date = :a_first_gift + --where + -- :a_act_average_months = 0 and + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; + --update + -- identity collection a_descriptives d + --set + -- :a_act_average_months = convert(integer, round(datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5)), + -- :a_act_average_amount = isnull((select condition( + -- count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, + -- convert(money, null), + -- sum(:i:amount) / (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) + -- ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key), $0), + -- :a_act_average_interval = (select condition( + -- count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, + -- convert(double, null), + -- (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) / convert(double, count(1)) + -- ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key) + --where + -- :d:a_act_average_interval is null and + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; - -- Update a_act_average_xyz for a_descriptives_hist - update - identity collection a_descriptives_hist dh - set - :a_act_average_amount = condition( - datediff(day, :dh:a_first_gift, :dh:a_last_gift) < 16, - convert(money, null), - (:dh:a_total - :dh:a_amount) / round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) - ), - :a_act_average_months = convert(integer, round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5)), - :a_act_average_interval = condition( - :dh:a_count = 1, - convert(double, null), - (datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) / (:dh:a_total / :dh:a_amount - 1) - ) - where - (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - :dh:is_approximate = 0 - ; + ---- Update a_act_average_xyz for a_descriptives_hist + --update + -- identity collection a_descriptives_hist dh + --set + -- :a_act_average_amount = condition( + -- datediff(day, :dh:a_first_gift, :dh:a_last_gift) < 16, + -- convert(money, null), + -- (:dh:a_total - :dh:a_amount) / round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) + -- ), + -- :a_act_average_months = convert(integer, round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5)), + -- :a_act_average_interval = condition( + -- :dh:a_count = 1, + -- convert(double, null), + -- (datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) / (:dh:a_total / :dh:a_amount - 1) + -- ) + --where + -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + -- :dh:is_approximate = 0 + --; - -- Update past/current flags on histories and on descriptives. For a history entry with - -- more than one gift, we base the current/lapsed indicator on the gift interval compared to - -- the most recent gift and current date. For a one-gift history entry, we look back 13 - -- months, considering the gift a onetime/annual gift. - -- - -- a_is_current Values: 0 = lapsed, 1 = due / possibly current, 2 = current - -- a_lapsed_days Values: negative = days before the statistical next gift date, - -- positive = days after the next statistical gift date. - print 'Setting lapsed stats at ' + dateformat(getdate(), 'hh:mm:ss'); - update - identity collection a_descriptives_hist dh - set - :a_is_current = condition(:a_count >= 2, - condition(:a_act_average_interval < 7.0, - condition(:a_last_gift > dateadd(day, -7 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), - 2, - condition(:a_last_gift > dateadd(day, -30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), - 1, - 0), - ), - condition(:a_last_gift > dateadd(day, 30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), - 2, - condition(:a_last_gift > dateadd(day, -30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), - 1, - 0), - ) - ), - condition(:a_last_gift > dateadd(day, -334, getdate()), - 2, - condition(:a_last_gift > dateadd(day, -396, getdate()), - 1, - 0) - ) - ), - :a_lapsed_days = condition(:a_count >= 2, - datediff(day, dateadd(day, convert(integer, round(30.5 * :a_act_average_interval)), :a_last_gift), getdate()), - datediff(day, dateadd(day, 365, :a_last_gift), getdate()) - ) - where - (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; + ---- Update past/current flags on histories and on descriptives. For a history entry with + ---- more than one gift, we base the current/lapsed indicator on the gift interval compared to + ---- the most recent gift and current date. For a one-gift history entry, we look back 13 + ---- months, considering the gift a onetime/annual gift. + ---- + ---- a_is_current Values: 0 = lapsed, 1 = due / possibly current, 2 = current + ---- a_lapsed_days Values: negative = days before the statistical next gift date, + ---- positive = days after the next statistical gift date. + --print 'Setting lapsed stats at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- identity collection a_descriptives_hist dh + --set + -- :a_is_current = condition(:a_count >= 2, + -- condition(:a_act_average_interval < 7.0, + -- condition(:a_last_gift > dateadd(day, -7 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), + -- 2, + -- condition(:a_last_gift > dateadd(day, -30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), + -- 1, + -- 0), + -- ), + -- condition(:a_last_gift > dateadd(day, 30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), + -- 2, + -- condition(:a_last_gift > dateadd(day, -30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), + -- 1, + -- 0), + -- ) + -- ), + -- condition(:a_last_gift > dateadd(day, -334, getdate()), + -- 2, + -- condition(:a_last_gift > dateadd(day, -396, getdate()), + -- 1, + -- 0) + -- ) + -- ), + -- :a_lapsed_days = condition(:a_count >= 2, + -- datediff(day, dateadd(day, convert(integer, round(30.5 * :a_act_average_interval)), :a_last_gift), getdate()), + -- datediff(day, dateadd(day, 365, :a_last_gift), getdate()) + -- ) + --where + -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) + --; - -- For the past/current flag for the overall giving to a fund, we use the history entry - -- with the most recent gift in it, ignoring other history entries even if they seem to - -- still be current. - update - identity collection a_descriptives d - set - :a_is_current = (select first(:dh:a_is_current) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), - :a_lapsed_days = (select first(:dh:a_lapsed_days) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) - where - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ---- For the past/current flag for the overall giving to a fund, we use the history entry + ---- with the most recent gift in it, ignoring other history entries even if they seem to + ---- still be current. + --update + -- identity collection a_descriptives d + --set + -- :a_is_current = (select first(:dh:a_is_current) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), + -- :a_lapsed_days = (select first(:dh:a_lapsed_days) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) + --where + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; - -- Identify increases in giving. An increase is when the history entry has a larger monthly - -- average than the immediately previous history entry that is not a one-time-extra gift. We - -- also require that the history entry and its predecessor both have at least two gifts, - -- unless the giving interval is greater than 7 (effectively yearly), in which case any - -- number of gifts for either produces useful results. - print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); - update - identity collection a_descriptives_hist dh - set - :dh:a_increase_pct = condition( - :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)) - 1.0, - condition( - :dh:a_count > 1, - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select :dh2:a_amount / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))) - 1.0, - null - ) - ) - where - (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - not isnull(:dh:a_is_extra, 0) != 0 - ; - update - identity collection a_descriptives_hist dh, - collection a_descriptives d - set - :dh:a_increase_pct = condition((:dh:a_count > 1 or :d:a_act_average_interval > 1.8) and :dh:a_increase_pct > 0.001, :dh:a_increase_pct, null) - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key and - (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; + ---- Identify increases in giving. An increase is when the history entry has a larger monthly + ---- average than the immediately previous history entry that is not a one-time-extra gift. We + ---- also require that the history entry and its predecessor both have at least two gifts, + ---- unless the giving interval is greater than 7 (effectively yearly), in which case any + ---- number of gifts for either produces useful results. + --print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- identity collection a_descriptives_hist dh + --set + -- :dh:a_increase_pct = condition( + -- :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), + -- convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)) - 1.0, + -- condition( + -- :dh:a_count > 1, + -- convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select :dh2:a_amount / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))) - 1.0, + -- null + -- ) + -- ) + --where + -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + -- not isnull(:dh:a_is_extra, 0) != 0 + --; + --update + -- identity collection a_descriptives_hist dh, + -- collection a_descriptives d + --set + -- :dh:a_increase_pct = condition((:dh:a_count > 1 or :d:a_act_average_interval > 1.8) and :dh:a_increase_pct > 0.001, :dh:a_increase_pct, null) + --where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key and + -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) + --; - -- Identify decreases in giving. A decrease is when the history entry has a smaller monthly - -- average than the immediately previous history entry that is not a one-time-extra gift. We - -- also require that the history entry and its predecessor both have at least two gifts, - -- unless the giving interval is greater than 7 (effectively yearly), in which case any - -- number of gifts for either produces useful results. - print 'Identifying decreases at ' + dateformat(getdate(), 'hh:mm:ss'); - update - identity collection a_descriptives_hist dh - set - :dh:a_decrease_pct = condition( - :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), - 1.0 - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)), - condition( - :dh:a_count > 1, - 1.0 - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select convert(double, :dh2:a_amount) / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))), - null - ) - ) - where - (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - not isnull(:dh:a_is_extra, 0) != 0 - ; - update - identity collection a_descriptives_hist dh - set - :dh:a_decrease_pct = condition(:dh:a_decrease_pct > 0.001, :dh:a_decrease_pct, null) - where - (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; + ---- Identify decreases in giving. A decrease is when the history entry has a smaller monthly + ---- average than the immediately previous history entry that is not a one-time-extra gift. We + ---- also require that the history entry and its predecessor both have at least two gifts, + ---- unless the giving interval is greater than 7 (effectively yearly), in which case any + ---- number of gifts for either produces useful results. + --print 'Identifying decreases at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- identity collection a_descriptives_hist dh + --set + -- :dh:a_decrease_pct = condition( + -- :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), + -- 1.0 - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)), + -- condition( + -- :dh:a_count > 1, + -- 1.0 - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select convert(double, :dh2:a_amount) / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))), + -- null + -- ) + -- ) + --where + -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + -- not isnull(:dh:a_is_extra, 0) != 0 + --; + --update + -- identity collection a_descriptives_hist dh + --set + -- :dh:a_decrease_pct = condition(:dh:a_decrease_pct > 0.001, :dh:a_decrease_pct, null) + --where + -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) + --; - -- Propagate informational flags/data to the overall giving descriptives - print 'Setting informational flags at ' + dateformat(getdate(), 'hh:mm:ss'); - update - identity collection a_descriptives d - set - :a_increase_pct = (select :dh:a_increase_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - :a_increase_date = (select condition(:dh:a_increase_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - :a_decrease_pct = (select :dh:a_decrease_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - :a_decrease_date = (select condition(:dh:a_decrease_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - :a_is_extra = (select max(:dh:a_is_extra) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3), - :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) - where - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ---- Propagate informational flags/data to the overall giving descriptives + --print 'Setting informational flags at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- identity collection a_descriptives d + --set + -- :a_increase_pct = (select :dh:a_increase_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + -- :a_increase_date = (select condition(:dh:a_increase_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + -- :a_decrease_pct = (select :dh:a_decrease_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + -- :a_decrease_date = (select condition(:dh:a_decrease_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + -- :a_is_extra = (select max(:dh:a_is_extra) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3), + -- :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) + --where + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; - -- Clear out any old descriptives - print 'Clearing old DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); - delete + --- Clear out any old descriptives + print 'Clearing old DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); + delete /apps/kardia/data/Kardia_DB/a_descriptives/rows d - where + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - delete + ; + delete /apps/kardia/data/Kardia_DB/a_descriptives_hist/rows dh - where + where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:ledger is null or :parameters:ledger = :dh:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; - - -- Save to database - print 'Saving DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); - insert + ; + --- Save to database + print 'Saving DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); + insert /apps/kardia/data/Kardia_DB/a_descriptives/rows - select + select * - from + from collection a_descriptives d - where + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - insert + ; + insert /apps/kardia/data/Kardia_DB/a_descriptives_hist/rows dh - select + select * - from + from collection a_descriptives_hist dh - where + where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:ledger is null or :parameters:ledger = :dh:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; - - -- Return the dataset to the caller - print 'Done at ' + dateformat(getdate(), 'hh:mm:ss'); - select - * - from - collection a_descriptives_hist d - where - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - order by - :a_hist_id desc + ; + ---- Return the dataset to the caller + --print 'Done at ' + dateformat(getdate(), 'hh:mm:ss'); + --select + -- * + --from + -- collection a_descriptives_hist d + --where + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --order by + -- :a_hist_id desc "; } From 420574f844c119e648a100cc933249c2eacccefa Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Tue, 22 Aug 2023 14:32:31 -0600 Subject: [PATCH 02/10] moved first hist_1_amount update into next update where clause --- .../modules/rcpt/update_descriptives_new.qy | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index c47fa8e0..abe1da53 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -319,17 +319,6 @@ update_descriptives_new "system/query" update identity collection a_descriptives d set - :a_hist_1_amount = (select - first(:dh:a_amount) - from - collection a_descriptives_hist dh - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key - order by - :dh:a_last_gift desc - ), :a_hist_1_last = (select first(:dh:a_last_gift) from @@ -351,12 +340,12 @@ update_descriptives_new "system/query" set :d:a_hist_1_count = :dh:a_count, :d:a_hist_1_first = :dh:a_first_gift, - :d:a_hist_1_last = :dh:a_last_gift + :d:a_hist_1_last = :dh:a_last_gift, + :d:a_hist_1_amount = :dh:a_amount where :d:a_ledger_number = :dh:a_ledger_number and :d:a_fund = :dh:a_fund and :d:p_donor_partner_key = :dh:p_donor_partner_key and - :d:a_hist_1_amount = :dh:a_amount and :d:a_hist_1_last = :dh:a_last_gift and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) From 094d4f431541557649b973ee8dc462aa39f80872 Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Tue, 22 Aug 2023 17:14:09 -0600 Subject: [PATCH 03/10] removed redundancy in a_is_current, a_lapsed_days descriptives flag --- .../rcpt/update_descriptives_merge_catchup.qy | 14 +-- .../modules/rcpt/update_descriptives_new.qy | 105 +++++++++--------- 2 files changed, 60 insertions(+), 59 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy index dcbdc295..aa175c1b 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy @@ -18,20 +18,20 @@ update_descriptives_merge_catchup "system/query" identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 set - :a_merged_id = 999999 + --:a_merged_id = 999999 --:a_merged_id = :dh2:a_hist_id - --:a_merged_id = isnull(:dh2:a_merged_id, :dh2:a_hist_id) + :a_merged_id = isnull(:dh2:a_merged_id, :dh2:a_hist_id) where (:parameters:fund is null or :parameters:fund = :dh2:a_fund) and (:parameters:donor is null or :parameters:donor = :dh2:p_donor_partner_key) and :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and - --:dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason - --(:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular - --round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup - --:dh2:a_count > :dh1:a_count --and -- regular has more gifts than catchup - :dh2:a_count >= 2 --and -- regular has at least 2 gifts + :dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason + (:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular + round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup + :dh2:a_count > :dh1:a_count --and -- regular has more gifts than catchup + --:dh2:a_count >= 2 --and -- regular has at least 2 gifts --:dh2:a_first_gift < :dh2:a_last_gift --and -- regular gifts are not on the same day --round(:dh2:a_act_average_interval, 0) != 0 --and -- regular has an established interval (non-zero) --(:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) --and -- gifts are adjacent diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index abe1da53..853f75d0 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -167,6 +167,7 @@ update_descriptives_new "system/query" select a_ledger_number = :ledger, p_donor_partner_key = :donor, + a_hist_id = row_number(), a_fund = :fund, a_amount = :amount, a_first_gift = last(:giftdate), @@ -222,11 +223,11 @@ update_descriptives_new "system/query" -- and may need to be repeated more for larger data sets. -- Merge in 'catch up' giving - -- print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); - -- exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - -- ; + print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy + fund=:parameters:fund, + donor=:parameters:donor + ; ---- Merge in adjacent giving segments --print 'Merge adjacent at ' + dateformat(getdate(), 'hh:mm:ss'); --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent.qy @@ -315,44 +316,44 @@ update_descriptives_new "system/query" --; ---- First history value - print 'Propagating history #1 at ' + dateformat(getdate(), 'hh:mm:ss'); - update - identity collection a_descriptives d - set - :a_hist_1_last = (select - first(:dh:a_last_gift) - from - collection a_descriptives_hist dh - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key - order by - :dh:a_last_gift desc - ) - where - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update - identity collection a_descriptives d, - collection a_descriptives_hist dh - set - :d:a_hist_1_count = :dh:a_count, - :d:a_hist_1_first = :dh:a_first_gift, - :d:a_hist_1_last = :dh:a_last_gift, - :d:a_hist_1_amount = :dh:a_amount - where - :d:a_ledger_number = :dh:a_ledger_number and - :d:a_fund = :dh:a_fund and - :d:p_donor_partner_key = :dh:p_donor_partner_key and - :d:a_hist_1_last = :dh:a_last_gift and - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - select :a_ledger_number, :a_fund, :p_donor_partner_key, :a_hist_1_amount, :a_hist_1_last, :a_hist_1_count, :a_hist_1_first from collection a_descriptives limit 100; - print ' ' - print 'After merging' + --print 'Propagating history #1 at ' + dateformat(getdate(), 'hh:mm:ss'); + --update + -- identity collection a_descriptives d + --set + -- :a_hist_1_last = (select + -- first(:dh:a_last_gift) + -- from + -- collection a_descriptives_hist dh + -- where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key + -- order by + -- :dh:a_last_gift desc + -- ) + --where + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; + --update + -- identity collection a_descriptives d, + -- collection a_descriptives_hist dh + --set + -- :d:a_hist_1_count = :dh:a_count, + -- :d:a_hist_1_first = :dh:a_first_gift, + -- :d:a_hist_1_last = :dh:a_last_gift, + -- :d:a_hist_1_amount = :dh:a_amount + --where + -- :d:a_ledger_number = :dh:a_ledger_number and + -- :d:a_fund = :dh:a_fund and + -- :d:p_donor_partner_key = :dh:p_donor_partner_key and + -- :d:a_hist_1_last = :dh:a_last_gift and + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --; + --select :a_ledger_number, :a_fund, :p_donor_partner_key, :a_hist_1_amount, :a_hist_1_last, :a_hist_1_count, :a_hist_1_first from collection a_descriptives limit 100; + --print ' ' + --print 'After merging' ---- Second history value --print 'Propagating history #2 at ' + dateformat(getdate(), 'hh:mm:ss'); @@ -586,15 +587,15 @@ update_descriptives_new "system/query" ---- For the past/current flag for the overall giving to a fund, we use the history entry ---- with the most recent gift in it, ignoring other history entries even if they seem to ---- still be current. - --update - -- identity collection a_descriptives d - --set - -- :a_is_current = (select first(:dh:a_is_current) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), - -- :a_lapsed_days = (select first(:dh:a_lapsed_days) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) - --where - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; + update + identity collection a_descriptives d + set + :a_is_current = (select first(:dh:a_is_current) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc ), + :a_lapsed_days = (select first(:dh:a_lapsed_days) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc ) + where + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; ---- Identify increases in giving. An increase is when the history entry has a larger monthly ---- average than the immediately previous history entry that is not a one-time-extra gift. We From 045eff8688f4d1781105274959abcd9b5a9eb0f7 Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Wed, 23 Aug 2023 09:47:47 -0600 Subject: [PATCH 04/10] Simplifed isnull(a_is_extra,0)=0 check into a_is_extra is null check --- .../modules/rcpt/update_descriptives_new.qy | 154 +++++++++--------- 1 file changed, 76 insertions(+), 78 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index 853f75d0..c1f5a51d 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -602,84 +602,82 @@ update_descriptives_new "system/query" ---- also require that the history entry and its predecessor both have at least two gifts, ---- unless the giving interval is greater than 7 (effectively yearly), in which case any ---- number of gifts for either produces useful results. - --print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives_hist dh - --set - -- :dh:a_increase_pct = condition( - -- :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), - -- convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)) - 1.0, - -- condition( - -- :dh:a_count > 1, - -- convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select :dh2:a_amount / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))) - 1.0, - -- null - -- ) - -- ) - --where - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - -- not isnull(:dh:a_is_extra, 0) != 0 - --; - --update - -- identity collection a_descriptives_hist dh, - -- collection a_descriptives d - --set - -- :dh:a_increase_pct = condition((:dh:a_count > 1 or :d:a_act_average_interval > 1.8) and :dh:a_increase_pct > 0.001, :dh:a_increase_pct, null) - --where - -- :d:a_ledger_number = :dh:a_ledger_number and - -- :d:a_fund = :dh:a_fund and - -- :d:p_donor_partner_key = :dh:p_donor_partner_key and - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - --; - - ---- Identify decreases in giving. A decrease is when the history entry has a smaller monthly - ---- average than the immediately previous history entry that is not a one-time-extra gift. We - ---- also require that the history entry and its predecessor both have at least two gifts, - ---- unless the giving interval is greater than 7 (effectively yearly), in which case any - ---- number of gifts for either produces useful results. - --print 'Identifying decreases at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives_hist dh - --set - -- :dh:a_decrease_pct = condition( - -- :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), - -- 1.0 - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)), - -- condition( - -- :dh:a_count > 1, - -- 1.0 - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select convert(double, :dh2:a_amount) / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))), - -- null - -- ) - -- ) - --where - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - -- not isnull(:dh:a_is_extra, 0) != 0 - --; - --update - -- identity collection a_descriptives_hist dh - --set - -- :dh:a_decrease_pct = condition(:dh:a_decrease_pct > 0.001, :dh:a_decrease_pct, null) - --where - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - --; - - ---- Propagate informational flags/data to the overall giving descriptives - --print 'Setting informational flags at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives d - --set - -- :a_increase_pct = (select :dh:a_increase_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - -- :a_increase_date = (select condition(:dh:a_increase_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - -- :a_decrease_pct = (select :dh:a_decrease_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - -- :a_decrease_date = (select condition(:dh:a_decrease_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - -- :a_is_extra = (select max(:dh:a_is_extra) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3), - -- :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) - --where - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; + print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives_hist dh + set + :dh:a_increase_pct = condition( + :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), + convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and (:dh2:a_is_extra is null) and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)) - 1.0, + condition( + :dh:a_count > 1, + convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select :dh2:a_amount / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and (:dh2:a_is_extra is null) and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))) - 1.0, + null + ) + ) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + (:dh:a_is_extra is null) + ; + update + identity collection a_descriptives_hist dh, + collection a_descriptives d + set + :dh:a_increase_pct = condition((:dh:a_count > 1 or :d:a_act_average_interval > 1.8) and :dh:a_increase_pct > 0.001, :dh:a_increase_pct, null) + where + :d:a_ledger_number = :dh:a_ledger_number and + :d:a_fund = :dh:a_fund and + :d:p_donor_partner_key = :dh:p_donor_partner_key and + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) + ; +-- Identify decreases in giving. A decrease is when the history entry has a smaller monthly + -- average than the immediately previous history entry that is not a one-time-extra gift. We + -- also require that the history entry and its predecessor both have at least two gifts, + -- unless the giving interval is greater than 7 (effectively yearly), in which case any + -- number of gifts for either produces useful results. + print 'Identifying decreases at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives_hist dh + set + :dh:a_decrease_pct = condition( + :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), + 1.0 - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and (:dh2:a_is_extra is null) and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)), + condition( + :dh:a_count > 1, + 1.0 - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select convert(double, :dh2:a_amount) / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and (:dh2:a_is_extra is null) and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))), + null + ) + ) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + (:dh:a_is_extra is null) + ; + update + identity collection a_descriptives_hist dh + set + :dh:a_decrease_pct = condition(:dh:a_decrease_pct > 0.001, :dh:a_decrease_pct, null) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) + ; + -- Propagate informational flags/data to the overall giving descriptives + print 'Setting informational flags at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives d + set + :a_increase_pct = (select :dh:a_increase_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + :a_increase_date = (select condition(:dh:a_increase_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + :a_decrease_pct = (select :dh:a_decrease_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + :a_decrease_date = (select condition(:dh:a_decrease_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + :a_is_extra = (select max(:dh:a_is_extra) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3), + :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) + where + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; --- Clear out any old descriptives print 'Clearing old DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); From 13982ebea21f1f3c9216bc99ff8c0d9bc7e2af07 Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Thu, 24 Aug 2023 12:27:04 -0600 Subject: [PATCH 05/10] Temporarily fixed NULL(Integer -> datetime) conversion error --- .../rcpt/update_descriptives_merge_approx.qy | 16 +- .../rcpt/update_descriptives_merge_catchup.qy | 34 +- .../modules/rcpt/update_descriptives_new.qy | 862 +++++++++--------- 3 files changed, 470 insertions(+), 442 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy b/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy index 9cb851a7..a85efa6d 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy @@ -74,7 +74,7 @@ update_descriptives_merge_approx "system/query" amount = null, first = min(:t1:first), last = max(:t1:last), - ntl = null, + ntl = convert(datetime, null), count = sum(:t1:count), total = sum(:t1:total), prev_end = condition(last(isnull(:t1:prev_end, 'null')) = 'null', null, last(:t1:prev_end)), @@ -110,7 +110,13 @@ update_descriptives_merge_approx "system/query" (:parameters:fund is null or :parameters:fund = :t:fund) and (:parameters:donor is null or :parameters:donor = :t:donor) ; - + --select :p_donor_partner_key, :a_fund, :a_first_gift, :a_last_gift, :a_ntl_gift from collection a_descriptives_hist; + + + --select :giftdate from collection tmp_gift_items limit 100; + --select :donor, :fund,:ntl from collection tmp_aggregated_segments; + --print 'TESTED FIRST' + -------------------------error #1----------------------------------- -- Step Four: Merge update identity collection a_descriptives_hist dh2, @@ -119,7 +125,7 @@ update_descriptives_merge_approx "system/query" :dh2:a_amount = :t:amount, :dh2:a_first_gift = :t:first, :dh2:a_last_gift = :t:last, - :dh2:a_ntl_gift = condition(:t:count > 1, :t:ntl, null), + :dh2:a_ntl_gift = condition(:t:count > 1, :t:ntl, convert(datetime, null)), :dh2:a_count = :t:count, :dh2:a_total = :t:total, :dh2:a_is_approximate = condition(:t:merge_count > 1, 1, 0), @@ -145,7 +151,9 @@ update_descriptives_merge_approx "system/query" :t:donor = :dh2:p_donor_partner_key and :t:merged_id = :dh2:a_hist_id ; - + ---------------------------error #1-------------------------------------- + + print ' TESTED SECOND' -- Step Five: Delete Merged delete collection a_descriptives_hist dh1 diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy index dcbdc295..dcad63d0 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy @@ -10,7 +10,7 @@ update_descriptives_merge_catchup "system/query" -- the regular giving. -- Step One: Locate - print 'donghwanHwang' + --print 'donghwanHwang' --select * from collection a_descriptives_hist; @@ -18,33 +18,33 @@ update_descriptives_merge_catchup "system/query" identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 set - :a_merged_id = 999999 + --:a_merged_id = 999999 --:a_merged_id = :dh2:a_hist_id - --:a_merged_id = isnull(:dh2:a_merged_id, :dh2:a_hist_id) + :a_merged_id = isnull(:dh2:a_merged_id, :dh2:a_hist_id) where (:parameters:fund is null or :parameters:fund = :dh2:a_fund) and (:parameters:donor is null or :parameters:donor = :dh2:p_donor_partner_key) and :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and - --:dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason - --(:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular - --round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup - --:dh2:a_count > :dh1:a_count --and -- regular has more gifts than catchup - :dh2:a_count >= 2 --and -- regular has at least 2 gifts - --:dh2:a_first_gift < :dh2:a_last_gift --and -- regular gifts are not on the same day - --round(:dh2:a_act_average_interval, 0) != 0 --and -- regular has an established interval (non-zero) - --(:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) --and -- gifts are adjacent - --datediff(month, condition(:dh1:a_last_gift < :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift), condition(:dh1:a_first_gift > :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) <= 14 and - -- checks that the average giving stays similar after the merge would occur - --(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and - --(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) + :dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason + (:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular + round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup + :dh2:a_count > :dh1:a_count and -- regular has more gifts than catchup + :dh2:a_count >= 2 and -- regular has at least 2 gifts + :dh2:a_first_gift < :dh2:a_last_gift and -- regular gifts are not on the same day + round(:dh2:a_act_average_interval, 0) != 0 and -- regular has an established interval (non-zero) + (:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) and -- gifts are adjacent + datediff(month, condition(:dh1:a_last_gift < :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift), condition(:dh1:a_first_gift > :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) <= 14 and + --checks that the average giving stays similar after the merge would occur + (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and + (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) ; --print 'before Merged' - select * from collection a_descriptives_hist; + --select * from collection a_descriptives_hist; - print 'FULLY TESTED' + --print 'FULLY TESTED' -- Step Two: Merge update identity collection a_descriptives_hist dh2, diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index c47fa8e0..3a62f2b7 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -124,7 +124,7 @@ update_descriptives_new "system/query" a_first_gift_amount = last(:amount), a_last_gift = first(:giftdate), a_last_gift_amount = first(:amount), - a_ntl_gift = nth(:giftdate, 2), + a_ntl_gift = (nth(:giftdate, 2)), a_ntl_gift_amount = nth(:amount, 2), -- Bootstrap the analysis process using the data we have. @@ -168,10 +168,11 @@ update_descriptives_new "system/query" a_ledger_number = :ledger, p_donor_partner_key = :donor, a_fund = :fund, + a_hist_id = row_number(), a_amount = :amount, a_first_gift = last(:giftdate), a_last_gift = first(:giftdate), - a_ntl_gift = nth(:giftdate, 2), + a_ntl_gift = isnull(nth(:giftdate, 2), convert(datetime, '1900-00-00 00:00:00')), a_count = count(1), a_total = sum(:amount), a_act_average_months = convert(integer, round(datediff(day, last(:giftdate), first(:giftdate)) / 30.5)), @@ -203,18 +204,31 @@ update_descriptives_new "system/query" :seg, :giftdate desc ; + --select :p_donor_partner_key,:a_fund, :a_ntl_gift from collection a_descriptives_hist; + --print 'BEFORE' + update + collection a_descriptives_hist + set + :a_ntl_gift = null + where + :a_ntl_gift = convert(datetime, '1900-00-00 00:00:00') + ; + + --select :p_donor_partner_key,:a_fund, :a_ntl_gift from collection a_descriptives_hist; + ---print 'AFTER' + -- Update previous/next interval start/end dates - --print 'Updating segment start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- collection a_descriptives_hist dh1 - --set - -- :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift order by :dh2:a_last_gift desc limit 1), - -- :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift order by :dh2:a_first_gift asc limit 1) - --where - -- (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) - --; + print 'Updating segment start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); + update + collection a_descriptives_hist dh1 + set + :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift order by :dh2:a_last_gift desc limit 1), + :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift order by :dh2:a_first_gift asc limit 1) + where + (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) + ; -- These are the merges that allow the gift histories to come out -- very neat and clean. Some of them are repeated, which is a @@ -222,97 +236,101 @@ update_descriptives_new "system/query" -- and may need to be repeated more for larger data sets. -- Merge in 'catch up' giving - -- print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); - -- exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - -- ; - ---- Merge in adjacent giving segments - --print 'Merge adjacent at ' + dateformat(getdate(), 'hh:mm:ss'); - --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - --; - ---- Merge in 'catch up' giving that includes extra gifts - --print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); - --exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - --; - ---- Merge in extra giving - --print 'Merge extra amounts at ' + dateformat(getdate(), 'hh:mm:ss'); - --exec /apps/kardia/modules/rcpt/update_descriptives_merge_extra.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - --; - ---- Merge in adjacent giving segments - --print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - --; - ---- Merge in adjacent single-gift segments - --print 'Merge adjacent singles at ' + dateformat(getdate(), 'hh:mm:ss'); - --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_singles.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - --; - ---- Merge in adjacent giving segments - --print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - --; - ---- Merge in adjacent giving segments - --print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - --exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - --; - ---- Approximate segments that did not fit in anywhere else - --print 'Merge approximates at ' + dateformat(getdate(), 'hh:mm:ss'); - --exec /apps/kardia/modules/rcpt/update_descriptives_merge_approx.qy - -- fund=:parameters:fund, - -- donor=:parameters:donor - --; + print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy + fund=:parameters:fund, + donor=:parameters:donor + ; + -- Merge in adjacent giving segments + print 'Merge adjacent at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent.qy + fund=:parameters:fund, + donor=:parameters:donor + ; + -- Merge in 'catch up' giving that includes extra gifts + print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy + fund=:parameters:fund, + donor=:parameters:donor + ; + -- Merge in extra giving + print 'Merge extra amounts at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_extra.qy + fund=:parameters:fund, + donor=:parameters:donor + ; + -- Merge in adjacent giving segments + print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + fund=:parameters:fund, + donor=:parameters:donor + ; + -- Merge in adjacent single-gift segments + print 'Merge adjacent singles at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_singles.qy + fund=:parameters:fund, + donor=:parameters:donor + ; + -- Merge in adjacent giving segments + print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + fund=:parameters:fund, + donor=:parameters:donor + ; + -- Merge in adjacent giving segments + print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + fund=:parameters:fund, + donor=:parameters:donor + ; + -- Approximate segments that did not fit in anywhere else + print 'Merge approximates at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_approx.qy + fund=:parameters:fund, + donor=:parameters:donor + ; - ---- Correct previous/next interval start/end dates - ---- Extra gifts are all linked together, and regular gifts are - ---- linked together. - --print 'Correcting start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- collection a_descriptives_hist dh1 - --set - -- :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and not :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), - -- :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and not :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) - --where - -- (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - -- not :dh1:a_is_extra - --; - --update - -- collection a_descriptives_hist dh1 - --set - -- :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), - -- :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) - --where - -- (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - -- :dh1:a_is_extra - --; + -- Correct previous/next interval start/end dates + -- Extra gifts are all linked together, and regular gifts are + -- linked together. + print 'Correcting start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); + update + collection a_descriptives_hist dh1 + set + :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and not :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), + :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and not :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) + where + (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and + not :dh1:a_is_extra + ; + update + collection a_descriptives_hist dh1 + set + :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), + :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) + where + (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and + :dh1:a_is_extra + ; - ---- Correct NTL Tracking - --print 'Fixing next-to-last tracking at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- collection a_descriptives_hist dh1 - --set - -- :dh1:a_ntl_gift = (select nth(:t:giftdate, 2) from collection tmp_gift_items t where :t:ledger = :dh1:a_ledger_number and :t:fund = :dh1:a_fund and :t:donor = :dh1:p_donor_partner_key and :t:giftdate <= :dh1:a_last_gift order by :t:giftdate desc) - --where - -- (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - -- :dh1:a_ntl_gift is null and - -- :dh1:a_count > 1 - --; + -------------------error #2 -------------------------------------------- + -- Correct NTL Tracking + print 'Fixing next-to-last tracking at ' + dateformat(getdate(), 'hh:mm:ss'); + update + collection a_descriptives_hist dh1 + set + :dh1:a_ntl_gift = (select nth(:t:giftdate, 2) from collection tmp_gift_items t where :t:ledger = :dh1:a_ledger_number and :t:fund = :dh1:a_fund and :t:donor = :dh1:p_donor_partner_key and :t:giftdate <= :dh1:a_last_gift order by :t:giftdate desc) + where + (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and + :dh1:a_ntl_gift is null and + :dh1:a_count > 1 + ; + + --------------------error #2 ----------------------------------------- + print 'REACHED?' ---- First history value print 'Propagating history #1 at ' + dateformat(getdate(), 'hh:mm:ss'); @@ -361,335 +379,335 @@ update_descriptives_new "system/query" (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) ; - select :a_ledger_number, :a_fund, :p_donor_partner_key, :a_hist_1_amount, :a_hist_1_last, :a_hist_1_count, :a_hist_1_first from collection a_descriptives limit 100; - print ' ' - print 'After merging' + --select :a_ledger_number, :a_fund, :p_donor_partner_key, :a_hist_1_amount, :a_hist_1_last, :a_hist_1_count, :a_hist_1_first from collection a_descriptives limit 100; + --print ' ' + --print 'After merging' - ---- Second history value - --print 'Propagating history #2 at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives d - --set - -- :a_hist_2_amount = (select - -- first(:dh:a_amount) - -- from - -- collection a_descriptives_hist dh - -- where - -- :d:a_ledger_number = :dh:a_ledger_number and - -- :d:a_fund = :dh:a_fund and - -- :d:p_donor_partner_key = :dh:p_donor_partner_key and - -- :dh:a_last_gift < :d:a_hist_1_last - -- order by - -- :dh:a_last_gift desc - -- ), - -- :a_hist_2_last = (select - -- first(:dh:a_last_gift) - -- from - -- collection a_descriptives_hist dh - -- where - -- :d:a_ledger_number = :dh:a_ledger_number and - -- :d:a_fund = :dh:a_fund and - -- :d:p_donor_partner_key = :dh:p_donor_partner_key and - -- :dh:a_last_gift < :d:a_hist_1_last - -- order by - -- :dh:a_last_gift desc - -- ) - --where - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; - --update - -- identity collection a_descriptives d, - -- collection a_descriptives_hist dh - --set - -- :d:a_hist_2_count = :dh:a_count, - -- :d:a_hist_2_first = :dh:a_first_gift - --where - -- :d:a_ledger_number = :dh:a_ledger_number and - -- :d:a_fund = :dh:a_fund and - -- :d:p_donor_partner_key = :dh:p_donor_partner_key and - -- :d:a_hist_2_amount = :dh:a_amount and - -- :d:a_hist_2_last = :dh:a_last_gift and - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; + -- Second history value + print 'Propagating history #2 at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives d + set + :a_hist_2_amount = (select + first(:dh:a_amount) + from + collection a_descriptives_hist dh + where + :d:a_ledger_number = :dh:a_ledger_number and + :d:a_fund = :dh:a_fund and + :d:p_donor_partner_key = :dh:p_donor_partner_key and + :dh:a_last_gift < :d:a_hist_1_last + order by + :dh:a_last_gift desc + ), + :a_hist_2_last = (select + first(:dh:a_last_gift) + from + collection a_descriptives_hist dh + where + :d:a_ledger_number = :dh:a_ledger_number and + :d:a_fund = :dh:a_fund and + :d:p_donor_partner_key = :dh:p_donor_partner_key and + :dh:a_last_gift < :d:a_hist_1_last + order by + :dh:a_last_gift desc + ) + where + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; + update + identity collection a_descriptives d, + collection a_descriptives_hist dh + set + :d:a_hist_2_count = :dh:a_count, + :d:a_hist_2_first = :dh:a_first_gift + where + :d:a_ledger_number = :dh:a_ledger_number and + :d:a_fund = :dh:a_fund and + :d:p_donor_partner_key = :dh:p_donor_partner_key and + :d:a_hist_2_amount = :dh:a_amount and + :d:a_hist_2_last = :dh:a_last_gift and + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; - ---- Third history value - --print 'Propagating history #3 at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives d - --set - -- :a_hist_3_amount = (select - -- first(:dh:a_amount) - -- from - -- collection a_descriptives_hist dh - -- where - -- :d:a_ledger_number = :dh:a_ledger_number and - -- :d:a_fund = :dh:a_fund and - -- :d:p_donor_partner_key = :dh:p_donor_partner_key and - -- :dh:a_last_gift < :d:a_hist_1_last and - -- :dh:a_last_gift < :d:a_hist_2_last - -- order by - -- :dh:a_last_gift desc - -- ), - -- :a_hist_3_last = (select - -- first(:dh:a_last_gift) - -- from - -- collection a_descriptives_hist dh - -- where - -- :d:a_ledger_number = :dh:a_ledger_number and - -- :d:a_fund = :dh:a_fund and - -- :d:p_donor_partner_key = :dh:p_donor_partner_key and - -- :dh:a_last_gift < :d:a_hist_1_last and - -- :dh:a_last_gift < :d:a_hist_2_last - -- order by - -- :dh:a_last_gift desc - -- ) - --where - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; - --update - -- identity collection a_descriptives d, - -- collection a_descriptives_hist dh - --set - -- :d:a_hist_3_count = :dh:a_count, - -- :d:a_hist_3_first = :dh:a_first_gift - --where - -- :d:a_ledger_number = :dh:a_ledger_number and - -- :d:a_fund = :dh:a_fund and - -- :d:p_donor_partner_key = :dh:p_donor_partner_key and - -- :d:a_hist_3_amount = :dh:a_amount and - -- :d:a_hist_3_last = :dh:a_last_gift and - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; + -- Third history value + print 'Propagating history #3 at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives d + set + :a_hist_3_amount = (select + first(:dh:a_amount) + from + collection a_descriptives_hist dh + where + :d:a_ledger_number = :dh:a_ledger_number and + :d:a_fund = :dh:a_fund and + :d:p_donor_partner_key = :dh:p_donor_partner_key and + :dh:a_last_gift < :d:a_hist_1_last and + :dh:a_last_gift < :d:a_hist_2_last + order by + :dh:a_last_gift desc + ), + :a_hist_3_last = (select + first(:dh:a_last_gift) + from + collection a_descriptives_hist dh + where + :d:a_ledger_number = :dh:a_ledger_number and + :d:a_fund = :dh:a_fund and + :d:p_donor_partner_key = :dh:p_donor_partner_key and + :dh:a_last_gift < :d:a_hist_1_last and + :dh:a_last_gift < :d:a_hist_2_last + order by + :dh:a_last_gift desc + ) + where + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; + update + identity collection a_descriptives d, + collection a_descriptives_hist dh + set + :d:a_hist_3_count = :dh:a_count, + :d:a_hist_3_first = :dh:a_first_gift + where + :d:a_ledger_number = :dh:a_ledger_number and + :d:a_fund = :dh:a_fund and + :d:p_donor_partner_key = :dh:p_donor_partner_key and + :d:a_hist_3_amount = :dh:a_amount and + :d:a_hist_3_last = :dh:a_last_gift and + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; - ---- Overall adjustments to a_act_average_xyz for a_descriptives - ---- if the interval is biannual or less, lookback slightly less than a year from the most recent gift - ---- the lookahead date is half an interval past the most recent gift - --print 'Setting general stats at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives d - --set - -- :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), - -- :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), - -- :a_act_average_interval = convert(double, null), - -- :a_act_average_months = null, - -- :a_act_average_amount = convert(money, null) - --where - -- :a_act_average_interval < 7 and - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; - ---- if the interval is greater than bi-annually, check back the last three intervals - --update - -- identity collection a_descriptives d - --set - -- :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), - -- :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), - -- :a_act_average_interval = convert(double, null), - -- :a_act_average_months = null, - -- :a_act_average_amount = convert(money, null) - --where - -- :a_act_average_interval >= 7 and - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; - ---- this handles one time gifts (and giving that falls all within half a month) - --update - -- identity collection a_descriptives d - --set - -- :a_act_average_interval = convert(double, null), - -- :a_act_average_months = null, - -- :a_act_average_amount = convert(money, null), - -- :a_act_lookahead_date = condition(getdate() < dateadd(day, 366, :a_last_gift), getdate(), dateadd(day, 366, :a_last_gift)), - -- :a_act_lookback_date = :a_first_gift - --where - -- :a_act_average_months = 0 and - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; - --update - -- identity collection a_descriptives d - --set - -- :a_act_average_months = convert(integer, round(datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5)), - -- :a_act_average_amount = isnull((select condition( - -- count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, - -- convert(money, null), - -- sum(:i:amount) / (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) - -- ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key), $0), - -- :a_act_average_interval = (select condition( - -- count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, - -- convert(double, null), - -- (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) / convert(double, count(1)) - -- ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key) - --where - -- :d:a_act_average_interval is null and - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; + -- Overall adjustments to a_act_average_xyz for a_descriptives + -- if the interval is biannual or less, lookback slightly less than a year from the most recent gift + -- the lookahead date is half an interval past the most recent gift + print 'Setting general stats at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives d + set + :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), + :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), + :a_act_average_interval = convert(double, null), + :a_act_average_months = null, + :a_act_average_amount = convert(money, null) + where + :a_act_average_interval < 7 and + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; + -- if the interval is greater than bi-annually, check back the last three intervals + update + identity collection a_descriptives d + set + :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), + :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), + :a_act_average_interval = convert(double, null), + :a_act_average_months = null, + :a_act_average_amount = convert(money, null) + where + :a_act_average_interval >= 7 and + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; + -- this handles one time gifts (and giving that falls all within half a month) + update + identity collection a_descriptives d + set + :a_act_average_interval = convert(double, null), + :a_act_average_months = null, + :a_act_average_amount = convert(money, null), + :a_act_lookahead_date = condition(getdate() < dateadd(day, 366, :a_last_gift), getdate(), dateadd(day, 366, :a_last_gift)), + :a_act_lookback_date = :a_first_gift + where + :a_act_average_months = 0 and + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; + update + identity collection a_descriptives d + set + :a_act_average_months = convert(integer, round(datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5)), + :a_act_average_amount = isnull((select condition( + count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, + convert(money, null), + sum(:i:amount) / (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) + ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key), $0), + :a_act_average_interval = (select condition( + count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, + convert(double, null), + (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) / convert(double, count(1)) + ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key) + where + :d:a_act_average_interval is null and + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; - ---- Update a_act_average_xyz for a_descriptives_hist - --update - -- identity collection a_descriptives_hist dh - --set - -- :a_act_average_amount = condition( - -- datediff(day, :dh:a_first_gift, :dh:a_last_gift) < 16, - -- convert(money, null), - -- (:dh:a_total - :dh:a_amount) / round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) - -- ), - -- :a_act_average_months = convert(integer, round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5)), - -- :a_act_average_interval = condition( - -- :dh:a_count = 1, - -- convert(double, null), - -- (datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) / (:dh:a_total / :dh:a_amount - 1) - -- ) - --where - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - -- :dh:is_approximate = 0 - --; + -- Update a_act_average_xyz for a_descriptives_hist + update + identity collection a_descriptives_hist dh + set + :a_act_average_amount = condition( + datediff(day, :dh:a_first_gift, :dh:a_last_gift) < 16, + convert(money, null), + (:dh:a_total - :dh:a_amount) / round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) + ), + :a_act_average_months = convert(integer, round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5)), + :a_act_average_interval = condition( + :dh:a_count = 1, + convert(double, null), + (datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) / (:dh:a_total / :dh:a_amount - 1) + ) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + :dh:is_approximate = 0 + ; - ---- Update past/current flags on histories and on descriptives. For a history entry with - ---- more than one gift, we base the current/lapsed indicator on the gift interval compared to - ---- the most recent gift and current date. For a one-gift history entry, we look back 13 - ---- months, considering the gift a onetime/annual gift. - ---- - ---- a_is_current Values: 0 = lapsed, 1 = due / possibly current, 2 = current - ---- a_lapsed_days Values: negative = days before the statistical next gift date, - ---- positive = days after the next statistical gift date. - --print 'Setting lapsed stats at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives_hist dh - --set - -- :a_is_current = condition(:a_count >= 2, - -- condition(:a_act_average_interval < 7.0, - -- condition(:a_last_gift > dateadd(day, -7 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), - -- 2, - -- condition(:a_last_gift > dateadd(day, -30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), - -- 1, - -- 0), - -- ), - -- condition(:a_last_gift > dateadd(day, 30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), - -- 2, - -- condition(:a_last_gift > dateadd(day, -30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), - -- 1, - -- 0), - -- ) - -- ), - -- condition(:a_last_gift > dateadd(day, -334, getdate()), - -- 2, - -- condition(:a_last_gift > dateadd(day, -396, getdate()), - -- 1, - -- 0) - -- ) - -- ), - -- :a_lapsed_days = condition(:a_count >= 2, - -- datediff(day, dateadd(day, convert(integer, round(30.5 * :a_act_average_interval)), :a_last_gift), getdate()), - -- datediff(day, dateadd(day, 365, :a_last_gift), getdate()) - -- ) - --where - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - --; + -- Update past/current flags on histories and on descriptives. For a history entry with + -- more than one gift, we base the current/lapsed indicator on the gift interval compared to + -- the most recent gift and current date. For a one-gift history entry, we look back 13 + -- months, considering the gift a onetime/annual gift. + -- + -- a_is_current Values: 0 = lapsed, 1 = due / possibly current, 2 = current + -- a_lapsed_days Values: negative = days before the statistical next gift date, + -- positive = days after the next statistical gift date. + print 'Setting lapsed stats at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives_hist dh + set + :a_is_current = condition(:a_count >= 2, + condition(:a_act_average_interval < 7.0, + condition(:a_last_gift > dateadd(day, -7 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), + 2, + condition(:a_last_gift > dateadd(day, -30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), + 1, + 0), + ), + condition(:a_last_gift > dateadd(day, 30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), + 2, + condition(:a_last_gift > dateadd(day, -30 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), + 1, + 0), + ) + ), + condition(:a_last_gift > dateadd(day, -334, getdate()), + 2, + condition(:a_last_gift > dateadd(day, -396, getdate()), + 1, + 0) + ) + ), + :a_lapsed_days = condition(:a_count >= 2, + datediff(day, dateadd(day, convert(integer, round(30.5 * :a_act_average_interval)), :a_last_gift), getdate()), + datediff(day, dateadd(day, 365, :a_last_gift), getdate()) + ) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) + ; - ---- For the past/current flag for the overall giving to a fund, we use the history entry - ---- with the most recent gift in it, ignoring other history entries even if they seem to - ---- still be current. - --update - -- identity collection a_descriptives d - --set - -- :a_is_current = (select first(:dh:a_is_current) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), - -- :a_lapsed_days = (select first(:dh:a_lapsed_days) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) - --where - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; + -- For the past/current flag for the overall giving to a fund, we use the history entry + -- with the most recent gift in it, ignoring other history entries even if they seem to + -- still be current. + update + identity collection a_descriptives d + set + :a_is_current = (select first(:dh:a_is_current) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), + :a_lapsed_days = (select first(:dh:a_lapsed_days) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) + where + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; - ---- Identify increases in giving. An increase is when the history entry has a larger monthly - ---- average than the immediately previous history entry that is not a one-time-extra gift. We - ---- also require that the history entry and its predecessor both have at least two gifts, - ---- unless the giving interval is greater than 7 (effectively yearly), in which case any - ---- number of gifts for either produces useful results. - --print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives_hist dh - --set - -- :dh:a_increase_pct = condition( - -- :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), - -- convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)) - 1.0, - -- condition( - -- :dh:a_count > 1, - -- convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select :dh2:a_amount / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))) - 1.0, - -- null - -- ) - -- ) - --where - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - -- not isnull(:dh:a_is_extra, 0) != 0 - --; - --update - -- identity collection a_descriptives_hist dh, - -- collection a_descriptives d - --set - -- :dh:a_increase_pct = condition((:dh:a_count > 1 or :d:a_act_average_interval > 1.8) and :dh:a_increase_pct > 0.001, :dh:a_increase_pct, null) - --where - -- :d:a_ledger_number = :dh:a_ledger_number and - -- :d:a_fund = :dh:a_fund and - -- :d:p_donor_partner_key = :dh:p_donor_partner_key and - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - --; + -- Identify increases in giving. An increase is when the history entry has a larger monthly + -- average than the immediately previous history entry that is not a one-time-extra gift. We + -- also require that the history entry and its predecessor both have at least two gifts, + -- unless the giving interval is greater than 7 (effectively yearly), in which case any + -- number of gifts for either produces useful results. + print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives_hist dh + set + :dh:a_increase_pct = condition( + :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), + convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)) - 1.0, + condition( + :dh:a_count > 1, + convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select :dh2:a_amount / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))) - 1.0, + null + ) + ) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + not isnull(:dh:a_is_extra, 0) != 0 + ; + update + identity collection a_descriptives_hist dh, + collection a_descriptives d + set + :dh:a_increase_pct = condition((:dh:a_count > 1 or :d:a_act_average_interval > 1.8) and :dh:a_increase_pct > 0.001, :dh:a_increase_pct, null) + where + :d:a_ledger_number = :dh:a_ledger_number and + :d:a_fund = :dh:a_fund and + :d:p_donor_partner_key = :dh:p_donor_partner_key and + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) + ; - ---- Identify decreases in giving. A decrease is when the history entry has a smaller monthly - ---- average than the immediately previous history entry that is not a one-time-extra gift. We - ---- also require that the history entry and its predecessor both have at least two gifts, - ---- unless the giving interval is greater than 7 (effectively yearly), in which case any - ---- number of gifts for either produces useful results. - --print 'Identifying decreases at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives_hist dh - --set - -- :dh:a_decrease_pct = condition( - -- :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), - -- 1.0 - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)), - -- condition( - -- :dh:a_count > 1, - -- 1.0 - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select convert(double, :dh2:a_amount) / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))), - -- null - -- ) - -- ) - --where - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - -- not isnull(:dh:a_is_extra, 0) != 0 - --; - --update - -- identity collection a_descriptives_hist dh - --set - -- :dh:a_decrease_pct = condition(:dh:a_decrease_pct > 0.001, :dh:a_decrease_pct, null) - --where - -- (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - --; + -- Identify decreases in giving. A decrease is when the history entry has a smaller monthly + -- average than the immediately previous history entry that is not a one-time-extra gift. We + -- also require that the history entry and its predecessor both have at least two gifts, + -- unless the giving interval is greater than 7 (effectively yearly), in which case any + -- number of gifts for either produces useful results. + print 'Identifying decreases at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives_hist dh + set + :dh:a_decrease_pct = condition( + :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), + 1.0 - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)), + condition( + :dh:a_count > 1, + 1.0 - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select convert(double, :dh2:a_amount) / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))), + null + ) + ) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + not isnull(:dh:a_is_extra, 0) != 0 + ; + update + identity collection a_descriptives_hist dh + set + :dh:a_decrease_pct = condition(:dh:a_decrease_pct > 0.001, :dh:a_decrease_pct, null) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) + ; - ---- Propagate informational flags/data to the overall giving descriptives - --print 'Setting informational flags at ' + dateformat(getdate(), 'hh:mm:ss'); - --update - -- identity collection a_descriptives d - --set - -- :a_increase_pct = (select :dh:a_increase_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - -- :a_increase_date = (select condition(:dh:a_increase_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - -- :a_decrease_pct = (select :dh:a_decrease_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - -- :a_decrease_date = (select condition(:dh:a_decrease_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), - -- :a_is_extra = (select max(:dh:a_is_extra) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3), - -- :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) - --where - -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and - -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - --; + -- Propagate informational flags/data to the overall giving descriptives + print 'Setting informational flags at ' + dateformat(getdate(), 'hh:mm:ss'); + update + identity collection a_descriptives d + set + :a_increase_pct = (select :dh:a_increase_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + :a_increase_date = (select condition(:dh:a_increase_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + :a_decrease_pct = (select :dh:a_decrease_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + :a_decrease_date = (select condition(:dh:a_decrease_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + :a_is_extra = (select max(:dh:a_is_extra) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3), + :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) + where + (:parameters:fund is null or :parameters:fund = :d:a_fund) and + (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + ; --- Clear out any old descriptives print 'Clearing old DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); @@ -720,6 +738,8 @@ update_descriptives_new "system/query" (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) ; + + print ' Saving Descriptives Hist Table'+ dateformat(getdate(), 'hh:mm:ss'); insert /apps/kardia/data/Kardia_DB/a_descriptives_hist/rows dh select From 290a29322f2628a75240976efb4ee07e7fbdda80 Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Thu, 24 Aug 2023 14:54:50 -0600 Subject: [PATCH 06/10] removed redundant check in merge_catchup condition --- .../modules/rcpt/update_descriptives_merge_catchup.qy | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy index dcad63d0..10ad3f98 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy @@ -37,14 +37,13 @@ update_descriptives_merge_catchup "system/query" (:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) and -- gifts are adjacent datediff(month, condition(:dh1:a_last_gift < :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift), condition(:dh1:a_first_gift > :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) <= 14 and --checks that the average giving stays similar after the merge would occur - (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and - (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) + (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) and + (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) ; - --print 'before Merged' - - --select * from collection a_descriptives_hist; + select :p_donor_partner_key, :a_fund, :a_hist_id, :a_merged_id from collection a_descriptives_hist; + - --print 'FULLY TESTED' + print 'FULLY TESTED' -- Step Two: Merge update identity collection a_descriptives_hist dh2, From 835e65cf71e8f7fb533c550d4cf689e02a67a12f Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Thu, 24 Aug 2023 17:24:10 -0600 Subject: [PATCH 07/10] merge_adjacent change (dh1 ntl gift setting while aggregating tmp_merge_aggregates) --- kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy | 4 ++-- kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy index 91fb5def..78daefb7 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy @@ -72,7 +72,7 @@ update_descriptives_merge_adjacent "system/query" next_start_dh1 = condition(first(isnull(:dh1:a_next_start, 'null')) = 'null', null, first(:dh1:a_next_start)), first_dh1_gift = min(:dh1:a_first_gift), last_dh1_gift = max(:dh1:a_last_gift), - ntl_dh1_gift = condition(first(:dh1:a_count) >=2, first(:dh1:a_ntl_gift), nth(:dh1:a_first_gift, 2)), + ntl_dh1_gift = condition(first(:dh1:a_count) >=2, first(:dh1:a_ntl_gift), nth(:dh1:a_last_gift, 2)), merge_next = :dh2:a_merged_id from collection a_descriptives_hist dh1, @@ -95,7 +95,7 @@ update_descriptives_merge_adjacent "system/query" :dh1:a_fund, :dh1:a_hist_id ; - + select :first_dh1_gift, :ntl_dh1_gift, :last_dh1_gift from collection tmp_merge_aggregates; -- Step Three: Merge dh1 into dh2, but only if it is safe -- Due to extra merges, it should always be safe. However, -- this is not guaranteed for exceptionally large data sets diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy index 10ad3f98..3705a761 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy @@ -40,7 +40,7 @@ update_descriptives_merge_catchup "system/query" (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) and (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) ; - select :p_donor_partner_key, :a_fund, :a_hist_id, :a_merged_id from collection a_descriptives_hist; + --select :p_donor_partner_key, :a_fund, :a_hist_id, :a_merged_id from collection a_descriptives_hist; print 'FULLY TESTED' From 7ea75b05ea13c3b5d1418a5b20d14eccef8a5741 Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Fri, 25 Aug 2023 09:21:25 -0600 Subject: [PATCH 08/10] Fixed a_is_current and a_lapsed_days first to limit 1 in order to match other formats --- .../rcpt/update_descriptives_merge_catchup_with_extra.qy | 2 ++ kardia-app/modules/rcpt/update_descriptives_new.qy | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy index 81f23880..e3dc1f4e 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy @@ -75,6 +75,8 @@ update_descriptives_merge_catchup_with_extra "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_merged_id = :dh2:a_hist_id ; + print 'Start printing collection' + select :merged_id, :regular_id, :catchup_amount from collection tmp_intermediate_vals; -- Step Three: Merge update diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index 29334559..ae15b00e 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -620,8 +620,8 @@ update_descriptives_new "system/query" update identity collection a_descriptives d set - :a_is_current = (select first(:dh:a_is_current) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc ), - :a_lapsed_days = (select first(:dh:a_lapsed_days) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc ) + :a_is_current = (select :dh:a_is_current from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), + :a_lapsed_days = (select :dh:a_lapsed_days from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) From 52e03471cb8e4f91e4944175a9449b644e942882 Mon Sep 17 00:00:00 2001 From: Donghwan Hwang - Kardia Date: Fri, 25 Aug 2023 10:46:06 -0600 Subject: [PATCH 09/10] Commit before rollback --- .../rcpt/update_descriptives_merge_adjacent.qy | 9 +++++++-- .../rcpt/update_descriptives_merge_catchup.qy | 6 ++++-- ...te_descriptives_merge_catchup_with_extra.qy | 4 ++-- .../modules/rcpt/update_descriptives_new.qy | 18 +++++++++--------- limit | 0 5 files changed, 22 insertions(+), 15 deletions(-) create mode 100644 limit diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy index 78daefb7..fdb96730 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy @@ -56,7 +56,9 @@ update_descriptives_merge_adjacent "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_merged_id = :dh2:a_hist_id ; - + -- print ' ' + --print 'MEREGINGING' + --select :a_merged_id from collection a_descriptives_hist; -- Step Two : Aggregate fields that are going to be merged into the same history insert collection tmp_merge_aggregates @@ -95,7 +97,10 @@ update_descriptives_merge_adjacent "system/query" :dh1:a_fund, :dh1:a_hist_id ; - select :first_dh1_gift, :ntl_dh1_gift, :last_dh1_gift from collection tmp_merge_aggregates; + -- print ' PRINTING TMP_MERGE_AGGREGATES' + + -- select :last_dh1_gift from collection tmp_merge_aggregates; + -- Step Three: Merge dh1 into dh2, but only if it is safe -- Due to extra merges, it should always be safe. However, -- this is not guaranteed for exceptionally large data sets diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy index 3705a761..4319e726 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy @@ -13,7 +13,8 @@ update_descriptives_merge_catchup "system/query" --print 'donghwanHwang' --select * from collection a_descriptives_hist; - + select :p_donor_partner_key,:a_fund, :a_ntl_gift from collection a_descriptives_hist; + print 'BEFORE' update identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 @@ -40,7 +41,8 @@ update_descriptives_merge_catchup "system/query" (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) and (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) ; - --select :p_donor_partner_key, :a_fund, :a_hist_id, :a_merged_id from collection a_descriptives_hist; + print 'MERGE CATHCUOPCAHTCOUO' + select :p_donor_partner_key, :a_fund, :a_hist_id, :a_merged_id from collection a_descriptives_hist; print 'FULLY TESTED' diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy index e3dc1f4e..6ac6d792 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy @@ -75,8 +75,8 @@ update_descriptives_merge_catchup_with_extra "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_merged_id = :dh2:a_hist_id ; - print 'Start printing collection' - select :merged_id, :regular_id, :catchup_amount from collection tmp_intermediate_vals; + --print 'Start printing collection' + --select :merged_id, :regular_id, :catchup_amount from collection tmp_intermediate_vals; -- Step Three: Merge update diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index ae15b00e..e6b29e47 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -169,7 +169,6 @@ update_descriptives_new "system/query" p_donor_partner_key = :donor, a_hist_id = row_number(), a_fund = :fund, - a_hist_id = row_number(), a_amount = :amount, a_first_gift = last(:giftdate), a_last_gift = first(:giftdate), @@ -205,15 +204,14 @@ update_descriptives_new "system/query" :seg, :giftdate desc ; - --select :p_donor_partner_key,:a_fund, :a_ntl_gift from collection a_descriptives_hist; - --print 'BEFORE' - update + + update collection a_descriptives_hist - set - :a_ntl_gift = null - where - :a_ntl_gift = convert(datetime, '1900-00-00 00:00:00') - ; + set + :a_ntl_gift = null + where + :a_ntl_gift = convert(datetime, '1900-00-00 00:00:00') + ; --select :p_donor_partner_key,:a_fund, :a_ntl_gift from collection a_descriptives_hist; ---print 'AFTER' @@ -237,6 +235,8 @@ update_descriptives_new "system/query" -- and may need to be repeated more for larger data sets. -- Merge in 'catch up' giving + + print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy fund=:parameters:fund, diff --git a/limit b/limit new file mode 100644 index 00000000..e69de29b From 8750ef97aa4204c3595fedbdd5403414d7cd5f8b Mon Sep 17 00:00:00 2001 From: donghwanHwang Date: Mon, 18 Sep 2023 01:13:42 -0400 Subject: [PATCH 10/10] Updated all the comments --- .../update_descriptives_merge_adjacent.qy | 24 ++++-- .../update_descriptives_merge_adjacent_2.qy | 3 + ...ate_descriptives_merge_adjacent_singles.qy | 1 + .../rcpt/update_descriptives_merge_approx.qy | 42 ++++++---- .../rcpt/update_descriptives_merge_catchup.qy | 22 ++--- ...e_descriptives_merge_catchup_with_extra.qy | 20 +++-- .../rcpt/update_descriptives_merge_extra.qy | 10 +++ .../modules/rcpt/update_descriptives_new.qy | 81 +++++++++++++------ 8 files changed, 134 insertions(+), 69 deletions(-) diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy index fdb96730..6e07fdbe 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy @@ -9,9 +9,14 @@ update_descriptives_merge_adjacent "system/query" delete collection tmp_merge_aggregates; + -- Step One: Locate adjacent segments -- compare the combined segments average to the -- longer segment's average. + + -- dh1 is the adjacent giving with the same amount + -- dh2 is the regular giving. + update identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 @@ -25,11 +30,20 @@ update_descriptives_merge_adjacent "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_amount = :dh1:a_amount and :dh1:a_hist_id < :dh2:a_hist_id and + --adjacent giving more recent than regular (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) ; -- Extra recurssions to ensure a merge does not occur -- into a history that is also getting merged away + + -- If you don't perform these redundant merges, rows with a non-null a_merged_id will be deleted after the query runs. + -- Consider a chain 1->2->3->4->5->6. After one query run, the chain updates to 1->3, 2->4, 3->5, 4->6, 5->6. + -- Merges occur only if a_merged_id of the recipient (dh2) is null. In this example, only 4->6 and 5->6 merge, because 6 has a_merged_id == null. + -- The next step, "Delete Merged," removes all rows with a non-null a_merged_id. So, 1, 2, 3, 4, 5 are deleted. + -- Though 4 and 5 are safely merged into 6, data from 1, 2, and 3 are lost as they're deleted without merging. + -- This issue could have be avoided with two more merges, hence the redundant query. + update identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 @@ -56,10 +70,9 @@ update_descriptives_merge_adjacent "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_merged_id = :dh2:a_hist_id ; - -- print ' ' - --print 'MEREGINGING' - --select :a_merged_id from collection a_descriptives_hist; - -- Step Two : Aggregate fields that are going to be merged into the same history + + + -- Step Two : Aggregate fields that are going to be merged into the same history insert collection tmp_merge_aggregates select @@ -97,9 +110,6 @@ update_descriptives_merge_adjacent "system/query" :dh1:a_fund, :dh1:a_hist_id ; - -- print ' PRINTING TMP_MERGE_AGGREGATES' - - -- select :last_dh1_gift from collection tmp_merge_aggregates; -- Step Three: Merge dh1 into dh2, but only if it is safe -- Due to extra merges, it should always be safe. However, diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy index 0f6d2313..a29b099f 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy @@ -33,6 +33,7 @@ update_descriptives_merge_adjacent_2 "system/query" -- check how far out you are merging in (datediff(day, condition(:dh2:a_last_gift < :dh1:a_first_gift, :dh2:a_last_gift, :dh1:a_last_gift), condition(:dh2:a_last_gift < :dh1:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) / 30.5 <= (:dh1:a_amount / :dh2:a_amount * 2 + 2)) and -- don't merge if both are established giving patterns of different amounts with nice intervals + -- checking if at least one of the merging sets is not regular giving (:dh2:a_amount = :dh1:a_amount or :dh2:a_count <= 5 or :dh1:a_count <= 5 or :dh2:a_act_average_months <= 5 or :dh1:a_act_average_months <= 5 or abs(round(:dh2:a_act_average_interval) - :dh2:a_act_average_interval) >= 0.15 or abs(round(:dh1:a_act_average_interval) - :dh1:a_act_average_interval) >= 0.15) ; -- Extra recurssions to ensure a merge does not occur @@ -114,11 +115,13 @@ update_descriptives_merge_adjacent_2 "system/query" :dh2:a_total = :dh2:a_total + :t:sum_dh1_a_total, :dh2:a_prev_end = condition( :t:prev_end_dh1 is null, + -- t is the first null, condition(:t:prev_end_dh1 < :dh2:a_prev_end, :t:prev_end_dh1, :dh2:a_prev_end) ), :dh2:a_next_start = condition( :t:next_start_dh1 is null, + -- t is the last null, condition(:t:next_start_dh1 > :dh2:a_next_start, :t:next_start_dh1, :dh2:a_next_start) ), diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy index b008c17e..df29a205 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy @@ -24,6 +24,7 @@ update_descriptives_merge_adjacent_singles "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_amount = :dh1:a_amount and :dh1:a_hist_id < :dh2:a_hist_id and + --dh1 is more recent than dh2 :dh1:a_count = 1 and :dh2:a_count = 1 and datediff(month, :dh2:a_first_gift, :dh1:a_first_gift) < 37 diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy b/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy index a85efa6d..b1222503 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy @@ -21,14 +21,20 @@ update_descriptives_merge_approx "system/query" :dh:a_is_approximate is null and (:dh:a_count < 5 or :dh:a_act_average_months < 5) and -- these try to discount regular giving patterns from being approximated --(:dh:a_prev_end is not null or :dh:a_count != 1) and -- first gifts are special - (:dh:a_hist_id != 1 or :dh:a_is_extra != 1) and-- don't merge most recent extra gifts, as there isn't enough info about them yet + (:dh:a_hist_id != 1 or :dh:a_is_extra != 1) and --exclude (most recent + extra gift) + -- don't merge most recent extra gifts, as there isn't enough info about them yet not datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 334 and -- this prevents annual gifts from being merged together (select count(1) from collection a_descriptives_hist dh2 - where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and (:dh2:a_hist_id = :dh:a_hist_id + 1 or :dh2:a_hist_id = :dh:a_hist_id - 1) and (:dh2:a_count < 5 or :dh2:a_act_average_months < 5) and (not isnull(:dh:a_is_extra, 0) or not isnull(:dh2:a_is_extra, 0)) - ) >= 1, - 2, + where :dh2:a_ledger_number = :dh:a_ledger_number and + :dh2:a_fund = :dh:a_fund and + :dh2:p_donor_partner_key = :dh:p_donor_partner_key and + (:dh2:a_hist_id = :dh:a_hist_id + 1 or :dh2:a_hist_id = :dh:a_hist_id - 1) and --gift history adjacent to each other? + (:dh2:a_count < 5 or :dh2:a_act_average_months < 5) and -- these try to discount regular giving patterns from being approximated + (not isnull(:dh:a_is_extra, 0) or not isnull(:dh2:a_is_extra, 0)) + ) >= 1, + 2,--intermediate value to distinguish(different conditions) testing purpose, being able to identify the descriptive 0 ) where @@ -64,6 +70,7 @@ update_descriptives_merge_approx "system/query" -- Step Three: Aggregate Each Segment insert collection tmp_aggregated_segments + --merge into dh2 where earliest giving segments and isapproximate select ledger = :t1:ledger, fund = :t1:fund, @@ -101,22 +108,20 @@ update_descriptives_merge_approx "system/query" -- This is so that other descriptives come out cleaner and -- because a real value for the amount provides better descriptives than -- an averged value. In case of a tie, this instead uses the avg of the amounts + + --select mode or average for amount based on total count update collection tmp_aggregated_segments t set + -- if the mode of giving count is 1 , select average amount. + -- else, select mode. :t:amount = condition((select sum(:count) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1) = 1, (select avg(:amount) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg), (select :amount from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1)), :t:ntl = (select nth(:giftdate, 2) from collection tmp_gift_items where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :giftdate <= :t:last order by :giftdate desc) where (:parameters:fund is null or :parameters:fund = :t:fund) and (:parameters:donor is null or :parameters:donor = :t:donor) ; - --select :p_donor_partner_key, :a_fund, :a_first_gift, :a_last_gift, :a_ntl_gift from collection a_descriptives_hist; - - - --select :giftdate from collection tmp_gift_items limit 100; - --select :donor, :fund,:ntl from collection tmp_aggregated_segments; - --print 'TESTED FIRST' - -------------------------error #1----------------------------------- + -- Step Four: Merge update identity collection a_descriptives_hist dh2, @@ -129,6 +134,8 @@ update_descriptives_merge_approx "system/query" :dh2:a_count = :t:count, :dh2:a_total = :t:total, :dh2:a_is_approximate = condition(:t:merge_count > 1, 1, 0), + --filtering from isapproximate=2 to find where merge_count bigger than 1 + --if merge_count =1 , set it to 0 :dh2:a_prev_end = :t:prev_end, :dh2:a_next_start = :t:next_start, :dh2:a_act_average_amount = condition( @@ -150,10 +157,7 @@ update_descriptives_merge_approx "system/query" :t:fund = :dh2:a_fund and :t:donor = :dh2:p_donor_partner_key and :t:merged_id = :dh2:a_hist_id - ; - ---------------------------error #1-------------------------------------- - - print ' TESTED SECOND' + ; -- Step Five: Delete Merged delete collection a_descriptives_hist dh1 @@ -162,9 +166,13 @@ update_descriptives_merge_approx "system/query" (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and :dh1:a_is_approximate = 2 ; - + -- Merge extra gifts if there are only extras and an approximate history -- Step One : Identify + + -- From descriptives_hist dh, + -- set dh1:merged_id into dh2:hist:id where dh1 is extra and dh2 is approximate count >1, + -- only if there isn't anywhere dh is both not extra and is_approximate count =1 update identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 @@ -225,6 +233,8 @@ update_descriptives_merge_approx "system/query" update collection tmp_merge_aggregates t set + -- if the mode of giving count is 1 , select average amount. + -- else, select mode. :t:amount = condition((select sum(:count) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1) = 1, (select avg(:amount) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg), (select :amount from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1)), :t:ntl = (select nth(:giftdate, 2) from collection tmp_gift_items where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :giftdate <= :t:last order by :giftdate desc) where diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy index 4319e726..aa8953be 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy @@ -6,21 +6,15 @@ update_descriptives_merge_catchup "system/query" sql = " declare collection tmp_gift_items scope application; - -- Merge in 'catch up' giving. dh1 is the larger catch up amount, and dh2 is - -- the regular giving. - + -- Merge in 'catch up' giving. + -- dh1 is the larger catch up amount, and dh2 is the regular giving. + -- Step One: Locate - --print 'donghwanHwang' - - --select * from collection a_descriptives_hist; - select :p_donor_partner_key,:a_fund, :a_ntl_gift from collection a_descriptives_hist; - print 'BEFORE' update + -- identity refers to the collection that is going to be updated identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 set - --:a_merged_id = 999999 - --:a_merged_id = :dh2:a_hist_id :a_merged_id = isnull(:dh2:a_merged_id, :dh2:a_hist_id) where (:parameters:fund is null or :parameters:fund = :dh2:a_fund) and @@ -28,7 +22,7 @@ update_descriptives_merge_catchup "system/query" :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and - :dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason + :dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason, arbitrary number for analysis (:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup :dh2:a_count > :dh1:a_count and -- regular has more gifts than catchup @@ -40,12 +34,8 @@ update_descriptives_merge_catchup "system/query" --checks that the average giving stays similar after the merge would occur (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) and (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) - ; - print 'MERGE CATHCUOPCAHTCOUO' - select :p_donor_partner_key, :a_fund, :a_hist_id, :a_merged_id from collection a_descriptives_hist; - + ; - print 'FULLY TESTED' -- Step Two: Merge update identity collection a_descriptives_hist dh2, diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy index 6ac6d792..8c3e4857 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy @@ -40,7 +40,8 @@ update_descriptives_merge_catchup_with_extra "system/query" -- how much is a part of regular giving (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 3.00 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and - (select count(1) from collection tmp_gift_items where :donor = :dh1:p_donor_partner_key and :ledger = :dh1:a_ledger_number and :fund = :dh1:a_fund and :giftdate > dateadd(day, convert(integer, round(-0.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift) and :giftdate < dateadd(day, convert(integer, round(.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift)) = 1 --and -- only 1 gift within +/- half an interval + (select count(1) from collection tmp_gift_items where :donor = :dh1:p_donor_partner_key and :ledger = :dh1:a_ledger_number and :fund = :dh1:a_fund and :giftdate > dateadd(day, convert(integer, round(-0.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift) and :giftdate < dateadd(day, convert(integer, round(.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift)) = 1 + -- check if only 1 gift within +/- half an interval of extra giving --((:dh1:a_amount / :dh2:a_amount) != round(:dh1:a_amount / :dh2:a_amount, 0) or :dh1:a_hist_id != 1) -- first history value could be a pay-in-advance gift ; @@ -53,12 +54,17 @@ update_descriptives_merge_catchup_with_extra "system/query" donor = :dh1:p_donor_partner_key, merged_id = :dh1:a_hist_id, regular_id = :dh2:a_hist_id, + --this catchup_amount logic is not fully functional. + --It either needs improvement on the logic or needs different method to check for catchup_amount inside extra giving. + --changing "<=" in front of "truncate(:dh1:a_amount / :dh2:a_amount))" to ">=" in order to improve the logic, merges few more rows. + --However, some interval values shifts greatly after changing the condition + -- needs more analysis on the datasets to improve the logic - -- The interval shift is to compensate for merging into a giving pattern - -- that is monthly, but due to a short timespan, the giving interval deviates - -- greatly from the more appropriate value (See sample donor 100400) - -- This also ensures that the catchup_amount won't cause a segment to go negative + --if the extra gift contain bigger or equal to gift amount expected in the time interval, + --it is considered catch_up gift and truncated as an catchup gift. + --Else, if it is false, it is given a value of 0. catchup_amount = condition(round((datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh1:a_prev_end), condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_next_start, :dh1:a_first_gift)) / 30.5) / ((round(:dh2:a_act_average_interval) + :dh2:a_act_average_interval) / 2) <= truncate(:dh1:a_amount / :dh2:a_amount)), round((datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh1:a_prev_end), condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_next_start, :dh1:a_first_gift)) / 30.5) / ((round(:dh2:a_act_average_interval) + :dh2:a_act_average_interval) / 2) <= truncate(:dh1:a_amount / :dh2:a_amount)), truncate(:dh1:a_amount / :dh2:a_amount)) * :dh2:a_amount, + --use this condition to filter catchup gift from extra giving to update the count for regular giving count = condition( catchup_amount = :dh1:a_amount, :dh1:a_count, @@ -75,8 +81,6 @@ update_descriptives_merge_catchup_with_extra "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_merged_id = :dh2:a_hist_id ; - --print 'Start printing collection' - --select :merged_id, :regular_id, :catchup_amount from collection tmp_intermediate_vals; -- Step Three: Merge update @@ -137,6 +141,7 @@ update_descriptives_merge_catchup_with_extra "system/query" :dh1:a_amount = :dh1:a_amount - :t:catchup_amount, :dh1:a_merged_id = null, :dh1:a_is_extra = 1 + --extra giving that has been merged already where (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and @@ -154,6 +159,7 @@ update_descriptives_merge_catchup_with_extra "system/query" (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and :dh1:a_is_extra and + --same as a_is_extra =2 :dh1:a_amount = 0 "; } diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_extra.qy b/kardia-app/modules/rcpt/update_descriptives_merge_extra.qy index 39fdfe9d..fb9ec8f2 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_extra.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_extra.qy @@ -12,6 +12,8 @@ update_descriptives_merge_extra "system/query" -- Step One: Identify -- Identify one-time-extra gifts. We consider a gift as one-time-extra if it is a single -- larger gift occurring within a regular giving pattern. + + -- dh is extra and dh2 is regular giving update identity collection a_descriptives_hist dh, collection a_descriptives d @@ -53,6 +55,7 @@ update_descriptives_merge_extra "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_is_extra = 2 and :dh1:a_hist_id != 1 and + --not most recent :dh2:a_count > 1 and :dh2:a_first_gift <= :dh1:a_first_gift and :dh2:a_last_gift >= :dh1:a_last_gift @@ -72,6 +75,7 @@ update_descriptives_merge_extra "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_is_extra = 2 and :dh1:a_hist_id = 1 and + --it is the most recent gift :dh2:a_count > 1 and :dh2:a_last_gift = :dh1:a_prev_end ; @@ -89,6 +93,8 @@ update_descriptives_merge_extra "system/query" -- The fuzziness here can be tweaked. If there are no other gifts -- within +/- half an interval of the extra gift, the gift is assumed -- to contain the regular giving amount lumped with the gift amount. + + --detecting if the extra gift is all on its own includes_regular_giving = ((select count(1) from collection tmp_gift_items where :donor = :dh1:p_donor_partner_key and :ledger = :dh1:a_ledger_number and :fund = :dh1:a_fund and :giftdate > dateadd(day, convert(integer, round(-0.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift) and :giftdate < dateadd(day, convert(integer, round(.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift) and :dh1:a_amount > :dh2:a_amount) = 1) from collection a_descriptives_hist dh1, @@ -108,6 +114,8 @@ update_descriptives_merge_extra "system/query" collection a_descriptives_hist dh1, collection tmp_intermediate_vals t set + --if we subtract it from some place we have to add it into other place + --moving money from step 3 to step 4 :dh2:a_total = condition( :t:includes_regular_giving, :dh2:a_total + :dh2:a_amount, @@ -193,6 +201,7 @@ update_descriptives_merge_extra "system/query" ), :dh1:a_merged_id = null, :dh1:a_is_extra = 1 + --succeeded in merging where (:parameters:fund is null or :parameters:fund = :dh2:a_fund) and (:parameters:donor is null or :parameters:donor = :dh2:p_donor_partner_key) and @@ -216,5 +225,6 @@ update_descriptives_merge_extra "system/query" (:parameters:fund is null or :parameters:fund = :a_fund) and (:parameters:donor is null or :parameters:donor = :p_donor_partner_key) and :a_is_extra = 2 + --failed in merging "; } diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index e6b29e47..082721c9 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -37,7 +37,12 @@ update_descriptives_new "system/query" amount = :i:a_amount, fund = :i:a_fund, ledger = :i:a_ledger_number, + --row_number() is an advanced sql function in order to provide structure for rows within a table + --row_number() assigns row_number incrementally throughout each rows id = row_number(), + --row_number() with parameters assigns row_number incrementally only if all the parameters are equal + --Else, it resets to zero and starts the process again + --Therefore, Based on the calculation, seg column outputs same segmentation value for parameter columns with same values seg = row_number() - row_number(:i:a_ledger_number, :i:a_fund, :i:p_dn_donor_partner_id, :i:a_amount) from /apps/kardia/data/Kardia_DB/a_subtrx_gift_item/rows i @@ -54,9 +59,11 @@ update_descriptives_new "system/query" :i:p_dn_donor_partner_id, isnull(:i:a_dn_gift_postmark_date, :i:a_dn_gift_received_date) desc ; - + -- Handling gifts that have been reimbursed by Lightsys -- Delete negative giving values and cancel out a -- corresponding positive gift + -- EX) negative gift : scholarship to donor , cancelled gift: donation gift + -- t1 is the negative gift, t2 is the gift to be -- cancelled out, and t3 is the intermediate value print 'Correcting for gift returns at ' + dateformat(getdate(), 'hh:mm:ss'); @@ -66,6 +73,7 @@ update_descriptives_new "system/query" ledger = :t1:ledger, fund = :t1:fund, donor = :t1:donor, + -- find the minimum date difference among gifts that are zero-sum to each other min_diff = (select min(abs(datediff(day, :t2:giftdate, :t1:giftdate))) from collection tmp_gift_items t2 where :t1:ledger = :t2:ledger and :t1:donor = :t2:donor and :t1:fund = :t2:fund and :t2:amount = -1 * :t1:amount group by :t2:ledger, :t2:donor, :t2:fund), id = null from @@ -74,12 +82,16 @@ update_descriptives_new "system/query" (:parameters:fund is null or :parameters:fund = :t1:fund) and (:parameters:donor is null or :parameters:donor = :t1:donor) and :t1:amount < 0 + -- t1:amount < 0 means gift given back to donor ; + --setting id if date difference among gifts are minimum and zero-sum to each other update identity collection tmp_intermediate_vals t3, collection tmp_gift_items t1 set :t3:id = (select :t2:id from collection tmp_gift_items t2 where :t1:ledger = :t2:ledger and :t1:donor = :t2:donor and :t1:fund = :t2:fund and :t2:amount = -1 * :t1:amount and :t3:min_diff = datediff(day, :t2:giftdate, :t1:giftdate) limit 1) + --checking if minimum date difference among gifts are zero-sum to each other + --this condition used since gifts and reimbursements are paired to each other consequently. where (:parameters:fund is null or :parameters:fund = :t1:fund) and (:parameters:donor is null or :parameters:donor = :t1:donor) and @@ -167,31 +179,39 @@ update_descriptives_new "system/query" select a_ledger_number = :ledger, p_donor_partner_key = :donor, + --needed to manually generate a_hist_id column since we are using temporally collection a_hist_id = row_number(), a_fund = :fund, a_amount = :amount, a_first_gift = last(:giftdate), a_last_gift = first(:giftdate), + -- arbitrary datetime value to represent null '1900-00-00 00:00:00' a_ntl_gift = isnull(nth(:giftdate, 2), convert(datetime, '1900-00-00 00:00:00')), a_count = count(1), a_total = sum(:amount), a_act_average_months = convert(integer, round(datediff(day, last(:giftdate), first(:giftdate)) / 30.5)), + -- ( 366 (days in year) / 12 (months in year) ) = 30.5 a_act_average_interval = condition( count(1) < 2 or datediff(day, last(:giftdate), first(:giftdate)) = 0, convert(double, null), (datediff(day, last(:giftdate), first(:giftdate)) / 30.5) / convert(double, count(1) - 1) ), + -- total months / interval == interval month average + -- we subtract 1 from (convert(double, count(1)) in order to match interval denominator and numerator + -- it is same as :amount * (count(1) - 1) a_act_average_amount = condition( datediff(day, last(:giftdate), first(:giftdate)) < 16, convert(money, null), (sum(:amount) - :amount) / round(datediff(day, last(:giftdate), first(:giftdate)) / 30.5) ), + -- in centralix SQL , when all the amounts are equal, we do not have to specify a row for a column s_date_created = getdate(), s_created_by = user_name(), s_date_modified = getdate(), s_modified_by = user_name() from collection tmp_gift_items + --segmenting based on duplicate gifts for descriptives_hist group by :ledger, :fund, @@ -204,7 +224,8 @@ update_descriptives_new "system/query" :seg, :giftdate desc ; - + -- manually updating null dates into datetime value by using intermediate date value + -- convert(datetime, null) is not working for some reason update collection a_descriptives_hist set @@ -213,17 +234,17 @@ update_descriptives_new "system/query" :a_ntl_gift = convert(datetime, '1900-00-00 00:00:00') ; - --select :p_donor_partner_key,:a_fund, :a_ntl_gift from collection a_descriptives_hist; - ---print 'AFTER' - -- Update previous/next interval start/end dates print 'Updating segment start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); update collection a_descriptives_hist dh1 set + --pick last gift from closest segment (amount) :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift order by :dh2:a_last_gift desc limit 1), + --pick first gift from closest segment (amount) :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift order by :dh2:a_first_gift asc limit 1) + --linking previous and next to every descriptives_hist to link all the segments together where (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) @@ -234,14 +255,12 @@ update_descriptives_new "system/query" -- known limitation. These repeats provide good data, but are finite -- and may need to be repeated more for larger data sets. - -- Merge in 'catch up' giving - - - print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy + -- Merge in 'catch up' giving + print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy fund=:parameters:fund, donor=:parameters:donor - ; + ; -- Merge in adjacent giving segments print 'Merge adjacent at ' + dateformat(getdate(), 'hh:mm:ss'); @@ -296,6 +315,8 @@ update_descriptives_new "system/query" -- Extra gifts are all linked together, and regular gifts are -- linked together. print 'Correcting start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); + + --link Regular Giving that are segmented based on same gift amount to track time sequence between segments update collection a_descriptives_hist dh1 set @@ -306,6 +327,9 @@ update_descriptives_new "system/query" (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and not :dh1:a_is_extra ; + + --link Extra Giving that are segmented based on same gift amount to track time sequence between segments + update collection a_descriptives_hist dh1 set @@ -317,7 +341,6 @@ update_descriptives_new "system/query" :dh1:a_is_extra ; - -------------------error #2 -------------------------------------------- -- Correct NTL Tracking print 'Fixing next-to-last tracking at ' + dateformat(getdate(), 'hh:mm:ss'); update @@ -331,9 +354,6 @@ update_descriptives_new "system/query" :dh1:a_count > 1 ; - --------------------error #2 ----------------------------------------- - print 'REACHED?' - ---- First history value print 'Propagating history #1 at ' + dateformat(getdate(), 'hh:mm:ss'); update @@ -381,9 +401,6 @@ update_descriptives_new "system/query" (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) ; - --select :a_ledger_number, :a_fund, :p_donor_partner_key, :a_hist_1_amount, :a_hist_1_last, :a_hist_1_count, :a_hist_1_first from collection a_descriptives limit 100; - --print ' ' - --print 'After merging' -- Second history value print 'Propagating history #2 at ' + dateformat(getdate(), 'hh:mm:ss'); @@ -486,8 +503,9 @@ update_descriptives_new "system/query" ; -- Overall adjustments to a_act_average_xyz for a_descriptives - -- if the interval is biannual or less, lookback slightly less than a year from the most recent gift - -- the lookahead date is half an interval past the most recent gift + -- the lookahead date is half an interval past the most recent gift + -- if the interval is biannual or less, lookback date slightly less than a year from the most recent gift + -- else the lookback date is half an interval past the oldest gift print 'Setting general stats at ' + dateformat(getdate(), 'hh:mm:ss'); update identity collection a_descriptives d @@ -502,13 +520,16 @@ update_descriptives_new "system/query" (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) ; - -- if the interval is greater than bi-annually, check back the last three intervals + -- if the interval is greater than bi-annually, check three intervals before last_gift date + -- if first_gift date more recent than three intervals before last_gift_date, just use first_gift date update identity collection a_descriptives d set :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), - :a_act_average_interval = convert(double, null), + -- between half interval past the first_gift date and three intervals before (half interval past the last_gift date), choose the bigger date + -- between first_gift date and three interval before last_gift date, choose more recent date + :a_act_average_interval = convert(double, null), :a_act_average_months = null, :a_act_average_amount = convert(money, null) where @@ -620,7 +641,9 @@ update_descriptives_new "system/query" update identity collection a_descriptives d set + --filtering to appropriate range :a_is_current = (select :dh:a_is_current from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), + --filtering to appropriate range :a_lapsed_days = (select :dh:a_lapsed_days from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) where (:parameters:fund is null or :parameters:fund = :d:a_fund) and @@ -632,6 +655,9 @@ update_descriptives_new "system/query" -- also require that the history entry and its predecessor both have at least two gifts, -- unless the giving interval is greater than 7 (effectively yearly), in which case any -- number of gifts for either produces useful results. + + --calculating the increased percentage from most recent previous gift to current gift + print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); update identity collection a_descriptives_hist dh @@ -650,10 +676,12 @@ update_descriptives_new "system/query" (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and not isnull(:dh:a_is_extra, 0) != 0 ; + ---removing meaningless descriptives_pct value for history segments update identity collection a_descriptives_hist dh, collection a_descriptives d set + --filtering to appropriate range :dh:a_increase_pct = condition((:dh:a_count > 1 or :d:a_act_average_interval > 1.8) and :dh:a_increase_pct > 0.001, :dh:a_increase_pct, null) where :d:a_ledger_number = :dh:a_ledger_number and @@ -689,6 +717,7 @@ update_descriptives_new "system/query" update identity collection a_descriptives_hist dh set + --filtering to appropriate range :dh:a_decrease_pct = condition(:dh:a_decrease_pct > 0.001, :dh:a_decrease_pct, null) where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and @@ -699,12 +728,16 @@ update_descriptives_new "system/query" update identity collection a_descriptives d set + --getting increase pct of most recent gift :a_increase_pct = (select :dh:a_increase_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + --getting the most recent increase date :a_increase_date = (select condition(:dh:a_increase_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), :a_decrease_pct = (select :dh:a_decrease_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), :a_decrease_date = (select condition(:dh:a_decrease_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + --tracking the extra from most recent three last gifts :a_is_extra = (select max(:dh:a_is_extra) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3), - :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) + --tracking the approximate from most recent three last gifts + :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) @@ -753,6 +786,8 @@ update_descriptives_new "system/query" (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) ; ---- Return the dataset to the caller + --- This gets printed on the console (optional) + --print 'Done at ' + dateformat(getdate(), 'hh:mm:ss'); --select -- *