imp: generate auto postings on forecast transactions by default

hledger-lib/Hledger/Data/Journal.hs

@@ -785,13 +785,16 @@ journalUntieTransactions t@Transaction{tpostings=ps} = t{tpostings=map (\p -> p{
 -- postings to transactions, eg). Or if a modifier rule fails to parse,
 -- return the error message. A reference date is provided to help interpret
 -- relative dates in transaction modifier queries.
-journalModifyTransactions :: Day -> Journal -> Either String Journal
-journalModifyTransactions d j =
-    case modifyTransactions (journalAccountType j) (journalInheritedAccountTags j) (journalCommodityStyles j) d (jtxnmodifiers j) (jtxns j) of
-      Right ts -> Right j{jtxns=ts}
-      Left err -> Left err
-
---
+-- The first argument selects whether to modify only generated (--forecast) transactions (False),
+-- or all transactions (True).
+journalModifyTransactions :: Bool -> Day -> Journal -> Either String Journal
+journalModifyTransactions alltxns d j =
+  case modifyTransactions predfn (journalAccountType j) (journalInheritedAccountTags j) (journalCommodityStyles j) d (jtxnmodifiers j) (jtxns j) of
+    Right ts -> Right j{jtxns=ts}
+    Left err -> Left err
+  where
+    predfn = if alltxns then const True else isgenerated
+    isgenerated = matchesTransaction (Tag (toRegex' "_generated-transaction") Nothing)
 
 -- | Choose and apply a consistent display style to the posting
 -- amounts in each commodity (see journalCommodityStyles).

hledger-lib/Hledger/Data/TransactionModifier.hs

@@ -25,7 +25,8 @@ import Hledger.Data.Transaction (txnTieKnot)
 import Hledger.Query (Query, filterQuery, matchesAmount, matchesPostingExtra,
                       parseQuery, queryIsAmt, queryIsSym, simplifyQuery)
 import Hledger.Data.Posting (commentJoin, commentAddTag, postingAddTags, postingApplyCommodityStyles)
-import Hledger.Utils (dbg6, wrap)
+import Hledger.Utils (wrap)
+import Hledger.Utils.Debug
 
 -- $setup
 -- >>> :set -XOverloadedStrings
@@ -33,25 +34,27 @@ import Hledger.Utils (dbg6, wrap)
 -- >>> import Hledger.Data.Transaction
 -- >>> import Hledger.Data.Journal
 
--- | Apply all the given transaction modifiers, in turn, to each transaction.
+-- | Apply all the given transaction modifiers, in turn, to each transaction
+-- for which the given predicate is true.
 -- Or if any of them fails to be parsed, return the first error. A reference
 -- date is provided to help interpret relative dates in transaction modifier
 -- queries.
-modifyTransactions :: (AccountName -> Maybe AccountType)
+modifyTransactions :: (Transaction -> Bool)
+                   -> (AccountName -> Maybe AccountType)
                    -> (AccountName -> [Tag])
                    -> M.Map CommoditySymbol AmountStyle
                    -> Day -> [TransactionModifier] -> [Transaction]
                    -> Either String [Transaction]
-modifyTransactions atypes atags styles d tmods ts = do
+modifyTransactions predfn atypes atags styles d tmods ts = do
   fs <- mapM (transactionModifierToFunction atypes atags styles d) tmods  -- convert modifiers to functions, or return a parse error
   let
-    modifytxn t = t''
+    maybemodifytxn t = if predfn t then t'' else t
       where
         t' = foldr (flip (.)) id fs t  -- apply each function in turn
         t'' = if t' == t  -- and add some tags if it was changed
               then t'
               else t'{tcomment=tcomment t' `commentAddTag` ("modified",""), ttags=("modified","") : ttags t'}
-  Right $ map modifytxn ts
+  Right $ map maybemodifytxn ts
 
 -- | Converts a 'TransactionModifier' to a 'Transaction'-transforming function
 -- which applies the modification(s) specified by the TransactionModifier.

hledger-lib/Hledger/Read/Common.hs

@@ -324,9 +324,9 @@ journalFinalise iopts@InputOpts{..} f txt pj = do
       &   journalApplyCommodityStyles                    -- Infer and apply commodity styles - should be done early
       <&> journalAddForecast (forecastPeriod iopts pj)   -- Add forecast transactions if enabled
       <&> journalPostingsAddAccountTags                  -- Add account tags to postings, so they can be matched by auto postings.
-      >>= (if auto_ && not (null $ jtxnmodifiers pj)
-              then journalAddAutoPostings _ioDay balancingopts_  -- Add auto postings if enabled, and account tags if needed
-              else pure)
+      >>= (if not (null $ jtxnmodifiers pj)
+            then journalAddAutoPostings auto_ _ioDay balancingopts_  -- Add auto postings if enabled, and account tags if needed
+            else pure)
       -- >>= Right . dbg0With (concatMap (T.unpack.showTransaction).jtxns)  -- debug
       >>= journalMarkRedundantCosts                      -- Mark redundant costs, to help journalBalanceTransactions ignore them
       >>= journalBalanceTransactions balancingopts_                         -- Balance all transactions and maybe check balance assertions.
@@ -346,14 +346,15 @@ journalFinalise iopts@InputOpts{..} f txt pj = do
     return j
 
 -- | Apply any auto posting rules to generate extra postings on this journal's transactions.
-journalAddAutoPostings :: Day -> BalancingOpts -> Journal -> Either String Journal
-journalAddAutoPostings d bopts =
+-- With a true first argument, applies them to all transactions, otherwise only to generated transactions.
+journalAddAutoPostings :: Bool -> Day -> BalancingOpts -> Journal -> Either String Journal
+journalAddAutoPostings alltxns d bopts =
     -- Balance all transactions without checking balance assertions,
     journalBalanceTransactions bopts{ignore_assertions_=True}
     -- then add the auto postings
     -- (Note adding auto postings after balancing means #893b fails;
     -- adding them before balancing probably means #893a, #928, #938 fail.)
-    >=> journalModifyTransactions d
+    >=> journalModifyTransactions alltxns d
 
 -- | Generate periodic transactions from all periodic transaction rules in the journal.
 -- These transactions are added to the in-memory Journal (but not the on-disk file).

hledger/Hledger/Cli/Commands/Rewrite.hs

@@ -41,7 +41,7 @@ rewrite opts@CliOpts{rawopts_=rawopts,reportspec_=rspec} j@Journal{jtxns=ts} = d
   -- rewrite matched transactions
   let today = _rsDay rspec
   let modifiers = transactionModifierFromOpts opts : jtxnmodifiers j
-  let j' = j{jtxns=either error' id $ modifyTransactions (journalAccountType j) (journalInheritedAccountTags j) mempty today modifiers ts}  -- PARTIAL:
+  let j' = j{jtxns=either error' id $ modifyTransactions (const True) (journalAccountType j) (journalInheritedAccountTags j) mempty today modifiers ts}  -- PARTIAL:
   -- run the print command, showing all transactions, or show diffs
   printOrDiff rawopts opts{reportspec_=rspec{_rsQuery=Any}} j j'
 

hledger/hledger.m4.md

@@ -2436,27 +2436,26 @@ So,
 - Do write two spaces between your period expression and your transaction description, if any.
 - Don't accidentally write two spaces in the middle of your period expression.
 
-## Other syntax
+## Auto postings
 
-hledger journal format supports quite a few other features,
-mainly to make interoperating with or converting from Ledger easier.
-Note some of the features below are powerful and can be useful in special cases,
-but in general, features in this section are considered less important
-or even not recommended for most users.
-Downsides are mentioned to help you decide if you want to use them.
+The `=` directive declares a rule for generating temporary extra postings 
+on transactions. Wherever the rule matches an existing posting, it can
+add one or more companion postings below that one, optionally influenced
+by the matched posting's amount. This can be useful for generating
+tax postings with a standard percentage, for example.
 
-### Auto postings
+By default, these auto posting rules are applied to transactions generated
+with --forecast (since 1.30), but not to transactions recorded in the journal.
+This means you can use `~` (periodic transaction) and `=` (auto posting) rules
+together to generate forecast transactions, and when such a transaction actually occurs,
+you can save the generated entry to the journal, finalising it.
 
-The `=` directive declares a rule for automatically adding
-temporary extra postings (visible in reports, not in the journal file)
-to all transactions matched by a certain query,
-when you use the `--auto` flag.
-
-Downsides: depending on generated data for your reports makes
-your financial data less portable, less future-proof,
-and less trustworthy in an audit. Also, because the feature
-is optional, other features like balance assertions can break
-depending on whether it is on or off.
+If instead you want to apply auto posting rules to recorded transactions
+as well, then use the `--auto` flag.
+This is not the default behaviour because depending on generated data
+is not ideal for financial records (it's less portable, less future-proof,
+less auditable, and less robust, since other features like balance assertions
+will be affected by the use or non-use of `--auto`.)
 
 An auto posting rule looks a bit like a transaction:
 ```journal
@@ -2562,6 +2561,14 @@ Also, any transaction that has been changed by auto posting rules will have thes
 - `modified:` - this transaction was modified
 - `_modified:` - a hidden tag not appearing in the comment; this transaction was modified "just now".
 
+## Other syntax
+
+hledger journal format supports quite a few other features,
+mainly to make interoperating with or converting from Ledger easier.
+Note some of the features below are powerful and can be useful in special cases,
+but in general, features in this section are considered less important
+or even not recommended for most users.
+Downsides are mentioned to help you decide if you want to use them.
 
 ### Balance assignments
 
@@ -4985,6 +4992,12 @@ When --forecast is not doing what you expect, one of these tips should help:
 - Consult [Forecast period, in detail](#forecast-period-in-detail), above.
 - Check inside the engine: add `--debug=2` (eg).
 
+## Forecast and auto postings
+
+Forecast transactions have one more feature: when they are generated,
+any applicable [auto posting rules](#auto-postings) will also be applied to them,
+generating additional postings. These are described below.
+
 # Budgeting
 
 With the balance command's [`--budget` report](#budget-report),

hledger/test/journal/auto-postings.test

@@ -241,7 +241,7 @@ $ hledger -f- print --auto
 #
 
 
-## Transaction modifiers affect forecast transactions (#959)
+## Transaction modifiers affect only forecast transactions by default:
 <
 = ^income
   (liabilities:tax)  *.33  ; income tax
@@ -251,15 +251,15 @@ $ hledger -f- print --auto
     income:donations         $-15
     assets:bank
 
-2016/1/3 withdraw
-    assets:cash             $20
-    assets:bank
+2016/1/3
+    assets:cash             $100
+    income:gifts
 
 # 13.
-$ hledger print -f- --auto --forecast -b 2016-01 -e 2016-03
-2016-01-03 withdraw
-    assets:cash             $20
-    assets:bank
+$ hledger print -f- --forecast -b 2016-01 -e 2016-03
+2016-01-03
+    assets:cash             $100
+    income:gifts
 
 2016-02-01 paycheck
     ; generated-transaction: ~ monthly from 2016-01, modified: 
@@ -271,16 +271,19 @@ $ hledger print -f- --auto --forecast -b 2016-01 -e 2016-03
 
 >=
 
-# 14. and they don't force --auto on
-$ hledger print -f- --forecast -b 2016-01 -e 2016-03
-2016-01-03 withdraw
-    assets:cash             $20
-    assets:bank
+# 14. With --auto, they affect all transactions:
+$ hledger print -f- --auto --forecast -b 2016-01 -e 2016-03
+2016-01-03  ; modified:
+    assets:cash                  $100
+    income:gifts
+    (liabilities:tax)            $-33  ; income tax, generated-posting: = ^income
 
 2016-02-01 paycheck
-    ; generated-transaction: ~ monthly from 2016-01
+    ; generated-transaction: ~ monthly from 2016-01, modified: 
     income:remuneration           $-100
+    (liabilities:tax)              $-33  ; income tax, generated-posting: = ^income
     income:donations               $-15
+    (liabilities:tax)            $-4.95  ; income tax, generated-posting: = ^income
     assets:bank
 
 >=