print: --new shows only transactions added since last time

First cut, error messages could be refined etc.

hledger-lib/Hledger/Read.hs

@@ -42,18 +42,20 @@ import qualified Control.Exception as C
 import Control.Monad.Except
 import Data.List
 import Data.Maybe
+import Data.Ord
 import Data.Text (Text)
 import qualified Data.Text as T
+import Data.Time (Day)
 import Safe
 import System.Directory (doesFileExist, getHomeDirectory)
 import System.Environment (getEnv)
 import System.Exit (exitFailure)
-import System.FilePath ((</>), takeExtension)
-import System.IO (stderr)
+import System.FilePath
+import System.IO
 import Test.HUnit
 import Text.Printf
 
-import Hledger.Data.Dates (getCurrentDay)
+import Hledger.Data.Dates (getCurrentDay, parsedate, showDate)
 import Hledger.Data.Types
 import Hledger.Read.Common
 import qualified Hledger.Read.JournalReader   as JournalReader
@@ -259,7 +261,7 @@ tryReaders readers mrulesfile assrt path t = firstSuccessOrFirstError [] readers
     path' = fromMaybe "(string)" path
 
 
---- New versions of readJournal* with easier arguments
+--- New versions of readJournal* with easier arguments, and --new/last-seen handling.
 
 readJournalFilesWithOpts :: InputOpts -> [FilePath] -> IO (Either String Journal)
 readJournalFilesWithOpts iopts =
@@ -275,7 +277,67 @@ readJournalFileWithOpts iopts prefixedfile = do
     (mfmt, f) = splitReaderPrefix prefixedfile
     iopts' = iopts{mformat_=firstJust [mfmt, mformat_ iopts]}
   requireJournalFileExists f
-  readFileOrStdinAnyLineEnding f >>= readJournalWithOpts iopts' (Just f)
+  t <- readFileOrStdinAnyLineEnding f
+  ej <- readJournalWithOpts iopts' (Just f) t
+  case ej of
+    Left e  -> return $ Left e
+    Right j | new_ iopts -> do
+      lastdates <- lastSeen f
+      let (newj, newlastdates) = journalFilterSinceLastDates lastdates j
+      when (not $ null newlastdates) $ saveLastSeen newlastdates f
+      return $ Right newj
+    Right j -> return $ Right j
+
+-- | Given zero or more date values (all the same, representing the
+-- latest previously seen transaction date, and how many transactions
+-- were seen on that date), remove transactions with earlier dates
+-- from the journal, and the same number of transactions on the
+-- latest date, if any, leaving only transactions that we can assume
+-- are newer. Also returns the new last dates of the new journal.
+journalFilterSinceLastDates :: [Day] -> Journal -> (Journal, [Day])
+journalFilterSinceLastDates [] j       = (j,  latestDates $ map tdate $ jtxns j)
+journalFilterSinceLastDates ds@(d:_) j = (j', ds')
+  where
+    samedateorlaterts     = filter ((>= d).tdate) $ jtxns j
+    (samedatets, laterts) = span ((== d).tdate) $ sortBy (comparing tdate) samedateorlaterts
+    newsamedatets         = drop (length ds) samedatets
+    j'                    = j{jtxns=newsamedatets++laterts}
+    ds'                   = latestDates $ map tdate $ samedatets++laterts
+
+-- | Get all instances of the latest date in an unsorted list of dates.
+-- Ie, if the latest date appears once, return it in a one-element list,
+-- if it appears three times (anywhere), return three of it.
+latestDates :: [Day] -> [Day]
+latestDates = headDef [] . take 1 . group . reverse . sort
+
+-- | Where to save last-seen transactions info for the given file path
+-- (.FILE.seen).
+seenFileFor :: FilePath -> FilePath
+seenFileFor f = dir </> fname' <.> "seen"
+  where
+    (dir, fname) = splitFileName f
+    fname' | "." `isPrefixOf` fname = fname
+           | otherwise              = '.':fname 
+
+-- | What were the latest transaction dates seen the last time this 
+-- journal file was read ? If there were multiple transactions on the
+-- latest date, that number of dates is returned, otherwise just one.
+-- Or none if no transactions were seen. 
+lastSeen :: FilePath -> IO [Day]
+lastSeen f = do
+  let seenfile = seenFileFor f
+  exists <- doesFileExist seenfile
+  if exists
+  then map (parsedate . strip) . lines . strip . T.unpack <$> readFileStrictly seenfile
+  else return []
+
+readFileStrictly :: FilePath -> IO Text
+readFileStrictly f = readFile' f >>= \t -> C.evaluate (T.length t) >> return t
+
+-- | Remember that these transaction dates were the latest seen when
+-- reading this journal file.
+saveLastSeen :: [Day] -> FilePath -> IO () 
+saveLastSeen dates f = writeFile (seenFileFor f) $ unlines $ map showDate dates
 
 readJournalWithOpts :: InputOpts -> Maybe FilePath -> Text -> IO (Either String Journal)
 readJournalWithOpts iopts mfile txt =

hledger-lib/Hledger/Read/Common.hs

@@ -55,13 +55,14 @@ data InputOpts = InputOpts {
     ,aliases_           :: [String]             -- ^ account name aliases to apply
     ,anon_              :: Bool                 -- ^ do light anonymisation/obfuscation of the data 
     ,ignore_assertions_ :: Bool                 -- ^ don't check balance assertions
+    ,new_               :: Bool                 -- ^ read only new transactions since this file was last read
     ,pivot_             :: String               -- ^ use the given field's value as the account name 
  } deriving (Show, Data) --, Typeable)
 
 instance Default InputOpts where def = definputopts
 
 definputopts :: InputOpts
-definputopts = InputOpts def def def def def def
+definputopts = InputOpts def def def def def def def
 
 rawOptsToInputOpts :: RawOpts -> InputOpts
 rawOptsToInputOpts rawopts = InputOpts{
@@ -71,6 +72,7 @@ rawOptsToInputOpts rawopts = InputOpts{
   ,aliases_           = map (T.unpack . stripquotes . T.pack) $ listofstringopt "alias" rawopts
   ,anon_              = boolopt "anon" rawopts
   ,ignore_assertions_ = boolopt "ignore-assertions" rawopts
+  ,new_               = boolopt "new" rawopts
   ,pivot_             = stringopt "pivot" rawopts
   }
 

hledger/Hledger/Cli/Commands/Print.hs

@@ -31,13 +31,13 @@ printmode = (defCommandMode $ ["print"] ++ aliases) {
   modeHelp = "show transaction journal entries, sorted by date. With --date2, sort by secondary date instead." `withAliases` aliases
  ,modeGroupFlags = Group {
      groupUnnamed = [
-        let matcharg = "STR"
-        in
-         flagReq  ["match","m"] (\s opts -> Right $ setopt "match" s opts) matcharg
-         ("show the transaction whose description is most similar to "++matcharg
-          ++ ", and is most recent"),
-        flagNone ["explicit","x"] (setboolopt "explicit")
-         "show all amounts explicitly"
+         let arg = "STR" in
+         flagReq  ["match","m"] (\s opts -> Right $ setopt "match" s opts) arg
+          ("show the transaction whose description is most similar to "++arg++", and is most recent")
+        ,flagNone ["explicit","x"] (setboolopt "explicit")
+          "show all amounts explicitly"
+        ,flagNone ["new"] (setboolopt "new")
+          "show only more recent transactions added to each file since last run"
         ]
         ++ outputflags
     ,groupHidden = []

hledger/doc/commands.m4.md

@@ -464,12 +464,15 @@ Print all market prices from the journal.
 ## print
 Show transactions from the journal. Aliases: p, txns.
 
-`-x     --explicit`
-: show all amounts explicitly
-
 `-m STR --match=STR             `
 : show the transaction whose description is most similar to STR, and is most recent
 
+`       --new`
+: show only more recent transactions added to each file since last run
+
+`-x     --explicit`
+: show all amounts explicitly
+
 `-O FMT --output-format=FMT     `
 : select the output format. Supported formats:
 txt, csv.
@@ -501,22 +504,36 @@ $ hledger print
     assets:bank:checking           $-1
 ```
 
-The print command displays full journal entries (transactions) from the journal file, tidily formatted.
+The print command displays full journal entries (transactions) from the journal file in date order, tidily formatted.
+print's output is always a valid [hledger journal](/journal.html).
+It preserves all transaction information, but it does not preserve directives or inter-transaction comments
 
-As of hledger 1.2, print's output is always a valid [hledger journal](/journal.html).
-However it may not preserve all original content, eg it does not print directives or inter-transaction comments.
-
-Normally, transactions' implicit/explicit amount style is preserved:
-when an amount is omitted in the journal, it will be omitted in the output.
-You can use the `-x/--explicit` flag to make all amounts explicit, which can be
+Normally, the journal entry's explicit or implicit amount style is preserved.
+Ie when an amount is omitted in the journal, it will be omitted in the output.
+You can use the `-x`/`--explicit` flag to make all amounts explicit, which can be
 useful for troubleshooting or for making your journal more readable and
 robust against data entry errors.
-Note, in this mode postings with a multi-commodity amount
-(possible with an implicit amount in a multi-commodity transaction)
+Note, `-x` will cause postings with a multi-commodity amount
+(these can arise when a multi-commodity transaction has an implicit amount)
 will be split into multiple single-commodity postings, for valid journal output.
 
-With -B/--cost, amounts with [transaction prices](/journal.html#transaction-prices)
-are converted to cost (using the transaction price).
+With `-B`/`--cost`, amounts with [transaction prices](/journal.html#transaction-prices)
+are converted to cost using that price.
+
+With `-m`/`--match` and a STR argument, print will show at most one transaction: the one 
+one whose description is most similar to STR, and is most recent. STR should contain at
+least two characters. If there is no similar-enough match, no transaction will be shown.
+
+With `--new`, for each FILE being read, hledger reads (and writes) a special .FILE.seen file in the same directory,
+containing the latest transaction date(s) that were seen last time FILE was read.
+When this file is found, only transactions with newer dates (and new transactions on the latest date) are printed.
+This is useful for ignoring already-seen entries in import data, such as downloaded CSV files.
+Eg:
+```console
+$ hledger -f bank1.csv print --new
+# shows transactions added since last print --new on this file
+```
+It assumes that only same-or-newer-dated transactions are added to FILE, and that the order of same-date transactions remains stable.   
 
 The print command also supports 
 [output destination](#output-destination)