;ci: add base-compat 0.14 to extra-deps

Work around ghc 9.10's extra newline, https://gitlab.haskell.org/ghc/ghc/-/issues/25116

.ghci

@@ -20,7 +20,7 @@ setEnv "NO_COLOR" "1"
 -- -- :reload and run commands in .ghci2
 -- :def R \_ -> return ":reload\n:script .ghci2"
 
--- Reload (to flush cached unsafe IO values) and run main with the given args
+-- Reload (to pick up code changes and flush cached unsafePerformIO values) and run main with the given args
 :def rmain \args -> return $ ":reload\n:main "<>args
 
 -- -- run commands from a file, .ghci2 by default

.github/ISSUE_TEMPLATE/a-bug.md

@@ -4,40 +4,9 @@ about: A weakness in the software, documentation, usability, or project
 labels: A BUG
 ---
 
-Thanks for reporting! Here are some tips (please delete them before submitting):
+Thanks for reporting! Here are some tips (please remove this text before submitting):
 
-- Have you checked the hledger manuals, and the right version ?
-  Eg for `hledger`: run `hledger help`, or go to
-  https://hledger.org/hledger.html and select your version at the top.
-
-- If you're not sure this is a bug, or if some discussion would help,
-  contact us on chat or the mail list first:
-  https://hledger.org/support.html
-
-- Not required, but any of these can help get issues resolved faster:
-  - A small reproducible example
-  - The output of hledger --version
-  - What platform you're on
-  - Links to any related docs that you found
-
-- If you have the access level to set labels, consider adding
-  - Any topic labels that seem appropriate
-  - Severity and impact labels estimating 
-
-    - How severe is this bug ?
-
-      - severity5: Data loss or privacy/security loss. A user would drop the product.
-      - severity4: Crash or bothersome regression or major usability or documentation issue. A user may look for an alternative product.
-      - severity3: Installability, packaging or new user experience issue. A potential user would fail to get started.
-      - severity2: Minor/moderate usability/doc issue. Easy to avoid or not a big deal.
-      - severity1: Cleanup/design/developer issue. Significant only to developers and design-minded users.
-
-    - Who is likely to be impacted by this bug ?
-
-      - impact5: All users.
-      - impact4: Most users.
-      - impact3: A minority of users.
-      - impact2: Only packagers or developers.
-      - impact1: Almost no one.
-
-(These are now in the issue tracker as severityN and impactN labels, keep synced.)
+- Have you checked (the right version of) the manual ? Eg https://hledger.org/hledger.html
+- If some discussion would help, just ask in chat or mail list: https://hledger.org/support.html
+- Any of these are very helpful: a small reproducible example, hledger --version output,
+  what platform you're running on, a link to to any relevant doc that you found.

.github/workflows/README

@@ -1,5 +1,8 @@
-hledger github actions workflows.
+The hledger project's github actions workflows.
 
+They have greppable one-line TRIGGER: and ACTION: comments near the top,
+summarising their current intended behaviour.
+These are carefully worded and should be kept up to date.
 
 docs:
 

.github/workflows/binaries-linux-arm32v7.yml

@@ -1,8 +1,7 @@
-# Runs on any push to binaries-linux-arm32v7.
-# Produces optimised static arm32v7 linux  binaries,
-# using GHC 8.10.4 and cabal.
-# Currently runs no tests.
-# Slow, will probably time out.
+# This was once used for certain raspberry pi hardware, may need update.
+# TRIGGER: Runs on any push to binaries-linux-arm32v7 branch.
+# ACTION: Builds and saves linux arm32v7 static binaries, using docker-arm32v7/Dockerfile and cabal and the ghc specified there.
+# XXX Slow, may time out.
 
 name: binaries-linux-arm32v7
 on:
@@ -35,6 +34,7 @@ jobs:
           docker rm -v $container_id
           cd tmp
           tar cvf hledger-linux-arm32v7.tar hledger hledger-ui hledger-web
+        # could add extras like hledger/shell-completion/hledger-completion.bash here
 
       # upload-artifact loses execute permissions, so we tar the binaries to preserve them.
       # github UI always zips artifacts when they are downloaded, so we don't bother compressing the tar.

.github/workflows/binaries-linux-x64-stack.yml

@@ -1,5 +1,6 @@
-# Runs on any push to binaries-linux-x64-stack.
-# Like binaries-linux-x64.yml except it builds with stack instead of cabal.
+# TRIGGER: Runs on any push to binaries-linux-x64-stack branch. Not normally used.
+# ACTION: Builds, unit-tests and saves mac x64 static binaries with stack and the default ghc. May not work,
+# the cabal-based binaries-linux-x64.yml is normally used instead.
 
 name: binaries-linux-x64-stack
 on:
@@ -98,26 +99,27 @@ jobs:
     - name: Install haskell tools with ghcup if needed
       run: |
         if [[ ! -x ~/.ghcup/bin/ghcup ]]; then mkdir -p ~/.ghcup/bin && curl https://downloads.haskell.org/~ghcup/x86_64-linux-ghcup > ~/.ghcup/bin/ghcup && chmod +x ~/.ghcup/bin/ghcup; fi; printf "ghcup: "; ghcup --version
-        if [[ ! -x ~/.ghcup/bin/stack ]]; then ~/.ghcup/bin/ghcup install stack 2.15.5   && ~/.ghcup/bin/ghcup set stack 2.15.5;   fi; printf "stack: "; stack --version
+        if [[ ! -x ~/.ghcup/bin/stack ]]; then ~/.ghcup/bin/ghcup install stack 3.1.1   && ~/.ghcup/bin/ghcup set stack 3.1.1;   fi; printf "stack: "; stack --version
 
     # --allow-different-user is needed because of #863 above (or because stack didn't notice we're in a docker container)
     - name: Install GHC with stack
       run: |
         stack --allow-different-user setup --install-ghc
 
-    - name: Build with stack
+    - name: List dep versions
       run: |
-        stack --allow-different-user build --ghc-options='-optl-static -fPIC' hledger      # || (echo "ERROR: building hledger failed"; false)
-        stack --allow-different-user build --ghc-options='-optl-static -fPIC' hledger-ui   # || (echo "ERROR: building hledger-ui failed"; false)
-        stack --allow-different-user build --ghc-options='-optl-static -fPIC' hledger-web  # || (echo "ERROR: building hledger-web failed"; false)
+        $stack exec -- ghc-pkg list
 
-    - name: Run built-in unit tests
+    - name: Build with stack and run unit tests
       run: |
-        stack exec -- hledger test
+        stack --allow-different-user build --test --ghc-options='-optl-static -fPIC' --ghc-options=-Werror hledger      # || (echo "ERROR: building hledger failed"; false)
+        stack --allow-different-user build --test --ghc-options='-optl-static -fPIC' --ghc-options=-Werror hledger-ui   # || (echo "ERROR: building hledger-ui failed"; false)
+        stack --allow-different-user build --test --ghc-options='-optl-static -fPIC' --ghc-options=-Werror hledger-web  # || (echo "ERROR: building hledger-web failed"; false)
 
     - name: Gather binaries
       run: |
         mkdir tmp
+        cp hledger/shell-completion/hledger-completion.bash hledger/embeddedfiles/*.{1,info} tmp
         cd tmp
         cp ~/.local/bin/hledger .
         cp ~/.local/bin/hledger-ui .
@@ -125,7 +127,7 @@ jobs:
         strip hledger
         strip hledger-ui
         strip hledger-web
-        tar cvf hledger-mac-x64.tar hledger hledger-ui hledger-web
+        tar cvf hledger-mac-x64.tar hledger hledger-ui hledger-web hledger-completion.bash
 
     # upload-artifact loses execute permissions, so we tar the binaries to preserve them.
     # github UI always zips artifacts when they are downloaded, so we don't bother compressing the tar.

.github/workflows/binaries-linux-x64.yml

@@ -1,6 +1,5 @@
-# Runs on any push to binaries-linux-x64 or binaries.
-# Produces optimised static x64 linux  binaries,
-# using the GHC version below and cabal and Alpine linux, 
+# TRIGGER: Runs on any push to binaries-linux-x64 or binaries branches.
+# ACTION: Builds, unit-tests and saves linux x64 static binaries with cabal and the ghc version below and Alpine linux,
 # which provides the statically-linkable musl.
 
 name: binaries-linux-x64
@@ -54,7 +53,7 @@ jobs:
       run: |
         if [[ ! -x ~/.ghcup/bin/ghcup ]];     then mkdir -p ~/.ghcup/bin && curl https://downloads.haskell.org/~ghcup/x86_64-linux-ghcup > ~/.ghcup/bin/ghcup && chmod +x ~/.ghcup/bin/ghcup; fi; printf "ghcup: "; ghcup --version
         if [[ ! -x ~/.ghcup/bin/ghc-9.8.2 ]]; then ~/.ghcup/bin/ghcup install ghc 9.8.2      && ~/.ghcup/bin/ghcup set ghc 9.8.2;      fi; printf "ghc: "; ghc --version
-        if [[ ! -x ~/.ghcup/bin/cabal ]];     then ~/.ghcup/bin/ghcup install cabal 3.10.3.0 && ~/.ghcup/bin/ghcup set cabal 3.10.3.0; fi; printf "cabal: "; cabal --version
+        if [[ ! -x ~/.ghcup/bin/cabal ]];     then ~/.ghcup/bin/ghcup install cabal 3.12.1.0 && ~/.ghcup/bin/ghcup set cabal 3.12.1.0; fi; printf "cabal: "; cabal --version
 
     - name: Update cabal package index
       run: |
@@ -62,9 +61,9 @@ jobs:
 
     - name: Build with cabal
       run: |
-        cabal build --enable-executable-static hledger || (echo "ERROR: building hledger failed"; false)
-        cabal build --enable-executable-static hledger-ui || (echo "ERROR: building hledger-ui failed"; false)
-        cabal build --enable-executable-static hledger-web || (echo "ERROR: building hledger-web failed"; false)
+        cabal build --enable-executable-static --ghc-options=-Werror hledger || (echo "ERROR: building hledger failed"; false)
+        cabal build --enable-executable-static --ghc-options=-Werror hledger-ui || (echo "ERROR: building hledger-ui failed"; false)
+        cabal build --enable-executable-static --ghc-options=-Werror hledger-web || (echo "ERROR: building hledger-web failed"; false)
 
     - name: Gather binaries
       run: |
@@ -72,11 +71,12 @@ jobs:
         cp dist-newstyle/build/x86_64-linux/ghc-*/hledger-*/x/hledger/build/hledger/hledger tmp
         cp dist-newstyle/build/x86_64-linux/ghc-*/hledger-ui-*/x/hledger-ui/build/hledger-ui/hledger-ui tmp
         cp dist-newstyle/build/x86_64-linux/ghc-*/hledger-web-*/x/hledger-web/build/hledger-web/hledger-web tmp
+        cp hledger/shell-completion/hledger-completion.bash hledger/embeddedfiles/*.1 hledger/embeddedfiles/*.info tmp
         cd tmp
         strip hledger
         strip hledger-ui
         strip hledger-web
-        tar cvf hledger-linux-x64.tar hledger hledger-ui hledger-web
+        tar cvf hledger-linux-x64.tar hledger hledger-ui hledger-web hledger-completion.bash
 
     # upload-artifact loses execute permissions, so we tar the binaries to preserve them.
     # github UI always zips artifacts when they are downloaded, so we don't bother compressing the tar.

.github/workflows/binaries-mac-arm64.yml

@@ -1,6 +1,5 @@
-# Runs on any push to binaries-mac-arm64 or binaries.
-# Produces optimised mac arm64 binaries and runs unit/doc/functional tests,
-# using the default stack.yaml's GHC version.
+# TRIGGER: Runs on any push to binaries-mac-arm64 or binaries branches.
+# ACTION: Builds, tests and saves mac arm64 dynamic binaries with stack and the default ghc.
 
 name: binaries-mac-arm64
 on:
@@ -12,7 +11,6 @@ jobs:
     # arm64
     runs-on: macos-14
     env:
-      ghc:   98
       stack: stack
     steps:
 
@@ -39,58 +37,58 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ~/.stack
-        key: ${{ runner.os }}-stack-global-from20220817-${{ hashFiles('**.yaml') }}
+        key: ${{ runner.os }}-arm64-stack-global-${{ hashFiles('**.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-stack-global-from20220817
+             ${{ runner.os }}-arm64-stack-global
 
     - name: process cache of stack-installed programs in ~/.local/bin
       id:   stack-programs
       uses: actions/cache@v4
       with:
         path: ~/.local/bin
-        key: ${{ runner.os }}-stack-programs-from20220817-${{ hashFiles('**.yaml') }}
+        key: ${{ runner.os }}-arm64-stack-programs-${{ hashFiles('**.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-stack-programs-from20220817
+             ${{ runner.os }}-arm64-stack-programs
 
     - name: process cache of .stack-work
       uses: actions/cache@v4
       with:
         path: .stack-work
-        key: ${{ runner.os }}-stack-work-from20220817-${{ hashFiles('**.yaml') }}
+        key: ${{ runner.os }}-arm64-stack-work-${{ hashFiles('**.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-stack-work-from20220817
+             ${{ runner.os }}-arm64-stack-work
 
     - name: process cache of hledger-lib/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-lib/.stack-work
-        key: ${{ runner.os }}-hledger-lib-stack-work-from20220817-${{ hashFiles('hledger-lib/package.yaml') }}
+        key: ${{ runner.os }}-arm64-hledger-lib-stack-work-${{ hashFiles('hledger-lib/package.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-hledger-lib-stack-work-from20220817
+             ${{ runner.os }}-arm64-hledger-lib-stack-work
 
     - name: process cache of hledger/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger/.stack-work
-        key: ${{ runner.os }}-hledger-stack-work-from20220817-${{ hashFiles('hledger/package.yaml') }}
+        key: ${{ runner.os }}-arm64-hledger-stack-work-${{ hashFiles('hledger/package.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-hledger-stack-work-from20220817
+             ${{ runner.os }}-arm64-hledger-stack-work
 
     - name: process cache of hledger-ui/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-ui/.stack-work
-        key: ${{ runner.os }}-hledger-ui-stack-work-from20220817-${{ hashFiles('hledger-ui/package.yaml') }}
+        key: ${{ runner.os }}-arm64-hledger-ui-stack-work-${{ hashFiles('hledger-ui/package.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-hledger-ui-stack-work-from20220817
+             ${{ runner.os }}-arm64-hledger-ui-stack-work
 
     - name: process cache of hledger-web/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-web/.stack-work
-        key: ${{ runner.os }}-hledger-web-stack-work-from20220817-${{ hashFiles('hledger-web/package.yaml') }}
+        key: ${{ runner.os }}-arm64-hledger-web-stack-work-${{ hashFiles('hledger-web/package.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-hledger-web-stack-work-from20220817
+             ${{ runner.os }}-arm64-hledger-web-stack-work
 
     # actions:
 
@@ -105,7 +103,7 @@ jobs:
         fi
         ghcup --version
         if [[ ! -x ~/.ghcup/bin/stack ]]; then
-          ~/.ghcup/bin/ghcup install stack 2.15.5 && ~/.ghcup/bin/ghcup set stack 2.15.5
+          ~/.ghcup/bin/ghcup install stack 3.1.1 && ~/.ghcup/bin/ghcup set stack 3.1.1
         fi
         stack --version
 
@@ -124,6 +122,10 @@ jobs:
         $stack build --test --only-dependencies --dry-run
         $stack build --test --only-dependencies
 
+    - name: List dep versions
+      run: |
+        $stack exec -- ghc-pkg list
+
     - name: Build hledger and test unit tests, doc tests
       run: |
         $stack install --test --force-dirty --ghc-options=-fforce-recomp --ghc-options=-Werror
@@ -155,6 +157,7 @@ jobs:
     - name: Gather binaries
       run: |
         mkdir tmp
+        cp hledger/shell-completion/hledger-completion.bash hledger/embeddedfiles/*.{1,info} tmp
         cd tmp
         cp ~/.local/bin/hledger .
         cp ~/.local/bin/hledger-ui .
@@ -162,11 +165,12 @@ jobs:
         strip hledger
         strip hledger-ui
         strip hledger-web
-        tar cvf hledger-mac-arm64.tar hledger hledger-ui hledger-web
+        tar cvf hledger-mac-arm64.tar hledger hledger-ui hledger-web hledger-completion.bash
 
     # upload-artifact loses execute permissions, so we tar the binaries to preserve them.
     # github UI always zips artifacts when they are downloaded, so we don't bother compressing the tar.
     # Unfortunately it means users must both unzip and untar.
+    # https://github.com/actions/upload-artifact?tab=readme-ov-file#limitations
     - name: Upload binaries artifact
       uses: actions/upload-artifact@v4
       with:

.github/workflows/binaries-mac-x64.yml

@@ -1,6 +1,5 @@
-# Runs on any push to binaries-mac-x64 or binaries.
-# Produces optimised mac x64 binaries and runs unit/doc/functional tests,
-# using the default stack.yaml's GHC version.
+# TRIGGER: Runs on any push to binaries-mac-x64 or binaries branches.
+# ACTION: Builds, tests and saves mac x64 dynamic binaries with stack and the default ghc.
 
 name: binaries-mac-x64
 on:
@@ -12,7 +11,6 @@ jobs:
     # x64
     runs-on: macos-13
     env:
-      ghc:   98
       stack: stack
     steps:
 
@@ -39,58 +37,58 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ~/.stack
-        key: ${{ runner.os }}-stack-global-from20220817-${{ hashFiles('**.yaml') }}
+        key: ${{ runner.os }}-x64-stack-global-${{ hashFiles('**.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-stack-global-from20220817
+             ${{ runner.os }}-x64-stack-global
 
     - name: process cache of stack-installed programs in ~/.local/bin
       id:   stack-programs
       uses: actions/cache@v4
       with:
         path: ~/.local/bin
-        key: ${{ runner.os }}-stack-programs-from20220817-${{ hashFiles('**.yaml') }}
+        key: ${{ runner.os }}-x64-stack-programs-${{ hashFiles('**.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-stack-programs-from20220817
+             ${{ runner.os }}-x64-stack-programs
 
     - name: process cache of .stack-work
       uses: actions/cache@v4
       with:
         path: .stack-work
-        key: ${{ runner.os }}-stack-work-from20220817-${{ hashFiles('**.yaml') }}
+        key: ${{ runner.os }}-x64-stack-work-${{ hashFiles('**.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-stack-work-from20220817
+             ${{ runner.os }}-x64-stack-work
 
     - name: process cache of hledger-lib/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-lib/.stack-work
-        key: ${{ runner.os }}-hledger-lib-stack-work-from20220817-${{ hashFiles('hledger-lib/package.yaml') }}
+        key: ${{ runner.os }}-x64-hledger-lib-stack-work-${{ hashFiles('hledger-lib/package.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-hledger-lib-stack-work-from20220817
+             ${{ runner.os }}-x64-hledger-lib-stack-work
 
     - name: process cache of hledger/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger/.stack-work
-        key: ${{ runner.os }}-hledger-stack-work-from20220817-${{ hashFiles('hledger/package.yaml') }}
+        key: ${{ runner.os }}-x64-hledger-stack-work-${{ hashFiles('hledger/package.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-hledger-stack-work-from20220817
+             ${{ runner.os }}-x64-hledger-stack-work
 
     - name: process cache of hledger-ui/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-ui/.stack-work
-        key: ${{ runner.os }}-hledger-ui-stack-work-from20220817-${{ hashFiles('hledger-ui/package.yaml') }}
+        key: ${{ runner.os }}-x64-hledger-ui-stack-work-${{ hashFiles('hledger-ui/package.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-hledger-ui-stack-work-from20220817
+             ${{ runner.os }}-x64-hledger-ui-stack-work
 
     - name: process cache of hledger-web/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-web/.stack-work
-        key: ${{ runner.os }}-hledger-web-stack-work-from20220817-${{ hashFiles('hledger-web/package.yaml') }}
+        key: ${{ runner.os }}-x64-hledger-web-stack-work-${{ hashFiles('hledger-web/package.yaml') }}
         restore-keys: |
-             ${{ runner.os }}-hledger-web-stack-work-from20220817
+             ${{ runner.os }}-x64-hledger-web-stack-work
 
     # actions:
 
@@ -98,10 +96,11 @@ jobs:
       run: |
         echo "$HOME/.ghcup/bin/" >> $GITHUB_PATH
 
+    # XXX occasionally we need to force a cache flush, or tools eventually become too old (Cabal tends to break old stack, eg)
     - name: Install haskell tools with ghcup if needed
       run: |
         if [[ ! -x ~/.ghcup/bin/ghcup ]];     then mkdir -p ~/.ghcup/bin && curl https://downloads.haskell.org/~ghcup/x86_64-apple-darwin-ghcup > ~/.ghcup/bin/ghcup && chmod +x ~/.ghcup/bin/ghcup; fi; printf "ghcup: "; ghcup --version
-        if [[ ! -x ~/.ghcup/bin/stack ]];     then ~/.ghcup/bin/ghcup install stack 2.15.5   && ~/.ghcup/bin/ghcup set stack 2.15.5;   fi; printf "stack: "; stack --version
+        if [[ ! -x ~/.ghcup/bin/stack ]];     then ~/.ghcup/bin/ghcup install stack 3.1.1   && ~/.ghcup/bin/ghcup set stack 3.1.1;   fi; printf "stack: "; stack --version
         #if [[ ! -x ~/.ghcup/bin/ghc-9.8.2 ]]; then ~/.ghcup/bin/ghcup install ghc 9.8.2      && ~/.ghcup/bin/ghcup set ghc 9.8.2;     fi; printf "ghc: "; ghc --version
 
     - name: Install GHC with stack
@@ -119,6 +118,10 @@ jobs:
         $stack build --test --only-dependencies --dry-run
         $stack build --test --only-dependencies
 
+    - name: List dep versions
+      run: |
+        $stack exec -- ghc-pkg list
+
     - name: Build hledger and test unit tests, doc tests
       run: |
         $stack install --test --force-dirty --ghc-options=-fforce-recomp --ghc-options=-Werror
@@ -141,6 +144,7 @@ jobs:
     - name: Gather binaries
       run: |
         mkdir tmp
+        cp hledger/shell-completion/hledger-completion.bash hledger/embeddedfiles/*.{1,info} tmp
         cd tmp
         cp ~/.local/bin/hledger .
         cp ~/.local/bin/hledger-ui .
@@ -148,7 +152,7 @@ jobs:
         strip hledger
         strip hledger-ui
         strip hledger-web
-        tar cvf hledger-mac-x64.tar hledger hledger-ui hledger-web
+        tar cvf hledger-mac-x64.tar hledger hledger-ui hledger-web hledger-completion.bash
 
     # upload-artifact loses execute permissions, so we tar the binaries to preserve them.
     # github UI always zips artifacts when they are downloaded, so we don't bother compressing the tar.

.github/workflows/binaries-windows-x64.yml

@@ -1,7 +1,5 @@
-# Runs on any push to binaries-windows-x64 or binaries.
-# Produces optimised windows binaries,
-# using the default stack.yaml's GHC version.
-# Currently runs no tests.
+# TRIGGER: Runs on any push to binaries-windows-x64 or binaries branches.
+# ACTION: Builds, unit-tests and saves windows x64 binaries with stack and the default ghc.
 
 name: binaries-windows-x64
 on:
@@ -120,24 +118,26 @@ jobs:
       run: |
         ./stack --no-terminal setup --install-ghc
 
+    - name: Install haskell deps
+      run: |
+        ./stack --no-terminal build --test --only-dependencies --dry-run
+        ./stack --no-terminal build --test --only-dependencies
+
+    - name: List dep versions
+      run: |
+        $stack exec -- ghc-pkg list
+
+    - name: Build all hledger modules warning free, optimised and minimised
+      run: |
+        ./stack --no-terminal install --test --force-dirty --ghc-options=-fforce-recomp --ghc-options=-Werror
+      # --ghc-options=-split-sections doesn't work on windows, "too many sections"
+      # --pedantic
+
     # - name: Install shelltestrunner
     ## - export PATH=~/.local/bin:$PATH
     # - if [[ ! -x ~/.local/bin/shelltest ]]; then stack install shelltestrunner-1.10; fi
     # - shelltest --version
 
-    - name: Install haskell deps
-      run: |
-        ./stack --no-terminal build --only-dependencies --dry-run
-        ./stack --no-terminal build --only-dependencies
-
-    # use whichever GHC is in default stack.yaml
-
-    - name: Build all hledger modules warning free, optimised and minimised
-      run: |
-        ./stack --no-terminal install --force-dirty --ghc-options=-fforce-recomp --ghc-options=-Werror
-      # --ghc-options=-split-sections doesn't work on windows, "too many sections"
-      # --pedantic
-
     # run hledger-lib/hledger functional tests, skipping the ones for addons
     ## - export PATH=~/.local/bin:$PATH
     #- COLUMNS=80 stack exec -- shelltest --execdir -j16 hledger/test -x /_ -x /addons -x ledger-compat/ledger-baseline -x ledger-compat/ledger-regress -x ledger-compat/ledger-collected
@@ -147,6 +147,7 @@ jobs:
     - name: Gather binaries
       run: |
         mkdir tmp
+        cp hledger/shell-completion/hledger-completion.bash hledger/embeddedfiles/*.{1,info} tmp
         cd tmp
         cp /C/Users/runneradmin/AppData/Roaming/local/bin/hledger.exe     .
         cp /C/Users/runneradmin/AppData/Roaming/local/bin/hledger-ui.exe  .

.github/workflows/ci.yml

@@ -1,10 +1,11 @@
-# The main hledger continuous integration test workflow.
-# Builds all packages expecting no warnings, runs lots of tests,
-# and on success, saves the binaries as an artifact.
-# Code must pass this successfully before it can be merged or pushed to master
-# (https://github.com/simonmichael/hledger/settings/branch_protection_rules/17386787).
+# The main hledger continuous integration tests.
+# Code must pass this successfully before it can be merged or pushed to master.
+# https://github.com/simonmichael/hledger/settings/branch_protection_rules/17386787
+# TRIGGER: Runs on any push to ci branch or any pull request against master.
+# ACTION: Builds, tests and saves linux x64 dynamic binaries with stack and the default ghc.
 
 name: ci
+
 on:
   # When manually triggered in github ui, it runs in master.
   workflow_dispatch:
@@ -41,33 +42,45 @@ on:
     #   - '!**.5'
     #   - '!**.info'
     #   - '!**.txt'
+
 jobs:
-  citest:
-    runs-on: ubuntu-latest
+  ci:
+    runs-on: ubuntu-24.04
     env:
-      ghc:   944
-      stack: stack --stack-yaml=stack9.4.yaml
-      # declare this to prevent "Context access might be invalid" warnings below
+      # This workflow uses github's preinstalled ghc & stack on ubuntu.
+      # Keep these synced with the latest ghc version at https://github.com/actions/runner-images/blob/ubuntu22/20240514.2/images/ubuntu/Ubuntu2404-Readme.md#haskell-tools
+      #
+      # caching id for this ghc's build artifacts:
+      # XXX ideally should match the default ghc in stack.yaml, though it's not critical
+      ghc: 910
+      # stack config for this ghc:
+      stack: stack --system-ghc
+
+      # flag for skipping later steps, declared here to prevent "Context access might be invalid" warnings
       do-all:
+
     steps:
 
     - name: Check out
       uses: actions/checkout@v4
       # have to fetch everything for git describe for hledger's --version
-      with: 
+      with:
         fetch-depth: 0
 
-    - name: Print some context for troubleshooting
-      env:
-        GITHUB_CONTEXT: ${{ toJson(github) }}
-      run: |
-        echo $GITHUB_CONTEXT
-      # echo "$GITHUB_SHA"
-      # echo "$GITHUB_REF"
-      # echo "$GITHUB_HEAD_REF"
-      # echo "$GITHUB_BASE_REF"
-      # git log "$GITHUB_BASE_REF"..
-      # tools/commitlint "$GITHUB_BASE_REF"..
+    # - name: Print some context for troubleshooting
+    #   env:
+    #     GITHUB_CONTEXT: ${{ toJson(github) }}
+    #   run: |
+    #     echo $GITHUB_CONTEXT
+    #   # echo "$GITHUB_SHA"
+    #   # echo "$GITHUB_REF"
+    #   # echo "$GITHUB_HEAD_REF"
+    #   # echo "$GITHUB_BASE_REF"
+    #   # git log "$GITHUB_BASE_REF"..
+    #   # tools/commitlint "$GITHUB_BASE_REF"..
+
+
+    # EARLY ACTIONS
 
     - name: Check commit messages
       # keep this step synced in all workflows which do it
@@ -112,28 +125,41 @@ jobs:
           && (grep -qE '^ *;' $$.gitlog || echo "do-all=true" >> $GITHUB_ENV)) \
           || ( echo "could not identify commit range, continuing CI steps"; echo "do-all=true" >> $GITHUB_ENV )
 
+
+    # Can't uncache to /usr/bin:
+    #  /usr/bin/tar -xf /home/runner/work/_temp/5cef703c-9831-41db-adb3-470b839f8a0e/cache.tzst -P -C /home/runner/work/hledger/hledger --use-compress-program unzstd
+    #  /usr/bin/tar: ../../../../../usr/bin/rg: Cannot open: Permission denied
+    # - name: Cache - extra tools (ripgrep) in /usr/bin
+    #   id:   extratools
+    #   uses: actions/cache@v4
+    #   with:
+    #     path: /usr/bin/rg
+    #     key: ${{ runner.os }}-extratools   # should have image version in there too
+    #   if: env.do-all
+
     - name: Check embedded files
       run: |
-        sudo apt install -y ripgrep
+        if [[ ! -x /usr/bin/rg ]]; then sudo apt install -y ripgrep; fi
         tools/checkembeddedfiles
       if: env.do-all
 
-    # things to be cached/restored:
 
-    - name: Uncache stack global package db
+    # CACHES
+
+    - name: Cache - stack global package db
       id:   stack-global
       uses: actions/cache@v4
       with:
         path: ~/.stack
         # XXX if stack.yaml is a symlink, this fails with
-        #  Error: The template is not valid. .github/workflows/push.yml (Line: 103, Col: 14): hashFiles('**.yaml') failed. 
+        #  Error: The template is not valid. .github/workflows/push.yml (Line: 103, Col: 14): hashFiles('**.yaml') failed.
         #  Fail to hash files under directory '/home/runner/work/hledger/hledger'
         key: ${{ runner.os }}-stack-global-$ghc-${{ hashFiles('**.yaml') }}
         restore-keys: |
              ${{ runner.os }}-stack-global-$ghc
       if: env.do-all
 
-    - name: Uncache stack-installed programs in ~/.local/bin
+    - name: Cache - stack-installed programs in ~/.local/bin
       id:   stack-programs
       uses: actions/cache@v4
       with:
@@ -143,7 +169,7 @@ jobs:
              ${{ runner.os }}-stack-programs-$ghc
       if: env.do-all
 
-    - name: Uncache .stack-work
+    - name: Cache - .stack-work
       uses: actions/cache@v4
       with:
         path: .stack-work
@@ -152,7 +178,7 @@ jobs:
              ${{ runner.os }}-stack-work-$ghc
       if: env.do-all
 
-    - name: Uncache hledger-lib/.stack-work
+    - name: Cache - hledger-lib/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-lib/.stack-work
@@ -161,7 +187,7 @@ jobs:
              ${{ runner.os }}-hledger-lib-stack-work-$ghc
       if: env.do-all
 
-    - name: Uncache hledger/.stack-work
+    - name: Cache - hledger/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger/.stack-work
@@ -170,7 +196,7 @@ jobs:
              ${{ runner.os }}-hledger-stack-work-$ghc
       if: env.do-all
 
-    - name: Uncache hledger-ui/.stack-work
+    - name: Cache - hledger-ui/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-ui/.stack-work
@@ -179,7 +205,7 @@ jobs:
              ${{ runner.os }}-hledger-ui-stack-work-$ghc
       if: env.do-all
 
-    - name: Uncache hledger-web/.stack-work
+    - name: Cache - hledger-web/.stack-work
       uses: actions/cache@v4
       with:
         path: hledger-web/.stack-work
@@ -188,39 +214,60 @@ jobs:
              ${{ runner.os }}-hledger-web-stack-work-$ghc
       if: env.do-all
 
-    # actions:
 
-    - name: Install stack
+    # ACTIONS
+    # in modular steps for faster & more focussed failures
+
+    # XXX slow, I feel this should happen less often
+    - name: Update package index
       run: |
-        mkdir -p ~/.local/bin
-        export PATH=~/.local/bin:$PATH
-        # curl -sL https://get.haskellstack.org/stable/linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C ~/.local/bin '*/stack'; chmod a+x ~/.local/bin/stack
-        if [[ ! -x ~/.local/bin/stack ]]; then curl -sL https://get.haskellstack.org/stable/linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C ~/.local/bin '*/stack'; chmod a+x ~/.local/bin/stack; fi
-        stack --version
+        $stack update
       if: env.do-all
 
-    - name: Install GHC
+    - name: Build deps of hledger-lib
       run: |
-        $stack setup --install-ghc
+        $stack build --test --bench hledger-lib --only-dependencies
       if: env.do-all
 
-    - name: Install haskell deps
+    - name: Build/test    hledger-lib
       run: |
-        $stack build --test --bench --only-dependencies --dry-run
-        $stack build --test --bench --only-dependencies
+        $stack install --test --bench hledger-lib --fast --ghc-options=-Werror
       if: env.do-all
 
-    # Packages are built one at a time to fail faster on error.
-    # Takes ~2m on a 2023 github worker.
-    - name: Build all hledger modules fast, warning free, run unit/doc/bench tests
+
+    - name: Build deps of hledger
       run: |
-        $stack install --fast --ghc-options=-Werror  --test --bench hledger-lib
-        $stack install --fast --ghc-options=-Werror  --test --bench hledger
-        $stack install --fast --ghc-options=-Werror  --test --bench hledger-ui
-        $stack install --fast --ghc-options=-Werror  --test --bench hledger-web
-        #  --ghc-options=-split-sections --no-terminal
+        $stack build --test --bench hledger --only-dependencies
       if: env.do-all
 
+    - name: Build/test    hledger
+      run: |
+        $stack install --test --bench hledger --fast --ghc-options=-Werror
+      if: env.do-all
+
+
+    - name: Build deps of hledger-ui
+      run: |
+        $stack build --test --bench hledger-ui --only-dependencies
+      if: env.do-all
+
+    - name: Build/test    hledger-ui
+      run: |
+        $stack install --test --bench hledger-ui --fast --ghc-options=-Werror
+      if: env.do-all
+
+
+    - name: Build deps of hledger-web
+      run: |
+        $stack build --test --bench hledger-web --only-dependencies
+      if: env.do-all
+
+    - name: Build/test    hledger-web
+      run: |
+        $stack install --test --bench hledger-web --fast --ghc-options=-Werror
+      if: env.do-all
+
+
     - name: Install shelltestrunner
       run: |
         export PATH=~/.local/bin:$PATH
@@ -249,6 +296,8 @@ jobs:
     #     # --no-print-missing-docs is 600% quieter
     #   if: env.do-all
 
+    # ARTIFACTS
+
     - name: Gather binaries
       id: exes
       run: |
@@ -266,7 +315,7 @@ jobs:
     # upload-artifact loses execute permissions, so we tar the binaries to preserve them.
     # github UI always zips artifacts when they are downloaded, so we don't bother compressing the tar.
     # Unfortunately it means users must both unzip and untar.
-    - name: Upload binaries artifact
+    - name: Upload binaries
       uses: actions/upload-artifact@v4
       with:
         name: hledger-linux-x64
@@ -275,7 +324,7 @@ jobs:
 
 
 
-    # snippets
+    # SNIPPETS
 
     # how to set a context variable, and an attempt to make a nice artifact version suffix:
     #   echo "::set-output name=version::$(git branch --show-current | sed 's/-.*//')-$(git rev-parse --short HEAD)"

.github/workflows/oldest.yml

@@ -0,0 +1,152 @@
+# TRIGGER: Runs on any push to oldest branch.
+# ACTION: Builds and tests with stack and the oldest supported ghc.
+
+name: oldest
+
+on:
+  # When there's a push to the oldest branch, it runs in that branch.
+  push:
+    branches: [ oldest ]
+  # If manually triggered in github ui, it runs in master.
+  workflow_dispatch:
+
+jobs:
+  oldest:
+    runs-on: ubuntu-24.04
+    env:
+      # This workflow uses github's preinstalled ghc & stack on ubuntu.
+      # Keep these synced with the latest ghc version at https://github.com/actions/runner-images/blob/ubuntu22/20240514.2/images/ubuntu/Ubuntu2404-Readme.md#haskell-tools
+      #
+      # caching id for this ghc's build artifacts:
+      ghc: 8107
+      # stack config for this ghc:
+      stack: stack --stack-yaml=stack8.10.yaml
+
+    steps:
+
+    - name: Check out
+      uses: actions/checkout@v4
+      # have to fetch everything for git describe for hledger's --version
+      with:
+        fetch-depth: 0
+
+
+    # CACHES
+
+    - name: Cache - stack global package db
+      id:   stack-global
+      uses: actions/cache@v4
+      with:
+        path: ~/.stack
+        # XXX if stack.yaml is a symlink, this fails with
+        #  Error: The template is not valid. .github/workflows/push.yml (Line: 103, Col: 14): hashFiles('**.yaml') failed.
+        #  Fail to hash files under directory '/home/runner/work/hledger/hledger'
+        key: ${{ runner.os }}-stack-global-$ghc-${{ hashFiles('**.yaml') }}
+        restore-keys: |
+             ${{ runner.os }}-stack-global-$ghc
+
+    - name: Cache - stack-installed programs in ~/.local/bin
+      id:   stack-programs
+      uses: actions/cache@v4
+      with:
+        path: ~/.local/bin
+        key: ${{ runner.os }}-stack-programs-$ghc-${{ hashFiles('**.yaml') }}
+        restore-keys: |
+             ${{ runner.os }}-stack-programs-$ghc
+
+    - name: Cache - .stack-work
+      uses: actions/cache@v4
+      with:
+        path: .stack-work
+        key: ${{ runner.os }}-stack-work-$ghc-${{ hashFiles('**.yaml') }}
+        restore-keys: |
+             ${{ runner.os }}-stack-work-$ghc
+
+    - name: Cache - hledger-lib/.stack-work
+      uses: actions/cache@v4
+      with:
+        path: hledger-lib/.stack-work
+        key: ${{ runner.os }}-hledger-lib-stack-work-$ghc-${{ hashFiles('hledger-lib/package.yaml') }}
+        restore-keys: |
+             ${{ runner.os }}-hledger-lib-stack-work-$ghc
+
+    - name: Cache - hledger/.stack-work
+      uses: actions/cache@v4
+      with:
+        path: hledger/.stack-work
+        key: ${{ runner.os }}-hledger-stack-work-$ghc-${{ hashFiles('hledger/package.yaml') }}
+        restore-keys: |
+             ${{ runner.os }}-hledger-stack-work-$ghc
+
+    - name: Cache - hledger-ui/.stack-work
+      uses: actions/cache@v4
+      with:
+        path: hledger-ui/.stack-work
+        key: ${{ runner.os }}-hledger-ui-stack-work-$ghc-${{ hashFiles('hledger-ui/package.yaml') }}
+        restore-keys: |
+             ${{ runner.os }}-hledger-ui-stack-work-$ghc
+
+    - name: Cache - hledger-web/.stack-work
+      uses: actions/cache@v4
+      with:
+        path: hledger-web/.stack-work
+        key: ${{ runner.os }}-hledger-web-stack-work-$ghc-${{ hashFiles('hledger-web/package.yaml') }}
+        restore-keys: |
+             ${{ runner.os }}-hledger-web-stack-work-$ghc
+
+
+    # ACTIONS
+    # in modular steps for faster & more focussed failures
+
+    - name: Install GHC
+      run: |
+        $stack setup --install-ghc
+
+
+    - name: Build deps of hledger-lib
+      run: |
+        $stack build --test --bench hledger-lib --only-dependencies
+
+    - name: Build/test    hledger-lib
+      run: |
+        $stack install --test --bench hledger-lib --fast --ghc-options=-Werror
+
+
+    - name: Build deps of hledger
+      run: |
+        $stack build --test --bench hledger --only-dependencies
+
+    - name: Build/test    hledger
+      run: |
+        $stack install --test --bench hledger --fast --ghc-options=-Werror
+
+
+    - name: Build deps of hledger-ui
+      run: |
+        $stack build --test --bench hledger-ui --only-dependencies
+
+    - name: Build/test    hledger-ui
+      run: |
+        $stack install --test --bench hledger-ui --fast --ghc-options=-Werror
+
+
+    - name: Build deps of hledger-web
+      run: |
+        $stack build --test --bench hledger-web --only-dependencies
+
+    - name: Build/test    hledger-web
+      run: |
+        $stack install --test --bench hledger-web --fast --ghc-options=-Werror
+
+
+    - name: Install shelltestrunner
+      run: |
+        export PATH=~/.local/bin:$PATH
+        if [[ ! -x ~/.local/bin/shelltest ]]; then $stack install shelltestrunner-1.10; fi
+        shelltest --version
+
+    - name: Test functional tests (excluding addons)
+      run: |
+        export PATH=~/.local/bin:$PATH
+        COLUMNS=80 $stack exec -- shelltest --execdir -j16 hledger/test -x /_ -x /addons -x ledger-compat/ledger-baseline -x ledger-compat/ledger-regress -x ledger-compat/ledger-collected
+        # XXX run the bin/ func tests corresponding to the GHC version enabled above, only

.github/workflows/release.yml

@@ -0,0 +1,223 @@
+# TRIGGER: Runs when a release tag like "1.*" is pushed to the repo.
+# XXX Triggers too much, eg for 1.x.99 dev tags; those releases must be deleted manually.
+# ACTION: Creates/updates a draft release with binaries from the latest successful binaries-* runs.
+# The main binaries* workflows should be completed before triggering this.
+
+name: release
+
+on:
+  push:
+    tags:
+      - '1.*'
+  workflow_dispatch:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Get artifact from latest successful binaries-windows-x64 run
+      uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe
+        # https://github.com/dawidd6/action-download-artifact v3.1.4, unverified so needs to be whitelisted in repo settings
+      with:
+        # all the settings, for reference. The other steps below will be more concise.
+        #
+        # Optional, GitHub token, a Personal Access Token with `public_repo` scope if needed
+        # Required, if the artifact is from a different repo
+        # Required, if the repo is private a Personal Access Token with `repo` scope is needed or GitHub token in a job where the permissions `action` scope set to `read`
+        # github_token: ${{secrets.GITHUB_TOKEN}}
+        # Optional, workflow file name or ID
+        # If not specified, will be inferred from run_id (if run_id is specified), or will be the current workflow
+        workflow: binaries-windows-x64.yml
+        # Optional, will use specified workflow run
+        # use ${{ github.event.workflow_run.id }} when your action runs in a workflow_run event
+        # and wants to download from the triggering workflow run
+        # run_id: 1122334455
+        # If no workflow is set and workflow_search set to true, then the most recent workflow matching
+        # all other criteria will be looked up instead of using the current workflow
+        workflow_search: false
+        # Optional, the status or conclusion of a completed workflow to search for
+        # Can be one of a workflow conclusion:
+        #   "failure", "success", "neutral", "cancelled", "skipped", "timed_out", "action_required"
+        # Or a workflow status:
+        #   "completed", "in_progress", "queued"
+        # Use the empty string ("") to ignore status or conclusion in the search
+        workflow_conclusion: success
+        # Optional, will get head commit SHA
+        # pr: ${{github.event.pull_request.number}}
+        # Optional, no need to specify if PR is
+        # commit: ${{github.event.pull_request.head.sha}}
+        # Optional, will use the specified branch. Defaults to all branches
+        # branch: binaries-linux-x64
+        # Optional, defaults to all types
+        # event: push
+        # Optional, run number from the workflow
+        # run_number: 34
+        # Optional, uploaded artifact name,
+        # will download all artifacts if not specified
+        # and extract them into respective subdirectories
+        # https://github.com/actions/download-artifact#download-all-artifacts
+        # is treated as a regular expression if input name_is_regexp is true
+        # will download only those artifacts with a name that matches this regular expression
+        # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions
+        # name: artifact_name
+        # Optional, name is treated as a regular expression if set true
+        # name_is_regexp: true
+        # Optional, a directory where to extract artifact(s), defaults to the current directory
+        path: artifacts
+        # Optional, defaults to current repo
+        # repo: ${{ github.repository }}
+        # Optional, check the workflow run to whether it has an artifact
+        # then will get the last available artifact from the previous workflow
+        # default false, just try to download from the last one
+        # check_artifacts: false
+        # Optional, search for the last workflow run whose stored an artifact named as in `name` input
+        # default false
+        # search_artifacts: false
+        # Optional, choose to skip unpacking the downloaded artifact(s)
+        # default false
+        # windows artifact is just zipped, no need to repack
+        skip_unpack: true
+        # Optional, choose how to exit the action if no artifact is found
+        # can be one of:
+        #  "fail", "warn", "ignore"
+        # default fail
+        # if_no_artifact_found: fail
+        # Optional, allow forks when searching for artifacts
+        # default true
+        allow_forks: false
+
+    - name: Get artifact from latest successful binaries-linux-x64 run
+      uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe
+      with:
+        workflow: binaries-linux-x64.yml
+        allow_forks: false
+        path: artifacts
+
+    - name: Get artifact from latest successful binaries-mac-x64 run
+      uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe
+      with:
+        workflow: binaries-mac-x64.yml
+        allow_forks: false
+        path: artifacts
+
+    - name: Get artifact from latest successful binaries-mac-arm64 run
+      uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe
+      with:
+        workflow: binaries-mac-arm64.yml
+        allow_forks: false
+        path: artifacts
+
+    - name: Inspect artifacts
+      shell: bash
+      run: |
+        ls -lRFh artifacts
+
+    # Artifacts are zip files because upload-artifact always zips.
+    # Here we can switch to more unix-standard gz.
+    - name: Repack unix artifacts with gz
+      shell: bash
+      run: |
+        cd artifacts
+        mv */*.tar .
+        gzip *.tar
+
+    - name: Inspect artifacts
+      shell: bash
+      run: |
+        ls -lRFh artifacts
+
+    - name: Generate github release notes
+      # ghrelnotes's argument should be the release's main tag name, eg "1.40".
+      # XXX Currently it is actually like "refs/tags/hledger-1.40.99", requiring manual fixup.
+      run: |
+        doc/ghrelnotes ${{ github.ref }} >ghrelnotes.md
+
+    - name: Create release
+      uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87  # https://github.com/softprops/action-gh-release 2.0.5
+      # permissions:
+      #   contents: write
+      with:
+        # https://github.com/softprops/action-gh-release?tab=readme-ov-file#-customizing
+        # body                     String   Text communicating notable changes in this release
+        # body_path                String   Path to load text communicating notable changes in this release
+        # draft                    Boolean  Indicator of whether or not this release is a draft
+        # prerelease               Boolean  Indicator of whether or not is a prerelease
+        # files                    String   Newline-delimited globs of paths to assets to upload for release
+        # name                     String   Name of the release. defaults to tag name
+        # tag_name                 String   Name of a tag. defaults to github.ref
+        # fail_on_unmatched_files  Boolean  Indicator of whether to fail if any of the files globs match nothing
+        # repository               String   Name of a target repository in <owner>/<repo> format. Defaults to GITHUB_REPOSITORY env variable
+        # target_commitish         String   Commitish value that determines where the Git tag is created from. Can be any branch or commit SHA. Defaults to repository default branch.
+        # token                    String   Secret GitHub Personal Access Token. Defaults to ${{ github.token }}
+        # discussion_category_name String   If specified, a discussion of the specified category is created and linked to the release. The value must be a category that already exists in the repository. For more information, see "Managing categories for discussions in your repository."
+        # generate_release_notes   Boolean  Whether to automatically generate the name and body for this release. If name is specified, the specified name will be used; otherwise, a name will be automatically generated. If body is specified, the body will be pre-pended to the automatically generated notes. See the GitHub docs for this feature for more information
+        # append_body              Boolean  Append to existing body instead of overwriting it
+        # make_latest              String   Specifies whether this release should be set as the latest release for the repository. Drafts and prereleases cannot be set as latest. Can be true, false, or legacy. Uses GitHub api defaults if not provided
+        #
+        body_path: ghrelnotes.md
+        files: |
+          artifacts/*.zip
+          artifacts/*.tar.gz
+        fail_on_unmatched_files: true
+        draft: true
+
+
+
+# snippets
+
+
+        # body: |
+        #   ${{ fromJSON(steps.<step-id>.outputs.assets)[0].browser_download_url }}
+
+# if you intend to run workflows on the release event (on: { release: { types: [published] } }),
+# you need to use a personal access token for this action, as the default secrets.GITHUB_TOKEN does not trigger another workflow.
+
+    # - name: Make tarball
+    #   shell: bash
+    #   run: |
+    #     outdir="target/${{ matrix.target }}/release"
+    #     staging="jj-${{ github.event.release.tag_name }}-${{ matrix.target }}"
+    #     mkdir "$staging"
+    #     cp {README.md,LICENSE} "$staging/"
+    #     if [ "${{ matrix.os }}" = "windows-2022" ]; then
+    #       cp "$outdir/jj.exe" "$staging/"
+    #       cd "$staging"
+    #       7z a "../$staging.zip" .
+    #       echo "ASSET=$staging.zip" >> $GITHUB_ENV
+    #     else
+    #       cp "$outdir/jj" "$staging/"
+    #       tar czf "$staging.tar.gz" -C "$staging" .
+    #       echo "ASSET=$staging.tar.gz" >> $GITHUB_ENV
+    #     fi
+
+
+# https://github.com/marketplace/actions/safe-download-workflow-artifact
+
+# https://github.com/actions/upload-artifact/issues/89#issuecomment-1194408215
+
+# https://www.eliostruyf.com/retrieving-artifact-previous-github-actions-workflow/
+
+# We have two workflows, one for building and one for releasing built artifacts upon a tag release.
+# They're both summoned from one push event, and the release job waits for the other job:
+#  https://github.com/dawidd6/action-download-artifact/issues/245
+
+
+    # - name: version
+    #   run: echo "::set-output name=version::$(./bin/azblogfilter --version)"
+    #   id: version
+
+    # - name: release
+    #   uses: actions/create-release@v1
+    #   id: create_release
+    #   with:
+    #     draft: false
+    #     prerelease: false
+    #     release_name: ${{ steps.version.outputs.version }}
+    #     tag_name: ${{ github.ref }}
+    #     body_path: CHANGELOG.md
+    #   env:
+    #     GITHUB_TOKEN: ${{ github.token }}

.gitignore

@@ -70,10 +70,10 @@ old
 !/bin/*.sh
 !/bin/*.md
 /.latest.*
-hledger/test/addons/hledger-*
+hledger/test/cli/addons
 tools/generatejournal
 tools/simplebench
-/examples/[1-9]0*.journal
+/examples/[1-9]*.journal
 *.webmanual.md
 *.m4-e
 stack*.yaml.lock
@@ -99,3 +99,14 @@ hledger-web/yesod-devel/
 /checks
 /headroom-templates
 .headroom.yaml
+*.prof
+/artifacts/
+/tools/docshelltest
+/hledger-lib/test/doctests
+/examples/edk-hledger-ui-mobile.gif
+/examples/tty-ui-1.gif
+/site
+/finance
+/examples/demo/
+/hledger-web/demo/
+/doc/ghrelnotes.md

.version

@@ -1 +1 @@
-1.33.99
+1.40.99

CHANGES.md

@@ -7,6 +7,7 @@
 |_|           |__/
 
 Docs
+(some overlap with hledger changelog; doc updates are mostly mentioned in that one since it's more visible)
 
 Scripts/addons
 
@@ -18,23 +19,79 @@ General changes in the hledger project.
 For package-specific changes, see the hledger package changelogs.
 
 
-# 0fedc35a9
+# 81167e81a
 
 Docs
 
-- move release notes from site repo to the main hledger repo
-- github release notes: improve windows install commands
-- dev doc updates; new Developer FAQ, Contributor Quick Start updates
-- examples: csv: vanguard, fidelity, paypal updates
+- REGRESSIONS: new table format; updates.
+- CODE: notes on the use of haddock [#2222]
+- Simplify github bug report template
+- Add man pages and info manuals to the release bindists on github
 
 Scripts/addons
 
+- hledger-install: fix installation of hledger-ui
+
 Infrastructure/Misc
 
-- Fully replace the main Makefile with Justfile
-- md-issue-refs: markdown issue links helper
-- relnotes.hs: generate release notes from changelogs
-- CI workflow updates
+- Add bash shell completion script to the release bindists ([#2223], gesh/hseg, Simon Michael)
+- hledger is now 35th among Github-starred haskell projects (up from 36th).
+
+
+# 1.40 2024-09-09
+
+Docs
+
+- In the hledger 1.29 release notes, Date adjustments has had some corrections.
+- Github release notes template cleanups; fix mac, linux install commands.
+- README: fixed contributors link.
+- RELEASING: updates
+
+Scripts/addons
+
+- hledger-install: cleanups, bump versions, perhaps fix hledger-interest install
+- hledger-install: clarify some stack/cabal setup messages
+
+Infrastructure/Misc
+
+- Shake.hs: fix partial warnings
+- Shake cmdhelp: renamed to cmddocs, and it now also updates the options
+  listed in the manuals, and shows progress output.  It should be run (at
+  some point) after changing commands' docs or options.
+- Shake txtmanuals: silence all but wide table warnings
+- just file cleanups; update to support just 1.28+
+- just twih: date fixes
+- just ghci: -fobject-code was a mistake, keep everything interpreted
+- just functest: try again to reduce rebuilding/slowdowns when testing
+- just installrel: update for .tar.gz
+- ci scripts: cleanup, fix a macos-ism
+
+
+# 1.34 2024-06-01
+
+Docs
+
+- move release notes from the hledger_site repo to the main hledger repo
+- github release notes: show the release notes, hide the install instructions by default
+- github release notes: improve windows install commands
+- github release notes: start mentioning github usernames, enabling the Contributors avatar list
+- dev docs: new Developer FAQ, Contributor Quick Start updates
+
+Scripts/addons
+
+- `hledger-install.sh` now uses stackage nightly, and a failure on non-Windows platforms has been fixed.
+
+Infrastructure/misc
+
+- A new `release` workflow creates github releases, uploads release binaries and generates release notes.
+- There is a new `oldest` workflow for testing the oldest GHC we support (currently 8.10.7).
+- The `binaries-mac-x64` workflow has been bumped from GHC 9.4 to 9.8.
+- The master branch's `ci` workflow has been updated to Ubuntu 24.04
+  and uses the preinstalled GHC & stack, saving some work.
+- `md-issue-refs` helps generate markdown issue links.
+- `relnotes.hs` helps generate release notes from changelogs.
+- The project `Makefile` has now been fully replaced by `Justfile`.
+
 
 # 1.33 2024-04-18
 

Justfile

@@ -14,17 +14,20 @@
 # is needed for efficiency, or when more powerful code is needed, use
 # Shake.hs instead of just.
 #
+#
 # Lines beginning with "# * ", "# ** ", etc are section headings,
-# foldable in Emacs outshine-mode. Some extra Emacs highlighting:
+# foldable in Emacs outshine-mode. Here's some more highlighting you can add
+# for readability:
 # (add-hook 'just-mode-hook (lambda ()
 #   (display-line-numbers-mode 1)
 #   (highlight-lines-matching-regexp "^# \\*\\*? " 'hi-yellow)  ; level 1-2 outshine headings
 #   (highlight-lines-matching-regexp "^@?\\w.*\\w:$" 'hi-pink) ; recipe headings (misses recipes with dependencies)
 #   ))
 #
-# This file is formatted by `just format`, which currently eats blank lines a bit (and commits).
+# This file is formatted by `just format`, which currently eats blank lines a bit.
+# (It also commits.)
 #
-# 'set export' makes constants and arguments available as $VAR as well as {{ VAR }}.
+# 'set export' below makes constants and arguments available as $VAR as well as {{ VAR }}.
 # $ makes just code more like shell code.
 # {{ }} handles multi-word values better and is fully evaluated in -n/--dry-run output.
 #
@@ -42,6 +45,7 @@
 # - hasktags (hackage, generates tag files for code navigation)
 # - profiterole (hackage/stackage, simplifies profiles)
 # - profiteur (hackage/stackage, renders profiles as html)
+# - dateround (from dateutils)
 
 # ** Helpers ------------------------------------------------------------
 HELPERS: help
@@ -60,7 +64,7 @@ WATCHEXEC := 'watchexec --timings'
 
 # list this justfile's recipes, optionally filtered by REGEX
 @help *REGEX:
-    if [[ '{{ REGEX }}' =~ '' ]]; then just -lu; else just -lu | rg -i '{{ REGEX }}'; true; fi
+    if [[ '{{ REGEX }}' =~ '' ]]; then just -ul; else just -ul | rg -i '{{ REGEX }}'; true; fi
 
 alias h := help
 
@@ -341,7 +345,7 @@ TESTING:
 
 # run ghci on hledger-lib + hledger
 @ghci *GHCIARGS:
-    $STACKGHCI exec -- $GHCI $BUILDFLAGS -fobject-code {{ GHCIARGS }} hledger/Hledger/Cli.hs
+    $STACKGHCI exec -- $GHCI $BUILDFLAGS {{ GHCIARGS }} hledger/Hledger/Cli.hs
 
 # run ghci on hledger-lib + hledger with profiling/call stack information
 @ghci-prof *GHCIARGS:
@@ -472,30 +476,31 @@ STACKTEST := STACK + ' test --fast'
 @unittest:
     ($STACK exec hledger test && echo $@ PASSED) || (echo $@ FAILED; false)
 
-SHELLTEST := 'COLUMNS=80 ' + STACK + ' exec -- shelltest --execdir --threads=64 --exclude=/_'
+SHELLTEST := 'COLUMNS=80 ' + STACK + ' exec -- shelltest --execdir --exclude=/_ --threads=32'
 
 #  --hide-successes
 
 # build hledger quickly and run functional tests, with any shelltest OPTS (requires mktestaddons)
 @functest *OPTS:
-    $STACK build --fast hledger
+    $STACK build hledger
     time (({{ SHELLTEST }} {{ if OPTS == '' { '' } else { OPTS } }} \
         hledger/test/ bin/ \
         -x ledger-compat/ledger-baseline -x ledger-compat/ledger-regress -x ledger-compat/ledger-extra \
         && echo $@ PASSED) || (echo $@ FAILED; false))
 
 ADDONEXTS := 'pl py rb sh hs lhs rkt exe com bat'
+ADDONSDIR := 'hledger/test/cli/addons'
 
 # generate dummy add-ons for testing the CLI
-@mktestaddons:
-    rm -rf hledger/test/addons/hledger-*
-    printf '#!/bin/sh\necho add-on: $0\necho args: $*\n' >hledger/test/addons/hledger-addon
-    for E in '' {{ ADDONEXTS }}; do \
-        cp hledger/test/addons/hledger-addon hledger/test/addons/hledger-addon.$E; done
-    for F in addon. addon2 addon2.hs addon3.exe addon3.lhs addon4.exe add reg; do \
-        cp hledger/test/addons/hledger-addon hledger/test/addons/hledger-$F; done
-    mkdir hledger/test/addons/hledger-addondir
-    chmod +x hledger/test/addons/hledger-*
+mktestaddons:
+    #!/usr/bin/env sh
+    rm -rf $ADDONSDIR
+    mkdir -p $ADDONSDIR $ADDONSDIR/hledger-addondir
+    cd $ADDONSDIR
+    printf '#!/bin/sh\necho add-on: $0\necho args: $@\n' > hledger-addon
+    for E in '' {{ ADDONEXTS }}; do cp hledger-addon hledger-addon.$E; done
+    for F in addon. addon2 addon2.hs addon3.exe addon3.lhs addon4.exe add reg; do cp hledger-addon hledger-$F; done
+    chmod +x hledger-*
 
 # compare hledger's and ledger's balance report
 compare-balance:
@@ -568,10 +573,15 @@ INSTALLING:
 #     make -C hledger/shell-completion/ clean-all all
 
 # On gnu/linux: can't interpolate GTAR here for some reason, and need the shebang line.
-# download github release VER binaries for OS (linux, mac, windows) and ARCH (x64, arm64) to bin/old/hledger*-VER
+# linux / mac only for now, does not handle the windows zip file.
+# download github release VER binaries for OS (linux, mac) and ARCH (x64, arm64) to bin/old/hledger*-VER
 @installrel VER OS ARCH:
     #!/usr/bin/env bash
-    cd bin/old && curl -L https://github.com/simonmichael/hledger/releases/download/{{ VER }}/hledger-{{ OS }}-{{ ARCH }}.zip | funzip | `type -P gtar || echo tar` xf - --transform 's/$/-{{ VER }}/'
+    # if [[ "$OS" == "windows" ]]; then
+    #   cd bin/old && curl -L https://github.com/simonmichael/hledger/releases/download/{{ VER }}/hledger-{{ OS }}-{{ ARCH }}.zip | funzip | `type -P gtar || echo tar` xf - --transform 's/$/-{{ VER }}/'
+    # else
+    # fi
+    cd bin/old && curl -L https://github.com/simonmichael/hledger/releases/download/{{ VER }}/hledger-{{ OS }}-{{ ARCH }}.tar.gz | `type -P gtar || echo tar` xzf - --transform 's/$/-{{ VER }}/'
 
 # # download recent versions of the hledger executables from github to bin/hledger*-VER
 # get-recent-binaries:
@@ -914,58 +924,73 @@ NEWS:
 # @_datearg *DATEARG:
 #     echo {{ if DATEARG == '' { `just reldate` } else { if DATEARG =~ '^\d+$' { `dateadd $(date +%Y-%m-%d) -$DATEARG` } else { DATEARG } } }}
 
-# If DATE is provided, return it, otherwise the date two fridays ago.
-@_dateortwofridaysago *DATE:
-    echo {{ if DATE == '' { `$GDATE -I -d 'last friday - 1 week'` } else { DATE } }}
+#dateround := 'dateround -n'
+dateround := 'dateround'
 
-# If DATE is provided, return today's date, otherwise last friday's.
-@_todayorlastfriday *DATE:
-    echo {{ if DATE == '' { `$GDATE -I -d 'last friday'` } else { `$GDATE -I` } }}
+# If DATE is provided, return it, otherwise the date two fridays ago.
+@_dateorsecondlatestfriday *DATE:
+    echo {{ if DATE == '' { `gdate -I -d "$($dateround today -- -fri) - 1 week"` } else { DATE } }}
+
+# If DATE is provided, return today's date, otherwise the most recent friday's (possibly today).
+@_todayorlatestfriday *DATE:
+    echo {{ if DATE == '' { `$dateround today -- -fri` } else { `$GDATE -I` } }}
 
 # If DATE is provided, return tomorrow's date, otherwise last friday's.
-@_tomorroworlastfriday *DATE:
-    echo {{ if DATE == '' { `$GDATE -I -d 'last friday'` } else { `$GDATE -I -d tomorrow` } }}
+@_tomorroworlatestfriday *DATE:
+    echo {{ if DATE == '' { `$dateround today -- -fri` } else { `$GDATE -I -d tomorrow` } }}
 
 # Show a draft This Week In Hledger post, with activity between the last two fridays (by default)
 twih:  # *DATE:
     #!/usr/bin/env osh
-    BEG=`just _dateortwofridaysago  $DATE`
-    END=`just _todayorlastfriday $DATE`
-    cat <<END
+    #BEG=`just _dateorsecondlatestfriday  $DATE`
+    END=`just _todayorlatestfriday $DATE`
+    cat <<EOS
+    == TWIH notes: ========================================
+
+    last release: `just rel`
+    
+    `gcal`
+    `just timelog $DATE`
+
+    `just worklog $DATE`
+
+    recent issue activity:
+    https://github.com/simonmichael/hledger/issues?q=sort:updated-desc
+
+
+    == TWIH draft (in clipboard) : ========================
+
+    EOS
+    (cat <<EOS
+    ---
+
     ## This Week In Hledger $END
     
     **sm**
-    
-    END
-    printf "DRAFT:\n\n"
-    just commitlog $DATE
-    printf "last release: `just rel`\n\n"
-    just worklog   $DATE
-    just timelog   $DATE
-    cat <<END
+    `just commitlog $DATE`
     
     **Misc**
-    
-    - for recent discussions, see <https://hledger.org/support.html>.
+    recent discussions: <https://hledger.org/support.html>
     
     **Quotes**
     
-    **
+    - **
     
-    **
+    - **
     
     <https://hledger.org/news.html#this-week-in-hledger-$END>
+
     
-    ---
-    END
+    EOS
+    ) | tee /dev/tty | pbcopy
 
 GITSHORTFMT := "--format='%ad %s' --date=short"
 
 # Show commits briefly in the three hledger repos between the last two fridays or since this date
 commitlog *DATE:
     #!/usr/bin/env osh
-    BEG=`just _dateortwofridaysago  $DATE`
-    END=`just _todayorlastfriday $DATE`
+    BEG=`just _dateorsecondlatestfriday $DATE`
+    END=`just _todayorlatestfriday $DATE`
     printf "** commits in $BEG..$END\n"
     printf "** hledger\n"
     git log {{ GITSHORTFMT }} --since $BEG --until $END --reverse | sed -E -e 's/ ;/ /'
@@ -986,13 +1011,13 @@ WORKLOG := "../../notes/CLOUD/hledger log.md"
 # Show hledger work logged since this date or days ago or last release
 worklog *DATE:
     #!/usr/bin/env osh
-    BEG=`just _dateortwofridaysago  $DATE`
-    END=`just _todayorlastfriday $DATE`
+    BEG=`just _dateorsecondlatestfriday  $DATE`
+    END=`just _todayorlatestfriday $DATE`
     # LOGGEDDATES=`just worklogdates`
     BEGLOGGED=`just worklogdates | $GHC -e "getContents >>= putStrLn . head . dropWhile (< \"$BEG\") . (++[\"9999-99-99\"]) . lines"`
     # ENDLOGGED=`just worklogdates | $GHC -e "getContents >>= putStrLn . head . takeWhile (< \"$END\") . (++[\"9999-99-99\"]) . dropWhile (< \"$BEG\") . (++[\"9999-99-99\"]) . lines"`
-    printf "** Work log in $BEG..\n"
-    # printf "** Work log in $BEGLOGGED..$ENDLOGGED\n"
+    printf "hledger work log in $BEG..:\n"
+    # printf "hledger work log in $BEGLOGGED..$ENDLOGGED:\n"
     awk "/^#### $BEGLOGGED/{p=1;print;next}; /^## /{p=0}; p" "$WORKLOG"
     # awk "/^#### $BEGLOGGED/{p=1;print;next}; /#### $ENDLOGGED/{p=0}; /^## /{p=0}; p" "$WORKLOG"
     echo
@@ -1000,14 +1025,14 @@ worklog *DATE:
 # Show hledger-related time logged between the last two fridays or since this date
 timelog *DATE:
     #!/usr/bin/env osh
-    BEG=`just _dateortwofridaysago  $DATE`
-    END=`just _todayorlastfriday $DATE`
-    END1=`just _tomorroworlastfriday $DATE`
-    printf "** Time log in $BEG..$END\n\n"
-    hledger -f $TIMELOG print hledger -b $BEG -e $END1 | rg '^2|hledger'
-    echo
+    BEG=`just _dateorsecondlatestfriday  $DATE`
+    END=`just _todayorlatestfriday $DATE`
+    END1=`just _tomorroworlatestfriday $DATE`
+    printf "hledger time logged in $BEG..$END:\n\n"
     hledger -f $TIMELOG bal -S --format '%-20(account) %12(total)' hledger -b $BEG -e $END1
     echo
+    hledger -f $TIMELOG print hledger -b $BEG -e $END1 | rg '^2|hledger'
+    echo
 
 # Copy some text to the system clipboard if possible
 @_clip TEXT:
@@ -1016,7 +1041,7 @@ timelog *DATE:
 # Show matrix chat since this date or days ago or last release
 chatlog *DATE:
     #!/usr/bin/env osh
-    DATE=`just _dateortwofridaysago $DATE`
+    DATE=`just _dateorsecondlatestfriday $DATE`
     JUMP="/jumptodate $DATE"
     just _clip "$JUMP"
     echo "** matrix:    https://matrix.hledger.org, $JUMP"
@@ -1025,7 +1050,7 @@ chatlog *DATE:
 # Show mail list discussion since this date or days ago or last release
 maillog *DATE:
     #!/usr/bin/env osh
-    DATE=`just _dateortwofridaysago $DATE`
+    DATE=`just _dateorsecondlatestfriday $DATE`
     DATE2=`$GDATE -d $DATE +"%b %-d"`
     echo "** mail list: https://list.hledger.org, since $DATE2 ($DATE)"
     echo
@@ -1033,7 +1058,7 @@ maillog *DATE:
 # Show /r/plaintextaccounting posts since this date or days ago or last release
 redditlog *DATE:
     #!/usr/bin/env osh
-    DATE=`just _dateortwofridaysago $DATE`
+    DATE=`just _dateorsecondlatestfriday $DATE`
     DAYS=`datediff $DATE now`
     echo "** reddit:    https://www.reddit.com/r/plaintextaccounting/new, since $DAYS days ago ($DATE)"
     echo
@@ -1041,7 +1066,7 @@ redditlog *DATE:
 # Show #hledger-tagged mastodon toots since this date or days ago or last release
 tootlog *DATE:
     #!/usr/bin/env osh
-    DATE=`just _dateortwofridaysago $DATE`
+    DATE=`just _dateorsecondlatestfriday $DATE`
     just _clip "#hledger after:$DATE"
     echo "** mastodon:  https://fosstodon.org/search, #hledger after:$DATE , #plaintextaccounting after:$DATE"
     echo
@@ -1078,7 +1103,7 @@ relprep VER:
     echo "Bumping all version strings to {{ VER }} ..."
     ./Shake setversion {{ VER }} $COMMIT
     echo "Updating all command help texts for embedding..."
-    ./Shake cmdhelp $COMMIT
+    ./Shake cmddocs $COMMIT
     echo "Updating all dates in man pages..."
     ./Shake mandates
     echo "Generating all the manuals in all formats...."
@@ -1180,10 +1205,6 @@ _gitSwitchAutoCreate BRANCH:
 # isclean-%:
 #     @$(ISCLEAN) $* || (echo "please clean these files first: $*"; false)
 
-# update all cabal files from latest package.yaml files using stack's built-in hpack
-cabalfiles:
-    {{ STACK }} build --dry-run --silent
-
 # # Update all cabal files based on latest package.yaml files using a specific hpack version.
 # # To avoid warnings, this should be the same version as stack's built-in hpack.
 # cabal-with-hpack-%:

README.md

@@ -113,7 +113,7 @@ More examples and screenshots: <https://hledger.org/#how-to-get-started>
 
 hledger is brought to you by
 [Simon Michael](http://joyful.com),
-[140+ contributors](CREDITS.md),
+[140+ contributors](doc/CREDITS.md),
 and the generous financial sponsors below.
 
 After enjoying some personal or organisational success with hledger,

Shake.hs

@@ -1,5 +1,5 @@
 #!/usr/bin/env stack
-{- stack script --resolver nightly-2024-05-01 --compile
+{- stack script --resolver nightly-2024-09-26 --compile
    --extra-include-dirs /Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/ffi
    --package base-prelude
    --package directory
@@ -22,7 +22,7 @@ Also uses tools like:
 - pandoc, groff, m4, makeinfo, sed, mv, cat, rm
 
 Some things that may be useful when working on this:
-- https://docs.haskellstack.org/en/stable/GUIDE/#script-interpreter
+- https://docs.haskellstack.org/en/stable/topics/scripts
 - watch Shake.hs for compile errors: make ghcid-shake
 - load Shake.hs in GHCI: make ghci-shake
 - rebuild things when files change with entr (file watcher), eg:
@@ -50,13 +50,14 @@ import "safe"         Safe
 import "shake"        Development.Shake
 import "shake"        Development.Shake.FilePath
 import "time"         Data.Time
+-- import Debug.Trace
 -- import "hledger-lib"  Hledger.Utils.Debug
 
 usage =
   let scriptname = "Shake" in replaceRe [re|/Shake|] ('/':scriptname) $
   unlines
     ---------------------------------------79--------------------------------------
-  ["Shake: heavy project scripting. See also: justfile, Makefile"
+  ["Shake: for heavy project scripting. See also: Justfile"
   ,"Usage:"
   ,"./Shake.hs [CMD [ARGS]]  run CMD, compiling this script first if needed"
   ,"./Shake    [CMD [ARGS]]  run CMD, using the compiled version of this script"
@@ -65,7 +66,9 @@ usage =
   ,"./Shake setversion [VER] [PKGS] [-c]"
   ,"                         update versions in source files to */.version or VER"
   ,"                         and update */*.cabal files"
-  ,"./Shake cmdhelp [-c]     update hledger CLI commands' help texts"
+  -- ,"./Shake optdocs [-c]  update options in hledger CLI command docs"
+  -- ,"                      (run after changing command flags)"
+  ,"./Shake cmddocs [-c]     update hledger command help after changing opts/docs"
   ,"./Shake mandates         update the date shown in some manual formats"
   ,"./Shake manuals [-c]     update all packages' txt/man/info/web manuals"
   -- ,"./Shake webmanuals       update just the web manuals"
@@ -141,9 +144,11 @@ main = do
   -- hledger manual also includes the markdown files from here:
   let commandsdir = "hledger/Hledger/Cli/Commands"
   commandmds <-
-    filter (not . ("README." `isPrefixOf`) . takeFileName) . filter (".md" `isSuffixOf`) . map (commandsdir </>)
+    sort . filter (not . ("README." `isPrefixOf`) . takeFileName) . filter (".md" `isSuffixOf`) . map (commandsdir </>)
     <$> S.getDirectoryContents commandsdir
-  let commandtxts = map (-<.> "txt") commandmds
+  let
+    commandmdsnew = map (<.> "new") commandmds
+    commandtxts = map (-<.> "txt") commandmds
 
   -- Run the shake rule selected by the first command line argument.
   -- Other arguments and some custom flags are set aside for the rule
@@ -322,14 +327,14 @@ main = do
       "hledger*/.version.m4" %> \out -> do
         let versionfile = takeDirectory out </> ".version"
         need [versionfile]
-        version <- ((head . words) <$>) $ liftIO $ readFile versionfile
+        version <- ((headDef (error $ "failed to read " <> versionfile) . words) <$>) $ liftIO $ readFile versionfile
         cmd_ Shell sed "-i -e" ("'s/(_version_}}, *)\\{\\{[^}]+/\\1{{"++version++"/;'") out
 
       -- PKG/package.yaml <- PKG/.version, just updates version strings
       "hledger*/package.yaml" %> \out -> do
         let versionfile = takeDirectory out </> ".version"
         need [versionfile]
-        version <- ((head . words) <$>) $ liftIO $ readFile versionfile
+        version <- ((headDef (error $ "failed to read " <> versionfile) . words) <$>) $ liftIO $ readFile versionfile
         let ma:jor:_ = splitOn "." version
             nextmajorversion = intercalate "." [ma, show $ read jor+1]
 
@@ -407,7 +412,7 @@ main = do
               ]
         when commit $
           commitIfChanged ";doc: update manuals" $
-            concat [packagemandatem4s, nroffmanuals, infomanuals, infodirentries, txtmanuals]  -- infodir
+            concat [commandmds, packagemandatem4s, nroffmanuals, infomanuals, infodirentries, txtmanuals]  -- infodir
 
       -- Update the dates to show in man pages, to the current month and year.
       -- Currently must be run manually when needed.
@@ -457,7 +462,10 @@ main = do
         -- remove with col -b, but it doesn't as can be seen with groff -V.)
         -- To get plain text, we run groff's lower-level commands (from -V) and add -cbuo.
         -- -Wall silences most troff warnings, remove to see them
-        cmd Shell "tbl" src "| eqn -Tascii | troff -Wall -mandoc -Tascii | grotty -cbuo >" out
+        -- XXX eqn complains on nonascii chars, not needed ?
+        -- cmd Shell "tbl" src "| eqn -Tascii | troff -Wall -mandoc -Tascii | grotty -cbuo >" out
+        -- XXX how to silence wide table warnings (generated by tbl, reported by troff) ?
+        cmd Shell "tbl" src "| troff -Wall -mandoc -Tascii | grotty -cbuo >" out
 
       -- Generate Info manuals suitable for viewing with info, from the .m4.md source.
       infomanuals |%> \out -> do -- hledger/hledger.info
@@ -577,22 +585,72 @@ main = do
       -- This may also update .cabal files from package.yaml files, and/or install haskell deps.
       phony "build" $ do
         let
-          pkgs | null args = packages
-               | otherwise = args
+          args' = drop 1 args
+          pkgs | null args' = packages
+               | otherwise = args'
         sequence_ [ do
           need $ fromMaybe [] $ lookup pkg embeddedFiles
           cmd Shell "stack build " pkg :: Action ()
           | pkg <- pkgs
           ]
 
-      -- regenerate Hledger/Cli/Commands/*.txt from the .md source files for CLI help
-      phony "cmdhelp" $ do
+      -- Regenerate Hledger/Cli/Commands/*.txt, rendering the corresponding .md files as plain text.
+      -- Also updates cmddocs first.
+      -- For commands' --help output.
+      -- NB this assumes the hledger executables are up to date. XXX
+      phony "cmddocs" $ do
+        -- need ["build"]  -- XXX circular dep, how would this work ?
+        liftIO $ putStrLn "please ensure the hledger build is up to date"  -- XXX never printed, why ?
         need commandtxts
-        when commit $ commitIfChanged ";doc: update command help" commandtxts
+        when commit $ commitIfChanged ";doc: update help" $ commandmds <> commandtxts
+
+      -- -- Update each Hledger/Cli/Commands/*.md, replacing the flags block with latest --help output,
+      -- -- or a placeholder if there are no command-specific flags.
+      -- -- For hledger manual and also for cmddocs below.
+      -- -- NB hledger executables should be up to date, see cmddocs
+      -- phony "optdocs" $ do
+      --   need commandmdsnew
+      --   when commit $ commitIfChanged ";doc: update command flag docs" commandmds
+
+      -- hledger/Hledger/Cli/Commands/CMD.md.new: a phony target that updates the flags doc
+      -- within hledger/Hledger/Cli/Commands/CMD.md. Runs "stack run -- hledger CMD -h" to get the latest.
+      -- If that fails, a warning is printed and it carries on, keeping the old flags doc.
+      -- NB this needs the hledger build to be up to date, see cmddocs.
+      phonys $ \out ->
+        if not $ "hledger/Hledger/Cli/Commands/" `isPrefixOf` out && ".md.new" `isSuffixOf` out
+        then Nothing
+        else Just $ do
+          let src = dropExtension out
+          need [src]
+          srcls <- fmap lines $ liftIO $ readFileStrictly src
+          let
+            (pre,rest) = break (=="```flags") srcls
+            (_,post)   = span  (/="```")     rest
+          let cmdname = map toLower $ takeBaseName src
+          do
+            let shellcmd = "stack exec -- hledger -h " <> cmdname
+            liftIO $ putStrLn $ "running " <> shellcmd <> " to get options help"
+            cmdhelp <- lines . fromStdout <$> (cmd Shell shellcmd :: Action (Stdout String))
+            let
+              cmdflagshelp = takeWhile (not.null) $ dropWhile (/="Flags:") cmdhelp
+              cmdflagshelp'
+                | null cmdflagshelp = ["Flags:","no command-specific flags"]
+                | otherwise         = cmdflagshelp
+            liftIO $ writeFile src $ unlines $ concat [pre, ["```flags"], cmdflagshelp', post]
+          -- This is supposed to print the error but otherwise ignore it, making this action a no-op,
+          -- in case hledger is not yet built/runnable.
+          `actionCatch` \(e::C.IOException) -> return ()
+          -- XXX should somehow control the output and elide the verbose "not found on path" errors
+            -- let elide err
+            --       | "path: [" `isInfixOf` err = takeWhile (/='[') err <> "..."
+            --       | otherwise = err
+            -- in \(e::C.IOException) -> liftIO $ hPutStrLn stderr $ elide $ show e  -- not used
 
       commandtxts |%> \out -> do
         let src = out -<.> "md"
-        need [src]
+        liftIO $ putStrLn ("generating " <> out)
+        need [src <.> "new"]  -- 1. update flags doc in src
+        need [src]            -- 2. depend on src
         cmd Shell
           pandoc fromsrcmd src "--lua-filter" "tools/pandoc-dedent-code-blocks.lua" "-t plain" ">" out
 
@@ -730,7 +788,7 @@ main = do
 
       -- Update all program-specific docs, eg after setversion.
       phony "docs" $ need [
-         "cmdhelp"
+         "cmddocs"
         ,"manuals"
         ,"changelogs"
         ]

bin/Justfile

@@ -1,6 +1,6 @@
 #!/usr/bin/env just -f
 # * financial reports/scripts, managed with https://github.com/casey/just
-# ** PUBLIC:  the scripts below can be shared in hledger's bin/justfile.
+# ** PUBLIC:  the scripts below can be shared in hledger's bin/Justfile.
 
 # *** PREAMBLE ------------------------------------------------------------
 
@@ -9,7 +9,7 @@ TODAY := `date +%Y-%m-%d`
 
 # list the commands available
 @help:
-    {{ just }} -lu --list-heading=$'{{ file_name(justfile()) }} commands:\n\
+    {{ just }} -ul --list-heading=$'{{ file_name(justfile()) }} commands:\n\
     ARGS can be added to customise reports.\n\
     '
 # XXX we don't quote ARGS properly, so each one must be free of spaces

bin/README.md

@@ -5,35 +5,44 @@
 <!-- toc -->
 </div>
 
-This document is the README in the hledger repo's [bin] directory, 
-and is also published as [Scripts and add-ons] on hledger.org.
+(This is the README in the hledger repo's `bin/` directory,
+also published as the [Scripts and add-ons] page on hledger.org.)
 
-[Add-on commands](hledger.html#add-on-commands) are executable script files or compiled programs
-named `hledger-*`, which show up in hledger's commands list.
-Some notable add-ons are listed [in the hledger manual](https://hledger.org/dev/hledger.html#add-ons). <!--  > PART 4. COMMANDS > ADD-ONS -->
-
-The rest of this page lists smaller scripts and add-ons which are collected in bin/,
-grouped by how closely they work with hledger:
-
-To be clear: you don't need any of these when starting out with hledger.
-hledger comes with many built-in commands, and you may want to get familiar with those first.
-
-
-<!-- This page can be viewed on github or hledger.org, so use absolute urls. -->
-[bin]:                https://github.com/simonmichael/hledger/tree/master/bin
+<!-- This page can be viewed on github or hledger.org, so use absolute urls here: -->
 [Scripts and add-ons]: https://hledger.org/scripts.html
-[Scripting hledger]:  https://hledger.org/scripting.html
+[Scripting hledger]:   https://hledger.org/scripting.html
+[bin]:                 https://github.com/simonmichael/hledger/tree/master/bin
 
-## HLEDGER-RELATED
+A *script* is a program you can run immediately without needing to compile it first.
+They are often small and defined in a single file or shell alias or shell function.
+You can create your own simple or complex scripts which enhance hledger.
+Eg you might script a complicated report so you don't have to remember the detailed command(s).
 
-These scripts don't use hledger directly, but are complementary and might be useful to hledger users.
-([plaintextaccounting.org](https://plaintextaccounting.org) has a longer list of PTA tools.)
+A hledger *add-on command* is any program whose name begins with "hledger-".
+Add-on commands found in PATH will appear in the commands list (shown when you run `hledger` with no arguments).
+Some of the scripts below are add-ons.
+Some add-ons are written in Haskell and can use hledger's full power, like builtin commands.
 
-### hledger-pricehist
+Below are some existing scripts you can use or learn from.
+Most of these are collected in [hledger's bin/ directory][bin],
+which you can get by [cloning](https://hledger.org/scripts.html#install-scripts) the hledger source.
+Compiled add-ons are also listed below, in their own section.
 
-[`hledger-pricehist`](https://github.com/simonmichael/hledger/blob/master/bin/hledger-pricehist)
-is just an alias for the market price downloader [pricehist](https://pypi.org/project/pricehist),
-so that it shows up in the hledger commands list.
+Note, you don't need any of these extras if you are new to hledger -
+except possibly hledger-ui and hledger-web, which can be nice to have at the start.
+
+
+## Related scripts
+
+Here are some scripts which don't use hledger directly, but might be useful to hledger users.
+(For more, see also: [plaintextaccounting.org > Software](https://plaintextaccounting.org#software)).
+
+
+### pricehist
+
+[pricehist](https://pypi.org/project/pricehist) is useful for downloading market prices / conversion rates; recommended.
+And [`hledger-pricehist`](https://github.com/simonmichael/hledger/blob/master/bin/hledger-pricehist)
+is a small script to make it show up in the hledger commands list.
 
 ### paypaljson
 
@@ -46,14 +55,9 @@ downloads the last 30 days of Paypal transactions (requires a free developer acc
 converts `paypaljson`'s output to CSV, with format similar to Paypal's manually-downloaded CSV.
 
 
-## HLEDGER-RUNNING
+## hledger command line scripts
 
-These scripts run hledger via its CLI,
-eg to help you produce a particular report without needing to remember a complicated command line. 
-They might also consume its text or CSV or JSON output.
-They can be
-small shell aliases or functions (typically defined in shell startup files like ~/.bashrc)
-or individual script files written in shell or another language (typically kept in ~/bin/ or elsewhere in $PATH).
+These scripts use hledger's command line interface, or process one of its output formats.
 
 ### bashrc
 
@@ -70,7 +74,7 @@ $ fin        # list the scripts available
 
 [`ft`](https://github.com/simonmichael/hledger/blob/master/bin/ft)
 is a way to organise your finance-related reports and scripts using standard bash.
-(See also [justfile](#justfile) below.)
+(See also [Justfile](#justfile) below.)
 
 ```cli
 $ alias f=~/src/hledger/bin/ft
@@ -138,22 +142,22 @@ OTHERCMD        [ARGS] run other hledger commands on $TIMELOG
 Add hledger options to customise reports.
 ```
 
-### justfile
+### Justfile
 
-<https://github.com/casey/just> is like [make](https://en.wikipedia.org/wiki/Make_(software)), but easier and more suitable for running commands.
-It is a nice tool for organising financial reports and scripts!
-More on [hledger and just](just.md).
+<https://github.com/casey/just> is a nice tool for organising financial reports and scripts,
+similar to `make`, but more robust for this use case. I can recommend it.
+See also [hledger and just](just.md).
 
-Here is a [justfile](https://github.com/simonmichael/hledger/blob/master/bin/justfile)
+Here is a [Justfile](https://github.com/simonmichael/hledger/blob/master/bin/Justfile)
 reimplementing the `ft` and `tt` scripts more simply:
 
 ```cli
 $ brew install just  # eg
 $ alias j=just
 $ cd ~/finance
-$ cp ~/src/hledger/bin/justfile .  # or start from scratch: just --init
+$ cp ~/src/hledger/bin/Justfile .  # or start from scratch: just --init
 $ j
-justfile commands:
+Justfile commands:
     watch CMD                      # rerun the given command with watchexec whenever local files change
     get-csv                        # download auto-downloadable CSVs (paypal)
     import-dry                     # import new downloaded transactions to the main journal, dry run
@@ -294,6 +298,7 @@ EDITOR='perl -pi -e "s|Cost:Food|Cost:Food:Fast Food|g"' hledger edit tag:locati
 [![asciicast](https://asciinema.org/a/549559.svg)](https://asciinema.org/a/549559)
 
 ### hledger-plot
+
 The [hledger-utils python package](https://pypi.org/project/hledger-utils/) provides
 a `hledger-plot` command for generating charts with matplotlib.
 
@@ -330,10 +335,12 @@ $ hledger lots list
 is a custom compound report done in shell. See also hledger-report1.hs.
 
 
-## HLEDGER-INTEGRATED
+## hledger haskell scripts
 
-These Haskell scripts use the hledger-lib API for maximum power and robustness;
-they can do anything hledger's built-in commands can do.
+These scripts are written in Haskell and use hledger's haskell API (by importing the `hledger` or `hledger-lib` haskell libraries).
+They are often [stack scripts](https://docs.haskellstack.org/en/stable/topics/scripts).
+They can do anything hledger's builtin commands can do, and are usually more robust than command line scripts.
+Some builtin commands were first developed as standalone haskell scripts.
 
 ### hledger-script-example
 
@@ -430,15 +437,43 @@ is a custom compound report done in haskell. See also hledger-report1.sh.
 is a mixture of a balance report and a register report; it shows each account's transactions
 under the account's balance.
 
-## HOW TO
+## Add-ons
+
+These are some official and third-party add-ons you can install as compiled programs:
+
+### hledger-ui
+
+[hledger-ui](hledger-ui.html) is hledger's official terminal UI. It allows faster browsing of your accounts and transactions.
+
+### hledger-web
+
+[hledger-web](hledger-web.html) is hledger's official web UI. It allows data entry and simple reports in a web browser. It's good for non-command-line users.
+
+### hledger-iadd
+
+[hledger-iadd](https://hackage.haskell.org/package/hledger-iadd) is a popular alternative to the builtin `add` command.
+
+### hledger-interest
+
+[hledger-interest](https://hackage.haskell.org/package/hledger-interest) generates interest transactions.
+
+
+### Other
+
+- [hledger-stockquotes](https://hackage.haskell.org/package/hledger-stockquotes) fetches market prices. Not widely used, consider pricehist instead.
+- [hledger-diff](https://hackage.haskell.org/package/hledger-diff) compares two journal files. It's now built in to hledger, so you don't need it.
+
+
+
+## How to...
 
 ### Install scripts
 
 To use these bin scripts you must ensure they are in your $PATH and runnable:
 
-- Shell scripts: you may need [bash], or to adapt the scripts for your shell.
+- Shell scripts: you may need [bash](https://www.gnu.org/software/bash), or to adapt the scripts for your shell.
 - Python scripts: you'll need python 3 and pip. 
-- Haskell scripts: you'll need stack (<https://www.haskell.org/get-started>).
+- Haskell scripts: you'll need [stack](http://haskellstack.org).
 Or if you know how, you can make them cabal scripts, or install their dependencies manually and use runghc/ghc.
 
 Here's a suggested install procedure:
@@ -470,17 +505,8 @@ $ pip install -U hledger-lots
 # Check that hledger's command list now includes the bin scripts.
 # Eg "check-fancyassertions" and "swap-dates" should be listed:
 $ hledger
-
-
 ```
 
-[bash]: https://www.gnu.org/software/bash
-[stack]: https://haskellstack.org
-[stack]: https://www.fpcomplete.com/haskell/get-started
-[stack scripts]: https://docs.haskellstack.org/en/stable/GUIDE/#script-interpreter
-[add-on commands]: https://hledger.org/dev/hledger.html#add-on-commands
-[cabal]: https://www.haskell.org/cabal
-
 ### Create a new script
 
 To create a new hledger-integrated script, copy hledger-script-example.hs.

bin/bashrc

@@ -1,42 +1,154 @@
-# Some hledger/PTA-related bash scripts.
+# Some hledger/PTA-related bash scripts. See also Justfile.
+
+alias hl='hledger'
+alias acc='hledger accounts'
+alias act='hledger activity'
+alias add='hledger add'
+alias areg='hledger aregister'
+alias bal='hledger balance'
+alias bar='hledger bar'
+alias bs='hledger balancesheet'
+alias bse='hledger balancesheetequity'
+alias budget='hledger balance --budget'
+alias cf='hledger cashflow'
+alias check='hledger check'
+alias close='hledger close'
+alias codes='hledger codes'
+alias comm='hledger commodities'
+alias desc='hledger descriptions'
+alias files='hledger files'
+alias iadd='hledger-iadd'
+alias interest='hledger interest'
+alias import='hledger import'
+alias is='hledger incomestatement'
+alias lots='hledger lots'
+alias notes='hledger notes'
+alias payees='hledger payees'
+alias plot='hledger plot'
+alias prices='hledger prices'
+alias print='hledger print'
+alias reg='hledger register'
+alias rewrite='hledger rewrite'
+alias roi='hledger roi'
+alias stats='hledger stats'
+alias tags='hledger tags'
+alias ui='hledger ui'
+alias web='hledger web'
 
 export FINDIR=~/finance
-export LEDGER_FILE=$FINDIR/2022.journal
+export LEDGER_FILE=$FINDIR/2024/2024.journal
+alias f=$FINDIR/justfile
+alias bser='hledger -f $LEDGER_FILE -f <(hledger close --retain) bse'
+alias all='hledger -f $FINDIR/all.journal'
+alias 2020='hledger -f $FINDIR/2020/2020.journal'
+alias 2021='hledger -f $FINDIR/2021/2021.journal'
+alias 2022='hledger -f $FINDIR/2022/2022.journal'
+alias 2023='hledger -f $FINDIR/2023/2023.journal'
+alias 2024='hledger -f $FINDIR/2024/2024.journal'
+alias jan="hledger -p jan"
+alias feb="hledger -p feb"
+alias mar="hledger -p mar"
+alias apr="hledger -p apr"
+alias may="hledger -p may"
+alias jun="hledger -p jun"
+alias jul="hledger -p jul"
+alias aug="hledger -p aug"
+alias sep="hledger -p sep"
+alias oct="hledger -p oct"
+alias nov="hledger -p nov"
+alias dec="hledger -p dec"
 
-fin() {  # fin [PAT] - list financial scripts in $FINDIR/bin/[bashrc] (default: ~/finance)
-    (cd ${FINDIR:-~/finance} || exit; bin "$@")
+# query hledger with sqlite
+hq() {
+    (hledger print -O sql; echo "$1") | sqlite3 -column -header;
 }
 
-bin() {  # bin [PAT] - list aliases, functions, scripts in ./bin/[bashrc]
-    PAT="${1-.}"
-    BINDIR=./bin
-    BASHRC=$BINDIR/bashrc
-    ( [[ -e $BASHRC ]] && grep -E '^(alias|function|\w+\(\))' $BASHRC \
-	  | gnused -E -e 's/^alias *//' -e 's/^(function )?(\w+) *\(\) *\{/\2()/' -e 's/#/\t#/' \
-		   # -e "s/=('[^']+'|\"[^\"]+\"|\w+)/=/"   # uncomment to hide alias definitions
-      [[ -d $BINDIR ]] && for F in "$BINDIR"/*; do
-	  [[ -x $F ]] || continue
-	  printf '%s  ' "$(basename "$F")"
-	  (grep -IE '^(#|--) ' "$F" 2>/dev/null | gnused -E 's/(#|--)/\t#/'; echo) | head -1
-      done
-    ) | grep -iE "$PAT" \
-      #| sort -b -k2    # uncomment to sort by name
+# list likely hledger-readable files in current directory
+hledgerfiles() {
+    ls $@ *.{journal,j,timelog,csv,ledger,lgr,dat} 2>/dev/null
 }
 
-gnused() {  # GNU sed, called gsed on mac
-    if hash gsed 2>/dev/null; then gsed "$@"; else sed "$@"; fi
+# helpers for working with yearly files.
+
+# yearfiles [N] - print the paths of all or the last N yearly journals.
+# Adjust to suit your files.
+yearfiles() {
+    FIRSTYEAR=2006
+    N="$1"; shift
+    YEAR=$(date +%Y)
+    if [[ -n "$N" ]]; then
+	START=$(( "$YEAR" - "$N" + 1))
+    else
+	START=$FIRSTYEAR
+    fi
+    for ((y="$START"; y<="$YEAR"; y++)); do echo "$FINDIR/$y/$y.journal"; done
 }
 
-gnudate() {  # GNU date, called gdate on mac
-    if hash gdate 2>/dev/null; then gdate "$@"; else date "$@"; fi
+# yearopts [N] - print -f options for all or the last N yearly journals
+yearopts() {
+    for y in $(yearfiles "$1"); do
+	echo -f"$y"
+    done
 }
 
+# years [N] CMD.. - run hledger CMD on all or just the last N yearly journals combined
+# eg:
+# years stats
+# years 2 stats
+years() {
+    N="$1"
+    if [[ "$N" =~ ^[0-9]+$ ]]; then
+	shift
+    else
+	N=
+    fi
+    # shellcheck disable=SC2046
+    hledger $(yearopts "$N" | xargs) "$@"
+}
 
+alias 10y='years 10'
+alias 9y='years 9'
+alias 5y='years 5'
+alias 2y='years 2'
+
+# eachyear [N] [n|p|P] "SHELLCMD" - run SHELLCMD with $LEDGER_FILE set,
+# for each or just the last N yearly journals,
+# optionally printing the file name with 0, 1 or 2 line breaks.
+# Accepts shell commands, extra quoting may be needed.
+# eg:
+# eachyear 10 hledger bal -0 -N cur:\\\\$
+# eachyear p files
+# eachyear P 'files | wc -l'
+# eachyear 5 P 5 'hledger stats | tail -1'
+# eachyear 7 p 'comm | rg ^...$'
+eachyear() {
+    N="$1"
+    if [[ "$N" =~ ^[0-9]+$ ]]; then
+	shift
+    else
+	N=
+    fi
+    P="$1"
+    if [[ "$P" =~ ^[npP]$ ]]; then
+	shift
+    else
+	P=
+    fi
+    for f in $(yearfiles "$N"); do
+	if [[ -n "$P" ]]; then
+            if [[ $P == P ]]; then echo; fi
+            printf "%s: " "$(basename "$f")"
+            if [[ $P != n ]]; then echo; fi
+        fi
+	bash -ic "(LEDGER_FILE=$f; $*)"  # XXX loses some quoting
+    done
+}
 
 # time
 
-# all in one big journal:
-#export TIMELOG=~/personal/time.journal
+#export TIMELOG=$FINDIR/time.journal
+export TIMELOG=$FINDIR/time/time-2024.journal
+export TIMEDOT=$FINDIR/time/time-2024.timedot
 
 alias hours="hledger -f $TIMELOG"
 alias today='hours -p today'
@@ -59,66 +171,6 @@ alias sephours="hours -p sep"
 alias octhours="hours -p oct"
 alias novhours="hours -p nov"
 alias dechours="hours -p dec"
-alias 2008hours="hours -p 2008"
-alias 2009hours="hours -p 2009"
-alias 2010hours="hours -p 2010"
-alias 2011hours="hours -p 2011"
-alias 2012hours="hours -p 2012"
-alias 2013hours="hours -p 2013"
-alias 2014hours="hours -p 2014"
-alias 2015hours="hours -p 2015"
-alias weeklyhours="hours -p 'weekly this year' register --empty"
-alias monthlyhours="hours -p 'monthly this year' register --empty"
-alias weeklybillablehours="weeklyhours jobs not:unbilled --depth 3"
-alias monthlybillablehours="monthlyhours jobs not:unbilled --depth 3"
-
-# money
-
-# one journal per year, included by current and all journals:
-#export LEDGER_FILE=~/personal/current.journal
-#export LEDGER_FILE=~/personal/all.journal
-
-alias jan="hledger -p jan"
-alias feb="hledger -p feb"
-alias mar="hledger -p mar"
-alias apr="hledger -p apr"
-alias may="hledger -p may"
-alias jun="hledger -p jun"
-alias jul="hledger -p jul"
-alias aug="hledger -p aug"
-alias sep="hledger -p sep"
-alias oct="hledger -p oct"
-alias nov="hledger -p nov"
-alias dec="hledger -p dec"
-alias 2006='hledger -f ~/personal/2006.journal'
-alias 2007='hledger -f ~/personal/2007.journal'
-alias 2008='hledger -f ~/personal/2008.journal'
-alias 2009='hledger -f ~/personal/2009.journal'
-alias 2010='hledger -f ~/personal/2010.journal'
-alias 2011='hledger -f ~/personal/2011.journal'
-alias 2012='hledger -f ~/personal/2012.journal'
-alias 2013='hledger -f ~/personal/2013.journal'
-alias 2014='hledger -f ~/personal/2014.journal'
-alias 2015='hledger -f ~/personal/2015.journal'
-alias all='hledger -f ~/personal/all.journal'
-alias household='hledger -f ~/personal/household.journal'
-alias add='hledger add'
-
-alias bankbalances='hledger bal assets:bank liabilities:credit -E'
-alias cashflow="hledger balance '^assets:(bank|cash)'"
-alias incexp="hledger balance '(^income|^expenses|^equity:draw)'"
-# show activity in these accounts this and last month
-alias cash="hledger -d 'd>=[last month]' reg 'assets:cash' -B"
-alias checking="hledger -d 'd>=[last month]' reg 'assets:bank:wells fargo:checking' -B"
-# show daily cleared checking balance - for reconciling with online bank statement
-alias checkingcleared="checking --cleared --period 'daily to tomorrow'"
-# show checking balance from today forward
-alias checkingfuture="checking -d 'd>=[yesterday]'"
-
-# generate a chart and view it in emacs
-chart () {
-  hledger chart $* && emacsclient -n hledger.png
-}
 
 # old ledger 2.6 scripts
 

bin/gsheet-csv.hs

@@ -96,6 +96,6 @@ exec ghcid         - watch for compile errors
 
 4. Declare which package each import is from, keep synced with --package; might as well be clear
 
-5. Doc: https://docs.haskellstack.org/en/stable/GUIDE/#script-interpreter
+5. Doc: https://docs.haskellstack.org/en/stable/topics/scripts
 
 -}

bin/hledger-check-fancyassertions.hs

@@ -11,7 +11,7 @@
 --
 -- You could make this a standalone script that runs from anywhere and
 -- recompiles itself when changed, by replacing "runghc" above with
--- "script --compile --resolver lts-16" (eg). However this uses the
+-- "script --compile --resolver lts-22" (eg). However this uses the
 -- hledger version from that stackage resolver, so in this case you
 -- should check out the corresponding release-tagged version of this
 -- script for compatibility (eg: git checkout 1.18.1).

bin/hledger-register-max.hs

@@ -1,5 +1,7 @@
 #!/usr/bin/env stack
--- stack script --compile --resolver lts-20.13 --verbosity error --package hledger-lib --package hledger --package text --package safe
+-- stack runghc
+-- (requires hledger > 1.34)
+-- -- stack script --compile --resolver nightly-2024-09-26 --verbosity error --package hledger-lib --package hledger --package text --package safe
 
 -- hledger-register-max - runs "hledger register" and prints the posting with largest running total/balance.
 -- Usage:
@@ -25,9 +27,7 @@ import qualified "text" Data.Text as T
 import qualified Data.Text.IO as T
 import Safe
 import System.Environment
-import Hledger
-import Hledger.Cli
-import Hledger.Cli (argsToCliOpts)
+import Hledger.Cli.Script
 
 -- XXX needs --help, see hledger-addon-example.hs
 
@@ -37,8 +37,9 @@ main = do
   withJournalDo opts $ \j -> do
     let
       r = postingsReport (reportspec_ opts) j
-      maxbal = fifth5 $ maximumBy (comparing fifth5) r
-      is = filter ((== maxbal).fifth5) r
+      getamt = pamount.fourth5
+      maxbal = getamt $ maximumBy (comparing getamt) r
+      is = filter ((== maxbal).getamt) r
     mapM_ printItem is
 
 printItem (_, _, _, p, bal) = do

bin/hledger-register-max2.hs

@@ -1,8 +1,8 @@
 #!/usr/bin/env stack
 -- stack runghc --verbosity error --package hledger
 -- stack runghc --verbosity error --package hledger --package hledger-lib --package text --package safe 
--- stack script --compile --resolver lts-20.13 --verbosity error --package hledger --package text
--- stack script --compile --resolver lts-20.13 --verbosity error --package hledger --package hledger-lib --package text --package safe
+-- stack script --compile --resolver nightly-2024-09-26 --verbosity error --package hledger --package text
+-- stack script --compile --resolver nightly-2024-09-26 --verbosity error --package hledger --package hledger-lib --package text --package safe
 -- The topmost stack command above is used to run this script.
 -- stack script uses released hledger, stack runghc uses local hledger source.
 -- This script currently requires local hledger source, for Hledger.Cli.Script.

bin/hledger-report1.hs

@@ -1,7 +1,7 @@
 #!/usr/bin/env stack
 -- stack runghc --verbosity error --package hledger --package hledger-lib --package text --package safe 
 -- (use the local hledger source)
--- -- stack script --compile --resolver nightly-2023-10-13 --verbosity info --package hledger --package text
+-- -- stack script --compile --resolver nightly-2024-09-26 --verbosity info --package hledger --package text
 -- -- (use a released hledger from stackage)
 
 -- A custom compound report - like incomestatement but with different,

bin/hledger-script-example.hs

@@ -1,11 +1,24 @@
 #!/usr/bin/env stack
 -- stack runghc --verbosity error --package hledger
--- stack runghc --verbosity error --package hledger --package hledger-lib --package text --package safe 
--- stack script --compile --resolver lts-20.13 --verbosity error --package hledger --package text
--- stack script --compile --resolver lts-20.13 --verbosity error --package hledger --package hledger-lib --package text --package safe
--- The topmost stack command above is used to run this script.
--- stack script uses released hledger, stack runghc uses local hledger source.
--- This script currently requires local hledger source, for Hledger.Cli.Script.
+
+-- This script is run by the "stack runghc" command above.
+-- This command expects to be running in a copy of the hledger source code.
+-- Its advantage is that it uses that latest hledger source version.
+-- To show more progress output, change --verbosity to info.
+-- The main haskell package needed for scripting is "hledger".
+-- To make more packages available for import, add more --package options.
+--
+-- For more robustness, use the "stack script" command below instead.
+-- This uses a released version of hledger, and does not need the hledger source code.
+-- The version of hledger (and haskell libraries, and ghc) is determined by
+-- the stackage snapshot (lts-X.Y or nightly-YYYY-MM-DD).
+--
+-- stack script --snapshot lts-22.27 --verbosity error --package hledger --package text
+--  (Uses the specified snapshot with its hledger and ghc, https://www.stackage.org/lts-22.27)
+--
+-- stack script --snapshot nightly-2024-09-26 --verbosity error --package hledger --package hledger-lib --package text --compile
+--  (Uses a newer snapshot, hledger and ghc. The --compile flag compiles the script to a faster binary.)
+--
 ------------------------------------78----------------------------------------
 
 {-# LANGUAGE OverloadedStrings, PackageImports #-}

bin/hledger-simplebal

@@ -6,7 +6,7 @@
 set -o pipefail
 
 VALUATION_COMMODITY="$"
-hledger bal -0 -N -X "${VALUATION_COMMODITY}" --infer-market-prices -c "${VALUATION_COMMODITY} 1000" --layout=bare "$@" | awk '{print $1}'
+hledger -n bal -0 -N -X "${VALUATION_COMMODITY}" --infer-market-prices -c "${VALUATION_COMMODITY} 1000" --layout=bare "$@" | awk '{print $1}'
 
 # Tired of complex financial reports ? This is a silly but fun and
 # occasionally useful script showing how to get "one number" semi-robustly

doc/ANNOUNCE

@@ -1,15 +1,15 @@
-I'm pleased to announce hledger 1.33.1, with several fixes,
-installability improvements, and doc updates.
+I'm pleased to announce hledger 1.40, with a new config file system,
+sortable register output, FODS output, and prettier tables.
 
-- https://github.com/simonmichael/hledger/releases/1.33.1
-- https://hledger.org/relnotes.html#2024-05-02-hledger-1331
+- https://github.com/simonmichael/hledger/releases/1.40
+- https://hledger.org/relnotes.html#2024-09-09-hledger-140
 - https://hledger.org/install
 
 Best,
 -Simon
 
 
-hledger is free, fast, robust, multicurrency, double-entry,
+hledger is free, robust, friendly, multicurrency, double-entry,
 plain text accounting software for unix, mac, windows, and the web.
 Written in Haskell for reliability and longevity,
 it is built around human-readable, version-controllable plain text files,

doc/ANNOUNCE.masto

@@ -1,13 +1,13 @@
-I'm pleased to announce hledger 1.33.1, with several fixes,
-installability improvements, and doc updates.
+I'm pleased to announce hledger 1.40, with a new config file system,
+sortable register output, FODS output, and prettier tables.
 
-- https://github.com/simonmichael/hledger/releases/1.33.1
-- https://hledger.org/relnotes.html#2024-05-02-hledger-1331
+- https://github.com/simonmichael/hledger/releases/1.40
+- https://hledger.org/relnotes.html#2024-09-09-hledger-140
 - https://hledger.org/install
 
-#hledger is free, fast, robust, multicurrency, double-entry,
+#hledger is free, robust, friendly, multicurrency, double-entry,
 #PlainTextAccounting software for unix, mac, windows, and the web,
 written in #Haskell for reliability and longevity.
-For help, join our chat or mail list: https://hledger.org/support
+For help, join the chat, forum or mail list: https://hledger.org/support
 
 #PersonalFinance

doc/CODE.md

@@ -8,7 +8,7 @@
 hledger is a suite of applications, tools and libraries.
 The main hledger code repository is [github.com/simonmichael/hledger](https://github.com/simonmichael/hledger)
 (shortcut url `code.hledger.org`).
-There are also various hledger addons maintained as separate projects with their own repos.
+There are also various hledger add-ons maintained as separate projects with their own repos.
 
 ## hledger packages
 
@@ -214,3 +214,51 @@ Relevant tools include:
   [log](https://ircbrowse.net/day/hledger/2015/10/11) -->
 
 
+## Code docs
+
+Haddock comments are the standard way of attaching docs (and sometimes small tests)
+to a haskell definition. They are used pervasively in the hledger codebase.
+
+From [#2222](https://github.com/simonmichael/hledger/pull/2222):
+
+I tend to add a line of english description to most things.
+They can be a big help to someone else debugging/improving code later.
+
+> *I use Haddock but I do not like to tell obvious things in Haddock comments.
+> (And if there are surprising things I try to eliminate the surprises instead.)*
+
+I hear you! This is of course a recurring debate among programmers.
+
+I agree that obviously redundant haddocks that add no value are to be avoided.
+
+In the hledger codebase, there's no hard requirement for haddocks.
+And sometimes the bar can be lower, eg in single-purpose less-frequently-developed modules like FODS.hs.
+
+But I believe it's always worth at least considering them when adding code, and it's something I consider when reviewing code.
+I'll give some reasons why I think so, for the record (and maybe they'll persuade you just a little).
+
+Haddocks help guide and anchor the developer while writing or changing code.
+They are the cheapest kind of doc, spec and test suite.
+They also create a place to add actual doctests, now or later.
+
+Haddocks can be helpful to contributors who are not expert haskellers, which happens quite often in the hledger project.
+The human language descriptions can complement the code, reducing cognitive effort and helping with mental chunking.
+I think people find the hledger's code, with its pervasive haddocks, above average in readability.
+(And I think we've had more successful contributions as a result.)
+
+Haddocks can also help experienced developers move faster.
+Code which seems quite clear and obvious when you are writing it is often less obvious to oneself a few weeks or years later.
+And certainly it can be less obvious to others than we might think.
+When debugging or coding we are often mentally stretched and we would prefer to conserve brainpower for the main task.
+
+In my experience debugging/writing/changing hledger code,
+
+- Number of times I've regretted seeing a haddock comment attached to some code: almost zero.
+- Effort to remove an excessive haddock: almost zero.
+- Number of times I've been trying to understand some contributed code and haddocks would have saved my time: quite a few.
+- Number of times I've appreciated code haddocks and found them helpful: many.
+- Number of times I've found errors or staleness in haddocks: not often.
+- Effort to fix wrong haddocks, or to write new ones: usually very low. And if it's high, it's usually very worthwhile because it alerts me to a confusion in the code or clarifies my thinking.
+
+hledger is a documentation-driven project, with docs in general being a top priority.
+I think this is one reason it has held together and kept improving over a long period.

doc/CONTRIBUTING.md

@@ -11,27 +11,27 @@ We invite you to jump in, and thank you!
 There are many ways to help. Browse the ideas below,
 and/or say hello in the [chat](support.md) and we'll help find you a useful job.
 
-### Visitor / passer-by ?
+## Visitor / passer-by ?
 
 - Give feedback on the site and your impressions of the project, small or large, good or bad. This is valuable.
 
-### New user ?
+## New user ?
 
 - Report your new user experiences, small or large, good or bad. This is valuable.
 
-### Tech supporter ?
+## Tech supporter ?
 
 - Share what you've learned so far to help others. This is a quadruple win -
   it helps them, improves your own understanding, builds community, and frees up maintainer time!
 
-### Funder ?
+## Funder ?
 
 - Become a financial backer: [Sponsor hledger](sponsor.md)
 - Contribute or pledge bounties on issues you care about
 - Ask your organization to contribute
 - Work on project [finance](FINANCE.md) - accounting, fundraising, sustainability..
 
-### Tester ?
+## Tester ?
 
 - Test installation on platforms you have access to
 - Test examples, advice, and links in the docs
@@ -42,7 +42,7 @@ and/or say hello in the [chat](support.md) and we'll help find you a useful job.
 - Test new releases, report regressions and collect regression finder bounties
 - Discuss and help analyse problems via chat/mail list/issue tracker
 
-### Bug wrangler ?
+## Bug wrangler ?
 
 - Respond to issue reports when needed, especially if they are from new reporters
 - Add appropriate labels to issues to categorise them
@@ -52,42 +52,42 @@ and/or say hello in the [chat](support.md) and we'll help find you a useful job.
 - Improve issues urls & dashboard(s)
 - Help ensure a consistently good bug-reporting and PR-contributing experience
 
-### Bug fixer ?
+## Bug fixer ?
 
 - Get familiar with issue tracker, issue labels, shortcut urls, issue dashboards..
 - Review open bug reports
 - Try to fix or help fix some
 - Fix regressions and collect regression fixer bounties
 
-### Developer ?
+## Developer ?
 
 - Give feedback on your experience using the hledger packages
 - Suggest API improvements
 - See the [Developer FAQ](DEVFAQ.md) and other [Developer docs](dev.md).
 
-### Technical writer ?
+## Technical writer ?
 
 - Get familiar with the documentation, website and online presence; review and test
 - Get familiar with the doc/site source files and generation process (see [Just, Make, Shake](JUST-MAKE-SHAKE.md))
 - Help improve user, contributor, process docs
 
-### Web designer / webmaster ?
+## Web designer / webmaster ?
 
 - Review and help improve our web presence
 
-### Graphic designer ?
+## Graphic designer ?
 
 - Review and improve logos, graphics, design language
 - Contribute illustrations, diagrams, cartoons, mockups
 
-### Packager ?
+## Packager ?
 
 - Start/test/improve hledger's packaging on various platforms
 - Find/assist/take over from existing packagers
 - Improve packaging-related docs/links
 - Develop mac or windows installers
 
-### Marketer / communicator ?
+## Marketer / communicator ?
 
 - Clarify project goals, value proposition, brand, mission, story
 - Monitor product-market fit
@@ -95,18 +95,18 @@ and/or say hello in the [chat](support.md) and we'll help find you a useful job.
 - Influence developer priorities
 - Spread the word!
 
-### Product designer ?
+## Product designer ?
 
 - Contribute design input to discussions in issue tracker and elsewhere
 - Develop your whole-system view of the hledger "product" (user software, docs, online presence, new user experience etc.)
 
-### Community builder/moderator ?
+## Community builder/moderator ?
 
 - Participate in [support](support.md) channels
 - As a regular member or moderator, help to resolve/report incidents
 - Help uphold and improve our community structures and dynamics
 
-### Project manager ?
+## Project manager ?
 
 - Monitor, report on project progress and performance
 - Research, compare and report on successful projects, related projects
@@ -114,7 +114,7 @@ and/or say hello in the [chat](support.md) and we'll help find you a useful job.
 - Assist with marketing, communication, outreach
 - Assist with maintainer tasks
 
-### Maintainer / co-maintainer ?
+## Maintainer / co-maintainer ?
 
 - Manage and ship releases
 - Manage the project roadmap

doc/CREDITS.md

@@ -36,7 +36,7 @@ and by the innumerable other benefactors making it all possible.
 | 161     | Dmitry Astapov                 | roi, files commands; --transpose; merge/improve --budget; generalise --forecast/--auto; docker packaging; improved CSV parsing, balancing, periodic transactions, close, parsing, docs, tests |
 | 81      | Vladimir Zhelezov              | new bash shell completions                                                                                                                                                                    |
 | 72      | Alex Chen                      | parsing improvements, code cleanups, better error messages; dep updates                                                                                                                       |
-| 52      | Mykola Orliuk                  | hledger-budget, hledger-prices addons; scientific number notation; print, hledger-equity, hledger-rewrite, --pivot, space, parsing improvements; code updates; GHC 8.0 support                |
+| 52      | Mykola Orliuk                  | hledger-budget, hledger-prices add-ons; scientific number notation; print, hledger-equity, hledger-rewrite, --pivot, space, parsing improvements; code updates; GHC 8.0 support                |
 | 51      | Jakob Schöttl                  | bash completions; register --invert; timeclock parsing improvements; code cleanups                                                                                                            |
 | 40      | Everett Hildenbrandt           | doc toolchain updates, switch from hakyll to pandoc; csv parser improvement                                                                                                                   |
 | 31      | Jakub Zárybnický               | hledger-web, hledger-ui improvements                                                                                                                                                          |

doc/DEVFAQ.md

@@ -6,7 +6,7 @@ This is just getting started. It will absorb some of the other [Developer docs](
 
 <!-- ## Developing hledger -->
 
-### How do I get/build the hledger source ?
+## How do I get/build the hledger source ?
 
 ```cli
 $ git clone https://github.com/simonmichael/hledger
@@ -15,7 +15,7 @@ $ stack build
 You can specify `hledger`, `hledger-ui` or `hledger-web` as an argument to build just that executable.
 Please see [Install > Build from source](install.md#build-from-source) for more details and other build methods.
 
-### What other repos are there ?
+## What other repos are there ?
 
 There are three official repos:
 - <https://github.com/simonmichael/hledger> - the main hledger repo, for hledger, hledger-ui and hledger-web. Shortcut url: <https://code.hledger.org>
@@ -24,7 +24,7 @@ There are three official repos:
 
 And third-party add-ons and tools (hledger-iadd, hledger-utils, full fledged hledger, hledger-flow, etc.) have their own repos.
 
-### How do I run a build in place ?
+## How do I run a build in place ?
 
 After building with stack,
 ```cli
@@ -36,7 +36,7 @@ Or after building with cabal,
 $ cabal exec -- hledger [ARGS]
 ```
 
-### How do I install a build in PATH ?
+## How do I install a build in PATH ?
 
 ```cli
 $ stack install
@@ -52,7 +52,7 @@ It will install executables to `~/.cabal/bin`:
 $ cabal install all:exes
 ```
 
-### How do I build/run with ghc-debug support ?
+## How do I build/run with ghc-debug support ?
 
 You might need to stop background builders like HLS, to avoid a fight over the build flag
 (in VS Code, run the command "Haskell: Stop Haskell LSP server").

doc/REGRESSIONS.md

@@ -43,69 +43,74 @@ To claim the bounty:
 ## Regressions reported
 
 - [Issue tracker: all regression reports](https://bugs.hledger.org/regressions)
-- [Opencollective: regression bounty requests](https://opencollective.com/hledger/expenses?amount=50-100)
+- [Opencollective: regression bounty requests](https://opencollective.com/hledger/expenses)  <!-- not ?amount=50-100 because other currencies -->
 - [Opencollective: regression bounty payments](https://opencollective.com/hledger/transactions?kind=EXPENSE&amount=50-100)
 
-| hledger version, bug report  | Reporter        | Bounty paid on
-|------------------------------|-----------------|------------------------------------------------------------------|
-| **1.19** 2020-09-01          | -               | -
-| [#1568]                      | jolmg           | pre bounty
-| [#1688]                      | Simon Michael   | N/A
-| [#1698]                      | David Lowe      | [2021-09-18](https://opencollective.com/hledger/expenses/50380)
-| [#1745]                      | Arne Schlüter   | [2021-11-02](https://opencollective.com/hledger/expenses/54446)
-| [#1800]                      | Chuck Holmes    | [2022-01-21](https://opencollective.com/hledger/expenses/61802)
-| **1.20** 2020-12-05          | -               | -
-| [#1439]                      | apauley         | pre bounty
-| [#1468]                      | Simon Michael   | N/A
-| **1.20.3** 2021-01-14        | -               | -
-| [#1566]                      | benwebber       | pre bounty
-| **1.21** 2021-03-10          | -               | -
-| [#1508]                      | edlanglois      | pre bounty
-| [#1523]                      | Simon Michael   | N/A
-| [#1526]                      | lestephane      | pre bounty
-| [#1527]                      | lestephane      | pre bounty
-| [#1656]                      | Stephen Morgan  | [2021-08-22](https://opencollective.com/hledger/expenses/48246)
-| **1.22** 2021-07-03          | -               | -
-| [#1597]                      | Simon Michael   | [2021-07-08](https://opencollective.com/hledger/expenses/44939)
-| [#1607]                      | Simon Michael   | [2021-07-16](https://opencollective.com/hledger/expenses/45547)
-| [#1625]                      | Julian Klode    | [2021-07-30](https://opencollective.com/hledger/expenses/46431)
-| [#1736]                      | Romain Gehrig   | [2021-11-14](https://opencollective.com/hledger/expenses/55510)
-| [#1851]                      | Eric Langlois   | [2022-04-11](https://opencollective.com/hledger/expenses/72187)
-| **1.22.1** 2021-08-02        | -               | -
-| [#1638]                      | Yann Büchau     | [2021-08-03](https://opencollective.com/hledger/expenses/46918)
-| [#1642]                      | Simon Michael   | N/A
-| **1.23** 2021-09-21          | -               | -
-| [#1933]                      | Simon Michael   | [2022-09-14](https://opencollective.com/hledger/expenses/95068)
-| [#2071]                      | William Pierce  | [2024-04-02](https://opencollective.com/hledger/expenses/195768)
-| **1.24** 2021-12-01          | -               | -
-| [#1782]                      | Simon Michael   | N/A
-| **1.25** 2022-03-04          | -               | -
-| [#2032]                      | Simon Michael   | [2023-05-03](https://opencollective.com/hledger/expenses/137410)
-| [#2196]                      | Pranesh Prakash | UNCLAIMED
-| **1.26** 2022-06-04          | -               | -
-| **1.27** 2022-09-01          | -               | -
-| [#1932]                      | Andras Fabian   | [2022-09-15](https://opencollective.com/hledger/expenses/95112)
-| [#2018]                      | Allan Odgaard   | [2023-03-28](https://opencollective.com/hledger/expenses/130591)
-| **1.28** 2022-12-01          | -               | -
-| **1.29** 2023-03-11          | -               | -
-| [#2012]                      | Simon Michael   | N/A
-| [#2020]                      | Pablo Mora      | [2023-03-31](https://opencollective.com/hledger/expenses/131350)
-| [#2023]                      | Simon Michael   | [2023-04-06](https://opencollective.com/hledger/expenses/132635)
-| [#2034]                      | Simon Michael   | N/A
-| [#2045]                      | Pranesh Prakash | [2023-10-17](https://opencollective.com/hledger/expenses/150171)
-| [#2153]                      | markokocic      | 2024-01-25, $50 donated
-| **1.30** 2023-06-01          | -               | -
-| [#2072]/[#2137]/[#2150]      | Simon Michael, usaAmch, ipvych |
-| **1.31** 2023-09-03          | -               | -
-| [#2091]                      | Petr Slansky    | [2023-10-16](https://opencollective.com/hledger/expenses/166632)
-| [#2115]                      | pepe_pecas      | 2023-12-15, $100 donated
-| **1.32** 2023-12-01          | -               | -
-| [#2125]                      | Simon Michael   | N/A
-| [#2127]                      | rajeevn1        | UNCLAIMED
-| [#2130]                      | Simon Michael   | N/A
-| [#2134]                      | pepe_pecas      | 2023-12-15, $100 donated
-| [#2156]                      | ishmaelavila    | UNCLAIMED
-| **1.33** 2024-04-18          | -               | -
+| hledger&nbsp;version, <br>regressions&nbsp;found | Finder&nbsp;bounties <br>(since 2021-06-14)                                                                         | Fixer&nbsp;bounties <br>(since 2024-01-01) <!-- some missing -->             |
+|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|
+| **1.19** 2020-09-01                              | ---                                                                                                                 | ---                                                                          |
+| [#1568]                                          | jolmg           : N/A                                                                                               |                                                                              |
+| [#1688]                                          | Simon Michael   : N/A                                                                                               |                                                                              |
+| [#1698]                                          | David Lowe      : [2021-09-18](https://opencollective.com/hledger/expenses/50380)                                   |                                                                              |
+| [#1745]                                          | Arne Schlüter   : [2021-11-02](https://opencollective.com/hledger/expenses/54446)                                   |                                                                              |
+| [#1800]                                          | Chuck Holmes    : [2022-01-21](https://opencollective.com/hledger/expenses/61802)                                   |                                                                              |
+| **1.20** 2020-12-05                              | ---                                                                                                                 | ---                                                                          |
+| [#1439]                                          | apauley         : N/A                                                                                               |                                                                              |
+| [#1468]                                          | Simon Michael   : N/A                                                                                               |                                                                              |
+| **1.20.3** 2021-01-14                            | ---                                                                                                                 | ---                                                                          |
+| [#1566]                                          | benwebber       : N/A                                                                                               |                                                                              |
+| **1.21** 2021-03-10                              | ---                                                                                                                 | ---                                                                          |
+| [#1508]                                          | edlanglois      : N/A                                                                                               |                                                                              |
+| [#1523]                                          | Simon Michael   : N/A                                                                                               |                                                                              |
+| [#1526]                                          | lestephane      : N/A                                                                                               |                                                                              |
+| [#1527]                                          | lestephane      : N/A                                                                                               |                                                                              |
+| [#1656]                                          | Stephen Morgan  : [2021-08-22](https://opencollective.com/hledger/expenses/48246)                                   |                                                                              |
+| **1.22** 2021-07-03                              | ---                                                                                                                 | ---                                                                          |
+| [#1597]                                          | Simon Michael   : [2021-07-08](https://opencollective.com/hledger/expenses/44939)                                   |                                                                              |
+| [#1607]                                          | Simon Michael   : [2021-07-16](https://opencollective.com/hledger/expenses/45547)                                   |                                                                              |
+| [#1625]                                          | Julian Klode    : [2021-07-30](https://opencollective.com/hledger/expenses/46431)                                   |                                                                              |
+| [#1736]                                          | Romain Gehrig   : [2021-11-14](https://opencollective.com/hledger/expenses/55510)                                   |                                                                              |
+| [#1851]                                          | Eric Langlois   : [2022-04-11](https://opencollective.com/hledger/expenses/72187)                                   |                                                                              |
+| **1.22.1** 2021-08-02                            | ---                                                                                                                 | ---                                                                          |
+| [#1638]                                          | Yann Büchau     : [2021-08-03](https://opencollective.com/hledger/expenses/46918)                                   |                                                                              |
+| [#1642]                                          | Simon Michael   : N/A                                                                                               |                                                                              |
+| **1.23** 2021-09-21                              | ---                                                                                                                 | ---                                                                          |
+| [#1933]                                          | Simon Michael   : [2022-09-14](https://opencollective.com/hledger/expenses/95068)                                   |                                                                              |
+| [#2071]                                          | William Pierce  : [2024-04-02](https://opencollective.com/hledger/expenses/195768)                                  |                                                                              |
+| **1.24** 2021-12-01                              | ---                                                                                                                 | ---                                                                          |
+| [#1782]                                          | Simon Michael   : N/A                                                                                               |                                                                              |
+| **1.25** 2022-03-04                              | ---                                                                                                                 | ---                                                                          |
+| [#2032]                                          | Simon Michael   : [2023-05-03](https://opencollective.com/hledger/expenses/137410)                                  |                                                                              |
+| [#2196]                                          | Pranesh Prakash : [2024-09-18](https://opencollective.com/hledger/expenses/220683)                                  | Simon Michael : **unclaimed**, <br>Bas van Dijk [#2224] : donated 2024-09-22 |
+| **1.26** 2022-06-04                              | ---                                                                                                                 | ---                                                                          |
+| **1.27** 2022-09-01                              | ---                                                                                                                 | ---                                                                          |
+| [#1932]                                          | Andras Fabian   : [2022-09-15](https://opencollective.com/hledger/expenses/95112)                                   |                                                                              |
+| [#2018]                                          | Allan Odgaard   : [2023-03-28](https://opencollective.com/hledger/expenses/130591)                                  |                                                                              |
+| **1.28** 2022-12-01                              | ---                                                                                                                 | ---                                                                          |
+| **1.29** 2023-03-11                              | ---                                                                                                                 | ---                                                                          |
+| [#2012]                                          | Simon Michael   : N/A                                                                                               |                                                                              |
+| [#2020]                                          | Pablo Mora      : [2023-03-31](https://opencollective.com/hledger/expenses/131350)                                  |                                                                              |
+| [#2023]                                          | Simon Michael   : [2023-04-06](https://opencollective.com/hledger/expenses/132635)                                  |                                                                              |
+| [#2034]                                          | Simon Michael   : N/A                                                                                               |                                                                              |
+| [#2045]                                          | Pranesh Prakash : [2023-10-17](https://opencollective.com/hledger/expenses/150171)                                  |                                                                              |
+| [#2153]                                          | markokocic      : donated 2024-01-25                                                                                | Simon Michael : N/A                                                          |
+| **1.30** 2023-06-01                              | ---                                                                                                                 | ---                                                                          |
+| [#2072]                                          | Simon Michael   : **unclaimed**, <br>usaAmch [#2137] : donated 2024-09-18, <br>ipvych [#2150] : paid 2024-09-25 | Simon Michael : **unclaimed**                                                |
+| **1.31** 2023-09-03                              | ---                                                                                                                 | ---                                                                          |
+| [#2091]                                          | Petr Slansky    : [2023-10-16](https://opencollective.com/hledger/expenses/166632)                                  | Simon Michael : N/A                                                          |
+| [#2115]                                          | usaAmch         : donated 2024-09-25                                                                                | Simon Michael : N/A                                                          |
+| **1.32** 2023-12-01                              | ---                                                                                                                 | ---                                                                          |
+| [#2125]                                          | Simon Michael   : N/A                                                                                               | Simon Michael : N/A                                                          |
+| [#2127]                                          | Rajeev N        : [2023-12-15](https://opencollective.com/hledger/expenses/177761)                                  | Simon Michael : **unclaimed**                                                |
+| [#2130]                                          | Simon Michael   : N/A                                                                                               | Simon Michael : N/A                                                          |
+| [#2134]                                          | pepe_pecas      : donated 2023-12-15                                                                                | Simon Michael : N/A                                                          |
+| [#2156]                                          | ishmaelavila    : **unclaimed**                                                                                     | Simon Michael : **unclaimed**                                                |
+| **1.33** 2024-04-18                              | ---                                                                                                                 | ---                                                                          |
+| [#2227]                                          | Henning Thielemann : **unclaimed**                                                                                  | Henning Thielemann : **unclaimed**                                           |
+| **1.34** 2024-06-01                              | ---                                                                                                                 | ---                                                                          |
+| **1.40** 2024-09-09                              | ---                                                                                                                 | ---                                                                          |
+| [#2225]                                          | Bas van Dijk    : donated 2024-09-22                                                                                | Henning Thielemann : **unclaimed**                                           |
+
 
 
 [#1439]: https://github.com/simonmichael/hledger/issues/1439
@@ -151,4 +156,6 @@ To claim the bounty:
 [#2153]: https://github.com/simonmichael/hledger/issues/2153
 [#2156]: https://github.com/simonmichael/hledger/issues/2156
 [#2196]: https://github.com/simonmichael/hledger/issues/2196
-
+[#2224]: https://github.com/simonmichael/hledger/issues/2224
+[#2225]: https://github.com/simonmichael/hledger/issues/2225
+[#2227]: https://github.com/simonmichael/hledger/issues/2227

doc/RELEASING.md

@@ -22,7 +22,7 @@ Notes for hledger release managers and maintainers.
 
 ## Release types
 
-hledger major releases happen each quarter, normally at the start of the third month.
+hledger major releases happen each quarter, normally at the start of the third month (see [past releases](relnotes.md)).
 Bugfix releases follow when needed.
 Preview releases may happen in the other months if wanted.
 
@@ -173,7 +173,7 @@ Branches named `MA.JOR-branch` in the hledger repo, eg `1.25-branch`. Releases a
 
 - When starting a release, save a copy of this file (RELEASING2.md) and update notes there until after release, to avoid obstructing git branch switching.
 
-- Use and update scripts. See `just` in the main repo -> RELEASING (and perhaps older stuff in `./Shake.hs` and `make`).
+- Use and update scripts, in `Justfile`, `Shake.hs`, `tools/` etc.
 
 - Do all releases from a release branch.
 
@@ -217,50 +217,51 @@ Higher things depend on lower things; when doing a release, work upward from the
 Here's an overview of a happy-path hledger release.
 These steps can be interleaved/reordered a little if needed.
 
-### 1 Prep software & docs
+### 1 Release prep
+
 In main repo, release branch:
 1. Check [release readiness](#check-dev-readiness)
 1. Create/switch to release branch, update versions/dates/docs: `just relprep NEW` (single-version releases; for mixed-version releases, take more care)
 1. If not the first release in this branch, cherry-pick changes from master: `magit l o ..master` (minor releases)
+1. Update shell completions: `make -C hledger/shell-completion`
 1. (Could start building/testing/fixing release binaries/workflows/caches here, it takes time: `just relbin`)
 1. Update install script: `hledger-install/hledger-install.sh`
-1. Update changelogs: `./Shake changelogs` & manually edit
-   (*TODO: fix Shake changelogs to not eat whitespace, add to Justfile*)
-1. Update release notes: `doc/relnotes*`
-   (*TODO: fix tools/relnotes.hs, fix md-issue-refs to uniquify, add to Justfile, automate github release notes*)
-1. Update announcements: `doc/ANNOUNCE*` (major releases)
+1. Update changelogs (`**/CHANGES.md`): `./Shake changelogs`, manually edit, `./Shake changelogs -c`
+1. Update release notes (`doc/relnotes*`): `tools/relnotes.hs`, select & transform with `md-issue-refs`, uniquify issue refs, add github nicks to authors, commit\
+   *TODO: fix tools/relnotes.hs (unwrap long lines..), fix md-issue-refs to uniquify, add to Justfile*
+1. Update announcements (`doc/ANNOUNCE*`) (major releases)
 1. Build/test release binaries: `just relbin`. Troubleshoot/repeat as needed.
 
 In site repo:
-1. [Update online manuals](#release-manuals): `site/Makefile`, `site/js/site.js`, `make -C site snapshot-NEW` (major releases)
 1. Update config in `hledger.org.caddy` (@oldmanpath, @unversionedmanpath, any new redirects) (major releases, usually)
+1. [Update online manuals](#release-manuals): bump versions in `site/Makefile`, `site/js/site.js`, `make -C site snapshot-NEW` (major releases)
+   *TODO: snapshot: don't switch to master, don't discard uncommitted changes, record git hash in commit message, clarify late update procedure*
 1. Update install page: `site/src/install.md`
-
-### 2 Prep release
-In main repo, release branch:
-1. Build final release binaries (`just relbin`) and tag the release (`just reltag`)
-  *(TODO: don't add the suggested sixth tag yet, it hinders pushing)*
-1. Download release binaries
-  *(In Safari: don't use the download button; use right-click, Download linked file)*
-1. Push release branch & tags to github: `git push --tags`
-1. Create a [github release draft](#github-release-draft)
+1. Don't push yet. Keep in local branch if needed.
 
 In main repo, master:
-1. Cherry-pick master-appropriate changes from release branch (including hledger-install): `magit l o LASTREL..REL-branch`
-1. Commit any process updates: `doc/RELEASING.md`
+1. Cherry-pick the hledger-install update, and other finished useful updates, from the release branch (maybe not release docs yet): `magit l o LASTREL..REL-branch`
 1. [Bump version](#bump-master-to-next-version) in master (major releases)
-  *(TODO: use the sixth tag command suggested above)*
+1. Add a new dev tag on the "bump version to ..." commit (magit `t t REL.99`)
+
+### 2 Release
 
-### 3 Release
 In main repo, release branch:
-1. Publish on hackage: `just hackageupload`
+1. Build final release binaries (`just relbin`) and tag the release (`just reltag`)
+1. Push release branch & tags (not more than 5 tags at once):\
+   `git push github HEAD REL hledger-REL hledger-lib-REL`\
+   `git push github HEAD hledger-ui-REL hledger-web-REL`\
+   Don't push all tags (don't push the dev tag; if you do, manually delete the corresponding draft release.)\
+1. release.yml creates a draft release when the REL tag is pushed. Check/fix its content.
+   *TODO: fix release.yml*
+1. Publish on hackage (final check): `just hackageupload`
 1. Publish github release
 
 In main repo, master:
 1. Push master: `just push`
 
 In site repo:
-1. Push to github: `git push github` or magit `P p`
+1. Push to github (& site): `git push github` or magit `P p`
 
 In hledger.org [cloudflare caching settings](https://dash.cloudflare.com/f629035917dd3b99b1e37ae20c15ff09/hledger.org/caching/configuration):
 1. Custom Purge `https://hledger.org/js/site.js`  (major release)
@@ -271,7 +272,8 @@ On hledger.org VPS:  (major release, usually)
 1. Test manuals are displaying and highlighting the new version
 1. If needed, `make buildall`
 
-### 4 Announce
+### 3 Announce
+
 (major releases, others if needed)
 1. Update hledger entry at https://plaintextaccounting.org/#pta-apps
 1. hledger matrix & irc chats
@@ -279,7 +281,10 @@ On hledger.org VPS:  (major release, usually)
 1. hledger mail list (& optionally haskell-cafe)
 1. mastodon with #hledger and #plaintextaccounting tags
 
-### 5 Post-release
+### 4 Release followup
+
+1. Cherry-pick any final useful updates from the release branch (eg release docs): `magit l o LASTREL..REL-branch`
+1. Add/commit any process updates: `doc/RELEASING.md`
 1. Monitor packaging status (stackage, brew, docker, linux, nix etc); keep install page updated
 1. Monitor, follow up on issues, especially regressions; keep doc/REGRESSIONS.md updated
 
@@ -309,8 +314,8 @@ Here's more detail of various steps.
 - if there are changes, `./Shake cabalfiles -c`
 
 #### Up-to-date help
-- `./Shake cmdhelp`
-- if there are changes, `./Shake cmdhelp -c`
+- `./Shake cmddocs`
+- if there are changes, `./Shake cmddocs -c`
 
 #### Up-to-date manuals
 - `./Shake mandates`
@@ -516,7 +521,7 @@ In release branch, update
 #### Bump master to next version
 (major release)
 - `./Shake setversion MA.JOR.99 -c`
-- `./Shake cmdhelp [-c]`  (might be empty)
+- `./Shake cmddocs [-c]`  (might be empty)
 - `./Shake mandates`
 - `./Shake manuals -c`
 

doc/ROADMAP.md

@@ -13,10 +13,10 @@ Related: [TODO](TODO.md).
 ## 2024
 
 **Targets:**
+- hledger 1.41, december
+- hledger 1.40, september
+- hledger 1.34, june
 - hledger 1.33, april
-- hledger 1.34, june ? TBD
-- hledger 1.35, september ?
-- hledger 1.36, december ?
 
 ## 2023
 

doc/common.m4

@@ -54,167 +54,88 @@ m4_define({{_timedot_}},   {{```timedot$1```}}        )m4_dnl
 m4_dnl
 m4_dnl Various lists of common command line options.
 m4_dnl Should be kept synced with CliOptions.hs etc.
-m4_define({{_helpoptions_}}, {{
-
-`-h --help`
-: show general or COMMAND help
-
-`--man`
-: show general or COMMAND user manual with man
-
-`--info`
-: show general or COMMAND user manual with info
-
-`--version`
-: show general or ADDONCMD version
-
-`--debug[=N]`
-: show debug output (levels 1-9, default: 1)
-
-}} )m4_dnl
-m4_dnl
-m4_define({{_inputoptions_}}, {{
-
-`-f FILE --file=FILE`
-: use a different input file. For stdin, use - (default: `$LEDGER_FILE` or `$HOME/.hledger.journal`)
-
-`--rules-file=RULESFILE`
-: Conversion rules file to use when reading CSV (default: FILE.rules)
-
-`--separator=CHAR`
-: Field separator to expect when reading CSV (default: ',')
-
-`--alias=OLD=NEW`
-: rename accounts named OLD to NEW
-
-`--pivot FIELDNAME`
-: use some other field or tag for the account name
-
-`-I --ignore-assertions`
-: disable balance assertion checks (note: does not disable balance assignments)
-
-`-s --strict`
-: do extra error checking (check that all posted accounts are declared)
-
-}} )m4_dnl
-m4_dnl
-m4_define({{_reportingoptions_}}, {{
-
-`-b --begin=DATE`
-: include postings/txns on or after this date
-  (will be adjusted to preceding subperiod start when using a report interval)
-
-`-e --end=DATE`
-: include postings/txns before this date
-  (will be adjusted to following subperiod end when using a report interval)
-
-`-D --daily`
-: multiperiod/multicolumn report by day
-
-`-W --weekly`
-: multiperiod/multicolumn report by week
-
-`-M --monthly`
-: multiperiod/multicolumn report by month
-
-`-Q --quarterly`
-: multiperiod/multicolumn report by quarter
-
-`-Y --yearly`
-: multiperiod/multicolumn report by year
-
-`-p --period=PERIODEXP`
-: set start date, end date, and/or reporting interval all at once using [period expressions](hledger.html#period-expressions) syntax
-
-`--date2`
-: match the secondary date instead (see command help for other effects)
-
-`--today=DATE`
-: override today's date (affects relative smart dates, for tests/examples)
-
-`-U --unmarked`
-: include only unmarked postings/txns (can combine with -P or -C)
-
-`-P --pending`
-: include only pending postings/txns
-
-`-C --cleared`
-: include only cleared postings/txns
-
-`-R --real`
-: include only non-virtual postings
-
-`-NUM --depth=NUM`
-: hide/aggregate accounts or postings more than NUM levels deep
-
-`-E --empty`
-: show items with zero amount, normally hidden (and vice-versa in hledger-ui/hledger-web)
-
-`-B --cost`
-: convert amounts to their cost/selling amount at transaction time
-
-`-V --market`
-: convert amounts to their market value in default valuation commodities
-
-`-X --exchange=COMM`
-: convert amounts to their market value in commodity COMM
-
-`--value`
-: convert amounts to cost or market value, more flexibly than -B/-V/-X
-
-`--infer-equity`
-: infer conversion equity postings from costs
-
-`--infer-costs`
-: infer costs from conversion equity postings
-
-`--infer-market-prices`
-: use costs as additional market prices, as if they were P directives
-
-`--forecast`
-: generate transactions from [periodic rules](hledger.html#periodic-transactions),
-: between the latest recorded txn and 6 months from today,
-: or during the specified PERIOD (= is required).
-: Auto posting rules will be applied to these transactions as well.
-: Also, in hledger-ui make future-dated transactions visible.
-
-`--auto`
-: generate extra postings by applying [auto posting rules](hledger.html#auto-postings) to all txns (not just forecast txns)
-
-`--verbose-tags`
-: add visible tags indicating transactions or postings which have been generated/modified
-
-`--commodity-style`
-: Override the commodity style in the output for the specified commodity. For example 'EUR1.000,00'.
-
-`--color=WHEN (or --colour=WHEN)`
-: Should color-supporting commands use ANSI color codes in text output.
-: 'auto' (default): whenever stdout seems to be a color-supporting terminal.
-: 'always' or 'yes': always, useful eg when piping output into 'less -R'.
-: 'never' or 'no': never.
-: A NO_COLOR environment variable overrides this.
-
-`--pretty[=WHEN]`
-: Show prettier output, e.g. using unicode box-drawing characters.
-: Accepts 'yes' (the default) or 'no' ('y', 'n', 'always', 'never' also work).
-: If you provide an argument you must use '=', e.g. '--pretty=yes'.
-
-When a reporting option appears more than once in the command line, the last one takes precedence.
-
-Some reporting options can also be written as [query arguments](hledger.html#queries).
-
-}} )m4_dnl
 m4_dnl
 m4_define({{_generaloptions_}}, {{
+```
+General input/data transformation flags:
+  -f --file=[FMT:]FILE      Read data from FILE, or from stdin if FILE is -,
+                            inferring format from extension or a FMT: prefix.
+                            Can be specified more than once. If not specified,
+                            reads from $LEDGER_FILE or $HOME/.hledger.journal.
+     --rules=RULESFILE      Use rules defined in this rules file for
+                            converting subsequent CSV/SSV/TSV files. If not
+                            specified, uses FILE.csv.rules for each FILE.csv.
+     --alias=A=B|/RGX/=RPL  transform account names from A to B, or by
+                            replacing regular expression matches
+     --auto                 generate extra postings by applying auto posting
+                            rules ("=") to all transactions
+     --forecast[=PERIOD]    Generate extra transactions from periodic rules
+                            ("~"), from after the latest ordinary transaction
+                            until 6 months from now. Or, during the specified
+                            PERIOD (the equals is required). Auto posting rules
+                            will also be applied to these transactions. In
+                            hledger-ui, also make future-dated transactions
+                            visible at startup.
+  -I --ignore-assertions    don't check balance assertions by default
+     --infer-costs          infer conversion equity postings from costs
+     --infer-equity         infer costs from conversion equity postings
+     --infer-market-prices  infer market prices from costs
+     --pivot=TAGNAME        use a different field or tag as account names
+  -s --strict               do extra error checks (and override -I)
+     --verbose-tags         add tags indicating generated/modified data
 
-_inputoptions_
-
-_reportingoptions_
-
-_helpoptions_
-
-_optionnotes_
+General output/reporting flags (supported by some commands):
+  -b --begin=DATE           include postings/transactions on/after this date
+  -e --end=DATE             include postings/transactions before this date
+                            (with a report interval, will be adjusted to
+                            following subperiod end)
+  -D --daily                multiperiod report with 1 day interval
+  -W --weekly               multiperiod report with 1 week interval
+  -M --monthly              multiperiod report with 1 month interval
+  -Q --quarterly            multiperiod report with 1 quarter interval
+  -Y --yearly               multiperiod report with 1 year interval
+  -p --period=PERIODEXP     set begin date, end date, and/or report interval,
+                            with more flexibility
+     --today=DATE           override today's date (affects relative dates)
+     --date2                match/use secondary dates instead (deprecated)
+  -U --unmarked             include only unmarked postings/transactions
+  -P --pending              include only pending postings/transactions
+  -C --cleared              include only cleared postings/transactions
+                            (-U/-P/-C can be combined)
+  -R --real                 include only non-virtual postings
+     --depth=NUM            or -NUM: show only top NUM levels of accounts
+  -E --empty                Show zero items, which are normally hidden.
+                            In hledger-ui & hledger-web, do the opposite.
+  -B --cost                 show amounts converted to their cost/sale amount
+  -V --market               Show amounts converted to their value at period
+                            end(s) in their default valuation commodity.
+                            Equivalent to --value=end.
+  -X --exchange=COMM        Show amounts converted to their value at period
+                            end(s) in the specified commodity.
+                            Equivalent to --value=end,COMM.
+     --value=WHEN[,COMM]    show amounts converted to their value on the
+                            specified date(s) in their default valuation
+                            commodity or a specified commodity. WHEN can be:
+                            'then':     value on transaction dates
+                            'end':      value at period end(s)
+                            'now':      value today
+                            YYYY-MM-DD: value on given date
+  -c --commodity-style=S    Override a commodity's display style.
+                            Eg: -c '$1000.' or -c '1.000,00 EUR'
+     --color=YN --colour    Use ANSI color codes in text output? Can be
+                            'y'/'yes'/'always', 'n'/'no'/'never' or 'auto'.
+     --pretty[=YN]          Use box-drawing characters in text output? Can be
+                            'y'/'yes' or 'n'/'no'.
+                            If YN is specified, the equals is required.
 
+General help flags:
+  -h --help                 show command line help
+     --tldr                 show command examples with tldr
+     --info                 show the manual with info
+     --man                  show the manual with man
+     --version              show version information
+     --debug=[1-9]          show this much debug output (default: 1)
+```
 }} )m4_dnl
 m4_dnl
 m4_dnl A standard description of hledger.

doc/dev.md

@@ -63,7 +63,7 @@ and then symlinked into the hledger_site repo for rendering on hledger.org.
 [codefund bounties](https://github.com/simonmichael/hledger/issues?utf8=✓&q=codefund)\
 [projects.hledger.org](http://projects.hledger.org)\
 [stars.hledger.org](http://stars.hledger.org) our rank among starred haskell projects:\
-2024:#36\
+2024:#35\
 2023:#32\
 2022:#34\
 2020:#36\

doc/ghrelnotes

@@ -1,77 +1,30 @@
-<!--
-       _              _             _            
-  __ _| |__  _ __ ___| |_ __   ___ | |_ ___  ___ 
- / _` | '_ \| '__/ _ \ | '_ \ / _ \| __/ _ \/ __|
-| (_| | | | | | |  __/ | | | | (_) | ||  __/\__ \
- \__, |_| |_|_|  \___|_|_| |_|\___/ \__\___||___/
- |___/                                           
+#!/usr/bin/env bash
+# ghrelnotes REL - generate github release notes for REL.   -*- markdown -*-
 
--->
-<details>
-<summary>
+cd "$(dirname $0)"
+cat <<'END'
 
 ## Release notes
 
-https://hledger.org/relnotes.html#2024-05-02-hledger-1331
+END
+export REL=$1
+../tools/getrelnotes $REL
+envsubst '$REL' <<'END'
+
+<details>
+<summary>
+
+## How to install
 
 </summary>
 
-
-### hledger 1.33.1
-
-- process >=1.6.19.0 seems not strictly needed and is no longer required,
-  improving installability.
-  [#2149]
-
-- `print` and `close` now show a trailing decimal mark on cost amounts also,
-  when needed to disambiguate a digit group mark.
-
-- The balance commands' HTML output now includes digit group marks when
-  appropriate (fixes a regression in 1.25).
-  [#2196]
-
-- The add command no longer shows ANSI escape codes in terminals that
-  don't support them.
-
-- Doc updates:
-  - import: Skipping -> Date skipping, discuss commodity styles more
-  - csv: Amount decimal places: expand, note import behaviour
-
-### hledger-ui 1.33.1
-
-- Require vty-windows-0.2.0.2+ to avoid display problems in recent
-  MS Terminal on Windows.
-
-- process >=1.6.19.0 seems not strictly needed and is no longer required,
-  improving installability.
-  [#2149]
-
-### hledger-web 1.33.1
-
-- Support base64 >=1.0
-
-### credits 1.33.1
-
-Simon Michael.
-
-[#2149]: https://github.com/simonmichael/hledger/issues/2149
-[#2196]: https://github.com/simonmichael/hledger/issues/2196
-
-
-</details>
-
-## Install
-
-This release may arrive in your local packaging system soon - look for green badges at <https://hledger.org/install>.
-Or you could build it yourself from source, as described there.
-Or if there are binaries for your OS and hardware at the bottom of this page, see instructions below.
-<!-- (On linux and mac, the double tar + zip packing is a Github workaround to preserve file permissions.) -->
+This release may arrive in your local packaging system soon - look for green badges at [hledger.org: Install](https://hledger.org/install.html).
+Or you can [build it from source](https://hledger.org/install.html#build-from-source), as described on that page.
+Or you can use the binaries below:
 <!--
 Updates to binaries:
 - YYYY-MM-DD: description. [#NNNN](https://github.com/simonmichael/hledger/issues/NNNN)
 -->
-Once installed, run `hledger`, or perhaps read [hledger.org > Quick start](https://hledger.org/#quick-start).
-
 
 <details>
 <summary>
@@ -84,14 +37,13 @@ At the command line,
 
 ```
 cd /usr/local/bin
-curl -LOC- https://github.com/simonmichael/hledger/releases/download/1.33/hledger-linux-x64.zip    # just rerun if interrupted
-unzip hledger-linux-x64.zip && tar xvf hledger-linux-x64.tar && rm -f hledger-linux-x64.{zip,tar}  # github workaround, preserves permissions
+curl -LOC- https://github.com/simonmichael/hledger/releases/download/$REL/hledger-linux-x64.tar.gz
+tar xvf hledger-linux-x64.tar.gz && rm -f hledger-linux-x64.tar.gz
 cd
-hledger --version    # should show the new version
+hledger --version    # should show $REL
 ```
 
 </details>
-
 <details>
 <summary>
 
@@ -100,7 +52,7 @@ hledger --version    # should show the new version
 </summary>
 
 In a terminal window, run these commands to download, unpack, authorise, and install the binaries in your command line PATH.
-(Don't use your web browser, it won't authorise the binaries.):
+Note, don't use your web browser; it won't authorise the binaries.
 <!--
 (Hopefully these commands are all installed by default; 
 if not, install [XCode Command Line Tools](https://mac.install.guide/commandlinetools/) 
@@ -111,19 +63,17 @@ and/or [Homebrew](https://brew.sh), and let me know.)
 cd /usr/local/bin
 
 # for ARM macs:
-curl -LOC- https://github.com/simonmichael/hledger/releases/download/1.33/hledger-mac-arm64.zip    # just rerun if interrupted
-unzip hledger-mac-arm64.zip && tar xvf hledger-mac-arm64.tar && rm -f hledger-mac-arm64.{zip,tar}  # github workaround, preserves permissions
+curl -LOC- https://github.com/simonmichael/hledger/releases/download/$REL/hledger-mac-arm64.tar.gz
+tar xvf hledger-mac-arm64.tar.gz && rm -f hledger-mac-arm64.tar.gz
 
 # or for Intel macs:
-curl -LOC- https://github.com/simonmichael/hledger/releases/download/1.33/hledger-mac-x64.zip
-unzip hledger-mac-x64.zip && tar xvf hledger-mac-x64.tar && rm -f hledger-mac-x64.{zip,tar}
-
+curl -LOC- https://github.com/simonmichael/hledger/releases/download/$REL/hledger-mac-x64.tar.gz
+tar xvf hledger-mac-x64.tar.gz && rm -f hledger-mac-x64.tar.gz
 cd
-hledger --version    # should show the new version
+hledger --version    # should show $REL
 ```
 
 </details>
-
 <details>
 <summary>
 
@@ -143,13 +93,13 @@ $ENV:PATH += ";"+$HOME+"\bin"
 2. Download and install the release binaries:
 ```
 cd $HOME\bin
-cp hledger.exe hledger.old.exe            # keep a backup of the old executables, if you care
+cp hledger.exe hledger.old.exe    # keep a backup of the old executables, if you like
 cp hledger-ui.exe hledger-ui.old.exe
 cp hledger-web.exe hledger-web.old.exe
-curl https://github.com/simonmichael/hledger/releases/download/1.33/hledger-windows-x64.zip -OutFile hledger-windows-x64.zip
+curl https://github.com/simonmichael/hledger/releases/download/$REL/hledger-windows-x64.zip -OutFile hledger-windows-x64.zip
 Expand-Archive hledger-windows-x64.zip -DestinationPath . -Force
 cd $HOME
-hledger --version      # should show the new version
+hledger --version      # should show $REL
 hledger-ui --version
 hledger-web --version
 ```
@@ -165,7 +115,6 @@ out-file -append -encoding ascii $HOME/.hledger.journal
 Once that journal file exists, you can start hledger-web by double-clicking on the icon if you wish.
 
 </details>
-
 <details>
 <summary>
 
@@ -183,7 +132,7 @@ Once that journal file exists, you can start hledger-web by double-clicking on t
 - for each icon: double-click, uncheck "Always ask before opening this file", click Run
 - close those Explorer windows
 - open a command window (press Windows-r, type CMD, press enter)
-- `hledger --version` should show the new version
+- `hledger --version` should show $REL
 - `echo # >> .hledger.journal` to ensure a default journal file exists. (Important: the doubled **>>** is needed to avoid overwriting existing data.)
 
 Problems:
@@ -194,3 +143,8 @@ Problems:
 
 </details>
 
+\
+Once installed, run `hledger`, and perhaps read [hledger.org: Quick start](https://hledger.org/#quick-start).
+
+</details>
+END

doc/relnotes.md

@@ -74,6 +74,290 @@ Changes in hledger-install.sh are shown
 
 
 
+
+
+## 2024-09-09 hledger-1.40
+
+
+**Config file support, sortable register, FODS output, prettier tables.**
+
+
+### hledger 1.40
+
+
+Fixes
+
+- Account tags (and type declarations) declared in multiple files are now combined correctly. [#2202]
+- Several kinds of report interval now choose a better start date:
+  - `every Nth day of month from DATE` with periodic transactions [#2218]
+  - `every M/D from DATE`
+  - `every Nth WEEKDAY from DATE`
+- The balance commands' html output no longer repeats the "Total" and "Net" headings when the totals row has multiple lines. And the layout has been improved and made more consistent with the text output.
+- The `--tldr` flag now also works with the `tealdeer` tldr client.
+
+Features
+
+- You can now save command line options in a [config file](https://hledger.org/hledger.html#config-files), to be added to your hledger commands either on demand or automatically. (This supersedes the older arguments files feature.) This has been a popular feature request. It has pros and cons, and is experimental; your testing and feedback is welcome. It changes the nature of hledger somewhat, which I have marked by giving this release a more memorable version number (1.40).
+- The balance commands can now output in FODS format, a spreadsheet file format accepted by LibreOffice Calc. If you use LibreOffice this is better than CSV because it works across locales (decimal point vs. decimal comma, character encoding stored in XML header), and it supports fixed header rows and columns, cell types (string vs. number vs. date), separation of number and currency, styles (bold), and borders. You can still extract CSV from FODS/ODS with the ods2csv utility from Hackage. (Henning Thielemann)
+- The `register` report can now be sorted by date, description, account, amount, absolute amount, or a combination of these. (Michael Rees, [#2211])
+
+Improvements
+
+- Command line processing has been overhauled and should be more robust in certain cases, with tweaked error messages and debug output. Command-specific flags can now optionally appear before the command name. (Though writing them afterward is usually more readable. Addon-specific flags must still come last, after `--`.)
+- The `--rules-file` option has been renamed to `--rules`. The old spelling is still supported as a hidden option.
+- Weekly reports' week headings are now more compact, especially in single-year balance reports. ([#2204], Victor Mihalache)
+- The `balance` command with no report interval, and also `balance --budget`, now support html output. (Henning Thielemann)
+- In balance commands' html and csv output, "Total:" and "Net:" headings are now capitalised consistently.
+- `bs`/`cf`/`is` reports now show the report interval in their title.
+- The balance commands' text output with the `--pretty` flag now shows an outer table border and inter-column borders.
+- The `check recentassertions` error message is now more readable.
+- Timedot format now allows comment lines to be indented.
+- When running the `tldr-node-client` client, auto-update of the tldr database is now suppressed.
+- When running a tldr client fails, the warning now mentions the required `--render` flag. [#2201]
+- The error message for unsupported regular expressions like `(?:foo)` has been improved.
+- `--debug` has moved to "General help flags", making it available in more situations.
+- Some verbose debug output from command line processing has been demoted to level 2.
+- Parsing timedot files now gives debug output at level 9.
+- Allow doclayout 0.5.
+
+Docs
+
+- The hledger/hledger-ui/hledger-web manuals now list all command options as shown by `--help`.
+- Added an example config file, `hledger.conf.sample`.
+- The `diff` and `prices` commands' help layout has been improved.
+- `add`'s doc described the effect of `D` wrongly, now fixed.
+- Date adjustments: rewrites and corrections
+- Period headings: added
+- Input: clarify that multiple -f options are allowed
+- Scripts and add-ons: edits, list add-ons again
+- Timeclock: edits, fix `ti`/`to` scripts
+- Fixed "hledger and Ledger" links [hledger_site#112]
+- examples/csv: Monzo CSV rules added
+- examples/csv: Tiller CSV rules added
+- examples/csv: Nordea CSV rules added (Arto Jonsson)
+
+Scripts/addons
+
+- `bin/bashrc` updates; add years, eachyear scripts
+- `bin/hledger-simplebal`: ignore config files
+- `bin/hledger-script-example`: explain shebang commands better
+- `bin/hledger-register-max`: update/fix
+
+
+### hledger-ui 1.40
+
+
+Improvements
+
+- The menu screen now supports the shift arrow and shift T keys, and its header shows any narrowed time period in effect, like other screens.
+- Support brick 2.4.
+
+Docs
+
+- The description of the shift-T key (set period to today) has been fixed.
+- The shift arrow keys and period narrowing have been clarified
+
+
+### hledger-web 1.40
+
+
+Improvements
+
+- We now guess a more robust base url when `--base-url` is not specified. Now relative links to js/css resources will use the same hostname etc. that the main page was requested from, making them work better when accessed via multiple IP addresses/hostnames without an explicit `--base-url` setting. (A followup to [#2099], [#2100] and [#2127].)
+- We now require a http[s] scheme in `--base-url`'s value. Previously it accepted just a hostname, and generated bad links.
+
+
+### project changes 1.40
+
+
+Docs
+
+- In the hledger 1.29 release notes, Date adjustments has had some corrections.
+- Github release notes template cleanups; fix mac, linux install commands.
+- README: fixed contributors link.
+- RELEASING: updates
+
+Scripts/addons
+
+- hledger-install: cleanups, bump versions, perhaps fix hledger-interest install
+- hledger-install: clarify some stack/cabal setup messages
+
+Infrastructure/Misc
+
+- Shake.hs: fix partial warnings
+- Shake cmdhelp: renamed to cmddocs, and it now also updates the options listed in the manuals, and shows progress output.  It should be run (at some point) after changing commands' docs or options.
+- Shake txtmanuals: silence all but wide table warnings
+- just file cleanups; update to support just 1.28+
+- just twih: date fixes
+- just ghci: -fobject-code was a mistake, keep everything interpreted
+- just functest: try again to reduce rebuilding/slowdowns when testing
+- just installrel: update for .tar.gz
+- ci scripts: cleanup, fix a macos-ism
+
+
+### credits 1.40
+
+Simon Michael (@simonmichael),
+Henning Thielemann (@thielema),
+Michael Rees (@reesmichael1),
+Arto Jonsson (@artoj),
+Victor Mihalache (@victormihalache).
+
+
+[#2099]: https://github.com/simonmichael/hledger/issues/2099
+[#2100]: https://github.com/simonmichael/hledger/issues/2100
+[#2127]: https://github.com/simonmichael/hledger/issues/2127
+[#2201]: https://github.com/simonmichael/hledger/issues/2201
+[#2202]: https://github.com/simonmichael/hledger/issues/2202
+[#2204]: https://github.com/simonmichael/hledger/issues/2204
+[#2211]: https://github.com/simonmichael/hledger/issues/2211
+[#2218]: https://github.com/simonmichael/hledger/issues/2218
+
+
+
+## 2024-06-01 hledger-1.34
+### hledger 1.34
+
+**--tldr (short command examples), reorganised commands list, ghc-debug support**
+
+
+Breaking changes
+
+- `check ordereddates` no longer supports `--date2`. Also (not a breaking change): `--date2` and secondary dates are now officially [deprecated](https://hledger.org/1.34/hledger.html#secondary-dates) in hledger, though kept for compatibility.
+
+Features
+
+- You can now get a quick list of example command lines for hledger or its most useful subcommands by adding the `--tldr` flag (or just `--tl`).  For best appearance you should install the [`tldr`][tldr] client, though it's not required.
+
+  These short "tldr pages" are a great counterbalance to verbose PTA docs. You can also use `tldr` without hledger to view the latest versions, or translations: `tldr hledger[-COMMAND]`. Or you can [browse tldr pages online](https://tldr.inbrowser.app/search?query=hledger+). Consider contributing translations! More tips at <https://github.com/simonmichael/hledger/tree/master/doc/tldr>.
+
+[tldr]: https://tldr.sh
+
+Improvements
+
+- The `hledger` commands list has been reorganised, with commands listed roughly in the order you'll need them.
+
+- The general flags descriptions in `--help` have been updated and grouped.
+
+- Correctness checks now run in a documented order.  `commodities` are now checked before `accounts`, and `tags` before `recentassertions`. When both `ordereddates` and `assertions` checks are enabled, `ordereddates` now runs first, giving more useful error messages.
+
+- `-I`/`--ignore-assertions` is now overridden by `-s`/`--strict` (or `check assertions`), enabling more flexible workflows. Eg you can `alias hl="hledger -I"` to delay balance assertions checking until you add `-s` to commands.
+
+- `--color` and `--pretty` now also accept `y` or `n` as argument.
+
+- When built with the `ghcdebug` flag and started with `--debug=-1`, hledger can be controlled by [ghc-debug] clients like ghc-debug-brick or a ghc-debug query script, for analysing memory/profile info.
+
+[ghc-debug]: https://gitlab.haskell.org/ghc/ghc-debug
+
+Fixes
+
+- `hledger COMMAND --man` and `hledger help TOPIC --man` now properly scroll the man page to the TOPIC or COMMAND heading. The exact/prefix matching behaviour has been clarified in `help --help`.
+
+- In journal files, `include` directives with trailing whitespace are now parsed correctly.
+
+- The help command's help flags are now consistent with other commands (and it has `--debug` as a hidden flag).
+
+- Build errors with GHC 8.10 have been fixed. [#2198]
+
+Docs
+
+- The tables of contents on hledger.org pages now just list top-level headings, (and the hledger manual structure has been adjusted for this). This makes the hledger manual on hledger.org more scannable and less scary.
+- add: drop lengthy transcript, add simpler example commands (from tldr)
+- Amount formatting: move down, it's not the best first topic
+- balance: mention the `--summary-only` flag
+- check: expand check descriptions
+- examples: CSV rules: vanguard, fidelity, paypal updates
+- Generating data: rewrite
+- JSON output: link to OpenAPI spec
+- manuals: synopsis, options cleanup/consistency
+- Options: correction, NO_COLOR does not override --color
+- PART 4: COMMANDS: reorganise into groups, like the CLI commands list.
+- Period expressions: mention last day of month adjusting [#2005]
+- Secondary dates: expand, and declare them deprecated.
+- Time periods cleanup, simplify markup
+- Unicode characters: mention UTF-8 on windows
+
+Scripts/addons
+
+- Added `hledger-pricehist`, an alias for the `pricehist` market price fetcher so that it can appear in hledger's commands list.
+
+[#2005]: https://github.com/simonmichael/hledger/issues/2005
+[#2198]: https://github.com/simonmichael/hledger/issues/2198
+
+
+### hledger-ui 1.34
+
+
+Features
+
+- You can now get a quick list of example command lines by running with `--tldr` (or just `--tl`). For best appearance, install the [`tldr`][tldr] client, though it's not required.
+
+Improvements
+
+- The general flags in `--help` have been updated and grouped, consistent with hledger.
+
+- When built with the `ghcdebug` flag and started with `--debug=-1`, hledger-ui can be controlled by [ghc-debug] clients like ghc-debug-brick or a ghc-debug query script, for analysing memory/profile info.
+
+[tldr]: https://tldr.sh
+[ghc-debug]: https://gitlab.haskell.org/ghc/ghc-debug
+
+
+### hledger-web 1.34
+
+
+Features
+
+- You can now get a quick list of example command lines by running with `--tldr` (or just `--tl`). For best appearance, install the [`tldr`][tldr] client, though it's not required.
+
+Improvements
+
+- The general flags in `--help` have been updated and grouped, consistent with hledger.
+
+- When built with the `ghcdebug` flag and started with `--debug=-1`, hledger-web can be controlled by [ghc-debug] clients like ghc-debug-brick or a ghc-debug query script, for analysing memory/profile info.
+
+Docs
+
+- A basic [OpenAPI specification][openapi.yaml] is provided for hledger-web's JSON-over-HTTP API. This is also applicable to `hledger print`'s JSON output format.
+
+[ghc-debug]: https://gitlab.haskell.org/ghc/ghc-debug
+[openapi.yaml]: https://github.com/simonmichael/hledger/blob/master/hledger-web/config/openapi.yaml
+[tldr]: https://tldr.sh
+
+
+### project changes 1.34
+
+
+Docs
+
+- move release notes from the hledger_site repo to the main hledger repo
+- github release notes: show the release notes, hide the install instructions by default
+- github release notes: improve windows install commands
+- github release notes: start mentioning github usernames, enabling the Contributors avatar list
+- dev docs: new Developer FAQ, Contributor Quick Start updates
+
+Scripts/addons
+
+- `hledger-install.sh` now uses stackage nightly, and a failure on non-Windows platforms has been fixed.
+
+Infrastructure/misc
+
+- A new `release` workflow creates github releases, uploads release binaries and generates release notes.
+- Github release binaries for mac and linux are now in .tar.gz format (no longer tarred and zipped). 
+- There is a new `oldest` workflow for testing the oldest GHC we support (currently 8.10.7).
+- The `binaries-mac-x64` workflow has been bumped from GHC 9.4 to 9.8.
+- The master branch's `ci` workflow has been updated to Ubuntu 24.04 and uses the preinstalled GHC & stack, saving some work.
+- `md-issue-refs` helps generate markdown issue links.
+- `relnotes.hs` helps generate release notes from changelogs.
+- The project `Makefile` has now been fully replaced by `Justfile`.
+
+
+### credits 1.34
+
+
+Simon Michael (@simonmichael)
+
+
+
 ## 2024-05-02 hledger-1.33.1
 
 ### hledger 1.33.1
@@ -111,9 +395,8 @@ Changes in hledger-install.sh are shown
 
 ### credits 1.33.1
 
-Simon Michael.
+- Simon Michael (@simonnmichael)
 
-[#2149]: https://github.com/simonmichael/hledger/issues/2149
 [#2149]: https://github.com/simonmichael/hledger/issues/2149
 [#2196]: https://github.com/simonmichael/hledger/issues/2196
 
@@ -1511,10 +1794,9 @@ Features
 
 - Periodic transactions and multi-period reports can now start on any date.
   To enable this while still maintaining pretty good backward compatibility,
-  hledger now treats inferred dates, and dates where the day is unspecified,
-  as "flexible" (which can be automatically adjusted to interval boundaries),
-  and dates specified to the day as "exact" (which can not).
-  Eg:
+  hledger now keeps explicitly specified start dates as they are,
+  but still automatically adjusts inferred start dates to interval boundaries.
+  This means, eg:
   
   - A periodic rule like `~ monthly from 2023-01-15` now works as
     you'd expect instead of raising an error. This also improves
@@ -1523,13 +1805,10 @@ Features
   - Period options like `-p 'monthly from 2023/1/15'` or `-M -b 2023/1/15`
     now start the report on exactly 1/15 instead of being adjusted to 1/1.
   
-  Note: periods using `in` may look partial but are considered to specify exact dates.
-  So weekly reports such as `-p 'weekly in 2023-01'`, which previously
-  were adjusted to start on a monday, will now start exactly on 2023-01-01.
-  This can also cause more verbose column headings.
-	To guarantee simple week headings, you must now start such reports
-  exactly on a monday, eg `-p 'weekly from 2022-12-26 to 2023-02'`.
-  ([#1982])
+  - `-p 'weekly in 2023-01'`, which previously was adjusted to start on a monday,
+    now starts exactly on 2023-01-01. This can cause more verbose report headings.
+	To ensure simple week headings, now weekly reports must start on a monday,
+    eg `-p 'weekly from 2022-12-26 to 2023-02'`. ([#1982])
 
 - You can now freely combine @/@@ notation and conversion postings
   in a single transaction. This can help readability, and also allows

doc/tldr/README.md

@@ -0,0 +1,38 @@
+[tldr-pages](https://tldr.sh) provides minimal, example-focussed doc pages for command line tools and their subcommands.
+
+tldr pages are short but high value, and a great counterbalance to PTA's verbose docs.
+So they are worth prioritising and maintaining.
+gutjuri made the first hledger tldr page in 2022, and sm added more in 2024.
+
+This directory has local copies of all [hledger-related tldr pages](https://github.com/search?q=repo%3Atldr-pages%2Ftldr%20hledger&type=code)
+The tldr-pages repo has the master copies, so they should be copied here periodically (eg before release).
+These docs are crafted first to suit tldr and its style rules, but we'll reuse them where we can.
+Eg they are now embedded in the hledger tools and accessible with --tldr.
+
+<https://tldr.inbrowser.app> is an online (& offline) tldr viewer,
+where you can search for [hledger command examples](https://tldr.inbrowser.app/search?query=hledger+).
+
+You can also make a web browser keyword for it ([firefox & chrome][1]; [safari][2]),
+like `tldr CMD` = `https://tldr.inbrowser.app/search?query=CMD`.
+Then in your browser's address bar you can type
+`tldr`, `tldr hledger`, `tldr hledger-web`, `tldr hledger-balancesheet`, etc.
+
+In the search field's gear icon you can configure preferred languages.
+If you speak a language other than english, please consider making a few [translations](https://github.com/tldr-pages/tldr/blob/main/CONTRIBUTING.md#translations)!
+It is relatively easy. You can do it entirely in your web browser if you have a Github account:
+
+1. Starting with eg https://tldr.inbrowser.app/pages/common/hledger
+2. click "Find this page on GitHub"
+3. click the "Copy raw file" icon (two squares)
+4. navigate to eg https://github.com/tldr-pages/tldr/tree/main/pages.de/common
+5. click "Add file" > "Create new file"
+6. reuse the original file name, paste the original content
+7. translate the content
+8. commit
+9. send a pull request
+
+Here is the [tldr translations status](https://lukwebsforge.github.io/tldri18n).
+
+
+[1]: https://karl-voit.at/browser-keywords
+[2]: http://safarikeywordsearch.aurlien.net

doc/tldr/hledger-accounts.md

@@ -0,0 +1,36 @@
+# hledger accounts
+
+> List account names.
+> More information: <https://hledger.org/hledger.html#accounts>.
+
+- Show all accounts used or declared in the default journal file:
+
+`hledger accounts`
+
+- Show accounts used by transactions:
+
+`hledger accounts --used`
+
+- Show accounts declared with account directives:
+
+`hledger accounts --declared`
+
+- Add new account directives, for accounts used but not declared, to the journal:
+
+`hledger accounts --undeclared --directives >> {{2024-accounts.journal}}`
+
+- Show accounts with `asset` in their name, and their declared/inferred types:
+
+`hledger accounts asset --types`
+
+- Show accounts of the `Asset` type:
+
+`hledger accounts type:A`
+
+- Show the first two levels of the accounts hierarchy:
+
+`hledger accounts --tree --depth 2`
+
+- Short form of the above:
+
+`hledger acc -t -2`

doc/tldr/hledger-add.md

@@ -0,0 +1,24 @@
+# hledger add
+
+> Record new transactions with interactive prompting in the console.
+> More information: <https://hledger.org/hledger.html#add>.
+
+- Record new transactions, saving to the default journal file:
+
+`hledger add`
+
+- Add transactions to `2024.journal`, but also load `2023.journal` for completions:
+
+`hledger add --file {{path/to/2024.journal}} --file {{path/to/2023.journal}}`
+
+- Provide answers for the first four prompts:
+
+`hledger add {{today}} '{{best buy}}' {{expenses:supplies}} '{{$20}}'`
+
+- Show `add`'s options and documentation with `$PAGER`:
+
+`hledger add --help`
+
+- Show `add`'s documentation with `info` or `man` if available:
+
+`hledger help add`

doc/tldr/hledger-aregister.md

@@ -0,0 +1,20 @@
+# hledger aregister
+
+> Show the transactions and running balances in one account, with each transaction on one line.
+> More information: <https://hledger.org/hledger.html#aregister>.
+
+- Show transactions and running balance in the `assets:bank:checking` account:
+
+`hledger aregister assets:bank:checking`
+
+- Show transactions and running balance in the first account named `*savings*`:
+
+`hledger aregister savings`
+
+- Show the checking account's cleared transactions, with a specified width:
+
+`hledger aregister checking --cleared --width {{120}}`
+
+- Show the checking register, including transactions from forecast rules:
+
+`hledger aregister checking --forecast`

doc/tldr/hledger-balance.md

@@ -0,0 +1,37 @@
+# hledger balance
+
+> A flexible, general purpose "summing" report that shows accounts with some kind of numeric data.
+> This can be balance changes per period, end balances, budget performance, unrealised capital gains, etc.
+> More information: <https://hledger.org/hledger.html#balance>.
+
+- Show the balance change in all accounts from all postings over all time:
+
+`hledger balance`
+
+- Show the balance change in accounts named `*expenses*`, as a tree, summarising the top two levels only:
+
+`hledger balance {{expenses}} --tree --depth {{2}}`
+
+- Show expenses each month, and their totals and averages, sorted by total; and their monthly budget goals:
+
+`hledger balance {{expenses}} --monthly --row-total --average --sort-amount --budget`
+
+- Similar to the above, shorter form, matching accounts by `Expense` type, as a two level tree without squashing boring accounts:
+
+`hledger bal type:{{X}} -MTAS --budget -t -{{2}} --no-elide`
+
+- Show end balances (including from postings before the start date), quarterly in 2024, in accounts named `*assets*` or `*liabilities*`:
+
+`hledger balance --historical --period '{{quarterly in 2024}}' {{assets}} {{liabilities}}`
+
+- Similar to the above, shorter form; also show zero balances, sort by total and summarise to three levels:
+
+`hledger bal -HQ date:{{2024}} type:{{AL}} -ES -{{3}}`
+
+- Show investment assets' market value in base currency at the end of each quarter:
+
+`hledger bal -HVQ {{assets:investments}}`
+
+- Show unrealised capital gains/losses from market price changes in each quarter, for non-cryptocurrency investment assets:
+
+`hledger bal --gain -Q {{assets:investments}} not:{{cryptocurrency}}`

doc/tldr/hledger-balancesheet.md

@@ -0,0 +1,33 @@
+# hledger balancesheet
+
+> Show the end balances in asset and liability accounts.
+> Amounts are shown with normal positive sign, as in conventional financial statements.
+> More information: <https://hledger.org/hledger.html#balancesheet>.
+
+- Show the current balances in `Asset` and `Liability` accounts, excluding zeros:
+
+`hledger balancesheet`
+
+- Show just the liquid assets (`Cash` account type):
+
+`hledger balancesheet type:C`
+
+- Include accounts with zero balances, and show the account hierarchy:
+
+`hledger balancesheet --empty --tree`
+
+- Show the balances at the end of each month:
+
+`hledger balancesheet --monthly`
+
+- Show the balances' market value in home currency at the end of each month:
+
+`hledger balancesheet --monthly -V`
+
+- Show quarterly balances, with just the top two levels of account hierarchy:
+
+`hledger balancesheet --quarterly --tree --depth 2`
+
+- Short form of the above, and generate HTML output in `bs.html`:
+
+`hledger bs -Qt -2 -o bs.html`

doc/tldr/hledger-import.md

@@ -0,0 +1,28 @@
+# hledger import
+
+> Import new transactions from one or more data files to the main journal.
+> More information: <https://hledger.org/hledger.html#import>.
+
+- Import new transactions from `bank.csv`, using `bank.csv.rules` to convert:
+
+`hledger import {{path/to/bank.csv}}`
+
+- Show what would be imported from these two files, without doing anything:
+
+`hledger import {{path/to/bank1.csv}} {{path/to/bank2.csv}} --dry-run`
+
+- Import new transactions from all CSV files, using the same rules for all:
+
+`hledger import --rules {{common.rules}} *.csv`
+
+- Show conversion errors or results while editing `bank.csv.rules`:
+
+`watchexec -- hledger -f {{path/to/bank.csv}} print`
+
+- Mark `bank.csv`'s current data as seen, as if already imported:
+
+`hledger import --catchup {{path/to/bank.csv}}`
+
+- Mark `bank.csv` as all new, as if not yet imported:
+
+`rm -f .latest.bank.csv`

doc/tldr/hledger-incomestatement.md

@@ -0,0 +1,21 @@
+# hledger incomestatement
+
+> Show revenue inflows and expense outflows during the report period.
+> Amounts are shown with normal positive sign, as in conventional financial statements.
+> More information: <https://hledger.org/hledger.html#incomestatement>.
+
+- Show revenues and expenses (changes in `Revenue` and `Expense` accounts):
+
+`hledger incomestatement`
+
+- Show revenues and expenses each month:
+
+`hledger incomestatement --monthly`
+
+- Show monthly revenues/expenses/totals, largest first, summarised to 2 levels:
+
+`hledger incomestatement --monthly --row-total --average --sort --depth 2`
+
+- Short form of the above, and generate HTML output in `is.html`:
+
+`hledger is -MTAS -2 -o is.html`

doc/tldr/hledger-print.md

@@ -0,0 +1,32 @@
+# hledger print
+
+> Show full journal entries, representing transactions.
+> More information: <https://hledger.org/hledger.html#print>.
+
+- Show all transactions in the default journal file:
+
+`hledger print`
+
+- Show transactions, with any implied amounts or costs made explicit:
+
+`hledger print --explicit --infer-costs`
+
+- Show transactions from two specified files, with amounts converted to cost:
+
+`hledger print --file {{path/to/2023.journal}} --file {{path/to/2024.journal}} --cost`
+
+- Show `$` transactions in `*food*` but not `*groceries*` accounts this month:
+
+`hledger print cur:\\$ food not:groceries date:thismonth`
+
+- Show transactions of amount 50 or more, with `whole foods` in their description:
+
+`hledger print amt:'>50' desc:'whole foods'`
+
+- Show cleared transactions, with `EUR` amounts rounded and with decimal commas:
+
+`hledger print --cleared --commodity '1000, EUR' --round hard`
+
+- Write transactions from `foo.journal` as a CSV file:
+
+`hledger print --file {{path/to/foo.journal}} --output-file {{path/to/output_file.csv}}`

doc/tldr/hledger-ui.md

@@ -0,0 +1,32 @@
+# hledger-ui
+
+> A terminal interface (TUI) for `hledger`, a robust, friendly plain text accounting app.
+> More information: <https://hledger.org/hledger-ui.html>.
+
+- Start in the main menu screen, reading from the default journal file:
+
+`hledger-ui`
+
+- Start with a different color theme:
+
+`hledger-ui --theme {{terminal|greenterm|dark}}`
+
+- Start in the balance sheet accounts screen, showing hierarchy down to level 3:
+
+`hledger-ui --bs --tree --depth 3`
+
+- Start in this account's screen, showing cleared transactions, and reload on change:
+
+`hledger-ui --register {{assets:bank:checking}} --cleared --watch`
+
+- Read two journal files, and show amounts as current value when known:
+
+`hledger-ui --file {{path/to/2024.journal}} --file {{path/to/2024-prices.journal}} --value now`
+
+- Show the manual in Info format, if possible:
+
+`hledger-ui --info`
+
+- Display help:
+
+`hledger-ui --help`

doc/tldr/hledger-web.md

@@ -0,0 +1,32 @@
+# hledger-web
+
+> A web interface and API for `hledger`, a robust, friendly plain text accounting app.
+> More information: <https://hledger.org/hledger-web.html>.
+
+- Start the web app, and a browser if possible, for local viewing and adding only:
+
+`hledger-web`
+
+- As above but with a specified file, and allow editing of existing data:
+
+`hledger-web --file {{path/to/file.journal}} --allow edit`
+
+- Start just the web app, and accept incoming connections to the specified host and port:
+
+`hledger-web --serve --host {{my.host.name}} --port 8000`
+
+- Start just the web app's JSON API, and allow only read access:
+
+`hledger-web --serve-api --host {{my.host.name}} --allow view`
+
+- Show amounts converted to current market value in your base currency when known:
+
+`hledger-web --value now --infer-market-prices`
+
+- Show the manual in Info format if possible:
+
+`hledger-web --info`
+
+- Display help:
+
+`hledger-web --help`

doc/tldr/hledger.md

@@ -0,0 +1,37 @@
+# hledger
+
+> A robust, friendly plain text accounting app (command line version).
+> See also: `hledger-ui` for TUI, `hledger-web` for web interface.
+> More information: <https://hledger.org/hledger.html>.
+
+- Record new transactions interactively, saving to the default journal file:
+
+`hledger add`
+
+- Import new transactions from `bank.csv`, using `bank.csv.rules` to convert:
+
+`hledger import {{path/to/bank.csv}}`
+
+- Print all transactions, reading from multiple specified journal files:
+
+`hledger print --file {{path/to/prices-2024.journal}} --file {{path/to/prices-2023.journal}}`
+
+- Show all accounts, as a hierarchy, and their types:
+
+`hledger accounts --tree --types`
+
+- Show asset and liability account balances, including zeros, hierarchically:
+
+`hledger balancesheet --empty --tree --no-elide`
+
+- Show monthly incomes/expenses/totals, largest first, summarised to 2 levels:
+
+`hledger incomestatement --monthly --row-total --average --sort --depth 2`
+
+- Show the `assets:bank:checking` account's transactions and running balance:
+
+`hledger aregister assets:bank:checking`
+
+- Show the amount spent on food from the `assets:cash` account:
+
+`hledger print assets:cash | hledger -f- -I aregister expenses:food`

docker-arm32v7/Dockerfile

@@ -33,7 +33,7 @@ RUN cabal update
 COPY . /hledger
 
 # Build static hledger binary
-RUN cd /hledger && cabal build --enable-executable-static --with-compiler=/usr/local/bin/ghc-8.10.4 hledger
+RUN cd /hledger && cabal build --enable-executable-static --with-compiler=/usr/local/bin/ghc-8.10.4 --ghc-options=-Werror hledger
 
 # Strip symbols from binary
 RUN cp /hledger/dist-newstyle/build/arm-linux/ghc-*/hledger-*/x/hledger/build/hledger/hledger /root/ && strip /root/hledger

examples/csv/monzo.csv.rules

@@ -0,0 +1,21 @@
+# Transaction ID,Date,Time,Type,Name,Emoji,Category,Amount,Currency,Local amount,Local currency,Notes and #tags,Address,Receipt,Description,Category split,Money Out,Money In,Balance,Balance currency
+# mm_0000AkoAEwpjaVPg6gKkT3,10/08/2024,01:09:44,Card payment,Berry Farm's Gosq.,,Personal Expenses,-13.75,GBP,-24.00,CAD,,Rr 1,,SQ *BERRY FARM'S GOSQ. BLACKFALDS    CAN,,-13.75,,569.97,GBP
+
+skip 1
+fields Transaction_ID,Date,Time,Type,Name,Emoji,Category,Amount,Currency,Local_amount,Local_currency,Notes_and_tags,Address,Receipt,Description,Category_split,Money_Out,Money_In,Balance,Balance_currency
+date-format %d/%m/%Y
+description %Name | %Type %Emoji %Description %Notes_and_tags
+comment \naddress: %Address
+account1 assets:monzo
+account2 %Category
+currency %Currency 
+
+if %category personal expenses
+ account2 expenses:personal
+
+if %category income
+ account2 revenues
+
+if %category travel
+ account2 expenses:travel
+

examples/csv/nordea.rules

@@ -0,0 +1,25 @@
+skip 1
+separator ;
+fields date_or_status, amount, from, to, name, note, code, balance, currency
+newest-first
+decimal-mark ,
+date-format %Y/%m/%d
+status *
+
+description %from | %to %note
+
+# Nordea's transaction CSV has two sections:
+#
+# 1. A list of pending transactions where the date field has the value
+#    "Varaus" (in the Finnish version of the Netbank).
+#
+# 2. List of cleared transactions.
+#
+# Skip the pending transactions and import only cleared.
+if %date_or_status Varaus
+	skip
+
+date %date_or_status
+
+if %currency EUR
+	currency €

examples/csv/tiller.csv.rules

@@ -0,0 +1,77 @@
+# hledger rules for CSV from Tiller's "raw data" spreadsheet, circa 2022,
+# with columns rearranged as follows:
+# Date,"Short Description",Description,"Full Description",Amount,"Category Hint",Institution,Account,"Account #","Check Number",Month,Week,"Transaction ID","Date Added","Dup Score","Dup Match"
+
+skip        1
+fields      Date,Short_Description,Description,Full_Description,Amount,Category_Hint,Institution,Account,Account_No,Check_Number,Month,Week,Transaction_ID,Date_Added,Dup_Score,Dup_Match
+date-format %-m/%-d/%Y
+newest-first
+# status      *
+# code        %Check_Number
+description %Description
+if ,Not Available,
+ description %Description
+comment     %Full_Description
+
+
+## account1
+
+# generic
+account1    sm:assets:%Institution:%Account:%Account_No
+
+# more specific
+if ,Wells Fargo - Bank,Business Checking,
+ account1 JS:assets:bank:wf:bchecking
+if ,Wells Fargo - Bank,Business Savings,
+ account1 JS:assets:bank:wf:bsavings:capital
+if ,Wells Fargo - Bank,Checking,
+ account1 sm:assets:bank:wf:pchecking
+if ,Wells Fargo - Bank,Savings,
+ account1 sm:assets:bank:wf:psavings
+if ,PayPal Balance,,
+ account1 sm:assets:online:paypal
+if ,Bank of Ireland.*,Current Account,
+ account1 sm:assets:bank:bi:ichecking
+
+## account2, etc.
+
+# use Tiller's category hint as default
+account2 sm:expenses:unknown:%Category_Hint?
+
+# Try to recognise and categorise transfers between accounts.
+# The descriptions here will not always be right.
+# We don't deduplicate; one imported txn will need to be removed manually.
+
+if 
+TRANSFER FROM BUSINESS CHECKING
+ account2 JS:assets:bank:wf:bchecking
+
+if
+TRANSFER TO CHECKING
+ account2 sm:assets:bank:wf:pchecking
+# description personal savings
+
+if ATM WITHDRAWAL
+ account2 sm:assets:cash:wallet
+
+## generic paypal cleanups
+# not using Tiller for paypal, it's too lossy (missing important fields, not currency aware)
+#
+# if ,Paypal,
+#  description paypal transfer
+#
+# if ,Paypal,.*,Checking,
+#  account2 sm:assets:online:paypal
+#
+# if PAYPAL INST XFER
+#  account2 sm:assets:online:paypal
+#
+# # the bank will generate this same txn, but this paypal one shows up quicker
+# if Transfer from Bank Account.*,PayPal
+#  account2 sm:assets:bank:wf:pchecking
+
+#
+
+# common categorising rules
+include common.rules
+

hledger-install/hledger-install.sh

@@ -1,18 +1,36 @@
 #!/bin/bash
 # Easy hledger installation script for POSIX systems, requiring bash and some other POSIX tools.
-# This is based on a snapshot of get-stack.sh which was copyright (c) 2015-2017, Stack contributors.
-
-# This was disabled as a workaround for https://github.com/simonmichael/hledger/issues/714
-# It has been left off so that one uninstallable tool doesn't block the others.
-# (XXX though, try_install is supposed to continue on failure)
-#set -e
-set -o pipefail
-
-# This install script's name (can't use $0 when it's piped into bash).
-HLEDGER_INSTALL_TOOL=hledger-install.sh
+# This was based on get-stack.sh which was copyright (c) 2015-2017, Stack contributors.
 
 # This install script's version.
-HLEDGER_INSTALL_VERSION=20240502
+HLEDGER_INSTALL_VERSION=20240912
+
+# Package versions to be installed by this install script.
+# Keep synced with the tools above.
+# When changing remember to also bump HLEDGER_INSTALL_VERSION.
+# Official:
+HLEDGER_LIB_VERSION=1.40
+HLEDGER_VERSION=1.40
+HLEDGER_UI_VERSION=1.40
+HLEDGER_WEB_VERSION=1.40
+# Third-party:
+HLEDGER_IADD_VERSION=1.3.21
+HLEDGER_INTEREST_VERSION=1.6.6
+HLEDGER_EDIT_VERSION=1.14.0
+HLEDGER_PLOT_VERSION=1.14.0
+HLEDGER_LOTS_VERSION=0.4.2
+PRICEHIST_VERSION=1.4.9
+
+# stackage snapshot to use when installing with stack.
+# You can try specifying a different stackage version here, or
+# commenting out this line to use your current global resolver,
+# to avoid unnecessary building.
+STACKAGE_SNAPSHOT=nightly-2024-09-04
+
+# If nny required haskell dependencies aren't in the above stackage snapshot,
+# list them here in this format: "PKG1-VER1 PKG2-VER2.."
+# (one line, don't break interpolation in commands below).
+STACK_EXTRA_DEPS="brick-2.4"
 
 # Tools to be installed by this install script, official tools first.
 # Keep synced with the package versions below.
@@ -36,21 +54,16 @@ cabal \
 pip \
 "
 
-# Package versions to be installed by this install script.
-# Keep synced with the tools above. 
-# When changing remember to also bump HLEDGER_INSTALL_VERSION.
-# Official:
-HLEDGER_LIB_VERSION=1.33.1
-HLEDGER_VERSION=1.33.1
-HLEDGER_UI_VERSION=1.33.1
-HLEDGER_WEB_VERSION=1.33.1
-# Third-party:
-HLEDGER_IADD_VERSION=1.3.21
-HLEDGER_INTEREST_VERSION=1.6.6
-HLEDGER_EDIT_VERSION=1.14.0
-HLEDGER_PLOT_VERSION=1.14.0
-HLEDGER_LOTS_VERSION=0.4.2
-PRICEHIST_VERSION=1.4.6
+##############################################################################
+
+# This was disabled as a workaround for https://github.com/simonmichael/hledger/issues/714
+# It has been left off so that one uninstallable tool doesn't block the others.
+# (XXX though, try_install is supposed to continue on failure)
+#set -e
+set -o pipefail
+
+# This install script's name (can't use $0 when it's piped into bash).
+HLEDGER_INSTALL_TOOL=hledger-install.sh
 
 # this script's one-line description
 HLEDGER_INSTALL_DESC="$HLEDGER_INSTALL_TOOL version $HLEDGER_INSTALL_VERSION to install hledger $HLEDGER_VERSION and related tools"
@@ -90,19 +103,6 @@ HERE
 # the oldest version of stack that might possibly work: perhaps 2.5.1
 STACK_MIN_VERSION=2.5.1
 
-# stackage snapshot to use when installing with stack.
-# You can try specifying a different stackage version here, or 
-# commenting out this line to use your current global resolver,
-# to avoid unnecessary building.
-STACK_RESOLVER="--resolver=lts-22.19"
-
-# Dependencies we require that aren't in the above stackage snapshot.
-# (Also requested when using cabal, but that's harmless.)
-# Be careful not to break interpolation in commands below, the format should be
-# STACK_EXTRA_DEPS="PKG1-VER1 PKG2-VER2 ..."
-# extra deps as in stack9.6.yaml:
-STACK_EXTRA_DEPS="vty-windows-0.2.0.1"
-
 #TODO? https://github.com/commercialhaskell/stack/issues/3055 https://github.com/haskell/hackage-security/issues/187
 #Updating package index Hackage (mirrored at https://s3.amazonaws.com/hackage.fpcomplete.com/) ...
 #   /Users/simon/.stack/indices/Hackage/hackage-security-lock: createDirectory: already exists (File exists)
@@ -898,7 +898,7 @@ quietly_run() {
 try_install() {
   cd  # ensure we install at user level, not in some project's stack/cabal setup
   if has_cmd stack ; then
-    try_info stack install --install-ghc $STACK_RESOLVER "$@" --verbosity="$STACK_VERBOSITY" || echo "Failed to install $@"
+    try_info stack install --install-ghc --resolver $STACKAGE_SNAPSHOT "$@" --verbosity="$STACK_VERBOSITY" || echo "Failed to install $@"
   elif has_cmd cabal ; then
     try_info cabal install "$@" --verbose="$CABAL_VERBOSITY" || echo "Failed to install $@"
   else
@@ -1015,7 +1015,7 @@ echo "Ensuring a Haskell build tool:"
 # if stack is installed, use stack
 # || [[ "$FORCE_INSTALL_STACK" == "true" ]]  #--force-install-stack
 if has_stack ; then
-  echo "stack $(cmd_version stack) is installed, using stack to install hledger in $HOME/.local/bin"
+  echo "stack $(cmd_version stack) is installed, and will be used to install hledger."
   # if it's too old, explain that we'll be installing the latest
   if ! has_good_stack ; then
     echo "Note: stack $(cmd_version stack) is too old, a newer version will be installed"
@@ -1026,13 +1026,13 @@ if has_stack ; then
   try_info stack update --verbosity=error
 # else if cabal is installed, use cabal
 elif has_cmd cabal ; then
-  echo "no stack installed, cabal $(cabal --numeric-version) installed; using cabal to install hledger in $HOME/.cabal/bin"
+  echo "stack is not installed, but cabal $(cabal --numeric-version) is installed, and will be used to install hledger."
   echo Using $(cabal --version)  # unquoted to squash cabal version to one line
   # run cabal update to make sure it knows about latest packages
   try_info cabal update -v0
 # else use stack
 else
-  echo "no stack or cabal installed; stack will be installed and used to install hledger in $HOME/.local/bin"
+  echo "Neither stack nor cabal is installed. stack will be installed and used to install hledger."
     # install stack now
   ensure_stack
 fi

hledger-lib/.date.m4

@@ -1,2 +1,2 @@
 m4_dnl Date to show in man pages. Updated by "Shake manuals"
-m4_define({{_monthyear_}}, {{April 2024}})m4_dnl
+m4_define({{_monthyear_}}, {{September 2024}})m4_dnl

hledger-lib/.version

@@ -1 +1 @@
-1.33.99
+1.40.99

hledger-lib/.version.m4

@@ -1,2 +1,2 @@
 m4_dnl Version number to show in manuals. Updated by "Shake setversion"
-m4_define({{_version_}}, {{1.33.99}})m4_dnl
+m4_define({{_version_}}, {{1.40.99}})m4_dnl

hledger-lib/CHANGES.md

@@ -21,17 +21,43 @@ Improvements
 Internal/api/developer-ish changes in the hledger-lib (and hledger) packages.
 For user-visible changes, see the hledger package changelog.
 
-# b7e5c05da
+
+# 81167e81a
 
 Breaking changes
 
+- New/refactored modules (Hledger.Write.*) and types (Spreadsheet) to help abstract the rendering of
+  tables in various output formats, eg HTML and FODS.
+  (Spreadsheet is in addition to the tabular package we already in use; there may be some overlap.)
+  (XXX Review module changes)
+
 Fixes
 
 Improvements
 
-- InputOpts has a new `_defer` flag for internal use instead of overusing `strict_`
+- dependency changes: 
 
-- moved journalCheckBalanceAssertions to JournalChecks
+
+# 1.40 2024-09-09
+
+Breaking changes
+
+- Some constructors of the Interval type have been renamed for clarity.
+- Hledger.Read.CsvUtils has moved to Hledger.Write.Csv. (Henning Thielemann)
+- Tabular report rendering code has been added/reworked to allow new output formats and more reuse. (Henning Thielemann)
+
+Improvements
+
+- Added `journalDbg` debug output helper.
+
+- Allow doclayout 0.5.
+
+# 1.34 2024-06-01
+
+Improvements
+
+- InputOpts has a new `_defer` flag for internal use instead of overusing `strict_`
+- journalCheckBalanceAssertions has moved to JournalChecks
 
 
 # 1.33.1 2024-05-02

hledger-lib/Hledger/Data/Account.hs

@@ -34,7 +34,10 @@ module Hledger.Data.Account
 
 import qualified Data.HashSet as HS
 import qualified Data.HashMap.Strict as HM
-import Data.List (find, foldl', sortOn)
+import Data.List (find, sortOn)
+#if !MIN_VERSION_base(4,20,0)
+import Data.List (foldl')
+#endif
 import Data.List.Extra (groupOn)
 import qualified Data.Map as M
 import Data.Ord (Down(..))

hledger-lib/Hledger/Data/Amount.hs

@@ -39,6 +39,7 @@ with similar amounts since it mostly ignores costss and commodity exchange rates
 
 -}
 
+{-# LANGUAGE CPP  #-}
 {-# LANGUAGE OverloadedStrings  #-}
 {-# LANGUAGE RecordWildCards    #-}
 {-# LANGUAGE NamedFieldPuns #-}
@@ -155,6 +156,7 @@ module Hledger.Data.Amount (
   showMixedAmountWithZeroCommodity,
   showMixedAmountB,
   showMixedAmountLinesB,
+  showMixedAmountLinesPartsB,
   wbToText,
   wbUnpack,
   mixedAmountSetPrecision,
@@ -174,7 +176,10 @@ import Data.Char (isDigit)
 import Data.Decimal (DecimalRaw(..), decimalPlaces, normalizeDecimal, roundTo)
 import Data.Default (Default(..))
 import Data.Foldable (toList)
-import Data.List (find, foldl', intercalate, intersperse, mapAccumL, partition)
+import Data.List (find, intercalate, intersperse, mapAccumL, partition)
+#if !MIN_VERSION_base(4,20,0)
+import Data.List (foldl')
+#endif
 import Data.List.NonEmpty (NonEmpty(..), nonEmpty)
 import qualified Data.Map.Strict as M
 import qualified Data.Set as S
@@ -763,7 +768,7 @@ instance Num MixedAmount where
     negate = maNegate
     (+)    = maPlus
     (*)    = error "error, mixed amounts do not support multiplication" -- PARTIAL:
-    abs    = error "error, mixed amounts do not support abs"
+    abs    = mapMixedAmount (\amt -> amt { aquantity = abs (aquantity amt)})
     signum = error "error, mixed amounts do not support signum"
 
 -- | Calculate the key used to store an Amount within a MixedAmount.
@@ -1120,10 +1125,17 @@ showMixedAmountB opts ma
 -- This returns the list of WideBuilders: one for each Amount, and padded/elided to the appropriate width.
 -- This does not honour displayOneLine; all amounts will be displayed as if displayOneLine were False.
 showMixedAmountLinesB :: AmountFormat -> MixedAmount -> [WideBuilder]
-showMixedAmountLinesB opts@AmountFormat{displayMaxWidth=mmax,displayMinWidth=mmin} ma =
-    map (adBuilder . pad) elided
+showMixedAmountLinesB opts ma =
+    map fst $ showMixedAmountLinesPartsB opts ma
+
+-- | Like 'showMixedAmountLinesB' but also returns
+-- the amounts associated with each text builder.
+showMixedAmountLinesPartsB :: AmountFormat -> MixedAmount -> [(WideBuilder, Amount)]
+showMixedAmountLinesPartsB opts@AmountFormat{displayMaxWidth=mmax,displayMinWidth=mmin} ma =
+    zip (map (adBuilder . pad) elided) amts
   where
-    astrs = amtDisplayList (wbWidth sep) (showAmountB opts) . orderedAmounts opts $
+    astrs = amtDisplayList (wbWidth sep) (showAmountB opts) amts
+    amts = orderedAmounts opts $
               if displayCost opts then ma else mixedAmountStripCosts ma
     sep   = WideBuilder (TB.singleton '\n') 0
     width = maximum $ map (wbWidth . adBuilder) elided

hledger-lib/Hledger/Data/Dates.hs

@@ -43,7 +43,7 @@ module Hledger.Data.Dates (
   showEFDate,
   showDateSpan,
   showDateSpanDebug,
-  showDateSpanMonthAbbrev,
+  showDateSpanAbbrev,
   elapsedSeconds,
   prevday,
   periodexprp,
@@ -139,8 +139,8 @@ showDateSpanDebug (DateSpan b e)= "DateSpan (" <> show b <> ") (" <> show e <> "
 
 -- | Like showDateSpan, but show month spans as just the abbreviated month name
 -- in the current locale.
-showDateSpanMonthAbbrev :: DateSpan -> Text
-showDateSpanMonthAbbrev = showPeriodMonthAbbrev . dateSpanAsPeriod
+showDateSpanAbbrev :: DateSpan -> Text
+showDateSpanAbbrev = showPeriodAbbrev . dateSpanAsPeriod
 
 -- | Get the current local date.
 getCurrentDay :: IO Day
@@ -188,14 +188,20 @@ spansSpan :: [DateSpan] -> DateSpan
 spansSpan spans = DateSpan (spanStartDate =<< headMay spans) (spanEndDate =<< lastMay spans)
 
 -- | Split a DateSpan into consecutive exact spans of the specified Interval.
--- If the first argument is true and the interval is Weeks, Months, Quarters or Years,
--- the start date will be adjusted backward if needed to nearest natural interval boundary
--- (a monday, first of month, first of quarter or first of year).
 -- If no interval is specified, the original span is returned.
 -- If the original span is the null date span, ie unbounded, the null date span is returned.
 -- If the original span is empty, eg if the end date is <= the start date, no spans are returned.
 --
--- ==== Examples:
+-- ==== Date adjustment
+-- Some intervals respect the "adjust" flag (years, quarters, months, weeks, every Nth weekday
+-- of month seem to be the ones that need it). This will move the start date earlier, if needed,
+-- to the previous natural interval boundary (first of year, first of quarter, first of month,
+-- monday, previous Nth weekday of month). Related: #1982 #2218
+--
+-- The end date is always moved later if needed to the next natural interval boundary,
+-- so that the last period is the same length as the others.
+--
+-- ==== Examples
 -- >>> let t i y1 m1 d1 y2 m2 d2 = splitSpan True i $ DateSpan (Just $ Flex $ fromGregorian y1 m1 d1) (Just $ Flex $ fromGregorian y2 m2 d2)
 -- >>> t NoInterval 2008 01 01 2009 01 01
 -- [DateSpan 2008]
@@ -212,38 +218,38 @@ spansSpan spans = DateSpan (spanStartDate =<< headMay spans) (spanEndDate =<< la
 -- >>> t (Months 2) 2008 01 01 2008 04 01
 -- [DateSpan 2008-01-01..2008-02-29,DateSpan 2008-03-01..2008-04-30]
 -- >>> t (Weeks 1) 2008 01 01 2008 01 15
--- [DateSpan 2007-12-31W01,DateSpan 2008-01-07W02,DateSpan 2008-01-14W03]
+-- [DateSpan 2007-W01,DateSpan 2008-W02,DateSpan 2008-W03]
 -- >>> t (Weeks 2) 2008 01 01 2008 01 15
 -- [DateSpan 2007-12-31..2008-01-13,DateSpan 2008-01-14..2008-01-27]
--- >>> t (DayOfMonth 2) 2008 01 01 2008 04 01
--- [DateSpan 2007-12-02..2008-01-01,DateSpan 2008-01-02..2008-02-01,DateSpan 2008-02-02..2008-03-01,DateSpan 2008-03-02..2008-04-01]
--- >>> t (WeekdayOfMonth 2 4) 2011 01 01 2011 02 15
+-- >>> t (MonthDay 2) 2008 01 01 2008   04 01
+-- [DateSpan 2008-01-02..2008-02-01,DateSpan 2008-02-02..2008-03-01,DateSpan 2008-03-02..2008-04-01]
+-- >>> t (NthWeekdayOfMonth 2 4) 2011 01 01 2011 02 15
 -- [DateSpan 2010-12-09..2011-01-12,DateSpan 2011-01-13..2011-02-09,DateSpan 2011-02-10..2011-03-09]
 -- >>> t (DaysOfWeek [2]) 2011 01 01 2011 01 15
 -- [DateSpan 2010-12-28..2011-01-03,DateSpan 2011-01-04..2011-01-10,DateSpan 2011-01-11..2011-01-17]
--- >>> t (DayOfYear 11 29) 2011 10 01 2011 10 15
--- [DateSpan 2010-11-29..2011-11-28]
--- >>> t (DayOfYear 11 29) 2011 12 01 2012 12 15
--- [DateSpan 2011-11-29..2012-11-28,DateSpan 2012-11-29..2013-11-28]
+-- >>> t (MonthAndDay 11 29) 2012 10 01 2013 10 15
+-- [DateSpan 2012-11-29..2013-11-28]
 --
 splitSpan :: Bool -> Interval -> DateSpan -> [DateSpan]
-splitSpan _ _ (DateSpan Nothing Nothing) = [DateSpan Nothing Nothing]
-splitSpan _ _ ds | isEmptySpan ds = []
-splitSpan _ _ ds@(DateSpan (Just s) (Just e)) | s == e = [ds]
-splitSpan _ NoInterval ds = [ds]
-splitSpan _ (Days n) ds = splitspan id addDays n                    ds
-splitSpan adjust (Weeks n)    ds = splitspan (if adjust then startofweek    else id) addDays (7*n)                ds
-splitSpan adjust (Months n)   ds = splitspan (if adjust then startofmonth   else id) addGregorianMonthsClip n     ds
-splitSpan adjust (Quarters n) ds = splitspan (if adjust then startofquarter else id) addGregorianMonthsClip (3*n) ds
-splitSpan adjust (Years n)    ds = splitspan (if adjust then startofyear    else id) addGregorianYearsClip n      ds
-splitSpan _ (DayOfMonth dom)  ds = splitspan (nthdayofmonthcontaining dom) (addGregorianMonthsToMonthday dom) 1 ds
-splitSpan _ (DayOfYear m n)   ds = splitspan (nthdayofyearcontaining m n) (addGregorianYearsClip) 1 ds
-splitSpan _ (WeekdayOfMonth n wd) ds = splitspan (nthweekdayofmonthcontaining n wd) advancemonths 1 ds
+splitSpan _      _                        (DateSpan Nothing Nothing) = [DateSpan Nothing Nothing]
+splitSpan _      _                        ds | isEmptySpan ds = []
+splitSpan _      _                        ds@(DateSpan (Just s) (Just e)) | s == e = [ds]
+splitSpan _      NoInterval               ds = [ds]
+splitSpan _      (Days n)                 ds = splitspan id addDays n ds
+splitSpan adjust (Weeks n)                ds = splitspan (if adjust then startofweek    else id) addDays                 (7*n) ds
+splitSpan adjust (Months n)               ds = splitspan (if adjust then startofmonth   else id) addGregorianMonthsClip  n     ds
+splitSpan adjust (Quarters n)             ds = splitspan (if adjust then startofquarter else id) addGregorianMonthsClip  (3*n) ds
+splitSpan adjust (Years n)                ds = splitspan (if adjust then startofyear    else id) addGregorianYearsClip   n     ds
+splitSpan adjust (NthWeekdayOfMonth n wd) ds = splitspan (if adjust then prevstart else nextstart) advancemonths          1     ds
   where
+    prevstart = prevNthWeekdayOfMonth n wd
+    nextstart = nextNthWeekdayOfMonth n wd
     advancemonths 0 = id
-    advancemonths w = advancetonthweekday n wd . startofmonth . addGregorianMonthsClip w
-splitSpan _ (DaysOfWeek [])         ds = [ds]
-splitSpan _ (DaysOfWeek days@(n:_)) ds = spansFromBoundaries e bdrys
+    advancemonths m = advanceToNthWeekday n wd . startofmonth . addGregorianMonthsClip m
+splitSpan _      (MonthDay dom)           ds = splitspan (nextnthdayofmonth dom) (addGregorianMonthsToMonthday dom) 1 ds
+splitSpan _      (MonthAndDay m d)        ds = splitspan (nextmonthandday m d)   (addGregorianYearsClip)            1 ds
+splitSpan _      (DaysOfWeek [])          ds = [ds]
+splitSpan _      (DaysOfWeek days@(n:_))  ds = spansFromBoundaries e bdrys
   where
     (s, e) = dateSpanSplitLimits (nthdayofweekcontaining n) nextday ds
     bdrys = concatMap (flip map starts . addDays) [0,7..]
@@ -260,16 +266,18 @@ addGregorianMonthsToMonthday dom n d =
   in fromGregorian y m dom
 
 -- Split the given span into exact spans using the provided helper functions:
--- 1. The start function is applied to the span's start date to get the first sub-span's start date.
--- 2. The addInterval function is used to calculate the subsequent spans' start dates,
--- possibly with stride increased by the mult multiplier.
--- It should adapt to spans of varying length, eg if splitting on "every 31st of month"
--- addInterval should adjust to 28/29/30 in short months but return to 31 in the long months.
+--
+-- 1. The start function is used to adjust the provided span's start date to get the first sub-span's start date.
+--
+-- 2. The next function is used to calculate subsequent sub-spans' start dates, possibly with stride increased by a multiplier.
+--    It should handle spans of varying length, eg when splitting on "every 31st of month",
+--    it adjusts to 28/29/30 in short months but returns to 31 in the long months.
+--
 splitspan :: (Day -> Day) -> (Integer -> Day -> Day) -> Int -> DateSpan -> [DateSpan]
-splitspan start addInterval mult ds = spansFromBoundaries e bdrys
+splitspan start next mult ds = spansFromBoundaries e bdrys
   where
-    (s, e) = dateSpanSplitLimits start (addInterval (toInteger mult)) ds
-    bdrys = mapM (addInterval . toInteger) [0,mult..] $ start s
+    (s, e) = dateSpanSplitLimits start (next (toInteger mult)) ds
+    bdrys = mapM (next . toInteger) [0,mult..] $ start s
 
 -- | Fill in missing start/end dates for calculating 'splitSpan'.
 dateSpanSplitLimits :: (Day -> Day) -> (Day -> Day) -> DateSpan -> (Day, Day)
@@ -490,8 +498,7 @@ fixSmartDateStr d s =
 fixSmartDateStrEither :: Day -> Text -> Either HledgerParseErrors Text
 fixSmartDateStrEither d = fmap showEFDate . fixSmartDateStrEither' d
 
-fixSmartDateStrEither'
-  :: Day -> Text -> Either HledgerParseErrors EFDay
+fixSmartDateStrEither' :: Day -> Text -> Either HledgerParseErrors EFDay
 fixSmartDateStrEither' d s = case parsewith smartdateonly (T.toLower s) of
                                Right sd -> Right $ fixSmartDate d sd
                                Left e -> Left e
@@ -621,7 +628,7 @@ startofquarter day = fromGregorian y (firstmonthofquarter m) 1
       firstmonthofquarter m2 = ((m2-1) `div` 3) * 3 + 1
 
 thisyear = startofyear
-prevyear = startofyear . addGregorianYearsClip (-1)
+-- prevyear = startofyear . addGregorianYearsClip (-1)
 nextyear = startofyear . addGregorianYearsClip 1
 startofyear day = fromGregorian y 1 1 where (y,_,_) = toGregorian day
 
@@ -633,65 +640,50 @@ intervalBoundaryBefore i d =
     (DateSpan (Just start) _:_) -> fromEFDay start
     _ -> d
 
--- | For given date d find year-long interval that starts on given
--- MM/DD of year and covers it.
--- The given MM and DD should be basically valid (1-12 & 1-31),
--- or an error is raised.
+-- | Find the next occurrence of the specified month and day of month, on or after the given date.
+-- The month should be 1-12 and the day of month should be 1-31, or an error will be raised.
 --
--- Examples: lets take 2017-11-22. Year-long intervals covering it that
--- starts before Nov 22 will start in 2017. However
--- intervals that start after Nov 23rd should start in 2016:
 -- >>> let wed22nd = fromGregorian 2017 11 22
--- >>> nthdayofyearcontaining 11 21 wed22nd
--- 2017-11-21
--- >>> nthdayofyearcontaining 11 22 wed22nd
+-- >>> nextmonthandday 11 21 wed22nd
+-- 2018-11-21
+-- >>> nextmonthandday 11 22 wed22nd
 -- 2017-11-22
--- >>> nthdayofyearcontaining 11 23 wed22nd
--- 2016-11-23
--- >>> nthdayofyearcontaining 12 02 wed22nd
--- 2016-12-02
--- >>> nthdayofyearcontaining 12 31 wed22nd
--- 2016-12-31
--- >>> nthdayofyearcontaining 1 1 wed22nd
--- 2017-01-01
-nthdayofyearcontaining :: Month -> MonthDay -> Day -> Day
-nthdayofyearcontaining m mdy date
+-- >>> nextmonthandday 11 23 wed22nd
+-- 2017-11-23
+nextmonthandday :: Month -> MonthDay -> Day -> Day
+nextmonthandday m n date
   -- PARTIAL:
-  | not (validMonth m)  = error' $ "nthdayofyearcontaining: invalid month "++show m
-  | not (validDay   mdy) = error' $ "nthdayofyearcontaining: invalid day "  ++show mdy
-  | mmddOfSameYear <= date = mmddOfSameYear
-  | otherwise = mmddOfPrevYear
-  where mmddOfSameYear = addDays (toInteger mdy-1) $ applyN (m-1) nextmonth s
-        mmddOfPrevYear = addDays (toInteger mdy-1) $ applyN (m-1) nextmonth $ prevyear s
-        s = startofyear date
+  | not (validMonth m) = error' $ "nextmonthandday: month should be 1..12, not "++show m
+  | not (validDay   n) = error' $ "nextmonthandday: day should be 1..31, not "  ++show n
+  | mdthisyear >= date = mdthisyear
+  | otherwise          = mdnextyear
+  where
+    s = startofyear date
+    advancetomonth = applyN (m-1) nextmonth
+    advancetoday = addDays (toInteger n-1)
+    mdthisyear = advancetoday $ advancetomonth s
+    mdnextyear = advancetoday $ advancetomonth $ nextyear s
 
--- | For a given date d find the month-long period that starts on day n of a month
--- that includes d. (It will begin on day n or either d's month or the previous month.)
--- The given day of month should be in the range 1-31, or an error will be raised.
+-- | Find the next occurrence of the specified day of month, on or after the given date.
+-- The day of month should be 1-31, or an error will be raised.
 --
--- Examples: lets take 2017-11-22. Month-long intervals covering it that
--- start on 1st-22nd of month will start in Nov. However
--- intervals that start on 23rd-30th of month should start in Oct:
 -- >>> let wed22nd = fromGregorian 2017 11 22
--- >>> nthdayofmonthcontaining 1 wed22nd
--- 2017-11-01
--- >>> nthdayofmonthcontaining 12 wed22nd
--- 2017-11-12
--- >>> nthdayofmonthcontaining 22 wed22nd
+-- >>> nextnthdayofmonth 21 wed22nd
+-- 2017-12-21
+-- >>> nextnthdayofmonth 22 wed22nd
 -- 2017-11-22
--- >>> nthdayofmonthcontaining 23 wed22nd
--- 2017-10-23
--- >>> nthdayofmonthcontaining 30 wed22nd
--- 2017-10-30
-nthdayofmonthcontaining :: MonthDay -> Day -> Day
-nthdayofmonthcontaining mdy date
+-- >>> nextnthdayofmonth 23 wed22nd
+-- 2017-11-23
+nextnthdayofmonth :: MonthDay -> Day -> Day
+nextnthdayofmonth n date
   -- PARTIAL:
-  | not (validDay mdy) = error' $ "nthdayofmonthcontaining: invalid day "  ++show mdy
-  | nthOfSameMonth <= date = nthOfSameMonth
-  | otherwise = nthOfPrevMonth
-  where nthOfSameMonth = nthdayofmonth mdy s
-        nthOfPrevMonth = nthdayofmonth mdy $ prevmonth s
-        s = startofmonth date
+  | not (validDay n)       = error' $ "nextnthdayofmonth: day should be 1..31, not "  ++show n
+  | nthofthismonth >= date = nthofthismonth
+  | otherwise              = nthofnextmonth
+  where
+    s = startofmonth date
+    nthofthismonth = nthdayofmonth n s
+    nthofnextmonth = nthdayofmonth n $ nextmonth s
 
 -- | For given date d find week-long interval that starts on nth day of week
 -- and covers it.
@@ -717,37 +709,66 @@ nthdayofweekcontaining n d | nthOfSameWeek <= d = nthOfSameWeek
           nthOfPrevWeek = addDays (toInteger n-1) $ prevweek s
           s = startofweek d
 
--- | For given date d find month-long interval that starts on nth weekday of month
--- and covers it.
---
--- Examples: 2017-11-22 is 3rd Wed of Nov. Month-long intervals that cover it and
--- start on 1st-4th Wed will start in Nov. However
--- intervals that start on 4th Thu or Fri or later should start in Oct:
--- >>> let wed22nd = fromGregorian 2017 11 22
--- >>> nthweekdayofmonthcontaining 1 3 wed22nd
--- 2017-11-01
--- >>> nthweekdayofmonthcontaining 3 2 wed22nd
--- 2017-11-21
--- >>> nthweekdayofmonthcontaining 4 3 wed22nd
--- 2017-11-22
--- >>> nthweekdayofmonthcontaining 4 4 wed22nd
--- 2017-10-26
--- >>> nthweekdayofmonthcontaining 4 5 wed22nd
--- 2017-10-27
-nthweekdayofmonthcontaining :: Int -> WeekDay -> Day -> Day
-nthweekdayofmonthcontaining n wd d | nthWeekdaySameMonth <= d  = nthWeekdaySameMonth
-                                   | otherwise = nthWeekdayPrevMonth
-    where nthWeekdaySameMonth = advancetonthweekday n wd $ startofmonth d
-          nthWeekdayPrevMonth = advancetonthweekday n wd $ prevmonth d
+-- -- | Find the next occurrence of some weekday, on or after the given date d.
+-- --
+-- -- >>> let wed22nd = fromGregorian 2017 11 22
+-- -- >>> nextnthdayofweek 1 wed22nd
+-- -- 2017-11-20
+-- -- >>> nextnthdayofweek 2 wed22nd
+-- -- 2017-11-21
+-- -- >>> nextnthdayofweek 3 wed22nd
+-- -- 2017-11-22
+-- -- >>> nextnthdayofweek 4 wed22nd
+-- -- 2017-11-16
+-- -- >>> nextnthdayofweek 5 wed22nd
+-- -- 2017-11-17
+-- nextdayofweek :: WeekDay -> Day -> Day
+-- nextdayofweek n d | nthOfSameWeek <= d = nthOfSameWeek
+--                            | otherwise = nthOfPrevWeek
+--     where nthOfSameWeek = addDays (toInteger n-1) s
+--           nthOfPrevWeek = addDays (toInteger n-1) $ prevweek s
+--           s = startofweek d
 
--- | Advance to nth weekday wd after given start day s
+-- | Find the next occurrence of some nth weekday of a month, on or after the given date d.
+--
+-- >>> let wed22nd = fromGregorian 2017 11 22
+-- >>> nextNthWeekdayOfMonth 3 3 wed22nd  -- next third wednesday
+-- 2017-12-20
+-- >>> nextNthWeekdayOfMonth 4 3 wed22nd  -- next fourth wednesday
+-- 2017-11-22
+-- >>> nextNthWeekdayOfMonth 5 3 wed22nd  -- next fifth wednesday
+-- 2017-11-29
+nextNthWeekdayOfMonth :: Int -> WeekDay -> Day -> Day
+nextNthWeekdayOfMonth n wd d
+  | nthweekdaythismonth >= d = nthweekdaythismonth
+  | otherwise                = nthweekdaynextmonth
+  where
+    nthweekdaythismonth = advanceToNthWeekday n wd $ startofmonth d
+    nthweekdaynextmonth = advanceToNthWeekday n wd $ nextmonth d
+
+-- | Find the previous occurrence of some nth weekday of a month, on or before the given date d.
+--
+-- >>> let wed22nd = fromGregorian 2017 11 22
+-- >>> prevNthWeekdayOfMonth 4 3 wed22nd
+-- 2017-11-22
+-- >>> prevNthWeekdayOfMonth 5 2 wed22nd
+-- 2017-10-31
+prevNthWeekdayOfMonth :: Int -> WeekDay -> Day -> Day
+prevNthWeekdayOfMonth n wd d
+  | nthweekdaythismonth <= d = nthweekdaythismonth
+  | otherwise                = nthweekdayprevmonth
+  where
+    nthweekdaythismonth = advanceToNthWeekday n wd $ startofmonth d
+    nthweekdayprevmonth = advanceToNthWeekday n wd $ prevmonth d
+
+-- | Advance to the nth occurrence of the given weekday, on or after the given date.
 -- Can call error.
-advancetonthweekday :: Int -> WeekDay -> Day -> Day
-advancetonthweekday n wd s =
+advanceToNthWeekday :: Int -> WeekDay -> Day -> Day
+advanceToNthWeekday n wd s =
   -- PARTIAL:
   maybe err (addWeeks (n-1)) $ firstMatch (>=s) $ iterate (addWeeks 1) $ firstweekday s
   where
-    err = error' "advancetonthweekday: should not happen"
+    err = error' "advanceToNthWeekday: should not happen"
     addWeeks k = addDays (7 * toInteger k)
     firstMatch p = headMay . dropWhile (not . p)
     firstweekday = addDays (toInteger wd-1) . startofweek
@@ -966,41 +987,41 @@ weekdaysp = fmap headErr . group . sort <$> sepBy1 weekday (string' ",")  -- PAR
 -- >>> p "every week to 2009"
 -- Right (Weeks 1,DateSpan ..2008-12-31)
 -- >>> p "every 2nd day of month"
--- Right (DayOfMonth 2,DateSpan ..)
+-- Right (MonthDay 2,DateSpan ..)
 -- >>> p "every 2nd day"
--- Right (DayOfMonth 2,DateSpan ..)
+-- Right (MonthDay 2,DateSpan ..)
 -- >>> p "every 2nd day 2009.."
--- Right (DayOfMonth 2,DateSpan 2009-01-01..)
+-- Right (MonthDay 2,DateSpan 2009-01-01..)
 -- >>> p "every 2nd day 2009-"
--- Right (DayOfMonth 2,DateSpan 2009-01-01..)
+-- Right (MonthDay 2,DateSpan 2009-01-01..)
 -- >>> p "every 29th Nov"
--- Right (DayOfYear 11 29,DateSpan ..)
+-- Right (MonthAndDay 11 29,DateSpan ..)
 -- >>> p "every 29th nov ..2009"
--- Right (DayOfYear 11 29,DateSpan ..2008-12-31)
+-- Right (MonthAndDay 11 29,DateSpan ..2008-12-31)
 -- >>> p "every nov 29th"
--- Right (DayOfYear 11 29,DateSpan ..)
+-- Right (MonthAndDay 11 29,DateSpan ..)
 -- >>> p "every Nov 29th 2009.."
--- Right (DayOfYear 11 29,DateSpan 2009-01-01..)
+-- Right (MonthAndDay 11 29,DateSpan 2009-01-01..)
 -- >>> p "every 11/29 from 2009"
--- Right (DayOfYear 11 29,DateSpan 2009-01-01..)
+-- Right (MonthAndDay 11 29,DateSpan 2009-01-01..)
 -- >>> p "every 11/29 since 2009"
--- Right (DayOfYear 11 29,DateSpan 2009-01-01..)
+-- Right (MonthAndDay 11 29,DateSpan 2009-01-01..)
 -- >>> p "every 2nd Thursday of month to 2009"
--- Right (WeekdayOfMonth 2 4,DateSpan ..2008-12-31)
+-- Right (NthWeekdayOfMonth 2 4,DateSpan ..2008-12-31)
 -- >>> p "every 1st monday of month to 2009"
--- Right (WeekdayOfMonth 1 1,DateSpan ..2008-12-31)
+-- Right (NthWeekdayOfMonth 1 1,DateSpan ..2008-12-31)
 -- >>> p "every tue"
 -- Right (DaysOfWeek [2],DateSpan ..)
 -- >>> p "every 2nd day of week"
 -- Right (DaysOfWeek [2],DateSpan ..)
 -- >>> p "every 2nd day of month"
--- Right (DayOfMonth 2,DateSpan ..)
+-- Right (MonthDay 2,DateSpan ..)
 -- >>> p "every 2nd day"
--- Right (DayOfMonth 2,DateSpan ..)
+-- Right (MonthDay 2,DateSpan ..)
 -- >>> p "every 2nd day 2009.."
--- Right (DayOfMonth 2,DateSpan 2009-01-01..)
+-- Right (MonthDay 2,DateSpan 2009-01-01..)
 -- >>> p "every 2nd day of month 2009.."
--- Right (DayOfMonth 2,DateSpan 2009-01-01..)
+-- Right (MonthDay 2,DateSpan 2009-01-01..)
 periodexprp :: Day -> TextParser m (Interval, DateSpan)
 periodexprp rdate = do
   skipNonNewlineSpaces
@@ -1029,9 +1050,9 @@ reportingintervalp = choice'
     , Months 2 <$ string' "bimonthly"
     , string' "every" *> skipNonNewlineSpaces *> choice'
         [ DaysOfWeek . pure <$> (nth <* skipNonNewlineSpaces <* string' "day" <* of_ "week")
-        , DayOfMonth <$> (nth <* skipNonNewlineSpaces <* string' "day" <* optOf_ "month")
-        , liftA2 WeekdayOfMonth nth $ skipNonNewlineSpaces *> weekday <* optOf_ "month"
-        , uncurry DayOfYear <$> (md <* optOf_ "year")
+        , MonthDay <$> (nth <* skipNonNewlineSpaces <* string' "day" <* optOf_ "month")
+        , liftA2 NthWeekdayOfMonth nth $ skipNonNewlineSpaces *> weekday <* optOf_ "month"
+        , uncurry MonthAndDay <$> (md <* optOf_ "year")
         , DaysOfWeek <$> weekdaysp
         , DaysOfWeek [1..5] <$ string' "weekday"
         , DaysOfWeek [6..7] <$ string' "weekendday"
@@ -1050,8 +1071,8 @@ reportingintervalp = choice'
     optOf_ period = optional . try $ of_ period
 
     nth = decimal <* choice (map string' ["st","nd","rd","th"])
-    d_o_y = runPermutation $ liftA2 DayOfYear (toPermutation $ (month <|> mon) <* skipNonNewlineSpaces)
-                                              (toPermutation $ nth <* skipNonNewlineSpaces)
+    d_o_y = runPermutation $ liftA2 MonthAndDay (toPermutation $ (month <|> mon) <* skipNonNewlineSpaces)
+                                                (toPermutation $ nth <* skipNonNewlineSpaces)
 
     -- Parse any of several variants of a basic interval, eg "daily", "every day", "every N days".
     tryinterval :: Text -> Text -> (Int -> Interval) -> TextParser m Interval

hledger-lib/Hledger/Data/Journal.hs

@@ -1,9 +1,10 @@
+{-# LANGUAGE CPP                 #-}
 {-# LANGUAGE NamedFieldPuns      #-}
 {-# LANGUAGE OverloadedStrings   #-}
 {-# LANGUAGE Rank2Types          #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TupleSections       #-}
-{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE RecordWildCards     #-}
 
 {-|
 
@@ -21,6 +22,7 @@ module Hledger.Data.Journal (
   addTransactionModifier,
   addPeriodicTransaction,
   addTransaction,
+  journalDbg,
   journalInferMarketPricesFromTransactions,
   journalInferCommodityStyles,
   journalStyleAmounts,
@@ -121,7 +123,10 @@ import Control.Monad.State.Strict (StateT)
 import Data.Char (toUpper, isDigit)
 import Data.Default (Default(..))
 import Data.Foldable (toList)
-import Data.List ((\\), find, foldl', sortBy, union, intercalate)
+import Data.List ((\\), find, sortBy, union, intercalate)
+#if !MIN_VERSION_base(4,20,0)
+import Data.List (foldl')
+#endif
 import Data.List.Extra (nubSort)
 import qualified Data.Map.Strict as M
 import Data.Maybe (catMaybes, fromMaybe, mapMaybe, maybeToList)
@@ -134,7 +139,6 @@ import Data.Time.Clock.POSIX (POSIXTime)
 import Data.Tree (Tree(..), flatten)
 import Text.Printf (printf)
 import Text.Megaparsec (ParsecT)
-import Text.Megaparsec.Custom (FinalParseError)
 
 import Hledger.Utils
 import Hledger.Data.Types
@@ -182,17 +186,39 @@ instance Show Journal where
              -- ++ (show $ journalTransactions l)
              where accounts = filter (/= "root") $ flatten $ journalAccountNameTree j
 
--- showJournalDebug j = unlines [
---                       show j
---                      ,show (jtxns j)
---                      ,show (jtxnmodifiers j)
---                      ,show (jperiodictxns j)
---                      ,show $ jparsetimeclockentries j
---                      ,show $ jpricedirectives j
---                      ,show $ jfinalcommentlines j
---                      ,show $ jparsestate j
---                      ,show $ map fst $ jfiles j
---                      ]
+journalDbg j@Journal{..} = chomp $ unlines $
+  ("Journal " ++ takeFileName (journalFilePath j)++":") :  --  ++ " {"
+  map (" "<>) [
+   "jparsedefaultyear: "         <> shw jparsedefaultyear
+  ,"jparsedefaultcommodity: "    <> shw jparsedefaultcommodity
+  ,"jparsedecimalmark: "         <> shw jparsedecimalmark
+  ,"jparseparentaccounts: "      <> shw jparseparentaccounts
+  ,"jparsealiases: "             <> shw jparsealiases
+  -- ,"jparsetimeclockentries: " <> shw jparsetimeclockentries
+  ,"jincludefilestack: "         <> shw jincludefilestack
+  ,"jdeclaredpayees: "           <> shw jdeclaredpayees
+  ,"jdeclaredtags: "             <> shw jdeclaredtags
+  ,"jdeclaredaccounts: "         <> shw jdeclaredaccounts
+  ,"jdeclaredaccounttags: "      <> shw jdeclaredaccounttags
+  ,"jdeclaredaccounttypes: "     <> shw jdeclaredaccounttypes
+  ,"jaccounttypes: "             <> shw jaccounttypes
+  ,"jglobalcommoditystyles: "    <> shw jglobalcommoditystyles
+  ,"jcommodities: "              <> shw jcommodities
+  ,"jinferredcommodities: "      <> shw jinferredcommodities
+  ,"jpricedirectives: "          <> shw jpricedirectives
+  ,"jinferredmarketprices: "     <> shw jinferredmarketprices
+  ,"jtxnmodifiers: "             <> shw jtxnmodifiers
+  -- ,"jperiodictxns: "          <> shw jperiodictxns
+  ,"jtxns: "                     <> shw jtxns
+  ,"jfinalcommentlines: "        <> shw jfinalcommentlines
+  ,"jfiles: "                    <> shw jfiles
+  ,"jlastreadtime: "             <> shw jlastreadtime
+  ]
+  -- ++ ["}"]
+  where
+    shw :: Show a => a -> String
+    shw = show
+    -- shw = pshow
 
 -- The semigroup instance for Journal is useful for two situations.
 --
@@ -232,12 +258,32 @@ journalConcat j1 j2 =
     ,jdeclaredpayees            = jdeclaredpayees            j1 <> jdeclaredpayees            j2
     ,jdeclaredtags              = jdeclaredtags              j1 <> jdeclaredtags              j2
     ,jdeclaredaccounts          = jdeclaredaccounts          j1 <> jdeclaredaccounts          j2
-    ,jdeclaredaccounttags       = jdeclaredaccounttags       j1 <> jdeclaredaccounttags       j2
-    ,jdeclaredaccounttypes      = jdeclaredaccounttypes      j1 <> jdeclaredaccounttypes      j2
-    ,jaccounttypes              = jaccounttypes              j1 <> jaccounttypes              j2
-    ,jglobalcommoditystyles     = jglobalcommoditystyles     j1 <> jglobalcommoditystyles     j2
-    ,jcommodities               = jcommodities               j1 <> jcommodities               j2
-    ,jinferredcommodities       = jinferredcommodities       j1 <> jinferredcommodities       j2
+    --
+    -- The next six fields are Maps, which need to be merged carefully for correct semantics,
+    -- especially the first two, which have list values. There may still be room for improvement here.
+    --
+    -- ,jdeclaredaccounttags   :: M.Map AccountName [Tag]
+    -- jdeclaredaccounttags can have multiple duplicated/conflicting values for an account's tag.
+    ,jdeclaredaccounttags       = M.unionWith (<>) (jdeclaredaccounttags j1) (jdeclaredaccounttags j2)
+    --
+    -- ,jdeclaredaccounttypes  :: M.Map AccountType [AccountName]
+    -- jdeclaredaccounttypes can have multiple duplicated/conflicting values for an account's type.
+    ,jdeclaredaccounttypes      = M.unionWith (<>) (jdeclaredaccounttypes j1) (jdeclaredaccounttypes j2)
+    --
+    -- ,jaccounttypes          :: M.Map AccountName AccountType
+    -- jaccounttypes has a single type for any given account. When it had multiple type declarations, the last/rightmost wins.
+    ,jaccounttypes              = M.unionWith (const id) (jaccounttypes j1) (jaccounttypes j2)
+    --
+    -- ,jglobalcommoditystyles :: M.Map CommoditySymbol AmountStyle
+    ,jglobalcommoditystyles     = (<>) (jglobalcommoditystyles j1) (jglobalcommoditystyles j2)
+    --
+    -- ,jcommodities           :: M.Map CommoditySymbol Commodity
+    ,jcommodities               = (<>) (jcommodities j1) (jcommodities j2)
+    --
+    -- ,jinferredcommodities   :: M.Map CommoditySymbol AmountStyle
+    ,jinferredcommodities       = (<>) (jinferredcommodities j1) (jinferredcommodities j2)
+    --
+    --
     ,jpricedirectives           = jpricedirectives           j1 <> jpricedirectives           j2
     ,jinferredmarketprices      = jinferredmarketprices      j1 <> jinferredmarketprices      j2
     ,jtxnmodifiers              = jtxnmodifiers              j1 <> jtxnmodifiers              j2

hledger-lib/Hledger/Data/JournalChecks.hs

@@ -289,18 +289,19 @@ findRecentAssertionError today ps = do
       "%s\n",
       "The recentassertions check is enabled, so accounts with balance assertions must",
       "have a balance assertion within %d days of their latest posting.",
-      "In account \"%s\", this posting is %d days later",
-      "than the last balance assertion, which was on %s.",
+      "",
+      "In %s,",
+      "this posting is %d days later than the balance assertion on %s.",
       "",
       "Consider adding a more recent balance assertion for this account. Eg:",
       "",
-      "%s\n    %s    %s0 = %s0  ; (adjust asserted amount)"
+      "%s\n    %s    %s0 = %sAMT"
       ])
     f
     l
     (textChomp ex)
     maxlag
-    acct
+    (bold' $ T.unpack acct)
     lag
     (showDate latestassertdate)
     (show today)

hledger-lib/Hledger/Data/Period.hs

@@ -15,7 +15,7 @@ module Hledger.Data.Period (
   ,isStandardPeriod
   ,periodTextWidth
   ,showPeriod
-  ,showPeriodMonthAbbrev
+  ,showPeriodAbbrev
   ,periodStart
   ,periodEnd
   ,periodNext
@@ -173,10 +173,10 @@ periodTextWidth = periodTextWidth' . simplifyPeriod
 -- | Render a period as a compact display string suitable for user output.
 --
 -- >>> showPeriod (WeekPeriod (fromGregorian 2016 7 25))
--- "2016-07-25W30"
+-- "2016-W30"
 showPeriod :: Period -> Text
 showPeriod (DayPeriod b)       = T.pack $ formatTime defaultTimeLocale "%F" b              -- DATE
-showPeriod (WeekPeriod b)      = T.pack $ formatTime defaultTimeLocale "%FW%V" b           -- STARTDATEWYEARWEEK
+showPeriod (WeekPeriod b)      = T.pack $ formatTime defaultTimeLocale "%0Y-W%V" b         -- YYYY-Www
 showPeriod (MonthPeriod y m)   = T.pack $ printf "%04d-%02d" y m                           -- YYYY-MM
 showPeriod (QuarterPeriod y q) = T.pack $ printf "%04dQ%d" y q                             -- YYYYQN
 showPeriod (YearPeriod y)      = T.pack $ printf "%04d" y                                  -- YYYY
@@ -186,13 +186,16 @@ showPeriod (PeriodFrom b)      = T.pack $ formatTime defaultTimeLocale "%F.." b
 showPeriod (PeriodTo e)        = T.pack $ formatTime defaultTimeLocale "..%F" (addDays (-1) e)    -- ..INCLUSIVEENDDATE
 showPeriod PeriodAll           = ".."
 
--- | Like showPeriod, but if it's a month period show just
--- the 3 letter month name abbreviation for the current locale.
-showPeriodMonthAbbrev :: Period -> Text
-showPeriodMonthAbbrev (MonthPeriod _ m)                           -- Jan
+-- | Like showPeriod, but if it's a month or week period show
+-- an abbreviated form.
+-- >>> showPeriodAbbrev (WeekPeriod (fromGregorian 2016 7 25))
+-- "W30"
+showPeriodAbbrev :: Period -> Text
+showPeriodAbbrev (MonthPeriod _ m)                                              -- Jan
   | m > 0 && m <= length monthnames = T.pack . snd $ monthnames !! (m-1)
   where monthnames = months defaultTimeLocale
-showPeriodMonthAbbrev p = showPeriod p
+showPeriodAbbrev (WeekPeriod b) = T.pack $ formatTime defaultTimeLocale "W%V" b -- Www
+showPeriodAbbrev p = showPeriod p
 
 periodStart :: Period -> Maybe Day
 periodStart p = fromEFDay <$> mb

hledger-lib/Hledger/Data/PeriodicTransaction.hs

@@ -114,10 +114,6 @@ instance Show PeriodicTransaction where
 -- <BLANKLINE>
 --
 -- >>> _ptgen "every 2nd day of month from 2017/02 to 2017/04"
--- 2017-01-02
---     ; generated-transaction: ~ every 2nd day of month from 2017/02 to 2017/04
---     a           $1.00
--- <BLANKLINE>
 -- 2017-02-02
 --     ; generated-transaction: ~ every 2nd day of month from 2017/02 to 2017/04
 --     a           $1.00
@@ -128,10 +124,6 @@ instance Show PeriodicTransaction where
 -- <BLANKLINE>
 --
 -- >>> _ptgen "every 30th day of month from 2017/1 to 2017/5"
--- 2016-12-30
---     ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5
---     a           $1.00
--- <BLANKLINE>
 -- 2017-01-30
 --     ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5
 --     a           $1.00
@@ -150,10 +142,6 @@ instance Show PeriodicTransaction where
 -- <BLANKLINE>
 --
 -- >>> _ptgen "every 2nd Thursday of month from 2017/1 to 2017/4"
--- 2016-12-08
---     ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4
---     a           $1.00
--- <BLANKLINE>
 -- 2017-01-12
 --     ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4
 --     a           $1.00
@@ -168,10 +156,6 @@ instance Show PeriodicTransaction where
 -- <BLANKLINE>
 --
 -- >>> _ptgen "every nov 29th from 2017 to 2019"
--- 2016-11-29
---     ; generated-transaction: ~ every nov 29th from 2017 to 2019
---     a           $1.00
--- <BLANKLINE>
 -- 2017-11-29
 --     ; generated-transaction: ~ every nov 29th from 2017 to 2019
 --     a           $1.00

hledger-lib/Hledger/Data/Posting.hs

@@ -1,4 +1,3 @@
-{-# LANGUAGE NamedFieldPuns #-}
 {-|
 
 A 'Posting' represents a change (by some 'MixedAmount') of the balance in
@@ -8,6 +7,8 @@ look up the date or description there.
 
 -}
 
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedStrings #-}
 
 module Hledger.Data.Posting (
@@ -82,7 +83,10 @@ import Data.Foldable (asum)
 import Data.Function ((&))
 import qualified Data.Map as M
 import Data.Maybe (fromMaybe, isJust, mapMaybe)
-import Data.List (foldl', sort, union)
+import Data.List (sort, union)
+#if !MIN_VERSION_base(4,20,0)
+import Data.List (foldl')
+#endif
 import qualified Data.Set as S
 import Data.Text (Text)
 import qualified Data.Text as T

hledger-lib/Hledger/Data/Types.hs

@@ -116,7 +116,7 @@ data DateSpan = DateSpan (Maybe EFDay) (Maybe EFDay) deriving (Eq,Ord,Generic)
 
 instance Default DateSpan where def = DateSpan Nothing Nothing
 
--- Typical report periods (spans of time), both finite and open-ended.
+-- Some common report subperiods, both finite and open-ended.
 -- A higher-level abstraction than DateSpan.
 data Period =
     DayPeriod Day
@@ -132,16 +132,8 @@ data Period =
 
 instance Default Period where def = PeriodAll
 
----- Typical report period/subperiod durations, from a day to a year.
---data Duration =
---    DayLong
---   WeekLong
---   MonthLong
---   QuarterLong
---   YearLong
---  deriving (Eq,Ord,Show,Generic)
-
--- Ways in which a period can be divided into subperiods.
+-- All the kinds of report interval allowed in a period expression
+-- (to generate periodic reports or periodic transactions).
 data Interval =
     NoInterval
   | Days Int
@@ -149,13 +141,10 @@ data Interval =
   | Months Int
   | Quarters Int
   | Years Int
-  | DayOfMonth Int
-  | WeekdayOfMonth Int Int
-  | DaysOfWeek [Int]
-  | DayOfYear Int Int -- Month, Day
-  -- WeekOfYear Int
-  -- MonthOfYear Int
-  -- QuarterOfYear Int
+  | NthWeekdayOfMonth Int Int  -- n,              weekday 1-7
+  | MonthDay Int               -- 1-31
+  | MonthAndDay Int Int        -- month 1-12,     monthday 1-31
+  | DaysOfWeek [Int]           -- [weekday 1-7]
   deriving (Eq,Show,Ord,Generic)
 
 instance Default Interval where def = NoInterval

hledger-lib/Hledger/Read/Common.hs

@@ -96,9 +96,10 @@ module Hledger.Read.Common (
   isSameLineCommentStart,
   multilinecommentp,
   emptyorcommentlinep,
+  emptyorcommentlinep2,
   followingcommentp,
   transactioncommentp,
-  commenttagsp,
+  commentlinetagsp,
   postingcommentp,
 
   -- ** bracketed dates
@@ -150,9 +151,6 @@ import System.FilePath (takeFileName)
 import Text.Megaparsec
 import Text.Megaparsec.Char (char, char', digitChar, newline, string)
 import Text.Megaparsec.Char.Lexer (decimal)
-import Text.Megaparsec.Custom
-  (FinalParseError, attachSource, finalErrorBundlePretty, parseErrorAt, parseErrorAtRegion)
--- import Text.Megaparsec.Debug (dbg)  -- from megaparsec 9.3+
 
 import Hledger.Data
 import Hledger.Query (Query(..), filterQuery, parseQueryTerm, queryEndDate, queryStartDate, queryIsDate, simplifyQuery)
@@ -211,7 +209,7 @@ rawOptsToInputOpts day rawopts =
     in definputopts{
        -- files_             = listofstringopt "file" rawopts
        mformat_           = Nothing
-      ,mrules_file_       = maybestringopt "rules-file" rawopts
+      ,mrules_file_       = maybestringopt "rules" rawopts
       ,aliases_           = listofstringopt "alias" rawopts
       ,anon_              = boolopt "obfuscate" rawopts
       ,new_               = boolopt "new" rawopts
@@ -361,7 +359,7 @@ journalFinalise iopts@InputOpts{auto_,balancingopts_,infer_costs_,infer_equity_,
        -- >>= Right . dbg0With (concatMap (T.unpack.showTransaction).jtxns)
        -- >>= \j -> deepseq (concatMap (T.unpack.showTransaction).jtxns $ j) (return j)
       <&> dbg9With (lbl "amounts after styling, forecasting, auto-posting".showJournalAmountsDebug)
-      >>= (\j -> if checkordereddates then journalCheckOrdereddates j <&> const j else Right j)  -- check ordereddates before assertions. The outer parentheses are needed.
+      >>= (\j -> if checkordereddates then journalCheckOrdereddates j $> j else Right j)  -- check ordereddates before assertions. The outer parentheses are needed.
       >>= journalBalanceTransactions balancingopts_{ignore_assertions_=not checkassertions}  -- infer balance assignments and missing amounts, and maybe check balance assertions.
       <&> dbg9With (lbl "amounts after transaction-balancing".showJournalAmountsDebug)
       -- <&> dbg9With (("journalFinalise amounts after styling, forecasting, auto postings, transaction balancing"<>).showJournalAmountsDebug)
@@ -409,6 +407,10 @@ setYear y = modify' (\j -> j{jparsedefaultyear=Just y})
 getYear :: JournalParser m (Maybe Year)
 getYear = fmap jparsedefaultyear get
 
+dp :: String -> TextParser m ()
+dp = const $ return ()  -- no-op
+-- dp = dbgparse 1  -- trace parse state at this --debug level
+
 -- | Get the decimal mark that has been specified for parsing, if any
 -- (eg by the CSV decimal-mark rule, or possibly a future journal directive).
 -- Return it as an AmountStyle that amount parsers can use.
@@ -1262,9 +1264,10 @@ multilinecommentp = startComment *> anyLine `skipManyTill` endComment
 
 -- | A blank or comment line in journal format: a line that's empty or
 -- containing only whitespace or whose first non-whitespace character
--- is semicolon, hash, or star.
+-- is semicolon, hash, or star. See also emptyorcommentlinep2.
 emptyorcommentlinep :: TextParser m ()
 emptyorcommentlinep = do
+  dp "emptyorcommentlinep"
   skipNonNewlineSpaces
   skiplinecommentp <|> void newline
   where
@@ -1277,6 +1280,19 @@ emptyorcommentlinep = do
 
 {-# INLINABLE emptyorcommentlinep #-}
 
+-- | A newer comment line parser.
+-- Parses a line which is empty, all blanks, or whose first non-blank character is one of those provided.
+emptyorcommentlinep2 :: [Char] -> TextParser m ()
+emptyorcommentlinep2 cs =
+  label ("empty line or comment line beginning with "++cs) $ do
+    dp "emptyorcommentlinep2"
+    skipNonNewlineSpaces
+    void newline <|> void commentp
+    where
+      commentp = do
+        choice (map (some.char) cs)
+        takeWhileP Nothing (/='\n') <* newline
+
 -- | Is this a character that, as the first non-whitespace on a line,
 -- starts a comment line ?
 isLineCommentStart :: Char -> Bool
@@ -1291,25 +1307,44 @@ isSameLineCommentStart :: Char -> Bool
 isSameLineCommentStart ';' = True
 isSameLineCommentStart _   = False
 
--- A parser for (possibly multiline) comments following a journal item.
+-- | Parse a comment following a journal item, possibly continued on multiple lines,
+-- and return the comment text.
 --
--- Comments following a journal item begin with a semicolon and extend to
--- the end of the line. They may span multiple lines; any comment lines 
--- not on the same line as the journal item must be indented (preceded by
--- leading whitespace).
+-- >>> rtp followingcommentp ""   -- no comment
+-- Right ""
+-- >>> rtp followingcommentp ";"    -- just a (empty) same-line comment. newline is added
+-- Right "\n"
+-- >>> rtp followingcommentp ";  \n"
+-- Right "\n"
+-- >>> rtp followingcommentp ";\n ;\n"  -- a same-line and a next-line comment
+-- Right "\n\n"
+-- >>> rtp followingcommentp "\n ;\n"  -- just a next-line comment. Insert an empty same-line comment so the next-line comment doesn't become a same-line comment.
+-- Right "\n\n"
 --
--- Like Ledger, we sometimes allow data to be embedded in comments. Eg,
--- comments on the account directive and on transactions can contain tags,
--- and comments on postings can contain tags and/or bracketed posting dates.
--- To handle these variations, this parser takes as parameter a subparser,
--- which should consume all input up until the next newline, and which can
--- optionally extract some kind of data from it.
--- followingcommentp' returns this data along with the full text of the comment.
+followingcommentp :: TextParser m Text
+followingcommentp =
+  fst <$> followingcommentpWith (void $ takeWhileP Nothing (/= '\n'))  -- XXX support \r\n ?
+
+{-# INLINABLE followingcommentp #-}
+
+-- | Parse a following comment, possibly continued on multiple lines,
+-- using the provided line parser to parse each line.
+-- This returns the comment text, and the combined results from the line parser.
 --
--- See followingcommentp for tests.
+-- Following comments begin with a semicolon and extend to the end of the line.
+-- They can optionally be continued on the next lines,
+-- where each next line begins with an indent and another semicolon.
+-- (This parser expects to see these semicolons and indents.)
 --
-followingcommentp' :: (Monoid a, Show a) => TextParser m a -> TextParser m (Text, a)
-followingcommentp' contentp = do
+-- Like Ledger, we sometimes allow data to be embedded in comments.
+-- account directive comments and transaction comments can contain tags,
+-- and posting comments can contain tags or bracketed posting dates.
+-- This helper lets us handle these variations. 
+-- The line parser should consume all input up until the next newline.
+-- See followingcommentp for some tests.
+--
+followingcommentpWith :: (Monoid a, Show a) => TextParser m a -> TextParser m (Text, a)
+followingcommentpWith contentp = do
   skipNonNewlineSpaces
   -- there can be 0 or 1 sameLine
   sameLine <- try headerp *> ((:[]) <$> match' contentp) <|> pure []
@@ -1330,25 +1365,35 @@ followingcommentp' contentp = do
   where
     headerp = char ';' *> skipNonNewlineSpaces
 
-{-# INLINABLE followingcommentp' #-}
+{-# INLINABLE followingcommentpWith #-}
 
--- | Parse the text of a (possibly multiline) comment following a journal item.
---
--- >>> rtp followingcommentp ""   -- no comment
--- Right ""
--- >>> rtp followingcommentp ";"    -- just a (empty) same-line comment. newline is added
--- Right "\n"
--- >>> rtp followingcommentp ";  \n"
--- Right "\n"
--- >>> rtp followingcommentp ";\n ;\n"  -- a same-line and a next-line comment
--- Right "\n\n"
--- >>> rtp followingcommentp "\n ;\n"  -- just a next-line comment. Insert an empty same-line comment so the next-line comment doesn't become a same-line comment.
--- Right "\n\n"
---
-followingcommentp :: TextParser m Text
-followingcommentp =
-  fst <$> followingcommentp' (void $ takeWhileP Nothing (/= '\n'))  -- XXX support \r\n ?
-{-# INLINABLE followingcommentp #-}
+
+-- Parse the tags from a single comment line, eg for use with followingcommentpWith.
+-- XXX what part of a comment line ? leading whitespace / semicolon or not ?
+commentlinetagsp :: TextParser m [Tag]
+commentlinetagsp = do
+  -- XXX sketchy
+  tagName <- (last . T.split isSpace) <$> takeWhileP Nothing (\c -> c /= ':' && c /= '\n')
+  atColon tagName <|> pure [] -- if not ':', then either '\n' or EOF
+
+  where
+    atColon :: Text -> TextParser m [Tag]
+    atColon name = char ':' *> do
+      if T.null name
+        then commentlinetagsp
+        else do
+          skipNonNewlineSpaces
+          val <- tagValue
+          let tag = (name, val)
+          (tag:) <$> commentlinetagsp
+
+    tagValue :: TextParser m Text
+    tagValue = do
+      val <- T.strip <$> takeWhileP Nothing (\c -> c /= ',' && c /= '\n')
+      _ <- optional $ char ','
+      pure val
+
+{-# INLINABLE commentlinetagsp #-}
 
 
 -- | Parse a transaction comment and extract its tags.
@@ -1377,33 +1422,9 @@ followingcommentp =
 -- leading and trailing whitespace.
 --
 transactioncommentp :: TextParser m (Text, [Tag])
-transactioncommentp = followingcommentp' commenttagsp
+transactioncommentp = followingcommentpWith commentlinetagsp
 {-# INLINABLE transactioncommentp #-}
 
-commenttagsp :: TextParser m [Tag]
-commenttagsp = do
-  tagName <- (last . T.split isSpace) <$> takeWhileP Nothing (\c -> c /= ':' && c /= '\n')
-  atColon tagName <|> pure [] -- if not ':', then either '\n' or EOF
-
-  where
-    atColon :: Text -> TextParser m [Tag]
-    atColon name = char ':' *> do
-      if T.null name
-        then commenttagsp
-        else do
-          skipNonNewlineSpaces
-          val <- tagValue
-          let tag = (name, val)
-          (tag:) <$> commenttagsp
-
-    tagValue :: TextParser m Text
-    tagValue = do
-      val <- T.strip <$> takeWhileP Nothing (\c -> c /= ',' && c /= '\n')
-      _ <- optional $ char ','
-      pure val
-
-{-# INLINABLE commenttagsp #-}
-
 
 -- | Parse a posting comment and extract its tags and dates.
 --
@@ -1457,7 +1478,7 @@ postingcommentp
   :: Maybe Year -> TextParser m (Text, [Tag], Maybe Day, Maybe Day)
 postingcommentp mYear = do
   (commentText, (tags, dateTags)) <-
-    followingcommentp' (commenttagsanddatesp mYear)
+    followingcommentpWith (commenttagsanddatesp mYear)
   let mdate  = snd <$> find ((=="date") .fst) dateTags
       mdate2 = snd <$> find ((=="date2").fst) dateTags
   pure (commentText, tags, mdate, mdate2)

hledger-lib/Hledger/Read/JournalReader.hs

@@ -91,7 +91,6 @@ import Data.Time.LocalTime
 import Safe
 import Text.Megaparsec hiding (parse)
 import Text.Megaparsec.Char
-import Text.Megaparsec.Custom
 import Text.Printf
 import System.FilePath
 import "Glob" System.FilePath.Glob hiding (match)

hledger-lib/Hledger/Read/RulesReader.hs

@@ -14,6 +14,7 @@ Most of the code for reading rules files and csv files is in this module.
 -- stack haddock hledger-lib --fast --no-haddock-deps --haddock-arguments='--ignore-all-exports' --open
 
 --- ** language
+{-# LANGUAGE CPP    #-}
 {-# LANGUAGE FlexibleInstances    #-}
 {-# LANGUAGE OverloadedStrings    #-}
 {-# LANGUAGE RecordWildCards      #-}
@@ -52,7 +53,10 @@ import Control.Monad.Trans.Class  (lift)
 import Data.Char                  (toLower, isDigit, isSpace, isAlphaNum, ord)
 import Data.Bifunctor             (first)
 import Data.Functor               ((<&>))
-import Data.List (elemIndex, foldl', mapAccumL, nub, sortOn)
+import Data.List (elemIndex, mapAccumL, nub, sortOn)
+#if !MIN_VERSION_base(4,20,0)
+import Data.List (foldl')
+#endif
 import Data.List.Extra (groupOn)
 import Data.Maybe (catMaybes, fromMaybe, isJust)
 import Data.MemoUgly (memo)
@@ -72,13 +76,12 @@ import qualified Data.ByteString.Lazy as BL
 import Data.Foldable (asum, toList)
 import Text.Megaparsec hiding (match, parse)
 import Text.Megaparsec.Char (char, newline, string, digitChar)
-import Text.Megaparsec.Custom (parseErrorAt)
 import Text.Printf (printf)
 
 import Hledger.Data
 import Hledger.Utils
-import Hledger.Read.Common (aliasesFromOpts, Reader(..), InputOpts(..), amountp, statusp, journalFinalise, accountnamep, commenttagsp )
-import Hledger.Read.CsvUtils
+import Hledger.Read.Common (aliasesFromOpts, Reader(..), InputOpts(..), amountp, statusp, journalFinalise, accountnamep, transactioncommentp, postingcommentp )
+import Hledger.Write.Csv
 import System.Directory (doesFileExist, getHomeDirectory)
 import Data.Either (fromRight)
 
@@ -113,7 +116,7 @@ getDownloadDir = do
 -- file's directory. When a glob pattern matches multiple files, the alphabetically
 -- last is used. (Eg in case of multiple numbered downloads, the highest-numbered
 -- will be used.)
--- The provided text, or a --rules-file option, are ignored by this reader.
+-- The provided text, or a --rules option, are ignored by this reader.
 -- Balance assertions are not checked.
 parse :: InputOpts -> FilePath -> Text -> ExceptT String IO Journal
 parse iopts f _ = do
@@ -368,7 +371,7 @@ FIELD-NAME: QUOTED-FIELD-NAME | BARE-FIELD-NAME
 
 QUOTED-FIELD-NAME: " (any CHAR except double-quote)+ "
 
-BARE-FIELD-NAME: any CHAR except space, tab, #, ;
+BARE-FIELD-NAME: (any CHAR except space, tab, #, ;)+
 
 FIELD-ASSIGNMENT: JOURNAL-FIELD ASSIGNMENT-SEPARATOR FIELD-VALUE
 
@@ -838,7 +841,7 @@ regexMatchValue rules record sgroup = let
                $ concatMap cbMatchers
                $ filter (isBlockActive rules record)
                $ rconditionalblocks rules
-               -- ^ XXX adjusted to not use memoized field as caller might be sending a subset of rules with just one CB (hacky)
+               -- XXX adjusted to not use memoized field as caller might be sending a subset of rules with just one CB (hacky)
   group = (read (T.unpack sgroup) :: Int) - 1 -- adjust to 0-indexing
   in atMay matchgroups group
 
@@ -884,7 +887,7 @@ _CSV_READING__________________________________________ = undefined
 -- 4. Return the transactions as a Journal.
 --
 readJournalFromCsv :: Maybe (Either CsvRules FilePath) -> FilePath -> Text -> Maybe SepFormat -> ExceptT String IO Journal
-readJournalFromCsv Nothing "-" _ _ = throwError "please use --rules-file when reading CSV from stdin"
+readJournalFromCsv Nothing "-" _ _ = throwError "please use --rules when reading CSV from stdin"
 readJournalFromCsv merulesfile csvfile csvtext sep = do
     -- for now, correctness is the priority here, efficiency not so much
 
@@ -1112,7 +1115,13 @@ transactionFromCsvRecord timesarezoned mtzin tzout sourcepos rules record = t
     code        = maybe "" singleline' $ fieldval "code"
     description = maybe "" singleline' $ fieldval "description"
     comment     = maybe "" unescapeNewlines $ fieldval "comment"
-    ttags       = fromRight [] $ rtp commenttagsp comment
+
+    -- Convert some parsed comment text back into following comment syntax,
+    -- with the semicolons and indents, so it can be parsed again for tags.
+    textToFollowingComment :: Text -> Text
+    textToFollowingComment = T.stripStart . T.unlines . map (" ;"<>) . T.lines
+
+    ttags       = fromRight [] $ fmap snd $ rtp transactioncommentp $ textToFollowingComment comment
     precomment  = maybe "" unescapeNewlines $ fieldval "precomment"
 
     singleline' = T.unwords . filter (not . T.null) . map T.strip . T.lines
@@ -1125,7 +1134,15 @@ transactionFromCsvRecord timesarezoned mtzin tzout sourcepos rules record = t
     p1IsVirtual = (accountNamePostingType <$> fieldval "account1") == Just VirtualPosting
     ps = [p | n <- [1..maxpostings]
          ,let cmt  = maybe "" unescapeNewlines $ fieldval ("comment"<> T.pack (show n))
-         ,let ptags = fromRight [] $ rtp commenttagsp cmt
+          -- Tags in the comment will be parsed and attached to the posting.
+          -- A posting date, in the date: tag or in brackets, will also be parsed and applied to the posting.
+          -- But it must have a year, or it will be ignored.
+          -- A secondary posting date will also be ignored.
+         ,let (tags,mdate) =
+                fromRight ([],Nothing) $
+                fmap (\(_,ts,md,_)->(ts,md)) $
+                rtp (postingcommentp Nothing) $
+                textToFollowingComment cmt
          ,let currency = fromMaybe "" (fieldval ("currency"<> T.pack (show n)) <|> fieldval "currency")
          ,let mamount  = getAmount rules record currency p1IsVirtual n
          ,let mbalance = getBalance rules record currency n
@@ -1133,12 +1150,13 @@ transactionFromCsvRecord timesarezoned mtzin tzout sourcepos rules record = t
          ,let acct' | not isfinal && acct==unknownExpenseAccount &&
                       fromMaybe False (mamount >>= isNegativeMixedAmount) = unknownIncomeAccount
                     | otherwise = acct
-         ,let p = nullposting{paccount          = accountNameWithoutPostingType acct'
+         ,let p = nullposting{pdate             = mdate
+                             ,paccount          = accountNameWithoutPostingType acct'
                              ,pamount           = fromMaybe missingmixedamt mamount
                              ,ptransaction      = Just t
                              ,pbalanceassertion = mkBalanceAssertion rules record <$> mbalance
                              ,pcomment          = cmt
-                             ,ptags             = ptags
+                             ,ptags             = tags
                              ,ptype             = accountNamePostingType acct
                              }
          ]
@@ -1311,9 +1329,7 @@ parseAmount rules record currency s =
       ,showRules rules record
       -- ,"the default-currency is: "++fromMaybe "unspecified" (getDirective "default-currency" rules)
       ,"the parse error is:      " <> T.pack (customErrorBundlePretty e)
-      ,"you may need to \
-        \change your amount*, balance*, or currency* rules, \
-        \or add or change your skip rule"
+      ,"you may need to change your amount*, balance*, or currency* rules, or add or change your skip rule"
       ]
 
 -- | Show the values assigned to each journal field.

hledger-lib/Hledger/Read/TimedotReader.hs

@@ -50,7 +50,7 @@ import Text.Megaparsec hiding (parse)
 import Text.Megaparsec.Char
 
 import Hledger.Data
-import Hledger.Read.Common hiding (emptyorcommentlinep)
+import Hledger.Read.Common
 import Hledger.Utils
 import Data.Decimal (roundTo)
 import Data.Functor ((<&>))
@@ -80,12 +80,10 @@ parse iopts fp t = initialiseAndParseJournal timedotp iopts fp t
 
 --- ** utilities
 
-traceparse, traceparse' :: String -> TextParser m ()
-traceparse  = const $ return ()
-traceparse' = const $ return ()
--- for debugging:
--- traceparse  s = traceParse (s++"?")
--- traceparse' s = trace s $ return ()
+-- Trace parser state above a certain --debug level ?
+tracelevel = 9
+dp :: String -> JournalParser m ()
+dp = if tracelevel >= 0 then lift . dbgparse tracelevel else const $ return ()
 
 --- ** parsers
 {-
@@ -113,9 +111,8 @@ timedotp = preamblep >> many dayp >> eof >> get
 
 preamblep :: JournalParser m ()
 preamblep = do
-  lift $ traceparse "preamblep"
-  many $ notFollowedBy datelinep >> (lift $ emptyorcommentlinep "#;*")
-  lift $ traceparse' "preamblep"
+  dp "preamblep"
+  void $ many $ notFollowedBy datelinep >> (lift $ emptyorcommentlinep2 "#;*")
 
 -- | Parse timedot day entries to multi-posting time transactions for that day.
 -- @
@@ -126,11 +123,13 @@ preamblep = do
 -- @
 dayp :: JournalParser m ()
 dayp = label "timedot day entry" $ do
-  lift $ traceparse "dayp"
+  dp "dayp"
   pos <- getSourcePos
   (date,desc,comment,tags) <- datelinep
+  dp "dayp1"
   commentlinesp
-  ps <- (many $ timedotentryp <* commentlinesp) <&> concat
+  dp "dayp2"
+  ps <- (many $ dp "dayp3" >> timedotentryp <* commentlinesp) <&> concat
   endpos <- getSourcePos
   let t = txnTieKnot $ nulltransaction{
     tsourcepos   = (pos, endpos),
@@ -145,7 +144,7 @@ dayp = label "timedot day entry" $ do
 
 datelinep :: JournalParser m (Day,Text,Text,[Tag])
 datelinep = do
-  lift $ traceparse "datelinep"
+  dp "datelinep"
   lift $ optional orgheadingprefixp
   date <- datep
   desc <- T.strip <$> lift descriptionp
@@ -156,16 +155,15 @@ datelinep = do
 -- or org headlines which do not start a new day.
 commentlinesp :: JournalParser m ()
 commentlinesp = do
-  lift $ traceparse "commentlinesp"
-  void $ many $ try $ lift $ emptyorcommentlinep "#;"
+  dp "commentlinesp"
+  void $ many $ try $ lift $ emptyorcommentlinep2 "#;"
 
 -- orgnondatelinep :: JournalParser m ()
 -- orgnondatelinep = do
---   lift $ traceparse "orgnondatelinep"
+--   dp "orgnondatelinep"
 --   lift orgheadingprefixp
 --   notFollowedBy datelinep
 --   void $ lift restofline
---   lift $ traceparse' "orgnondatelinep"
 
 orgheadingprefixp = skipSome (char '*') >> skipNonNewlineSpaces1
 
@@ -175,7 +173,7 @@ orgheadingprefixp = skipSome (char '*') >> skipNonNewlineSpaces1
 -- @
 timedotentryp :: JournalParser m [Posting]
 timedotentryp = do
-  lift $ traceparse "timedotentryp"
+  dp "timedotentryp"
   notFollowedBy datelinep
   lift $ optional $ choice [orgheadingprefixp, skipNonNewlineSpaces1]
   a <- modifiedaccountnamep
@@ -225,7 +223,7 @@ durationsp =
 -- @
 numericquantityp :: TextParser m Hours
 numericquantityp = do
-  -- lift $ traceparse "numericquantityp"
+  -- dp "numericquantityp"
   (q, _, _, _) <- numberp Nothing
   msymbol <- optional $ choice $ map (string . fst) timeUnits
   skipNonNewlineSpaces
@@ -257,7 +255,7 @@ timeUnits =
 -- @
 dotquantityp :: TextParser m Hours
 dotquantityp = do
-  -- lift $ traceparse "dotquantityp"
+  -- dp "dotquantityp"
   char '.'
   dots <- many (oneOf ['.', ' ']) <&> filter (not.isSpace)
   return $ fromIntegral (1 + length dots) / 4
@@ -267,7 +265,7 @@ dotquantityp = do
 -- ignoring any interspersed spaces after the first letter.
 letterquantitiesp :: TextParser m [(Hours, TagValue)]
 letterquantitiesp =
-  -- dbg "letterquantitiesp" $ 
+  -- dp "letterquantitiesp"
   do
     letter1 <- letterChar
     letters <- many (letterChar <|> spacenonewline) <&> filter (not.isSpace)
@@ -276,19 +274,3 @@ letterquantitiesp =
           | t@(c:_) <- group $ sort $ letter1:letters
           ]
     return groups
-
--- | XXX new comment line parser, move to Hledger.Read.Common.emptyorcommentlinep
--- Parse empty lines, all-blank lines, and lines beginning with any of the provided
--- comment-beginning characters.
-emptyorcommentlinep :: [Char] -> TextParser m ()
-emptyorcommentlinep cs =
-  label ("empty line or comment line beginning with "++cs) $ do
-    traceparse "emptyorcommentlinep" -- XXX possible to combine label and traceparse ?
-    skipNonNewlineSpaces
-    void newline <|> void commentp
-    traceparse' "emptyorcommentlinep"
-    where
-      commentp = do
-        choice (map (some.char) cs)
-        takeWhileP Nothing (/='\n') <* newline
-

hledger-lib/Hledger/Reports/BudgetReport.hs

@@ -1,5 +1,4 @@
 {-# LANGUAGE OverloadedStrings   #-}
-{-# LANGUAGE RecordWildCards     #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 
 module Hledger.Reports.BudgetReport (
@@ -10,9 +9,6 @@ module Hledger.Reports.BudgetReport (
   BudgetReportRow,
   BudgetReport,
   budgetReport,
-  budgetReportAsTable,
-  budgetReportAsText,
-  budgetReportAsCsv,
   -- * Helpers
   combineBudgetAndActual,
   -- * Tests
@@ -21,25 +17,16 @@ module Hledger.Reports.BudgetReport (
 where
 
 import Control.Applicative ((<|>))
-import Control.Arrow ((***))
-import Data.Decimal (roundTo)
-import Data.Function (on)
 import Data.HashMap.Strict (HashMap)
 import qualified Data.HashMap.Strict as HM
-import Data.List (find, partition, transpose, foldl', maximumBy, intercalate)
+import Data.List (find, partition, maximumBy, intercalate)
 import Data.List.Extra (nubSort)
-import Data.Maybe (fromMaybe, catMaybes, isJust)
+import Data.Maybe (fromMaybe, isJust)
 import Data.Map (Map)
 import qualified Data.Map as Map
 import qualified Data.Set as S
-import Data.Text (Text)
 import qualified Data.Text as T
-import qualified Data.Text.Lazy as TL
-import qualified Data.Text.Lazy.Builder as TB
 import Safe (minimumDef)
---import System.Console.CmdArgs.Explicit as C
---import Lucid as L
-import qualified Text.Tabular.AsciiWide as Tab
 
 import Hledger.Data
 import Hledger.Utils
@@ -62,17 +49,6 @@ type BudgetReportRow = PeriodicReportRow DisplayName BudgetCell
 -- | A full budget report table.
 type BudgetReport    = PeriodicReport    DisplayName BudgetCell
 
--- A BudgetCell's data values rendered for display - the actual change amount,
--- the budget goal amount if any, and the corresponding goal percentage if possible.
-type BudgetDisplayCell = (WideBuilder, Maybe (WideBuilder, Maybe WideBuilder))
--- | A row of rendered budget data cells.
-type BudgetDisplayRow  = [BudgetDisplayCell]
-
--- | An amount render helper for the budget report. Renders each commodity separately.
-type BudgetShowAmountsFn   = MixedAmount -> [WideBuilder]
--- | A goal percentage calculating helper for the budget report.
-type BudgetCalcPercentagesFn  = Change -> BudgetGoal -> [Maybe Percentage]
-
 _brrShowDebug :: BudgetReportRow -> String
 _brrShowDebug (PeriodicReportRow dname budgetpairs _tot _avg) =
   unwords [
@@ -279,280 +255,6 @@ combineBudgetAndActual ropts j
         totActualByPeriod = Map.fromList $ zip actualperiods actualtots :: Map DateSpan Change
         budget b = if mixedAmountLooksZero b then Nothing else Just b
 
--- | Render a budget report as plain text suitable for console output.
-budgetReportAsText :: ReportOpts -> BudgetReport -> TL.Text
-budgetReportAsText ropts@ReportOpts{..} budgetr = TB.toLazyText $
-    TB.fromText title <> TB.fromText "\n\n" <>
-      balanceReportTableAsText ropts (budgetReportAsTable ropts budgetr)
-  where
-    title = "Budget performance in " <> showDateSpan (periodicReportSpan budgetr)
-           <> (case conversionop_ of
-                 Just ToCost -> ", converted to cost"
-                 _           -> "")
-           <> (case value_ of
-                 Just (AtThen _mc)   -> ", valued at posting date"
-                 Just (AtEnd _mc)    -> ", valued at period ends"
-                 Just (AtNow _mc)    -> ", current value"
-                 Just (AtDate d _mc) -> ", valued at " <> showDate d
-                 Nothing             -> "")
-           <> ":"
-
--- | Build a 'Table' from a multi-column balance report.
-budgetReportAsTable :: ReportOpts -> BudgetReport -> Tab.Table Text Text WideBuilder
-budgetReportAsTable ReportOpts{..} (PeriodicReport spans items totrow) =
-  maybetransposetable $
-  addtotalrow $
-    Tab.Table
-      (Tab.Group Tab.NoLine $ map Tab.Header accts)
-      (Tab.Group Tab.NoLine $ map Tab.Header colheadings)
-      rows
-  where
-    maybetransposetable
-      | transpose_ = \(Tab.Table rh ch vals) -> Tab.Table ch rh (transpose vals)
-      | otherwise  = id
-
-    addtotalrow
-      | no_total_ = id
-      | otherwise = let rh = Tab.Group Tab.NoLine . replicate (length totalrows) $ Tab.Header ""
-                        ch = Tab.Header [] -- ignored
-                     in (flip (Tab.concatTables Tab.SingleLine) $ Tab.Table rh ch totalrows)
-
-    colheadings = ["Commodity" | layout_ == LayoutBare]
-                  ++ map (reportPeriodName balanceaccum_ spans) spans
-                  ++ ["  Total" | row_total_]
-                  ++ ["Average" | average_]
-
-    (accts, rows, totalrows) =
-      (accts'
-      ,maybecommcol itemscs  $ showcells  texts
-      ,maybecommcol totrowcs $ showtotrow totrowtexts)
-      where
-        -- If --layout=bare, prepend a commodities column.
-        maybecommcol :: [WideBuilder] -> [[WideBuilder]] -> [[WideBuilder]]
-        maybecommcol cs
-          | layout_ == LayoutBare = zipWith (:) cs
-          | otherwise             = id
-
-        showcells, showtotrow :: [[BudgetDisplayCell]] -> [[WideBuilder]]
-        (showcells, showtotrow) =
-          (maybetranspose . map (zipWith showBudgetDisplayCell widths)       . maybetranspose
-          ,maybetranspose . map (zipWith showBudgetDisplayCell totrowwidths) . maybetranspose)
-          where
-            -- | Combine a BudgetDisplayCell's rendered values into a "[PERCENT of GOAL]" rendering,
-            -- respecting the given widths.
-            showBudgetDisplayCell :: (Int, Int, Int) -> BudgetDisplayCell -> WideBuilder
-            showBudgetDisplayCell (actualwidth, budgetwidth, percentwidth) (actual, mbudget) =
-              flip WideBuilder (actualwidth + totalbudgetwidth) $
-                toPadded actual <> maybe emptycell showBudgetGoalAndPercentage mbudget
-
-              where
-                toPadded (WideBuilder b w) = (TB.fromText . flip T.replicate " " $ actualwidth - w) <> b
-
-                (totalpercentwidth, totalbudgetwidth) =
-                  let totalpercentwidth' = if percentwidth == 0 then 0 else percentwidth + 5
-                   in ( totalpercentwidth'
-                      , if budgetwidth == 0 then 0 else budgetwidth + totalpercentwidth' + 3
-                      )
-
-                emptycell :: TB.Builder
-                emptycell = TB.fromText $ T.replicate totalbudgetwidth " "
-
-                showBudgetGoalAndPercentage :: (WideBuilder, Maybe WideBuilder) -> TB.Builder
-                showBudgetGoalAndPercentage (goal, perc) =
-                  let perct = case perc of
-                        Nothing  -> T.replicate totalpercentwidth " "
-                        Just pct -> T.replicate (percentwidth - wbWidth pct) " " <> wbToText pct <> "% of "
-                   in TB.fromText $ " [" <> perct <> T.replicate (budgetwidth - wbWidth goal) " " <> wbToText goal <> "]"
-
-            -- | Build a list of widths for each column.
-            -- When --transpose is used, the totals row must be included in this list.
-            widths :: [(Int, Int, Int)]
-            widths = zip3 actualwidths budgetwidths percentwidths
-              where
-                actualwidths  = map (maximum' . map first3 ) $ cols
-                budgetwidths  = map (maximum' . map second3) $ cols
-                percentwidths = map (maximum' . map third3 ) $ cols
-                catcolumnwidths = foldl' (zipWith (++)) $ repeat []
-                cols = maybetranspose $ catcolumnwidths $ map (cellswidth . rowToBudgetCells) items ++ [cellswidth $ rowToBudgetCells totrow]
-
-                cellswidth :: [BudgetCell] -> [[(Int, Int, Int)]]
-                cellswidth row =
-                  let cs = budgetCellsCommodities row
-                      (showmixed, percbudget) = mkBudgetDisplayFns cs
-                      disp = showcell showmixed percbudget
-                      budgetpercwidth = wbWidth *** maybe 0 wbWidth
-                      cellwidth (am, bm) = let (bw, pw) = maybe (0, 0) budgetpercwidth bm in (wbWidth am, bw, pw)
-                   in map (map cellwidth . disp) row
-
-            totrowwidths :: [(Int, Int, Int)]
-            totrowwidths
-              | transpose_ = drop (length texts) widths
-              | otherwise = widths
-
-            maybetranspose
-              | transpose_ = transpose
-              | otherwise  = id
-
-        (accts', itemscs, texts) = unzip3 $ concat shownitems
-          where
-            shownitems :: [[(AccountName, WideBuilder, BudgetDisplayRow)]]
-            shownitems =
-              map (\i ->
-                let
-                  addacctcolumn = map (\(cs, cvals) -> (renderacct i, cs, cvals))
-                  isunbudgetedrow = displayFull (prrName i) == unbudgetedAccountName
-                in addacctcolumn $ showrow isunbudgetedrow $ rowToBudgetCells i)
-              items
-              where
-                -- FIXME. Have to check explicitly for which to render here, since
-                -- budgetReport sets accountlistmode to ALTree. Find a principled way to do
-                -- this.
-                renderacct row = case accountlistmode_ of
-                  ALTree -> T.replicate ((prrDepth row - 1)*2) " " <> prrDisplayName row
-                  ALFlat -> accountNameDrop (drop_) $ prrFullName row
-
-        (totrowcs, totrowtexts)  = unzip  $ concat showntotrow
-          where
-            showntotrow :: [[(WideBuilder, BudgetDisplayRow)]]
-            showntotrow = [showrow False $ rowToBudgetCells totrow]
-
-        -- | Get the data cells from a row or totals row, maybe adding 
-        -- the row total and/or row average depending on options.
-        rowToBudgetCells :: PeriodicReportRow a BudgetCell -> [BudgetCell]
-        rowToBudgetCells (PeriodicReportRow _ as rowtot rowavg) = as
-            ++ [rowtot | row_total_ && not (null as)]
-            ++ [rowavg | average_   && not (null as)]
-
-        -- | Render a row's data cells as "BudgetDisplayCell"s, and a rendered list of commodity symbols.
-        -- Also requires a flag indicating whether this is the special <unbudgeted> row.
-        -- (The types make that hard to check here.)
-        showrow :: Bool -> [BudgetCell] -> [(WideBuilder, BudgetDisplayRow)]
-        showrow isunbudgetedrow cells =
-          let
-            cs = budgetCellsCommodities cells
-            -- #2071 If there are no commodities - because there are no actual or goal amounts -
-            -- the zipped list would be empty, causing this row not to be shown.
-            -- But rows like this sometimes need to be shown to preserve the account tree structure.
-            -- So, ensure 0 will be shown as actual amount(s).
-            -- Unfortunately this disables boring parent eliding, as if --no-elide had been used.
-            -- (Just turning on --no-elide higher up doesn't work right.)
-            -- Note, no goal amount will be shown for these rows,
-            -- whereas --no-elide is likely to show a goal amount aggregated from children.
-            cs1 = if null cs && not isunbudgetedrow then [""] else cs
-            (showmixed, percbudget) = mkBudgetDisplayFns cs1
-          in
-            zip (map wbFromText cs1) $
-            transpose $
-            map (showcell showmixed percbudget)
-            cells
-
-        budgetCellsCommodities :: [BudgetCell] -> [CommoditySymbol]
-        budgetCellsCommodities = S.toList . foldl' S.union mempty . map budgetCellCommodities
-          where
-            budgetCellCommodities :: BudgetCell -> S.Set CommoditySymbol
-            budgetCellCommodities (am, bm) = f am `S.union` f bm
-              where f = maybe mempty maCommodities
-
-        -- | Render a "BudgetCell"'s amounts as "BudgetDisplayCell"s (one per commodity).
-        showcell :: BudgetShowAmountsFn -> BudgetCalcPercentagesFn -> BudgetCell -> BudgetDisplayRow
-        showcell showCommodityAmounts calcCommodityPercentages (mactual, mbudget) =
-          zip actualamts budgetinfos
-          where
-            actual = fromMaybe nullmixedamt mactual
-            actualamts = showCommodityAmounts actual
-            budgetinfos =
-              case mbudget of
-                Nothing   -> repeat Nothing
-                Just goal -> map Just $ showGoalAmountsAndPercentages goal
-                where
-                  showGoalAmountsAndPercentages :: MixedAmount -> [(WideBuilder, Maybe WideBuilder)]
-                  showGoalAmountsAndPercentages goal = zip amts mpcts
-                    where
-                      amts  = showCommodityAmounts goal
-                      mpcts = map (showrounded <$>) $ calcCommodityPercentages actual goal
-                        where showrounded = wbFromText . T.pack . show . roundTo 0
-
-        -- | Make budget info display helpers that adapt to --layout=wide.
-        mkBudgetDisplayFns :: [CommoditySymbol] -> (BudgetShowAmountsFn, BudgetCalcPercentagesFn)
-        mkBudgetDisplayFns cs = case layout_ of
-          LayoutWide width ->
-               ( pure . showMixedAmountB oneLineNoCostFmt{displayMaxWidth=width, displayColour=color_}
-               , \a -> pure . percentage a)
-          _ -> ( showMixedAmountLinesB noCostFmt{displayCommodity=layout_/=LayoutBare, displayCommodityOrder=Just cs, displayMinWidth=Nothing, displayColour=color_}
-               , \a b -> map (percentage' a b) cs)
-          where
-            -- | Calculate the percentage of actual change to budget goal to show, if any.
-            -- If valuing at cost, both amounts are converted to cost before comparing.
-            -- A percentage will not be shown if:
-            --
-            -- - actual or goal are not the same, single, commodity
-            --
-            -- - the goal is zero
-            --
-            percentage :: Change -> BudgetGoal -> Maybe Percentage
-            percentage actual budget =
-              case (costedAmounts actual, costedAmounts budget) of
-                ([a], [b]) | (acommodity a == acommodity b || amountLooksZero a) && not (amountLooksZero b)
-                    -> Just $ 100 * aquantity a / aquantity b
-                _   -> Nothing
-              where
-                costedAmounts = case conversionop_ of
-                    Just ToCost -> amounts . mixedAmountCost
-                    _           -> amounts
-
-            -- | Like percentage, but accept multicommodity actual and budget amounts,
-            -- and extract the specified commodity from both.
-            percentage' :: Change -> BudgetGoal -> CommoditySymbol -> Maybe Percentage
-            percentage' am bm c = case ((,) `on` find ((==) c . acommodity) . amounts) am bm of
-                (Just a, Just b) -> percentage (mixedAmount a) (mixedAmount b)
-                _                -> Nothing
-
--- XXX generalise this with multiBalanceReportAsCsv ?
--- | Render a budget report as CSV. Like multiBalanceReportAsCsv,
--- but includes alternating actual and budget amount columns.
-budgetReportAsCsv :: ReportOpts -> BudgetReport -> [[Text]]
-budgetReportAsCsv
-  ReportOpts{..}
-  (PeriodicReport colspans items totrow)
-  = (if transpose_ then transpose else id) $
-
-  -- heading row
-  ("Account" :
-  ["Commodity" | layout_ == LayoutBare ]
-   ++ concatMap (\spn -> [showDateSpan spn, "budget"]) colspans
-   ++ concat [["Total"  ,"budget"] | row_total_]
-   ++ concat [["Average","budget"] | average_]
-  ) :
-
-  -- account rows
-  concatMap (rowAsTexts prrFullName) items
-
-  -- totals row
-  ++ concat [ rowAsTexts (const "Total:") totrow | not no_total_ ]
-
-  where
-    flattentuples tups = concat [[a,b] | (a,b) <- tups]
-    showNorm = maybe "" (wbToText . showMixedAmountB oneLineNoCostFmt)
-
-    rowAsTexts :: (PeriodicReportRow a BudgetCell -> Text)
-               -> PeriodicReportRow a BudgetCell
-               -> [[Text]]
-    rowAsTexts render row@(PeriodicReportRow _ as (rowtot,budgettot) (rowavg, budgetavg))
-      | layout_ /= LayoutBare = [render row : map showNorm vals]
-      | otherwise =
-            joinNames . zipWith (:) cs  -- add symbols and names
-          . transpose                   -- each row becomes a list of Text quantities
-          . map (map wbToText . showMixedAmountLinesB dopts . fromMaybe nullmixedamt)
-          $ vals
-      where
-        cs = S.toList . foldl' S.union mempty . map maCommodities $ catMaybes vals
-        dopts = oneLineNoCostFmt{displayCommodity=layout_ /= LayoutBare, displayCommodityOrder=Just cs, displayMinWidth=Nothing}
-        vals = flattentuples as
-            ++ concat [[rowtot, budgettot] | row_total_]
-            ++ concat [[rowavg, budgetavg] | average_]
-
-        joinNames = map (render row :)
-
 -- tests
 
 tests_BudgetReport = testGroup "BudgetReport" [

hledger-lib/Hledger/Reports/MultiBalanceReport.hs

@@ -28,7 +28,6 @@ module Hledger.Reports.MultiBalanceReport (
   getPostings,
   startingPostings,
   generateMultiBalanceReport,
-  balanceReportTableAsText,
 
   -- -- * Tests
   tests_MultiBalanceReport
@@ -39,7 +38,7 @@ import Control.Monad (guard)
 import Data.Bifunctor (second)
 import Data.Foldable (toList)
 import Data.List (sortOn, transpose)
-import Data.List.NonEmpty (NonEmpty(..))
+import Data.List.NonEmpty (NonEmpty((:|)))
 import Data.HashMap.Strict (HashMap)
 import qualified Data.HashMap.Strict as HM
 import Data.Map (Map)
@@ -52,11 +51,6 @@ import qualified Data.Set as Set
 import Data.Time.Calendar (fromGregorian)
 import Safe (lastDef, minimumMay)
 
-import Data.Default (def)
-import qualified Data.Text as T
-import qualified Data.Text.Lazy.Builder as TB
-import qualified Text.Tabular.AsciiWide as Tab
-
 import Hledger.Data
 import Hledger.Query
 import Hledger.Utils hiding (dbg3,dbg4,dbg5)
@@ -594,33 +588,13 @@ periodChanges start amtmap =
 cumulativeSum :: Account -> Map DateSpan Account -> Map DateSpan Account
 cumulativeSum start = snd . M.mapAccum (\a b -> let s = sumAcct a b in (s, s)) start
 
--- | Given a table representing a multi-column balance report (for example,
--- made using 'balanceReportAsTable'), render it in a format suitable for
--- console output. Amounts with more than two commodities will be elided
--- unless --no-elide is used.
-balanceReportTableAsText :: ReportOpts -> Tab.Table T.Text T.Text WideBuilder -> TB.Builder
-balanceReportTableAsText ReportOpts{..} =
-    Tab.renderTableByRowsB def{Tab.tableBorders=False, Tab.prettyTable=pretty_} renderCh renderRow
-  where
-    renderCh
-      | layout_ /= LayoutBare || transpose_ = fmap (Tab.textCell Tab.TopRight)
-      | otherwise = zipWith ($) (Tab.textCell Tab.TopLeft : repeat (Tab.textCell Tab.TopRight))
-
-    renderRow (rh, row)
-      | layout_ /= LayoutBare || transpose_ =
-          (Tab.textCell Tab.TopLeft rh, fmap (Tab.Cell Tab.TopRight . pure) row)
-      | otherwise =
-          (Tab.textCell Tab.TopLeft rh, zipWith ($) (Tab.Cell Tab.TopLeft : repeat (Tab.Cell Tab.TopRight)) (fmap pure row))
-
-
-
 -- tests
 
 tests_MultiBalanceReport = testGroup "MultiBalanceReport" [
 
   let
-    amt0 = Amount {acommodity="$", aquantity=0, acost=Nothing, 
-      astyle=AmountStyle {ascommodityside = L, ascommodityspaced = False, asdigitgroups = Nothing, 
+    amt0 = Amount {acommodity="$", aquantity=0, acost=Nothing,
+      astyle=AmountStyle {ascommodityside = L, ascommodityspaced = False, asdigitgroups = Nothing,
       asdecimalmark = Just '.', asprecision = Precision 2, asrounding = NoRounding}}
     (rspec,journal) `gives` r = do
       let rspec' = rspec{_rsQuery=And [queryFromFlags $ _rsReportOpts rspec, _rsQuery rspec]}

hledger-lib/Hledger/Reports/PostingsReport.hs

@@ -15,15 +15,18 @@ module Hledger.Reports.PostingsReport (
   PostingsReportItem,
   postingsReport,
   mkpostingsReportItem,
+  SortSpec,
+  defsortspec,
 
   -- * Tests
   tests_PostingsReport
 )
 where
 
-import Data.List (nub, sortOn)
+import Data.List (nub, sortBy, sortOn)
 import Data.List.Extra (nubSort)
-import Data.Maybe (isJust, isNothing)
+import Data.Maybe (isJust, isNothing, fromMaybe)
+import Data.Ord
 import Data.Text (Text)
 import Data.Time.Calendar (Day)
 import Safe (headMay)
@@ -82,10 +85,12 @@ postingsReport rspec@ReportSpec{_rsReportOpts=ropts@ReportOpts{..}} j = items
           summariseps = summarisePostingsByInterval whichdate mdepth showempty colspans
           showempty = empty_ || average_
 
+      sortedps = if sortspec_ /= defsortspec then sortPostings ropts sortspec_ displayps else displayps
+
       -- Posting report items ready for display.
       items =
         dbg4 "postingsReport items" $
-        postingsReportItems displayps (nullposting,Nothing) whichdate mdepth startbal runningcalc startnum
+        postingsReportItems postings (nullposting,Nothing) whichdate mdepth startbal runningcalc startnum
         where
           -- In historical mode we'll need a starting balance, which we
           -- may be converting to value per hledger_options.m4.md "Effect
@@ -100,6 +105,10 @@ postingsReport rspec@ReportSpec{_rsReportOpts=ropts@ReportOpts{..}} j = items
 
           runningcalc = registerRunningCalculationFn ropts
           startnum = if historical then length precedingps + 1 else 1
+          postings | historical = if sortspec_ /= defsortspec 
+                        then error "--historical and --sort should not be used together" 
+                        else sortedps
+                   | otherwise = sortedps
 
 -- | Based on the given report options, return a function that does the appropriate
 -- running calculation for the register report, ie a running average or running total.
@@ -110,6 +119,34 @@ registerRunningCalculationFn ropts
   | average_ ropts = \i avg amt -> avg `maPlus` divideMixedAmount (fromIntegral i) (amt `maMinus` avg)
   | otherwise      = \_ bal amt -> bal `maPlus` amt
 
+-- | Sort two postings by the current list of value expressions (given in SortSpec).
+comparePostings :: ReportOpts -> SortSpec -> (Posting, Maybe Period) -> (Posting, Maybe Period) -> Ordering
+comparePostings _ [] _ _ = EQ
+comparePostings ropts (ex:es) (a, pa) (b, pb) = 
+    let 
+    getDescription p = 
+        let tx = ptransaction p 
+            description = fmap (\t -> tdescription t) tx
+        -- If there's no transaction attached, then use empty text for the description
+        in fromMaybe "" description
+    comparison = case ex of
+          AbsAmount' False -> compare (abs (pamount a)) (abs (pamount b))
+          Amount' False -> compare (pamount a) (pamount b)
+          Account' False -> compare (paccount a) (paccount b)
+          Date' False -> compare (postingDateOrDate2 (whichDate ropts) a) (postingDateOrDate2 (whichDate ropts) b)
+          Description' False -> compare (getDescription a) (getDescription b)
+          AbsAmount' True -> compare (Down (abs (pamount a))) (Down (abs (pamount b)))
+          Amount' True -> compare (Down (pamount a)) (Down (pamount b))
+          Account' True -> compare (Down (paccount a)) (Down (paccount b))
+          Date' True -> compare (Down (postingDateOrDate2 (whichDate ropts) a)) (Down (postingDateOrDate2 (whichDate ropts) b))
+          Description' True -> compare (Down (getDescription a)) (Down (getDescription b))
+    in 
+    if comparison == EQ then comparePostings ropts es (a, pa) (b, pb) else comparison
+
+-- | Sort postings by the current SortSpec.
+sortPostings :: ReportOpts -> SortSpec -> [(Posting, Maybe Period)] -> [(Posting, Maybe Period)]
+sortPostings ropts sspec = sortBy (comparePostings ropts sspec)
+
 -- | Find postings matching a given query, within a given date span,
 -- and also any similarly-matched postings before that date span.
 -- Date restrictions and depth restrictions in the query are ignored.

hledger-lib/Hledger/Reports/ReportOptions.hs

@@ -21,6 +21,9 @@ module Hledger.Reports.ReportOptions (
   HasReportOpts(..),
   ReportSpec(..),
   HasReportSpec(..),
+  SortField(..),
+  SortSpec,
+  sortKeysDescription,
   overEither,
   setEither,
   BalanceCalculation(..),
@@ -31,6 +34,7 @@ module Hledger.Reports.ReportOptions (
   defreportopts,
   rawOptsToReportOpts,
   defreportspec,
+  defsortspec,
   setDefaultConversionOp,
   reportOptsToSpec,
   updateReportSpec,
@@ -71,15 +75,13 @@ import Data.Char (toLower)
 import Data.Either (fromRight)
 import Data.Either.Extra (eitherToMaybe)
 import Data.Functor.Identity (Identity(..))
-import Data.List.Extra (find, isPrefixOf, nubSort)
+import Data.List.Extra (find, isPrefixOf, nubSort, stripPrefix)
 import Data.Maybe (fromMaybe, isJust, isNothing)
 import qualified Data.Text as T
 import Data.Time.Calendar (Day, addDays)
 import Data.Default (Default(..))
 import Safe (headMay, lastDef, lastMay, maximumMay, readMay)
 
-import Text.Megaparsec.Custom
-
 import Hledger.Data
 import Hledger.Query
 import Hledger.Utils
@@ -138,12 +140,15 @@ data ReportOpts = ReportOpts {
     ,no_elide_         :: Bool
     ,real_             :: Bool
     ,format_           :: StringFormat
+    ,balance_base_url_ :: Maybe T.Text
     ,pretty_           :: Bool
     ,querystring_      :: [T.Text]
     --
     ,average_          :: Bool
     -- for posting reports (register)
     ,related_          :: Bool
+    -- for sorting reports (register)
+    ,sortspec_             :: SortSpec
     -- for account transactions reports (aregister)
     ,txn_dates_        :: Bool
     -- for balance reports (bal, bs, cf, is)
@@ -195,10 +200,12 @@ defreportopts = ReportOpts
     , no_elide_         = False
     , real_             = False
     , format_           = def
+    , balance_base_url_ = Nothing
     , pretty_           = False
     , querystring_      = []
     , average_          = False
     , related_          = False
+    , sortspec_         = defsortspec 
     , txn_dates_        = False
     , balancecalc_      = def
     , balanceaccum_     = def
@@ -230,7 +237,7 @@ rawOptsToReportOpts d rawopts =
 
     let formatstring = T.pack <$> maybestringopt "format" rawopts
         querystring  = map T.pack $ listofstringopt "args" rawopts  -- doesn't handle an arg like "" right
-        pretty = fromMaybe False $ alwaysneveropt "pretty" rawopts
+        pretty = fromMaybe False $ ynopt "pretty" rawopts
 
         format = case parseStringFormat <$> formatstring of
             Nothing         -> defaultBalanceLineFormat
@@ -250,9 +257,11 @@ rawOptsToReportOpts d rawopts =
           ,no_elide_         = boolopt "no-elide" rawopts
           ,real_             = boolopt "real" rawopts
           ,format_           = format
+          ,balance_base_url_ = T.pack <$> maybestringopt "base-url" rawopts
           ,querystring_      = querystring
           ,average_          = boolopt "average" rawopts
           ,related_          = boolopt "related" rawopts
+          ,sortspec_         = getSortSpec rawopts
           ,txn_dates_        = boolopt "txn-dates" rawopts
           ,balancecalc_      = balancecalcopt rawopts
           ,balanceaccum_     = balanceaccumopt rawopts
@@ -329,8 +338,8 @@ balancecalcopt =
 balanceaccumopt :: RawOpts -> BalanceAccumulation
 balanceaccumopt = fromMaybe PerPeriod . balanceAccumulationOverride
 
-alwaysneveropt :: String -> RawOpts -> Maybe Bool
-alwaysneveropt opt rawopts = case maybestringopt opt rawopts of
+ynopt :: String -> RawOpts -> Maybe Bool
+ynopt opt rawopts = case maybestringopt opt rawopts of
     Just "always" -> Just True
     Just "yes"    -> Just True
     Just "y"      -> Just True
@@ -665,6 +674,47 @@ queryFromFlags ReportOpts{..} = simplifyQuery $ And flagsq
     consIf f b = if b then (f True:) else id
     consJust f = maybe id ((:) . f)
 
+-- Methods/types needed for --sort argument
+
+-- Possible arguments taken by the --sort command
+-- Each of these takes a bool, which shows if it has been inverted
+-- (True -> has been inverted, reverse the order)
+data SortField
+    = AbsAmount' Bool
+    | Account' Bool
+    | Amount' Bool
+    | Date' Bool
+    | Description' Bool
+    deriving (Show, Eq)
+type SortSpec = [SortField]
+
+-- By default, sort by date in ascending order
+defsortspec :: SortSpec
+defsortspec = [Date' False]
+
+-- Load a SortSpec from the argument given to --sort
+-- If there is no spec given, then sort by [Date' False] by default
+getSortSpec :: RawOpts -> SortSpec
+getSortSpec opts = 
+    let opt = maybestringopt "sort" opts
+        optParser s = 
+          let terms = map strip $ splitAtElement ',' s 
+              termParser t = case trimmed of
+                "date"        -> Date'        isNegated
+                "desc"        -> Description' isNegated
+                "description" -> Description' isNegated
+                "account"     -> Account'     isNegated
+                "amount"      -> Amount'      isNegated
+                "absamount"   -> AbsAmount'   isNegated
+                _ -> error' $ "unknown --sort key " ++ t ++ ". Supported keys are: " <> sortKeysDescription <> "."
+                where isNegated = isPrefixOf "-" t
+                      trimmed = fromMaybe t (stripPrefix "-" t)
+          in map termParser terms
+    in maybe defsortspec optParser opt 
+
+-- for option's help and parse error message
+sortKeysDescription = "date, desc, account, amount, absamount"  -- 'description' is also accepted
+
 -- Report dates.
 
 -- | The effective report span is the start and end dates specified by
@@ -766,7 +816,7 @@ reportPeriodOrJournalLastDay rspec j = reportPeriodLastDay rspec <|> journalOrPr
 reportPeriodName :: BalanceAccumulation -> [DateSpan] -> DateSpan -> T.Text
 reportPeriodName balanceaccumulation spans =
   case balanceaccumulation of
-    PerPeriod -> if multiyear then showDateSpan else showDateSpanMonthAbbrev
+    PerPeriod -> if multiyear then showDateSpan else showDateSpanAbbrev
       where
         multiyear = (>1) $ length $ nubSort $ map spanStartYear spans
     _ -> maybe "" (showDate . prevday) . spanEnd
@@ -819,7 +869,7 @@ makeHledgerClassyLenses ''ReportSpec
 -- >>> _rsQuery <$> setEither querystring ["assets"] defreportspec
 -- Right (Acct (RegexpCI "assets"))
 -- >>> _rsQuery <$> setEither querystring ["(assets"] defreportspec
--- Left "This regular expression is malformed, please correct it:\n(assets"
+-- Left "This regular expression is invalid or unsupported, please correct it:\n(assets"
 -- >>> _rsQuery $ set querystring ["assets"] defreportspec
 -- Acct (RegexpCI "assets")
 -- >>> _rsQuery $ set querystring ["(assets"] defreportspec

hledger-lib/Hledger/Utils.hs

@@ -3,6 +3,8 @@ Utilities used throughout hledger, or needed low in the module hierarchy.
 These are the bottom of hledger's module graph.
 -}
 
+{-# LANGUAGE CPP #-}
+
 module Hledger.Utils (
 
   -- * Functions
@@ -69,7 +71,10 @@ where
 
 import Data.Char (toLower)
 import Data.List (intersperse)
-import Data.List.Extra (chunksOf, foldl', foldl1', uncons, unsnoc)
+import Data.List.Extra (chunksOf, foldl1', uncons, unsnoc)
+#if !MIN_VERSION_base(4,20,0)
+import Data.List (foldl')
+#endif
 import qualified Data.Set as Set
 import qualified Data.Text as T (pack, unpack)
 import Data.Tree (foldTree, Tree (Node, subForest))

hledger-lib/Hledger/Utils/Debug.hs

@@ -53,7 +53,7 @@ In hledger, debug levels are used as follows:
 Debug level:  What to show:
 ------------  ---------------------------------------------------------
 0             normal command output only (no warnings, eg)
-1             useful warnings, most common troubleshooting info, eg valuation
+1             useful warnings, most common troubleshooting info (config file args, valuation..)
 2             common troubleshooting info, more detail
 3             report options selection
 4             report generation

hledger-lib/Hledger/Utils/IO.hs

@@ -97,8 +97,10 @@ import           Control.Monad (when, forM)
 import           Data.Colour.RGBSpace (RGB(RGB))
 import           Data.Colour.RGBSpace.HSL (lightness)
 import           Data.FileEmbed (makeRelativeToProject, embedStringFile)
+import           Data.Functor ((<&>))
 import           Data.List hiding (uncons)
 import           Data.Maybe (isJust)
+import           Data.Ord (comparing, Down (Down))
 import qualified Data.Text as T
 import qualified Data.Text.IO as T
 import qualified Data.Text.Lazy as TL
@@ -115,6 +117,7 @@ import           System.Console.Terminal.Size (Window (Window), size)
 import           System.Directory (getHomeDirectory, getModificationTime)
 import           System.Environment (getArgs, lookupEnv, setEnv)
 import           System.FilePath (isRelative, (</>))
+import "Glob"    System.FilePath.Glob (glob)
 import           System.IO
   (Handle, IOMode (..), hGetEncoding, hSetEncoding, hSetNewlineMode,
    openFile, stdin, stdout, stderr, universalNewlineMode, utf8_bom, hIsTerminalDevice)
@@ -127,15 +130,14 @@ import           Text.Pretty.Simple
   defaultOutputOptionsDarkBg, defaultOutputOptionsNoColor, pShowOpt, pPrintOpt)
 
 import Hledger.Utils.Text (WideBuilder(WideBuilder))
-import "Glob" System.FilePath.Glob (glob)
-import Data.Functor ((<&>))
+
 
 -- Pretty showing/printing with pretty-simple
 
 -- https://hackage.haskell.org/package/pretty-simple/docs/Text-Pretty-Simple.html#t:OutputOptions
 
 -- | pretty-simple options with colour enabled if allowed.
-prettyopts = 
+prettyopts =
   (if useColorOnStderr then defaultOutputOptionsDarkBg else defaultOutputOptionsNoColor)
     { outputOptionsIndentAmount = 2
     -- , outputOptionsCompact      = True  -- fills lines, but does not respect page width (https://github.com/cdepillabout/pretty-simple/issues/126)
@@ -225,7 +227,7 @@ progArgs = unsafePerformIO getArgs
 -- | Read the value of the -o/--output-file command line option provided at program startup,
 -- if any, using unsafePerformIO.
 outputFileOption :: Maybe String
-outputFileOption = 
+outputFileOption =
   -- keep synced with output-file flag definition in hledger:CliOptions.
   let args = progArgs in
   case dropWhile (not . ("-o" `isPrefixOf`)) args of
@@ -315,7 +317,7 @@ rgb' r g b  = ifAnsi (rgb r g b)
 -- | Read the value of the --color or --colour command line option provided at program startup
 -- using unsafePerformIO. If this option was not provided, returns the empty string.
 colorOption :: String
-colorOption = 
+colorOption =
   -- similar to debugLevel
   -- keep synced with color/colour flag definition in hledger:CliOptions
   let args = progArgs in
@@ -360,8 +362,8 @@ useColorOnHandle h = unsafePerformIO $ do
   no_color       <- isJust <$> lookupEnv "NO_COLOR"
   supports_color <- hSupportsANSIColor h
   let coloroption = colorOption
-  return $ coloroption `elem` ["always","yes"]
-       || (coloroption `notElem` ["never","no"] && not no_color && supports_color)
+  return $ coloroption `elem` ["always","yes","y"]
+       || (coloroption `notElem` ["never","no","n"] && not no_color && supports_color)
 
 -- | Wrap a string in ANSI codes to set and reset foreground colour.
 -- ColorIntensity is @Dull@ or @Vivid@ (ie normal and bold).
@@ -462,7 +464,7 @@ expandGlob curdir p = expandPath curdir p >>= glob <&> sort  -- PARTIAL:
 sortByModTime :: [FilePath] -> IO [FilePath]
 sortByModTime fs = do
   ftimes <- forM fs $ \f -> do {t <- getModificationTime f; return (t,f)}
-  return $ map snd $ reverse $ sort ftimes
+  return $ map snd $ sortBy (comparing Data.Ord.Down) ftimes
 
 -- | Like readFilePortably, but read all of the file before proceeding.
 readFileStrictly :: FilePath -> IO T.Text
@@ -516,3 +518,4 @@ getCurrentZonedTime = do
   tz <- getCurrentTimeZone
   return $ utcToZonedTime tz t
 
+

hledger-lib/Hledger/Utils/Parse.hs

@@ -1,8 +1,15 @@
+{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE FlexibleInstances #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE StandaloneDeriving #-}
 {-# LANGUAGE TypeFamilies #-}
 {-# LANGUAGE TypeOperators #-}
 
 module Hledger.Utils.Parse (
+
+  -- * Some basic hledger parser flavours
   SimpleStringParser,
   SimpleTextParser,
   TextParser,
@@ -15,6 +22,7 @@ module Hledger.Utils.Parse (
   sourcePosPretty,
   sourcePosPairPretty,
 
+  -- * Parsers and helpers
   choice',
   choiceInState,
   surroundedBy,
@@ -32,7 +40,6 @@ module Hledger.Utils.Parse (
   isNonNewlineSpace,
   restofline,
   eolof,
-
   spacenonewline,
   skipNonNewlineSpaces,
   skipNonNewlineSpaces1,
@@ -42,10 +49,44 @@ module Hledger.Utils.Parse (
   dbgparse,
   traceOrLogParse,
 
-  -- * re-exports
-  HledgerParseErrors,
+  -- * More helpers, previously in Text.Megaparsec.Custom
+
+  -- ** Custom parse error types
   HledgerParseErrorData,
+  HledgerParseErrors,
+
+  -- ** Failing with an arbitrary source position
+  parseErrorAt,
+  parseErrorAtRegion,
+
+  -- ** Re-parsing
+  SourceExcerpt,
+  getExcerptText,
+  excerpt_,
+  reparseExcerpt,
+
+  -- ** Pretty-printing custom parse errors
   customErrorBundlePretty,
+
+  -- ** "Final" parse errors
+  FinalParseError,
+  FinalParseError',
+  FinalParseErrorBundle,
+  FinalParseErrorBundle',
+
+  -- *** Constructing "final" parse errors
+  finalError,
+  finalFancyFailure,
+  finalFail,
+  finalCustomFailure,
+
+  -- *** Pretty-printing "final" parse errors
+  finalErrorBundlePretty,
+  attachSource,
+
+  -- *** Handling parse errors from include files with "final" parse errors
+  parseIncludeFile,
+
 )
 where
 
@@ -61,7 +102,15 @@ import Data.Functor.Identity (Identity(..))
 import Data.List
 import Data.Text (Text)
 import Text.Megaparsec.Char
-import Text.Megaparsec.Custom
+-- import Text.Megaparsec.Debug (dbg)  -- from megaparsec 9.3+
+
+import Control.Monad.Except (ExceptT, MonadError, catchError, throwError)
+-- import Control.Monad.State.Strict (StateT, evalStateT)
+import Control.Monad.Trans.Class (lift)
+import qualified Data.List.NonEmpty as NE
+import Data.Monoid (Alt(..))
+import qualified Data.Set as S
+
 import Hledger.Utils.Debug (debugLevel, traceOrLog)
 
 -- | A parser of string to some type.
@@ -200,3 +249,384 @@ skipNonNewlineSpaces' = True <$ skipNonNewlineSpaces1 <|> pure False
 
 eolof :: TextParser m ()
 eolof = void newline <|> eof
+
+
+
+-- A bunch of megaparsec helpers, eg for re-parsing (formerly in Text.Megaparsec.Custom).
+-- I think these are generic apart from the HledgerParseError name.
+
+--- * Custom parse error types
+
+-- | Custom error data for hledger parsers. Specialised for a 'Text' parse stream.
+-- ReparseableTextParseErrorData ?
+data HledgerParseErrorData
+  -- | Fail with a message at a specific source position interval. The
+  -- interval must be contained within a single line.
+  = ErrorFailAt Int -- Starting offset
+                Int -- Ending offset
+                String -- Error message
+  -- | Re-throw parse errors obtained from the "re-parsing" of an excerpt
+  -- of the source text.
+  | ErrorReparsing
+      (NE.NonEmpty (ParseError Text HledgerParseErrorData)) -- Source fragment parse errors
+  deriving (Show, Eq, Ord)
+
+-- | A specialised version of ParseErrorBundle: 
+-- a non-empty collection of hledger parse errors, 
+-- equipped with PosState to help pretty-print them.
+-- Specialised for a 'Text' parse stream.
+type HledgerParseErrors = ParseErrorBundle Text HledgerParseErrorData
+
+-- We require an 'Ord' instance for 'CustomError' so that they may be
+-- stored in a 'Set'. The actual instance is inconsequential, so we just
+-- derive it, but the derived instance requires an (orphan) instance for
+-- 'ParseError'. Hopefully this does not cause any trouble.
+
+deriving instance Ord (ParseError Text HledgerParseErrorData)
+
+-- Note: the pretty-printing of our 'HledgerParseErrorData' type is only partally
+-- defined in its 'ShowErrorComponent' instance; we perform additional
+-- adjustments in 'customErrorBundlePretty'.
+
+instance ShowErrorComponent HledgerParseErrorData where
+  showErrorComponent (ErrorFailAt _ _ errMsg) = errMsg
+  showErrorComponent (ErrorReparsing _) = "" -- dummy value
+
+  errorComponentLen (ErrorFailAt startOffset endOffset _) =
+    endOffset - startOffset
+  errorComponentLen (ErrorReparsing _) = 1 -- dummy value
+
+
+--- * Failing with an arbitrary source position
+
+-- | Fail at a specific source position, given by the raw offset from the
+-- start of the input stream (the number of tokens processed at that
+-- point).
+
+parseErrorAt :: Int -> String -> HledgerParseErrorData
+parseErrorAt offset = ErrorFailAt offset (offset+1)
+
+-- | Fail at a specific source interval, given by the raw offsets of its
+-- endpoints from the start of the input stream (the numbers of tokens
+-- processed at those points).
+--
+-- Note that care must be taken to ensure that the specified interval does
+-- not span multiple lines of the input source. This will not be checked.
+
+parseErrorAtRegion
+  :: Int    -- ^ Start offset
+  -> Int    -- ^ End end offset
+  -> String -- ^ Error message
+  -> HledgerParseErrorData
+parseErrorAtRegion startOffset endOffset msg =
+  if startOffset < endOffset
+    then ErrorFailAt startOffset endOffset msg'
+    else ErrorFailAt startOffset (startOffset+1) msg'
+  where
+    msg' = "\n" ++ msg
+
+
+--- * Re-parsing
+
+-- | A fragment of source suitable for "re-parsing". The purpose of this
+-- data type is to preserve the content and source position of the excerpt
+-- so that parse errors raised during "re-parsing" may properly reference
+-- the original source.
+
+data SourceExcerpt = SourceExcerpt Int  -- Offset of beginning of excerpt
+                                   Text -- Fragment of source file
+
+-- | Get the raw text of a source excerpt.
+
+getExcerptText :: SourceExcerpt -> Text
+getExcerptText (SourceExcerpt _ txt) = txt
+
+-- | 'excerpt_ p' applies the given parser 'p' and extracts the portion of
+-- the source consumed by 'p', along with the source position of this
+-- portion. This is the only way to create a source excerpt suitable for
+-- "re-parsing" by 'reparseExcerpt'.
+
+-- This function could be extended to return the result of 'p', but we don't
+-- currently need this.
+
+excerpt_ :: MonadParsec HledgerParseErrorData Text m => m a -> m SourceExcerpt
+excerpt_ p = do
+  offset <- getOffset
+  (!txt, _) <- match p
+  pure $ SourceExcerpt offset txt
+
+-- | 'reparseExcerpt s p' "re-parses" the source excerpt 's' using the
+-- parser 'p'. Parse errors raised by 'p' will be re-thrown at the source
+-- position of the source excerpt.
+--
+-- In order for the correct source file to be displayed when re-throwing
+-- parse errors, we must ensure that the source file during the use of
+-- 'reparseExcerpt s p' is the same as that during the use of 'excerpt_'
+-- that generated the source excerpt 's'. However, we can usually expect
+-- this condition to be satisfied because, at the time of writing, the
+-- only changes of source file in the codebase take place through include
+-- files, and the parser for include files neither accepts nor returns
+-- 'SourceExcerpt's.
+
+reparseExcerpt
+  :: Monad m
+  => SourceExcerpt
+  -> ParsecT HledgerParseErrorData Text m a
+  -> ParsecT HledgerParseErrorData Text m a
+reparseExcerpt (SourceExcerpt offset txt) p = do
+  (_, res) <- lift $ runParserT' p (offsetInitialState offset txt)
+  case res of
+    Right result -> pure result
+    Left errBundle -> customFailure $ ErrorReparsing $ bundleErrors errBundle
+
+  where
+    offsetInitialState :: Int -> s ->
+#if MIN_VERSION_megaparsec(8,0,0)
+      State s e
+#else
+      State s
+#endif
+    offsetInitialState initialOffset s = State
+      { stateInput  = s
+      , stateOffset = initialOffset
+      , statePosState = PosState
+        { pstateInput = s
+        , pstateOffset = initialOffset
+        , pstateSourcePos = initialPos ""
+        , pstateTabWidth = defaultTabWidth
+        , pstateLinePrefix = ""
+        }
+#if MIN_VERSION_megaparsec(8,0,0)
+      , stateParseErrors = []
+#endif
+      }
+
+--- * Pretty-printing custom parse errors
+
+-- | Pretty-print our custom parse errors. It is necessary to use this
+-- instead of 'errorBundlePretty' when custom parse errors are thrown.
+--
+-- This function intercepts our custom parse errors and applies final
+-- adjustments ('finalizeCustomError') before passing them to
+-- 'errorBundlePretty'. These adjustments are part of the implementation
+-- of the behaviour of our custom parse errors.
+--
+-- Note: We must ensure that the offset of the 'PosState' of the provided
+-- 'ParseErrorBundle' is no larger than the offset specified by a
+-- 'ErrorFailAt' constructor. This is guaranteed if this offset is set to
+-- 0 (that is, the beginning of the source file), which is the
+-- case for 'ParseErrorBundle's returned from 'runParserT'.
+
+customErrorBundlePretty :: HledgerParseErrors -> String
+customErrorBundlePretty errBundle =
+  let errBundle' = errBundle { bundleErrors =
+        NE.sortWith errorOffset $ -- megaparsec requires that the list of errors be sorted by their offsets
+        bundleErrors errBundle >>= finalizeCustomError }
+  in  errorBundlePretty errBundle'
+
+  where
+    finalizeCustomError
+      :: ParseError Text HledgerParseErrorData -> NE.NonEmpty (ParseError Text HledgerParseErrorData)
+    finalizeCustomError err = case findCustomError err of
+      Nothing -> pure err
+
+      Just errFailAt@(ErrorFailAt startOffset _ _) ->
+        -- Adjust the offset
+        pure $ FancyError startOffset $ S.singleton $ ErrorCustom errFailAt
+
+      Just (ErrorReparsing errs) ->
+        -- Extract and finalize the inner errors
+        errs >>= finalizeCustomError
+
+    -- If any custom errors are present, arbitrarily take the first one
+    -- (since only one custom error should be used at a time).
+    findCustomError :: ParseError Text HledgerParseErrorData -> Maybe HledgerParseErrorData
+    findCustomError err = case err of
+      FancyError _ errSet ->
+        finds (\case {ErrorCustom e -> Just e; _ -> Nothing}) errSet
+      _ -> Nothing
+
+    finds :: (Foldable t) => (a -> Maybe b) -> t a -> Maybe b
+    finds f = getAlt . foldMap (Alt . f)
+
+
+--- * "Final" parse errors
+--
+-- | A type representing "final" parse errors that cannot be backtracked
+-- from and are guaranteed to halt parsing. The anti-backtracking
+-- behaviour is implemented by an 'ExceptT' layer in the parser's monad
+-- stack, using this type as the 'ExceptT' error type.
+--
+-- We have three goals for this type:
+-- (1) it should be possible to convert any parse error into a "final"
+-- parse error,
+-- (2) it should be possible to take a parse error thrown from an include
+-- file and re-throw it in the parent file, and
+-- (3) the pretty-printing of "final" parse errors should be consistent
+-- with that of ordinary parse errors, but should also report a stack of
+-- files for errors thrown from include files.
+--
+-- In order to pretty-print a "final" parse error (goal 3), it must be
+-- bundled with include filepaths and its full source text. When a "final"
+-- parse error is thrown from within a parser, we do not have access to
+-- the full source, so we must hold the parse error until it can be joined
+-- with its source (and include filepaths, if it was thrown from an
+-- include file) by the parser's caller.
+--
+-- A parse error with include filepaths and its full source text is
+-- represented by the 'FinalParseErrorBundle' type, while a parse error in
+-- need of either include filepaths, full source text, or both is
+-- represented by the 'FinalParseError' type.
+
+data FinalParseError' e
+  -- a parse error thrown as a "final" parse error
+  = FinalError           (ParseError Text e)
+  -- a parse error obtained from running a parser, e.g. using 'runParserT'
+  | FinalBundle          (ParseErrorBundle Text e)
+  -- a parse error thrown from an include file
+  | FinalBundleWithStack (FinalParseErrorBundle' e)
+  deriving (Show)
+
+type FinalParseError = FinalParseError' HledgerParseErrorData
+
+-- We need a 'Monoid' instance for 'FinalParseError' so that 'ExceptT
+-- FinalParseError m' is an instance of Alternative and MonadPlus, which
+-- is needed to use some parser combinators, e.g. 'many'.
+--
+-- This monoid instance simply takes the first (left-most) error.
+
+instance Semigroup (FinalParseError' e) where
+  e <> _ = e
+
+instance Monoid (FinalParseError' e) where
+  mempty = FinalError $ FancyError 0 $
+            S.singleton (ErrorFail "default parse error")
+  mappend = (<>)
+
+-- | A type bundling a 'ParseError' with its full source text, filepath,
+-- and stack of include files. Suitable for pretty-printing.
+--
+-- Megaparsec's 'ParseErrorBundle' type already bundles a parse error with
+-- its full source text and filepath, so we just add a stack of include
+-- files.
+
+data FinalParseErrorBundle' e = FinalParseErrorBundle'
+  { finalErrorBundle :: ParseErrorBundle Text e
+  , includeFileStack :: [FilePath]
+  } deriving (Show)
+
+type FinalParseErrorBundle = FinalParseErrorBundle' HledgerParseErrorData
+
+
+--- * Constructing and throwing final parse errors
+
+-- | Convert a "regular" parse error into a "final" parse error.
+
+finalError :: ParseError Text e -> FinalParseError' e
+finalError = FinalError
+
+-- | Like megaparsec's 'fancyFailure', but as a "final" parse error.
+
+finalFancyFailure
+  :: (MonadParsec e s m, MonadError (FinalParseError' e) m)
+  => S.Set (ErrorFancy e) -> m a
+finalFancyFailure errSet = do
+  offset <- getOffset
+  throwError $ FinalError $ FancyError offset errSet
+
+-- | Like 'fail', but as a "final" parse error.
+
+finalFail
+  :: (MonadParsec e s m, MonadError (FinalParseError' e) m) => String -> m a
+finalFail = finalFancyFailure . S.singleton . ErrorFail
+
+-- | Like megaparsec's 'customFailure', but as a "final" parse error.
+
+finalCustomFailure
+  :: (MonadParsec e s m, MonadError (FinalParseError' e) m) => e -> m a
+finalCustomFailure = finalFancyFailure . S.singleton . ErrorCustom
+
+
+--- * Pretty-printing "final" parse errors
+
+-- | Pretty-print a "final" parse error: print the stack of include files,
+-- then apply the pretty-printer for parse error bundles. Note that
+-- 'attachSource' must be used on a "final" parse error before it can be
+-- pretty-printed.
+
+finalErrorBundlePretty :: FinalParseErrorBundle' HledgerParseErrorData -> String
+finalErrorBundlePretty bundle =
+     concatMap showIncludeFilepath (includeFileStack bundle)
+  <> customErrorBundlePretty (finalErrorBundle bundle)
+  where
+    showIncludeFilepath path = "in file included from " <> path <> ",\n"
+
+-- | Supply a filepath and source text to a "final" parse error so that it
+-- can be pretty-printed. You must ensure that you provide the appropriate
+-- source text and filepath.
+
+attachSource
+  :: FilePath -> Text -> FinalParseError' e -> FinalParseErrorBundle' e
+attachSource filePath sourceText finalParseError = case finalParseError of
+
+  -- A parse error thrown directly with the 'FinalError' constructor
+  -- requires both source and filepath.
+  FinalError err ->
+    let bundle = ParseErrorBundle
+          { bundleErrors = err NE.:| []
+          , bundlePosState = initialPosState filePath sourceText }
+    in  FinalParseErrorBundle'
+          { finalErrorBundle = bundle
+          , includeFileStack  = [] }
+
+  -- A 'ParseErrorBundle' already has the appropriate source and filepath
+  -- and so needs neither.
+  FinalBundle peBundle -> FinalParseErrorBundle'
+    { finalErrorBundle = peBundle
+    , includeFileStack = [] }
+
+  -- A parse error from a 'FinalParseErrorBundle' was thrown from an
+  -- include file, so we add the filepath to the stack.
+  FinalBundleWithStack fpeBundle -> fpeBundle
+    { includeFileStack = filePath : includeFileStack fpeBundle }
+
+
+--- * Handling parse errors from include files with "final" parse errors
+
+-- | Parse a file with the given parser and initial state, discarding the
+-- final state and re-throwing any parse errors as "final" parse errors.
+
+parseIncludeFile
+  :: Monad m
+  => StateT st (ParsecT HledgerParseErrorData Text (ExceptT FinalParseError m)) a
+  -> st
+  -> FilePath
+  -> Text
+  -> StateT st (ParsecT HledgerParseErrorData Text (ExceptT FinalParseError m)) a
+parseIncludeFile parser initialState filepath text =
+  catchError parser' handler
+  where
+    parser' = do
+      eResult <- lift $ lift $
+                  runParserT (evalStateT parser initialState) filepath text
+      case eResult of
+        Left parseErrorBundle -> throwError $ FinalBundle parseErrorBundle
+        Right result -> pure result
+
+    -- Attach source and filepath of the include file to its parse errors
+    handler e = throwError $ FinalBundleWithStack $ attachSource filepath text e
+
+
+--- * Helpers
+
+-- Like megaparsec's 'initialState', but instead for 'PosState'. Used when
+-- constructing 'ParseErrorBundle's. The values for "tab width" and "line
+-- prefix" are taken from 'initialState'.
+
+initialPosState :: FilePath -> Text -> PosState Text
+initialPosState filePath sourceText = PosState
+  { pstateInput      = sourceText
+  , pstateOffset     = 0
+  , pstateSourcePos  = initialPos filePath
+  , pstateTabWidth   = defaultTabWidth
+  , pstateLinePrefix = "" }

hledger-lib/Hledger/Utils/Regex.hs

@@ -1,7 +1,3 @@
-{-# LANGUAGE FlexibleInstances     #-}
-{-# LANGUAGE MultiParamTypeClasses #-}
-{-# LANGUAGE OverloadedStrings     #-}
-{-# LANGUAGE ScopedTypeVariables   #-}
 {-|
 
 Easy regular expression helpers, currently based on regex-tdfa. These should:
@@ -42,6 +38,12 @@ Current limitations:
 
 -}
 
+{-# LANGUAGE CPP                   #-}
+{-# LANGUAGE FlexibleInstances     #-}
+{-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE OverloadedStrings     #-}
+{-# LANGUAGE ScopedTypeVariables   #-}
+
 module Hledger.Utils.Regex (
   -- * Regexp type and constructors
    Regexp(reString)
@@ -66,7 +68,9 @@ import Control.Monad (foldM)
 import Data.Aeson (ToJSON(..), Value(String))
 import Data.Array ((!), elems, indices)
 import Data.Char (isDigit)
+#if !MIN_VERSION_base(4,20,0)
 import Data.List (foldl')
+#endif
 import Data.MemoUgly (memo)
 import Data.Text (Text)
 import qualified Data.Text as T
@@ -136,7 +140,7 @@ toRegexCI = memo $ \s -> mkRegexErr s (RegexpCI s <$> makeRegexOptsM defaultComp
 -- | Make a nice error message for a regexp error.
 mkRegexErr :: Text -> Maybe a -> Either RegexError a
 mkRegexErr s = maybe (Left errmsg) Right
-  where errmsg = T.unpack $ "This regular expression is malformed, please correct it:\n" <> s
+  where errmsg = T.unpack $ "This regular expression is invalid or unsupported, please correct it:\n" <> s
 
 -- Convert a Regexp string to a compiled Regex, throw an error
 toRegex' :: Text -> Regexp

hledger-lib/Hledger/Utils/String.hs

@@ -190,7 +190,7 @@ shellchars      = "<>(){}[]$7?#!~`"
 -- NB correctly handles "a'b" but not "''a''". Can raise an error if parsing fails.
 words' :: String -> [String]
 words' "" = []
-words' s  = map stripquotes $ fromparse $ parsewithString p s
+words' s  = map stripquotes $ fromparse $ parsewithString p s  -- PARTIAL
     where
       p = (singleQuotedPattern <|> doubleQuotedPattern <|> patterns) `sepBy` skipNonNewlineSpaces1
           -- eof

hledger-lib/Hledger/Utils/Test.hs

@@ -31,7 +31,9 @@ import Test.Tasty.HUnit
 -- import Test.Tasty.QuickCheck as QC
 -- import Test.Tasty.SmallCheck as SC
 import Text.Megaparsec
-import Text.Megaparsec.Custom
+
+import Hledger.Utils.IO (pshow)
+import Hledger.Utils.Parse
   ( HledgerParseErrorData,
     FinalParseError,
     attachSource,
@@ -39,8 +41,6 @@ import Text.Megaparsec.Custom
     finalErrorBundlePretty,
   )
 
-import Hledger.Utils.IO (pshow)
-
 -- * tasty helpers
 
 -- TODO: pretty-print values in failure messages

hledger-lib/Hledger/Write/Csv.hs

@@ -10,7 +10,7 @@ CSV utilities.
 {-# LANGUAGE OverloadedStrings    #-}
 
 --- ** exports
-module Hledger.Read.CsvUtils (
+module Hledger.Write.Csv (
   CSV, CsvRecord, CsvValue,
   printCSV,
   printTSV,
@@ -37,12 +37,12 @@ type CSV       = [CsvRecord]
 type CsvRecord = [CsvValue]
 type CsvValue  = Text
 
-printCSV :: [CsvRecord] -> TL.Text
+printCSV :: CSV -> TL.Text
 printCSV = TB.toLazyText . unlinesB . map printRecord
     where printRecord = foldMap TB.fromText . intersperse "," . map printField
           printField = wrap "\"" "\"" . T.replace "\"" "\"\""
 
-printTSV :: [CsvRecord] -> TL.Text
+printTSV :: CSV -> TL.Text
 printTSV = TB.toLazyText . unlinesB . map printRecord
     where printRecord = foldMap TB.fromText . intersperse "\t" . map printField
           printField = T.map replaceWhitespace

hledger-lib/Hledger/Write/Html.hs

@@ -0,0 +1,38 @@
+{-# LANGUAGE OverloadedStrings #-}
+{- |
+Common definitions for Html.Blaze and Html.Lucid
+-}
+module Hledger.Write.Html (
+    Lines(..),
+    borderStyles,
+    ) where
+
+import qualified Hledger.Write.Spreadsheet as Spr
+import Hledger.Write.Spreadsheet (Cell(..))
+
+import Data.Text (Text)
+
+
+borderStyles :: Lines border => Cell border text -> [Text]
+borderStyles cell =
+    let border field access =
+            map (field<>) $ borderLines $ access $ cellBorder cell in
+    let leftBorder   = border "border-left:"   Spr.borderLeft   in
+    let rightBorder  = border "border-right:"  Spr.borderRight  in
+    let topBorder    = border "border-top:"    Spr.borderTop    in
+    let bottomBorder = border "border-bottom:" Spr.borderBottom in
+    leftBorder++rightBorder++topBorder++bottomBorder
+
+
+class (Spr.Lines border) => Lines border where
+    borderLines :: border -> [Text]
+
+instance Lines () where
+    borderLines () = []
+
+instance Lines Spr.NumLines where
+    borderLines prop =
+        case prop of
+            Spr.NoLine -> []
+            Spr.SingleLine -> ["black"]
+            Spr.DoubleLine -> ["double black"]

hledger-lib/Hledger/Write/Html/Attribute.hs

@@ -0,0 +1,64 @@
+{-# LANGUAGE OverloadedStrings #-}
+{- |
+Helpers and CSS styles for HTML output.
+-}
+module Hledger.Write.Html.Attribute (
+    stylesheet,
+    concatStyles,
+    tableStylesheet,
+    tableStyle,
+    bold,
+    doubleborder,
+    topdoubleborder,
+    bottomdoubleborder,
+    alignright,
+    alignleft,
+    aligncenter,
+    collapse,
+    lpad,
+    rpad,
+    hpad,
+    vpad,
+    ) where
+
+import qualified Data.Text as Text
+import Data.Text (Text)
+
+
+stylesheet :: [(Text,Text)] -> Text
+stylesheet elstyles =
+    Text.unlines $
+        "" : [el<>" {"<>styles<>"}" | (el,styles) <- elstyles]
+
+concatStyles :: [Text] -> Text
+concatStyles = Text.intercalate "; "
+
+
+tableStylesheet :: Text
+tableStylesheet = stylesheet tableStyle
+
+tableStyle :: [(Text, Text)]
+tableStyle =
+  [("table", collapse),
+   ("th, td", lpad),
+   ("th.account, td.account", "padding-left:0;")]
+
+bold, doubleborder, topdoubleborder, bottomdoubleborder :: Text
+bold = "font-weight:bold"
+doubleborder = "double black"
+topdoubleborder    = "border-top:"<>doubleborder
+bottomdoubleborder = "border-bottom:"<>doubleborder
+
+alignright, alignleft, aligncenter :: Text
+alignright  = "text-align:right"
+alignleft   = "text-align:left"
+aligncenter = "text-align:center"
+
+collapse :: Text
+collapse = "border-collapse:collapse"
+
+lpad, rpad, hpad, vpad :: Text
+lpad = "padding-left:1em"
+rpad = "padding-right:1em"
+hpad = "padding-left:1em; padding-right:1em"
+vpad = "padding-top:1em;  padding-bottom:1em"

hledger-lib/Hledger/Write/Html/Blaze.hs

@@ -0,0 +1,78 @@
+{-# LANGUAGE OverloadedStrings #-}
+{- |
+Export spreadsheet table data as HTML table.
+
+This is derived from <https://hackage.haskell.org/package/classify-frog-0.2.4.3/src/src/Spreadsheet/Format.hs>
+-}
+module Hledger.Write.Html.Blaze (
+    printHtml,
+    formatRow,
+    formatCell,
+    ) where
+
+import qualified Hledger.Write.Html.Attribute as Attr
+import qualified Hledger.Write.Spreadsheet as Spr
+import Hledger.Write.Html (Lines, borderStyles)
+import Hledger.Write.Spreadsheet (Type(..), Style(..), Emphasis(..), Cell(..))
+
+import qualified Text.Blaze.Html4.Transitional.Attributes as HtmlAttr
+import qualified Text.Blaze.Html4.Transitional as Html
+import qualified Data.Text as Text
+import Text.Blaze.Html4.Transitional (Html, toHtml, (!))
+import Data.Foldable (traverse_)
+
+
+printHtml :: (Lines border) => [[Cell border Html]] -> Html
+printHtml table = do
+    Html.style $ toHtml $ Attr.tableStylesheet
+    Html.table $ traverse_ formatRow table
+
+formatRow:: (Lines border) => [Cell border Html] -> Html
+formatRow = Html.tr . traverse_ formatCell
+
+formatCell :: (Lines border) => Cell border Html -> Html
+formatCell cell =
+    let str = cellContent cell in
+    let content =
+            if Text.null $ cellAnchor cell
+                then str
+                else Html.a str !
+                        HtmlAttr.href (Html.textValue (cellAnchor cell)) in
+    let style =
+            case borderStyles cell of
+                [] -> []
+                ss -> [HtmlAttr.style $ Html.textValue $
+                        Attr.concatStyles ss] in
+    let class_ =
+            map (HtmlAttr.class_ . Html.textValue) $
+            filter (not . Text.null) [Spr.textFromClass $ cellClass cell] in
+    let span_ makeCell attrs =
+            case Spr.cellSpan cell of
+                Spr.NoSpan -> foldl (!) makeCell attrs
+                Spr.Covered -> pure ()
+                Spr.SpanHorizontal n ->
+                    foldl (!) makeCell
+                        (HtmlAttr.colspan (Html.stringValue $ show n) : attrs)
+                Spr.SpanVertical n ->
+                    foldl (!) makeCell
+                        (HtmlAttr.rowspan (Html.stringValue $ show n) : attrs)
+            in
+    case cellStyle cell of
+        Head -> span_ (Html.th content) (style++class_)
+        Body emph ->
+            let align =
+                    case cellType cell of
+                        TypeString -> []
+                        TypeDate -> []
+                        _ -> [HtmlAttr.align "right"]
+                valign =
+                    case Spr.cellSpan cell of
+                        Spr.SpanVertical n ->
+                            if n>1 then [HtmlAttr.valign "top"] else []
+                        _ -> []
+                withEmph =
+                    case emph of
+                        Item -> id
+                        Total -> Html.b
+            in  span_ (Html.td $ withEmph content) $
+                style++align++valign++class_

hledger-lib/Hledger/Write/Html/Lucid.hs

@@ -0,0 +1,78 @@
+{-# LANGUAGE OverloadedStrings #-}
+{- |
+Export spreadsheet table data as HTML table.
+
+This is derived from <https://hackage.haskell.org/package/classify-frog-0.2.4.3/src/src/Spreadsheet/Format.hs>
+-}
+module Hledger.Write.Html.Lucid (
+    printHtml,
+    formatRow,
+    formatCell,
+    ) where
+
+import qualified Hledger.Write.Html.Attribute as Attr
+import qualified Hledger.Write.Spreadsheet as Spr
+import Hledger.Write.Html (Lines, borderStyles)
+import Hledger.Write.Spreadsheet (Type(..), Style(..), Emphasis(..), Cell(..))
+
+import qualified Data.Text as Text
+import qualified Lucid.Base as HtmlBase
+import qualified Lucid as Html
+import Data.Foldable (traverse_)
+
+
+type Html = Html.Html ()
+
+printHtml :: (Lines border) => [[Cell border Html]] -> Html
+printHtml table = do
+    Html.link_ [Html.rel_ "stylesheet", Html.href_ "hledger.css"]
+    Html.style_ Attr.tableStylesheet
+    Html.table_ $ traverse_ formatRow table
+
+formatRow:: (Lines border) => [Cell border Html] -> Html
+formatRow = Html.tr_ . traverse_ formatCell
+
+formatCell :: (Lines border) => Cell border Html -> Html
+formatCell cell =
+    let str = cellContent cell in
+    let content =
+            if Text.null $ cellAnchor cell
+                then str
+                else Html.a_ [Html.href_ $ cellAnchor cell] str in
+    let style =
+            case borderStyles cell of
+                [] -> []
+                ss -> [Html.style_ $ Attr.concatStyles ss] in
+    let class_ =
+            map Html.class_ $
+            filter (not . Text.null) [Spr.textFromClass $ cellClass cell] in
+    let span_ makeCell attrs cont =
+            case Spr.cellSpan cell of
+                Spr.NoSpan -> makeCell attrs cont
+                Spr.Covered -> pure ()
+                Spr.SpanHorizontal n ->
+                    makeCell (Html.colspan_ (Text.pack $ show n) : attrs) cont
+                Spr.SpanVertical n ->
+                    makeCell (Html.rowspan_ (Text.pack $ show n) : attrs) cont
+            in
+    case cellStyle cell of
+        Head -> span_ Html.th_ (style++class_) content
+        Body emph ->
+            let align =
+                    case cellType cell of
+                        TypeString -> []
+                        TypeDate -> []
+                        _ -> [HtmlBase.makeAttribute "align" "right"]
+                valign =
+                    case Spr.cellSpan cell of
+                        Spr.SpanVertical n ->
+                            if n>1
+                                then [HtmlBase.makeAttribute "valign" "top"]
+                                else []
+                        _ -> []
+                withEmph =
+                    case emph of
+                        Item -> id
+                        Total -> Html.b_
+            in  span_ Html.td_ (style++align++valign++class_) $
+                withEmph content

hledger-lib/Hledger/Write/Ods.hs

@@ -0,0 +1,363 @@
+{- |
+Export table data as OpenDocument Spreadsheet
+<https://docs.oasis-open.org/office/OpenDocument/v1.3/>.
+This format supports character encodings, fixed header rows and columns,
+number formatting, text styles, merged cells, formulas, hyperlinks.
+Currently we support Flat ODS, a plain uncompressed XML format.
+
+This is derived from <https://hackage.haskell.org/package/classify-frog-0.2.4.3/src/src/Spreadsheet/Format.hs>
+
+-}
+module Hledger.Write.Ods (
+    printFods,
+    ) where
+
+import Prelude hiding (Applicative(..))
+import Control.Applicative (Applicative(..))
+
+import qualified Data.Text.Lazy as TL
+import qualified Data.Text as T
+import Data.Text (Text)
+
+import qualified Data.Foldable as Fold
+import qualified Data.List as List
+import qualified Data.Map as Map
+import qualified Data.Set as Set
+import Data.Foldable (fold)
+import Data.Map (Map)
+import Data.Set (Set)
+import Data.Maybe (catMaybes)
+
+import qualified System.IO as IO
+import Text.Printf (printf)
+
+import qualified Hledger.Write.Spreadsheet as Spr
+import Hledger.Write.Spreadsheet (Type(..), Style(..), Emphasis(..), Cell(..))
+import Hledger.Data.Types (CommoditySymbol, AmountPrecision(..))
+import Hledger.Data.Types (acommodity, aquantity, astyle, asprecision)
+
+printFods ::
+    IO.TextEncoding ->
+    Map Text ((Maybe Int, Maybe Int), [[Cell Spr.NumLines Text]]) -> TL.Text
+printFods encoding tables =
+    let fileOpen customStyles =
+          map (map (\c -> case c of '\'' -> '"'; _ -> c)) $
+          printf "<?xml version='1.0' encoding='%s'?>" (show encoding) :
+          "<office:document" :
+          "  office:mimetype='application/vnd.oasis.opendocument.spreadsheet'" :
+          "  office:version='1.3'" :
+          "  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'" :
+          "  xmlns:xsd='http://www.w3.org/2001/XMLSchema'" :
+          "  xmlns:text='urn:oasis:names:tc:opendocument:xmlns:text:1.0'" :
+          "  xmlns:style='urn:oasis:names:tc:opendocument:xmlns:style:1.0'" :
+          "  xmlns:meta='urn:oasis:names:tc:opendocument:xmlns:meta:1.0'" :
+          "  xmlns:config='urn:oasis:names:tc:opendocument:xmlns:config:1.0'" :
+          "  xmlns:xlink='http://www.w3.org/1999/xlink'" :
+          "  xmlns:fo='urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0'" :
+          "  xmlns:ooo='http://openoffice.org/2004/office'" :
+          "  xmlns:office='urn:oasis:names:tc:opendocument:xmlns:office:1.0'" :
+          "  xmlns:table='urn:oasis:names:tc:opendocument:xmlns:table:1.0'" :
+          "  xmlns:number='urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0'" :
+          "  xmlns:of='urn:oasis:names:tc:opendocument:xmlns:of:1.2'" :
+          "  xmlns:field='urn:openoffice:names:experimental:ooo-ms-interop:xmlns:field:1.0'" :
+          "  xmlns:form='urn:oasis:names:tc:opendocument:xmlns:form:1.0'>" :
+          "<office:styles>" :
+          "  <number:date-style style:name='iso-date'>" :
+          "    <number:year number:style='long'/>" :
+          "    <number:text>-</number:text>" :
+          "    <number:month number:style='long'/>" :
+          "    <number:text>-</number:text>" :
+          "    <number:day number:style='long'/>" :
+          "  </number:date-style>" :
+          customStyles ++
+          "</office:styles>" :
+          []
+
+        fileClose =
+          "</office:document>" :
+          []
+
+        tableConfig tableNames =
+          " <office:settings>" :
+          "  <config:config-item-set config:name='ooo:view-settings'>" :
+          "   <config:config-item-map-indexed config:name='Views'>" :
+          "    <config:config-item-map-entry>" :
+          "     <config:config-item-map-named config:name='Tables'>" :
+          (fold $
+           flip Map.mapWithKey tableNames $ \tableName (mTopRow,mLeftColumn) ->
+             printf "      <config:config-item-map-entry config:name='%s'>" tableName :
+             (flip foldMap mLeftColumn $ \leftColumn ->
+                "       <config:config-item config:name='HorizontalSplitMode' config:type='short'>2</config:config-item>" :
+                printf "       <config:config-item config:name='HorizontalSplitPosition' config:type='int'>%d</config:config-item>" leftColumn :
+                printf "       <config:config-item config:name='PositionRight' config:type='int'>%d</config:config-item>" leftColumn :
+                []) ++
+             (flip foldMap mTopRow $ \topRow ->
+                "       <config:config-item config:name='VerticalSplitMode' config:type='short'>2</config:config-item>" :
+                printf "       <config:config-item config:name='VerticalSplitPosition' config:type='int'>%d</config:config-item>" topRow :
+                printf "       <config:config-item config:name='PositionBottom' config:type='int'>%d</config:config-item>" topRow :
+                []) ++
+             "      </config:config-item-map-entry>" :
+             []) ++
+          "     </config:config-item-map-named>" :
+          "    </config:config-item-map-entry>" :
+          "   </config:config-item-map-indexed>" :
+          "  </config:config-item-set>" :
+          " </office:settings>" :
+          []
+
+        tableOpen name =
+          "<office:body>" :
+          "<office:spreadsheet>" :
+          printf "<table:table table:name='%s'>" name :
+          []
+
+        tableClose =
+          "</table:table>" :
+          "</office:spreadsheet>" :
+          "</office:body>" :
+          []
+
+    in  TL.unlines $ map (TL.fromStrict . T.pack) $
+        fileOpen
+          (let styles = cellStyles (foldMap (concat.snd) tables) in
+           (numberConfig =<< Set.toList (foldMap (numberParams.snd) styles))
+           ++
+           (cellConfig =<< Set.toList styles)) ++
+        tableConfig (fmap fst tables) ++
+        (Map.toAscList tables >>= \(name,(_,table)) ->
+            tableOpen name ++
+            (table >>= \row ->
+                "<table:table-row>" :
+                (row >>= formatCell) ++
+                "</table:table-row>" :
+                []) ++
+            tableClose) ++
+        fileClose
+
+
+dataStyleFromType :: Type -> DataStyle
+dataStyleFromType typ =
+    case typ of
+        TypeString -> DataString
+        TypeDate -> DataDate
+        TypeAmount amt -> DataAmount (acommodity amt) (asprecision $ astyle amt)
+        TypeMixedAmount -> DataMixedAmount
+
+cellStyles ::
+    (Ord border) =>
+    [Cell border Text] ->
+    Set ((Spr.Border border, Style), DataStyle)
+cellStyles =
+    Set.fromList .
+    map (\cell ->
+            ((cellBorder cell, cellStyle cell),
+             dataStyleFromType $ cellType cell))
+
+numberStyleName :: (CommoditySymbol, AmountPrecision) -> String
+numberStyleName (comm, prec) =
+    printf "%s-%s" comm $
+    case prec of
+        NaturalPrecision -> "natural"
+        Precision k -> show k
+
+numberParams :: DataStyle -> Set (CommoditySymbol, AmountPrecision)
+numberParams (DataAmount comm prec) = Set.singleton (comm, prec)
+numberParams _ = Set.empty
+
+numberConfig :: (CommoditySymbol, AmountPrecision) -> [String]
+numberConfig (comm, prec) =
+    let precStr =
+            case prec of
+                NaturalPrecision -> ""
+                Precision k -> printf " number:decimal-places='%d'" k
+        name = numberStyleName (comm, prec)
+    in
+    printf "  <number:number-style style:name='number-%s'>" name :
+    printf "    <number:number number:min-integer-digits='1'%s/>" precStr :
+    printf "    <number:text>%s%s</number:text>"
+      (if T.null comm then "" else " ") comm :
+    "  </number:number-style>" :
+    []
+
+emphasisName :: Emphasis -> String
+emphasisName emph =
+    case emph of
+        Item -> "item"
+        Total -> "total"
+
+cellStyleName :: Style -> String
+cellStyleName style =
+    case style of
+        Head -> "head"
+        Body emph -> emphasisName emph
+
+linesName :: Spr.NumLines -> Maybe String
+linesName prop =
+    case prop of
+        Spr.NoLine -> Nothing
+        Spr.SingleLine -> Just "single"
+        Spr.DoubleLine -> Just "double"
+
+linesStyle :: Spr.NumLines -> String
+linesStyle prop =
+    case prop of
+        Spr.NoLine -> "none"
+        Spr.SingleLine -> "1.5pt solid #000000"
+        Spr.DoubleLine -> "1.5pt double-thin #000000"
+
+borderLabels :: Spr.Border String
+borderLabels = Spr.Border "left" "right" "top" "bottom"
+
+borderName :: Spr.Border Spr.NumLines -> String
+borderName border =
+    (\bs ->
+        case bs of
+            [] -> "noborder"
+            _ ->
+                ("border="++) $ List.intercalate "," $
+                map (\(name,num) -> name ++ ':' : num) bs) $
+    catMaybes $ Fold.toList $
+    liftA2
+        (\name numLines -> (,) name <$> linesName numLines)
+        borderLabels
+        border
+
+borderStyle :: Spr.Border Spr.NumLines -> [String]
+borderStyle border =
+    if border == Spr.noBorder
+        then []
+        else (:[]) $
+            printf "    <style:table-cell-properties%s/>" $
+            (id :: String -> String) $ fold $
+            liftA2 (printf " fo:border-%s='%s'") borderLabels $
+            fmap linesStyle border
+
+data DataStyle =
+      DataString
+    | DataDate
+    | DataAmount CommoditySymbol AmountPrecision
+    | DataMixedAmount
+    deriving (Eq, Ord, Show)
+
+cellConfig :: ((Spr.Border Spr.NumLines, Style), DataStyle) -> [String]
+cellConfig ((border, cstyle), dataStyle) =
+    let boldStyle = "    <style:text-properties fo:font-weight='bold'/>"
+        alignTop =
+            "    <style:table-cell-properties style:vertical-align='top'/>"
+        alignParagraph =
+            printf "    <style:paragraph-properties fo:text-align='%s'/>"
+        moreStyles =
+            borderStyle border
+            ++
+            (
+            case cstyle of
+                Body Item ->
+                    alignTop :
+                    []
+                Body Total ->
+                    alignTop :
+                    boldStyle :
+                    []
+                Head ->
+                    alignParagraph "center" :
+                    boldStyle :
+                    []
+            )
+            ++
+            (
+            case dataStyle of
+                DataMixedAmount -> [alignParagraph "end"]
+                _ -> []
+            )
+        cstyleName = cellStyleName cstyle
+        bordName = borderName border
+        style :: String
+        style =
+            case dataStyle of
+                DataDate ->
+                    printf
+                      "style:name='%s-%s-date' style:data-style-name='iso-date'"
+                      cstyleName bordName
+                DataAmount comm prec ->
+                    let name = numberStyleName (comm, prec) in
+                    printf
+                      "style:name='%s-%s-%s' style:data-style-name='number-%s'"
+                      cstyleName bordName name name
+                _ -> printf "style:name='%s-%s'" cstyleName bordName
+    in
+    case moreStyles of
+        [] ->
+            printf "  <style:style style:family='table-cell' %s/>" style :
+            []
+        _ ->
+            printf "  <style:style style:family='table-cell' %s>" style :
+            moreStyles ++
+            "  </style:style>" :
+            []
+
+
+formatCell :: Cell Spr.NumLines Text -> [String]
+formatCell cell =
+    let style, valueType :: String
+        style = tableStyle styleName
+        cstyleName = cellStyleName $ cellStyle cell
+        bordName = borderName $ cellBorder cell
+        styleName :: String
+        styleName =
+            case dataStyleFromType $ cellType cell of
+                DataDate -> printf "%s-%s-date" cstyleName bordName
+                DataAmount comm prec ->
+                    let name = numberStyleName (comm, prec) in
+                    printf "%s-%s-%s" cstyleName bordName name
+                _ -> printf "%s-%s" cstyleName bordName
+        tableStyle = printf " table:style-name='%s'"
+
+        valueType =
+            case cellType cell of
+                TypeAmount amt ->
+                    printf
+                        "office:value-type='float' office:value='%s'"
+                        (show $ aquantity amt)
+                TypeDate ->
+                    printf
+                        "office:value-type='date' office:date-value='%s'"
+                        (cellContent cell)
+                _ -> "office:value-type='string'"
+
+        covered =
+            case cellSpan cell of
+                Spr.Covered -> "covered-"
+                _ -> ""
+
+        span_ =
+            case cellSpan cell of
+                Spr.SpanHorizontal n | n>1 ->
+                    printf " table:number-columns-spanned='%d'" n
+                Spr.SpanVertical n | n>1 ->
+                    printf " table:number-rows-spanned='%d'" n
+                _ -> ""
+
+        anchor text =
+            if T.null $ Spr.cellAnchor cell
+                then text
+                else printf "<text:a xlink:href='%s'>%s</text:a>"
+                        (escape $ T.unpack $ Spr.cellAnchor cell) text
+
+    in
+    printf "<table:%stable-cell%s%s %s>" covered style span_ valueType :
+    printf "<text:p>%s</text:p>"
+        (anchor $ escape $ T.unpack $ cellContent cell) :
+    printf "</table:%stable-cell>" covered :
+    []
+
+escape :: String -> String
+escape =
+    concatMap $ \c ->
+        case c of
+            '\n' -> "&#10;"
+            '&' -> "&amp;"
+            '<' -> "&lt;"
+            '>' -> "&gt;"
+            '"' -> "&quot;"
+            '\'' -> "&apos;"
+            _ -> [c]

hledger-lib/Hledger/Write/Spreadsheet.hs

@@ -0,0 +1,165 @@
+{- |
+Rich data type to describe data in a table.
+This is the basis for ODS and HTML export.
+-}
+module Hledger.Write.Spreadsheet (
+    Type(..),
+    Style(..),
+    Emphasis(..),
+    Cell(..),
+    Class(Class), textFromClass,
+    Span(..),
+    Border(..),
+    Lines(..),
+    NumLines(..),
+    noBorder,
+    defaultCell,
+    emptyCell,
+    transposeCell,
+    transpose,
+    ) where
+
+import Hledger.Data.Types (Amount)
+
+import qualified Data.List as List
+import Data.Text (Text)
+
+import Prelude hiding (span)
+
+
+data Type =
+      TypeString
+    | TypeAmount !Amount
+    | TypeMixedAmount
+    | TypeDate
+    deriving (Eq, Ord, Show)
+
+data Style = Body Emphasis | Head
+    deriving (Eq, Ord, Show)
+
+data Emphasis = Item | Total
+    deriving (Eq, Ord, Show)
+
+
+class Lines border where noLine :: border
+instance Lines () where noLine = ()
+instance Lines NumLines where noLine = NoLine
+
+{- |
+The same as Tab.Properties, but has 'Eq' and 'Ord' instances.
+We need those for storing 'NumLines' in 'Set's.
+-}
+data NumLines = NoLine | SingleLine | DoubleLine
+    deriving (Eq, Ord, Show)
+
+data Border lines =
+    Border {
+        borderLeft, borderRight,
+        borderTop, borderBottom :: lines
+    }
+    deriving (Eq, Ord, Show)
+
+instance Functor Border where
+    fmap f (Border left right top bottom) =
+        Border (f left) (f right) (f top) (f bottom)
+
+instance Applicative Border where
+    pure a = Border a a a a
+    Border fLeft fRight fTop fBottom <*> Border left right top bottom =
+        Border (fLeft left) (fRight right) (fTop top) (fBottom bottom)
+
+instance Foldable Border where
+    foldMap f (Border left right top bottom) =
+        f left <> f right <> f top <> f bottom
+
+noBorder :: (Lines border) => Border border
+noBorder = pure noLine
+
+transposeBorder :: Border lines -> Border lines
+transposeBorder (Border left right top bottom) =
+    Border top bottom left right
+
+
+newtype Class = Class Text
+
+textFromClass :: Class -> Text
+textFromClass (Class cls) = cls
+
+
+{- |
+* 'NoSpan' means a single unmerged cell.
+
+* 'Covered' is a cell if it is part of a horizontally or vertically merged cell.
+  We maintain these cells although they are ignored in HTML output.
+  In contrast to that, FODS can store covered cells
+  and allows to access the hidden cell content via formulas.
+  CSV does not support merged cells
+  and thus simply writes the content of covered cells.
+  Maintaining 'Covered' cells also simplifies transposing.
+
+* @'SpanHorizontal' n@ denotes the first cell in a row
+  that is part of a merged cell.
+  The merged cell contains @n@ atomic cells, including the first one.
+  That is @SpanHorizontal 1@ is actually like @NoSpan@.
+  The content of this cell is shown as content of the merged cell.
+
+* @'SpanVertical' n@ starts a vertically merged cell.
+
+The writer functions expect consistent data,
+that is, 'Covered' cells must actually be part of a merged cell
+and merged cells must only cover 'Covered' cells.
+-}
+data Span =
+      NoSpan
+    | Covered
+    | SpanHorizontal Int
+    | SpanVertical Int
+    deriving (Eq)
+
+transposeSpan :: Span -> Span
+transposeSpan span =
+    case span of
+        NoSpan -> NoSpan
+        Covered -> Covered
+        SpanHorizontal n -> SpanVertical n
+        SpanVertical n -> SpanHorizontal n
+
+data Cell border text =
+    Cell {
+        cellType :: Type,
+        cellBorder :: Border border,
+        cellStyle :: Style,
+        cellSpan :: Span,
+        cellAnchor :: Text,
+        cellClass :: Class,
+        cellContent :: text
+    }
+
+instance Functor (Cell border) where
+    fmap f (Cell typ border style span anchor class_ content) =
+        Cell typ border style span anchor class_ $ f content
+
+defaultCell :: (Lines border) => text -> Cell border text
+defaultCell text =
+    Cell {
+        cellType = TypeString,
+        cellBorder = noBorder,
+        cellStyle = Body Item,
+        cellSpan = NoSpan,
+        cellAnchor = mempty,
+        cellClass = Class mempty,
+        cellContent = text
+    }
+
+emptyCell :: (Lines border, Monoid text) => Cell border text
+emptyCell = defaultCell mempty
+
+transposeCell :: Cell border text -> Cell border text
+transposeCell cell =
+    cell {
+        cellBorder = transposeBorder $ cellBorder cell,
+        cellSpan = transposeSpan $ cellSpan cell
+    }
+
+transpose :: [[Cell border text]] -> [[Cell border text]]
+transpose = List.transpose . map (map transposeCell)

hledger-lib/Text/Megaparsec/Custom.hs

@@ -1,437 +0,0 @@
--- A bunch of megaparsec helpers for re-parsing etc.
--- I think these are generic apart from the HledgerParseError name.
-
-{-# LANGUAGE BangPatterns #-}
-{-# LANGUAGE CPP #-}
-{-# LANGUAGE FlexibleContexts #-}
-{-# LANGUAGE FlexibleInstances #-} -- new
-{-# LANGUAGE LambdaCase #-}
-{-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE StandaloneDeriving #-} -- new
-
-module Text.Megaparsec.Custom (
-  -- * Custom parse error types
-  HledgerParseErrorData,
-  HledgerParseErrors,
-
-  -- * Failing with an arbitrary source position
-  parseErrorAt,
-  parseErrorAtRegion,
-
-  -- * Re-parsing
-  SourceExcerpt,
-  getExcerptText,
-
-  excerpt_,
-  reparseExcerpt,
-
-  -- * Pretty-printing custom parse errors
-  customErrorBundlePretty,
-
-
-  -- * "Final" parse errors
-  FinalParseError,
-  FinalParseError',
-  FinalParseErrorBundle,
-  FinalParseErrorBundle',
-
-  -- * Constructing "final" parse errors
-  finalError,
-  finalFancyFailure,
-  finalFail,
-  finalCustomFailure,
-
-  -- * Pretty-printing "final" parse errors
-  finalErrorBundlePretty,
-  attachSource,
-
-  -- * Handling parse errors from include files with "final" parse errors
-  parseIncludeFile,
-)
-where
-
-import Control.Monad.Except (ExceptT, MonadError, catchError, throwError)
-import Control.Monad.State.Strict (StateT, evalStateT)
-import Control.Monad.Trans.Class (lift)
-import qualified Data.List.NonEmpty as NE
-import Data.Monoid (Alt(..))
-import qualified Data.Set as S
-import Data.Text (Text)
-import Text.Megaparsec
-
-
---- * Custom parse error types
-
--- | Custom error data for hledger parsers. Specialised for a 'Text' parse stream.
--- ReparseableTextParseErrorData ?
-data HledgerParseErrorData
-  -- | Fail with a message at a specific source position interval. The
-  -- interval must be contained within a single line.
-  = ErrorFailAt Int -- Starting offset
-                Int -- Ending offset
-                String -- Error message
-  -- | Re-throw parse errors obtained from the "re-parsing" of an excerpt
-  -- of the source text.
-  | ErrorReparsing
-      (NE.NonEmpty (ParseError Text HledgerParseErrorData)) -- Source fragment parse errors
-  deriving (Show, Eq, Ord)
-
--- | A specialised version of ParseErrorBundle: 
--- a non-empty collection of hledger parse errors, 
--- equipped with PosState to help pretty-print them.
--- Specialised for a 'Text' parse stream.
-type HledgerParseErrors = ParseErrorBundle Text HledgerParseErrorData
-
--- We require an 'Ord' instance for 'CustomError' so that they may be
--- stored in a 'Set'. The actual instance is inconsequential, so we just
--- derive it, but the derived instance requires an (orphan) instance for
--- 'ParseError'. Hopefully this does not cause any trouble.
-
-deriving instance Ord (ParseError Text HledgerParseErrorData)
-
--- Note: the pretty-printing of our 'HledgerParseErrorData' type is only partally
--- defined in its 'ShowErrorComponent' instance; we perform additional
--- adjustments in 'customErrorBundlePretty'.
-
-instance ShowErrorComponent HledgerParseErrorData where
-  showErrorComponent (ErrorFailAt _ _ errMsg) = errMsg
-  showErrorComponent (ErrorReparsing _) = "" -- dummy value
-
-  errorComponentLen (ErrorFailAt startOffset endOffset _) =
-    endOffset - startOffset
-  errorComponentLen (ErrorReparsing _) = 1 -- dummy value
-
-
---- * Failing with an arbitrary source position
-
--- | Fail at a specific source position, given by the raw offset from the
--- start of the input stream (the number of tokens processed at that
--- point).
-
-parseErrorAt :: Int -> String -> HledgerParseErrorData
-parseErrorAt offset = ErrorFailAt offset (offset+1)
-
--- | Fail at a specific source interval, given by the raw offsets of its
--- endpoints from the start of the input stream (the numbers of tokens
--- processed at those points).
---
--- Note that care must be taken to ensure that the specified interval does
--- not span multiple lines of the input source. This will not be checked.
-
-parseErrorAtRegion
-  :: Int    -- ^ Start offset
-  -> Int    -- ^ End end offset
-  -> String -- ^ Error message
-  -> HledgerParseErrorData
-parseErrorAtRegion startOffset endOffset msg =
-  if startOffset < endOffset
-    then ErrorFailAt startOffset endOffset msg'
-    else ErrorFailAt startOffset (startOffset+1) msg'
-  where
-    msg' = "\n" ++ msg
-
-
---- * Re-parsing
-
--- | A fragment of source suitable for "re-parsing". The purpose of this
--- data type is to preserve the content and source position of the excerpt
--- so that parse errors raised during "re-parsing" may properly reference
--- the original source.
-
-data SourceExcerpt = SourceExcerpt Int  -- Offset of beginning of excerpt
-                                   Text -- Fragment of source file
-
--- | Get the raw text of a source excerpt.
-
-getExcerptText :: SourceExcerpt -> Text
-getExcerptText (SourceExcerpt _ txt) = txt
-
--- | 'excerpt_ p' applies the given parser 'p' and extracts the portion of
--- the source consumed by 'p', along with the source position of this
--- portion. This is the only way to create a source excerpt suitable for
--- "re-parsing" by 'reparseExcerpt'.
-
--- This function could be extended to return the result of 'p', but we don't
--- currently need this.
-
-excerpt_ :: MonadParsec HledgerParseErrorData Text m => m a -> m SourceExcerpt
-excerpt_ p = do
-  offset <- getOffset
-  (!txt, _) <- match p
-  pure $ SourceExcerpt offset txt
-
--- | 'reparseExcerpt s p' "re-parses" the source excerpt 's' using the
--- parser 'p'. Parse errors raised by 'p' will be re-thrown at the source
--- position of the source excerpt.
---
--- In order for the correct source file to be displayed when re-throwing
--- parse errors, we must ensure that the source file during the use of
--- 'reparseExcerpt s p' is the same as that during the use of 'excerpt_'
--- that generated the source excerpt 's'. However, we can usually expect
--- this condition to be satisfied because, at the time of writing, the
--- only changes of source file in the codebase take place through include
--- files, and the parser for include files neither accepts nor returns
--- 'SourceExcerpt's.
-
-reparseExcerpt
-  :: Monad m
-  => SourceExcerpt
-  -> ParsecT HledgerParseErrorData Text m a
-  -> ParsecT HledgerParseErrorData Text m a
-reparseExcerpt (SourceExcerpt offset txt) p = do
-  (_, res) <- lift $ runParserT' p (offsetInitialState offset txt)
-  case res of
-    Right result -> pure result
-    Left errBundle -> customFailure $ ErrorReparsing $ bundleErrors errBundle
-
-  where
-    offsetInitialState :: Int -> s ->
-#if MIN_VERSION_megaparsec(8,0,0)
-      State s e
-#else
-      State s
-#endif
-    offsetInitialState initialOffset s = State
-      { stateInput  = s
-      , stateOffset = initialOffset
-      , statePosState = PosState
-        { pstateInput = s
-        , pstateOffset = initialOffset
-        , pstateSourcePos = initialPos ""
-        , pstateTabWidth = defaultTabWidth
-        , pstateLinePrefix = ""
-        }
-#if MIN_VERSION_megaparsec(8,0,0)
-      , stateParseErrors = []
-#endif
-      }
-
---- * Pretty-printing custom parse errors
-
--- | Pretty-print our custom parse errors. It is necessary to use this
--- instead of 'errorBundlePretty' when custom parse errors are thrown.
---
--- This function intercepts our custom parse errors and applies final
--- adjustments ('finalizeCustomError') before passing them to
--- 'errorBundlePretty'. These adjustments are part of the implementation
--- of the behaviour of our custom parse errors.
---
--- Note: We must ensure that the offset of the 'PosState' of the provided
--- 'ParseErrorBundle' is no larger than the offset specified by a
--- 'ErrorFailAt' constructor. This is guaranteed if this offset is set to
--- 0 (that is, the beginning of the source file), which is the
--- case for 'ParseErrorBundle's returned from 'runParserT'.
-
-customErrorBundlePretty :: HledgerParseErrors -> String
-customErrorBundlePretty errBundle =
-  let errBundle' = errBundle { bundleErrors =
-        NE.sortWith errorOffset $ -- megaparsec requires that the list of errors be sorted by their offsets
-        bundleErrors errBundle >>= finalizeCustomError }
-  in  errorBundlePretty errBundle'
-
-  where
-    finalizeCustomError
-      :: ParseError Text HledgerParseErrorData -> NE.NonEmpty (ParseError Text HledgerParseErrorData)
-    finalizeCustomError err = case findCustomError err of
-      Nothing -> pure err
-
-      Just errFailAt@(ErrorFailAt startOffset _ _) ->
-        -- Adjust the offset
-        pure $ FancyError startOffset $ S.singleton $ ErrorCustom errFailAt
-
-      Just (ErrorReparsing errs) ->
-        -- Extract and finalize the inner errors
-        errs >>= finalizeCustomError
-
-    -- If any custom errors are present, arbitrarily take the first one
-    -- (since only one custom error should be used at a time).
-    findCustomError :: ParseError Text HledgerParseErrorData -> Maybe HledgerParseErrorData
-    findCustomError err = case err of
-      FancyError _ errSet ->
-        finds (\case {ErrorCustom e -> Just e; _ -> Nothing}) errSet
-      _ -> Nothing
-
-    finds :: (Foldable t) => (a -> Maybe b) -> t a -> Maybe b
-    finds f = getAlt . foldMap (Alt . f)
-
-
---- * "Final" parse errors
---
--- | A type representing "final" parse errors that cannot be backtracked
--- from and are guaranteed to halt parsing. The anti-backtracking
--- behaviour is implemented by an 'ExceptT' layer in the parser's monad
--- stack, using this type as the 'ExceptT' error type.
---
--- We have three goals for this type:
--- (1) it should be possible to convert any parse error into a "final"
--- parse error,
--- (2) it should be possible to take a parse error thrown from an include
--- file and re-throw it in the parent file, and
--- (3) the pretty-printing of "final" parse errors should be consistent
--- with that of ordinary parse errors, but should also report a stack of
--- files for errors thrown from include files.
---
--- In order to pretty-print a "final" parse error (goal 3), it must be
--- bundled with include filepaths and its full source text. When a "final"
--- parse error is thrown from within a parser, we do not have access to
--- the full source, so we must hold the parse error until it can be joined
--- with its source (and include filepaths, if it was thrown from an
--- include file) by the parser's caller.
---
--- A parse error with include filepaths and its full source text is
--- represented by the 'FinalParseErrorBundle' type, while a parse error in
--- need of either include filepaths, full source text, or both is
--- represented by the 'FinalParseError' type.
-
-data FinalParseError' e
-  -- a parse error thrown as a "final" parse error
-  = FinalError           (ParseError Text e)
-  -- a parse error obtained from running a parser, e.g. using 'runParserT'
-  | FinalBundle          (ParseErrorBundle Text e)
-  -- a parse error thrown from an include file
-  | FinalBundleWithStack (FinalParseErrorBundle' e)
-  deriving (Show)
-
-type FinalParseError = FinalParseError' HledgerParseErrorData
-
--- We need a 'Monoid' instance for 'FinalParseError' so that 'ExceptT
--- FinalParseError m' is an instance of Alternative and MonadPlus, which
--- is needed to use some parser combinators, e.g. 'many'.
---
--- This monoid instance simply takes the first (left-most) error.
-
-instance Semigroup (FinalParseError' e) where
-  e <> _ = e
-
-instance Monoid (FinalParseError' e) where
-  mempty = FinalError $ FancyError 0 $
-            S.singleton (ErrorFail "default parse error")
-  mappend = (<>)
-
--- | A type bundling a 'ParseError' with its full source text, filepath,
--- and stack of include files. Suitable for pretty-printing.
---
--- Megaparsec's 'ParseErrorBundle' type already bundles a parse error with
--- its full source text and filepath, so we just add a stack of include
--- files.
-
-data FinalParseErrorBundle' e = FinalParseErrorBundle'
-  { finalErrorBundle :: ParseErrorBundle Text e
-  , includeFileStack :: [FilePath]
-  } deriving (Show)
-
-type FinalParseErrorBundle = FinalParseErrorBundle' HledgerParseErrorData
-
-
---- * Constructing and throwing final parse errors
-
--- | Convert a "regular" parse error into a "final" parse error.
-
-finalError :: ParseError Text e -> FinalParseError' e
-finalError = FinalError
-
--- | Like megaparsec's 'fancyFailure', but as a "final" parse error.
-
-finalFancyFailure
-  :: (MonadParsec e s m, MonadError (FinalParseError' e) m)
-  => S.Set (ErrorFancy e) -> m a
-finalFancyFailure errSet = do
-  offset <- getOffset
-  throwError $ FinalError $ FancyError offset errSet
-
--- | Like 'fail', but as a "final" parse error.
-
-finalFail
-  :: (MonadParsec e s m, MonadError (FinalParseError' e) m) => String -> m a
-finalFail = finalFancyFailure . S.singleton . ErrorFail
-
--- | Like megaparsec's 'customFailure', but as a "final" parse error.
-
-finalCustomFailure
-  :: (MonadParsec e s m, MonadError (FinalParseError' e) m) => e -> m a
-finalCustomFailure = finalFancyFailure . S.singleton . ErrorCustom
-
-
---- * Pretty-printing "final" parse errors
-
--- | Pretty-print a "final" parse error: print the stack of include files,
--- then apply the pretty-printer for parse error bundles. Note that
--- 'attachSource' must be used on a "final" parse error before it can be
--- pretty-printed.
-
-finalErrorBundlePretty :: FinalParseErrorBundle' HledgerParseErrorData -> String
-finalErrorBundlePretty bundle =
-     concatMap showIncludeFilepath (includeFileStack bundle)
-  <> customErrorBundlePretty (finalErrorBundle bundle)
-  where
-    showIncludeFilepath path = "in file included from " <> path <> ",\n"
-
--- | Supply a filepath and source text to a "final" parse error so that it
--- can be pretty-printed. You must ensure that you provide the appropriate
--- source text and filepath.
-
-attachSource
-  :: FilePath -> Text -> FinalParseError' e -> FinalParseErrorBundle' e
-attachSource filePath sourceText finalParseError = case finalParseError of
-
-  -- A parse error thrown directly with the 'FinalError' constructor
-  -- requires both source and filepath.
-  FinalError err ->
-    let bundle = ParseErrorBundle
-          { bundleErrors = err NE.:| []
-          , bundlePosState = initialPosState filePath sourceText }
-    in  FinalParseErrorBundle'
-          { finalErrorBundle = bundle
-          , includeFileStack  = [] }
-
-  -- A 'ParseErrorBundle' already has the appropriate source and filepath
-  -- and so needs neither.
-  FinalBundle peBundle -> FinalParseErrorBundle'
-    { finalErrorBundle = peBundle
-    , includeFileStack = [] }
-
-  -- A parse error from a 'FinalParseErrorBundle' was thrown from an
-  -- include file, so we add the filepath to the stack.
-  FinalBundleWithStack fpeBundle -> fpeBundle
-    { includeFileStack = filePath : includeFileStack fpeBundle }
-
-
---- * Handling parse errors from include files with "final" parse errors
-
--- | Parse a file with the given parser and initial state, discarding the
--- final state and re-throwing any parse errors as "final" parse errors.
-
-parseIncludeFile
-  :: Monad m
-  => StateT st (ParsecT HledgerParseErrorData Text (ExceptT FinalParseError m)) a
-  -> st
-  -> FilePath
-  -> Text
-  -> StateT st (ParsecT HledgerParseErrorData Text (ExceptT FinalParseError m)) a
-parseIncludeFile parser initialState filepath text =
-  catchError parser' handler
-  where
-    parser' = do
-      eResult <- lift $ lift $
-                  runParserT (evalStateT parser initialState) filepath text
-      case eResult of
-        Left parseErrorBundle -> throwError $ FinalBundle parseErrorBundle
-        Right result -> pure result
-
-    -- Attach source and filepath of the include file to its parse errors
-    handler e = throwError $ FinalBundleWithStack $ attachSource filepath text e
-
-
---- * Helpers
-
--- Like megaparsec's 'initialState', but instead for 'PosState'. Used when
--- constructing 'ParseErrorBundle's. The values for "tab width" and "line
--- prefix" are taken from 'initialState'.
-
-initialPosState :: FilePath -> Text -> PosState Text
-initialPosState filePath sourceText = PosState
-  { pstateInput      = sourceText
-  , pstateOffset     = 0
-  , pstateSourcePos  = initialPos filePath
-  , pstateTabWidth   = defaultTabWidth
-  , pstateLinePrefix = "" }

hledger-lib/Text/Tabular/AsciiWide.hs

@@ -27,7 +27,7 @@ module Text.Tabular.AsciiWide
 import Data.Bifunctor (bimap)
 import Data.Maybe (fromMaybe)
 import Data.Default (Default(..))
-import Data.List (intercalate, intersperse, transpose)
+import Data.List (intersperse, transpose)
 import Data.Semigroup (stimesMonoid)
 import Data.Text (Text)
 import qualified Data.Text as T
@@ -130,9 +130,9 @@ renderTableByRowsB topts@TableOpts{prettyTable=pretty, tableBorders=borders} fc
 
   -- maximum width for each column
   sizes = map (fromMaybe 0 . maximumMay . map cellWidth) $ transpose cells2
-  renderRs (Header s)   = [s]
-  renderRs (Group p hs) = intercalate sep $ map renderRs hs
-    where sep = renderHLine VM borders pretty sizes ch2 p
+  renderRs =
+    concatMap (either (renderHLine VM borders pretty sizes ch2) (:[])) .
+    flattenHeader
 
   -- borders and bars
   addBorders xs = if borders then bar VT SingleLine : xs ++ [bar VB SingleLine] else xs
@@ -145,9 +145,7 @@ renderRow topts = toLazyText . renderRowB topts
 
 -- | A version of renderRow which returns the underlying Builder.
 renderRowB:: TableOpts -> Header Cell -> Builder
-renderRowB topts h = renderColumns topts is h
-  where is = map cellWidth $ headerContents h
-
+renderRowB topts h = renderColumns topts ws h where ws = map cellWidth $ headerContents h
 
 verticalBar :: Bool -> Char
 verticalBar pretty = if pretty then '│' else '|'

hledger-lib/hledger-lib.cabal

@@ -1,11 +1,11 @@
 cabal-version: 1.12
 
--- This file has been generated from package.yaml by hpack version 0.36.0.
+-- This file has been generated from package.yaml by hpack version 0.37.0.
 --
 -- see: https://github.com/sol/hpack
 
 name:           hledger-lib
-version:        1.33.99
+version:        1.40.99
 synopsis:       A library providing the core functionality of hledger
 description:    This library contains hledger's core functionality.
                 It is used by most hledger* packages so that they support the same
@@ -80,12 +80,18 @@ library
       Hledger.Read
       Hledger.Read.Common
       Hledger.Read.CsvReader
-      Hledger.Read.CsvUtils
       Hledger.Read.InputOptions
       Hledger.Read.JournalReader
       Hledger.Read.RulesReader
       Hledger.Read.TimedotReader
       Hledger.Read.TimeclockReader
+      Hledger.Write.Csv
+      Hledger.Write.Ods
+      Hledger.Write.Html
+      Hledger.Write.Html.Attribute
+      Hledger.Write.Html.Blaze
+      Hledger.Write.Html.Lucid
+      Hledger.Write.Spreadsheet
       Hledger.Reports
       Hledger.Reports.ReportOptions
       Hledger.Reports.ReportTypes
@@ -104,9 +110,8 @@ library
       Hledger.Utils.Test
       Hledger.Utils.Text
       Text.Tabular.AsciiWide
-  other-modules:
-      Text.Megaparsec.Custom
       Text.WideString
+  other-modules:
       Paths_hledger_lib
   hs-source-dirs:
       ./
@@ -118,8 +123,9 @@ library
     , aeson-pretty
     , ansi-terminal >=0.9
     , array
-    , base >=4.14 && <4.20
-    , base-compat
+    , base >=4.14 && <4.21
+    , base-compat >=0.14.0
+    , blaze-html
     , blaze-markup >=0.5.1
     , bytestring
     , call-stack
@@ -131,11 +137,12 @@ library
     , data-default >=0.5
     , deepseq
     , directory
-    , doclayout >=0.3 && <0.5
+    , doclayout >=0.3 && <0.6
     , extra >=1.6.3
     , file-embed >=0.0.10
     , filepath
     , hashtables >=1.2.3.1
+    , lucid
     , megaparsec >=7.0.0 && <9.7
     , microlens >=0.4
     , microlens-th >=0.4
@@ -180,8 +187,9 @@ test-suite doctest
     , aeson-pretty
     , ansi-terminal >=0.9
     , array
-    , base >=4.14 && <4.20
-    , base-compat
+    , base >=4.14 && <4.21
+    , base-compat >=0.14.0
+    , blaze-html
     , blaze-markup >=0.5.1
     , bytestring
     , call-stack
@@ -193,12 +201,13 @@ test-suite doctest
     , data-default >=0.5
     , deepseq
     , directory
-    , doclayout >=0.3 && <0.5
+    , doclayout >=0.3 && <0.6
     , doctest >=0.18.1
     , extra >=1.6.3
     , file-embed >=0.0.10
     , filepath
     , hashtables >=1.2.3.1
+    , lucid
     , megaparsec >=7.0.0 && <9.7
     , microlens >=0.4
     , microlens-th >=0.4
@@ -245,8 +254,9 @@ test-suite unittest
     , aeson-pretty
     , ansi-terminal >=0.9
     , array
-    , base >=4.14 && <4.20
-    , base-compat
+    , base >=4.14 && <4.21
+    , base-compat >=0.14.0
+    , blaze-html
     , blaze-markup >=0.5.1
     , bytestring
     , call-stack
@@ -258,12 +268,13 @@ test-suite unittest
     , data-default >=0.5
     , deepseq
     , directory
-    , doclayout >=0.3 && <0.5
+    , doclayout >=0.3 && <0.6
     , extra >=1.6.3
     , file-embed >=0.0.10
     , filepath
     , hashtables >=1.2.3.1
     , hledger-lib
+    , lucid
     , megaparsec >=7.0.0 && <9.7
     , microlens >=0.4
     , microlens-th >=0.4

hledger-lib/package.yaml

@@ -1,5 +1,5 @@
 name: hledger-lib
-version: 1.33.99
+version: 1.40.99
 synopsis: A library providing the core functionality of hledger
 description: |
   This library contains hledger's core functionality.
@@ -39,13 +39,14 @@ extra-source-files:
 #data-files:
 
 dependencies:
-- base >=4.14 && <4.20
-- base-compat
+- base >=4.14 && <4.21
+- base-compat >=0.14.0
 - aeson >=1 && <2.3
 - aeson-pretty
 - ansi-terminal >=0.9
 - array
 - blaze-markup >=0.5.1
+- blaze-html
 - bytestring
 - call-stack
 - cmdargs >=0.10
@@ -57,10 +58,11 @@ dependencies:
 - deepseq
 - Decimal >=0.5.1
 - directory
-- doclayout >=0.3 && <0.5
+- doclayout >=0.3 && <0.6
 - file-embed >=0.0.10
 - filepath
 - hashtables >=1.2.3.1
+- lucid
 - megaparsec >=7.0.0 && <9.7
 - microlens >=0.4
 - microlens-th >=0.4
@@ -142,13 +144,19 @@ library:
   - Hledger.Read
   - Hledger.Read.Common
   - Hledger.Read.CsvReader
-  - Hledger.Read.CsvUtils
   - Hledger.Read.InputOptions
   - Hledger.Read.JournalReader
   - Hledger.Read.RulesReader
 #  - Hledger.Read.LedgerReader
   - Hledger.Read.TimedotReader
   - Hledger.Read.TimeclockReader
+  - Hledger.Write.Csv
+  - Hledger.Write.Ods
+  - Hledger.Write.Html
+  - Hledger.Write.Html.Attribute
+  - Hledger.Write.Html.Blaze
+  - Hledger.Write.Html.Lucid
+  - Hledger.Write.Spreadsheet
   - Hledger.Reports
   - Hledger.Reports.ReportOptions
   - Hledger.Reports.ReportTypes
@@ -167,6 +175,7 @@ library:
   - Hledger.Utils.Test
   - Hledger.Utils.Text
   - Text.Tabular.AsciiWide
+  - Text.WideString
 #  other-modules:
 #  - Ledger.Parser.Text
 

hledger-ui/.date.m4

@@ -1,2 +1,2 @@
 m4_dnl Date to show in man pages. Updated by "Shake manuals"
-m4_define({{_monthyear_}}, {{April 2024}})m4_dnl
+m4_define({{_monthyear_}}, {{September 2024}})m4_dnl

hledger-ui/.version

@@ -1 +1 @@
-1.33.99
+1.40.99

hledger-ui/.version.m4

@@ -1,2 +1,2 @@
 m4_dnl Version number to show in manuals. Updated by "Shake setversion"
-m4_define({{_version_}}, {{1.33.99}})m4_dnl
+m4_define({{_version_}}, {{1.40.99}})m4_dnl