shake: move changelog scripts to Shake, enhance

New shake targets:
./Shake changelogs
./Shake CHANGES.md
./Shake CHANGES.md-dry
./Shake PKG/CHANGES.md
./Shake PKG/CHANGES.md-dry

Enhancements:
- removes the changelog's previous top heading
- ignores commits like "changelog", "doc: update changelogs"..
- does not write temporary files
- dry run mode

[ci skip]

CONTRIBUTING.md

@@ -1234,22 +1234,10 @@ Project documentation lives in a number of places:
 
 How to prepare changelogs & release notes
 
-Draft:
-
-- `make changelog-draft >> doc/CHANGES.draft.org` (or `>` if this is the first draft)
-- open this org file and sort the nodes (`C-c ^ a`)
-- categorisation pass 1: go through and add topic prefixes where missing
-- sort the nodes again
-- categorisation pass 2: move significant items to the appropriate package subnode as appropriate; keep "soft" items that might appear in release notes; delete the rest
-- cleanup pass: combine/rewrite items for clarity
-
 Changelogs:
 
-- `CHANGES.md` in each package directory, and one in the top directory (project-wide changes, perhaps just for staging)
+- ./Shake changelogs
-- markdown format: - bullets, indented literal blocks
+- edit the new changelog items (identify, filter, move to correct changelog, deduplicate, rewrite, sort/group)
-- there's always a heading at the top whose first word is a release version or (between releases) a commit hash
-- `make changelogs` to add any new commits to the top of all changelogs
-- then clean those up manually: identify, filter, move to correct changelog, deduplicate, rewrite, sort/group
 
 Release notes:
 

Makefile

@@ -1109,98 +1109,6 @@ haddock: \
 # # 	cd site/api && \
 # # 	hoogle --convert=main.txt --output=default.hoo
 
-#
-
-# in subsequent rules, allow automatic variables to be used in prerequisites (use $$)
-.SECONDEXPANSION:
-
-########################
-# changelogs
-
-# -E for extended regular expressions
-SED=sed -E
-# GNU sed, when needed. It's gsed eg with homebrew. This test may print an error message
-GSED=$(notdir $(shell which gsed || which sed)) -E
-
-# --abbrev-commit for short commit hashes
-GITLOG=git log --abbrev-commit
-
-# git exclude pathspecs, https://git-scm.com/docs/gitglossary.html#gitglossary-aiddefpathspecapathspec
-EXCLUDEPKGDIRS=\
-	':!hledger-lib' \
-	':!hledger' \
-	':!hledger-ui' \
-	':!hledger-web' \
-	':!hledger-api' \
-	':!tests' \
-
-# XXX would like to include tests/ in hledger changelog
-
-# git log format suitable for changelogs/release notes
-# %s=subject, %an=author name, %n=newline if needed, %w=width/indent1/indent2, %b=body, %h=hash
-CHANGEFMT=--pretty=format:'- %s (%an)%n%w(0,2,2)%b'
-
-# git log format like the above plus hashes and --stat info
-VERBOSEFMT=--pretty=format:'- %s (%an)%n%w(0,2,2)%b%h' --stat
-
-# Format a git log message, with one of the formats above, as a changelog item:
-# ensure bullet lists in descriptions use hyphens not stars
-# strip maintainer's author name
-# strip [ci skip] lines
-# replace lines containing only spaces with empty lines
-# strip windows carriage returns (XXX can't edit this with IDEA right now, it rewrites the ^M)
-# replace consecutive newlines with one
-CHANGECLEANUP=$(SED) \
-		-e 's/^( )*\* /\1- /' \
-		-e 's/ \(Simon Michael\)//' \
-		-e 's/\[ci skip\]//' \
-		-e 's/^  $$//' \
-		-e 's/
//' \
-		-e '/./,/^$$/!d' \
-
-#CHANGECLEANUP=cat
-
-# periodically run this to add new commits to changelogs, then clean up manually
-changelogs: */CHANGES.md CHANGES.md \
-		$(call def-help,changelogs, update all changelogs with the latest commits )
-
-# inserts a blank line + heading + new items after line 2.
-# dry run: put echo before the last $(SED), ls Makefile | entr bash -c 'make hledger/CHANGES.md && cat hledger/CHANGES.md.new'
-# Needs GNU sed to do the r insertion.
-%/CHANGES.md: .FORCE \
-		$(call def-help,*/CHANGES.md, add commits to the specified changelog(s) since the tag/commit in the topmost heading )
-	$(eval PKGDIR=$(dir $@))
-	$(eval PKG=$(shell echo $(PKGDIR) | $(SED) -e "s/\///;"))
-	$(eval LAST=$(shell grep -E '^#' $@ | head -1 | cut -d' ' -f2 | $(SED) -e "s/(.*\..*)/$(PKG)-\1/;"))
-	$(eval HEAD=$(shell $(GITLOG) -1 --pretty=%h -- $(PKGDIR)))
-	@( [[ $(HEAD) == $(LAST) ]] \
-	   && echo "$@: up to date" \
-	   || ( \
-	        ( printf "\n# $(HEAD)\n\n"; \
-	          $(GITLOG) $(CHANGEFMT) $(LAST).. -- $(PKGDIR) | $(CHANGECLEANUP) \
-	        ) >$@.new ; \
-	        $(GSED) -i "2r $@.new" $@ ; \
-	        echo "$@: added $(LAST)..$(HEAD)" \
-	      ) \
-	 )
-
-CHANGES.md: .FORCE \
-		$(call def-help,CHANGES.md, add commits to the project-wide changelog since the tag/commit in the topmost heading )
-	$(eval LAST=$(shell grep -E '^#' $@ | head -1 | cut -d' ' -f2 | $(SED) -e "s/(.*\..*)/hledger-\1/;"))
-	$(eval HEAD=$(shell $(GITLOG) -1 --pretty=%h -- . $(EXCLUDEPKGDIRS)))
-	@( [[ $(HEAD) == $(LAST) ]] \
-		 && echo "$@: up to date" \
-		 || ( \
-					( printf "\n# $(HEAD)\n\n"; $(GITLOG) $(CHANGEFMT) $(LAST).. -- . $(EXCLUDEPKGDIRS) | $(CHANGECLEANUP) ) >$@.new ; \
-					$(SED) -i "2r $@.new" $@ ; \
-					echo "$@: added $(LAST)..$(HEAD)" \
-		    ) \
-	 )
-
-.FORCE:
-
-#LASTTAG=$(shell git describe --tags --abbrev=0)
-
 
 ###############################################################################
 $(call def-help-subheading,RELEASING:)

Shake.hs

@@ -39,11 +39,14 @@ multiple individually accessible wildcards
 not having to write :: Action ExitCode after a non-final cmd
 -}
 
-{-# LANGUAGE PackageImports, ScopedTypeVariables #-}
+{-# LANGUAGE MultiWayIf #-}
+{-# LANGUAGE PackageImports #-}
+{-# LANGUAGE ScopedTypeVariables #-}
 
 import                Prelude ()
 import "base-prelude" BasePrelude
--- keep imports synced with Makefile -> SHAKEDEPS
+import "base"         Control.Exception as C
+-- required packages, keep synced with Makefile -> SHAKEDEPS:
 import "directory"    System.Directory as S (getDirectoryContents)
 import "extra"        Data.List.Extra
 import "process"      System.Process
@@ -63,6 +66,7 @@ usage = unlines
   ,"./Shake build            # build all hledger packages, with awareness of embedded docs"
   ,"./Shake all              # generate everything"
   ,""
+  ,"./Shake changelogs       # update the changelogs with any new commits"
   ,"./Shake site/doc/VERSION/.snapshot   # save the checked-out web manuals as a versioned snapshot"
   ,"./Shake FILE             # build any individual file"
   ,"./Shake clean            # clean generated files"
@@ -124,6 +128,9 @@ main = do
         ,"hledger-api"
         ]
 
+      changelogs = "CHANGES.md" : map (</> "CHANGES.md") packages
+      changelogsdry = map (++"-dry") changelogs
+
       -- doc files (or related targets) that should be generated
       -- before building hledger packages.
       -- [(PKG, [TARGETS])]
@@ -375,6 +382,90 @@ main = do
         -- "m4 -P -DHELP -I" commandsdir lib src "|"
         pandoc fromsrcmd src "--lua-filter" "tools/pandoc-dedent-code-blocks.lua" "-t plain" ">" out
 
+    -- CHANGELOGS
+
+    let
+      -- git log showing short commit hashes
+      gitlog = "git log --abbrev-commit"
+
+      -- git log formats suitable for changelogs/release notes
+      -- %s=subject, %an=author name, %n=newline if needed, %w=width/indent1/indent2, %b=body, %h=hash
+      changelogGitFormat = "--pretty=format:'- %s (%an)%n%w(0,2,2)%b'"
+      -- changelogVerboseGitFormat = "--pretty=format:'- %s (%an)%n%w(0,2,2)%b%h' --stat"
+
+      -- Format a git log message, with one of the formats above, as a changelog item
+      changelogCleanupCmd = unwords [
+         "sed -E"
+        ,"-e 's/^( )*\\* /\1- /'"        --  ensure bullet lists in descriptions use hyphens not stars
+        ,"-e 's/ \\(Simon Michael\\)//'" --  strip maintainer's author name
+        ,"-e 's/^- (doc: *)?(updated? *)?changelogs?( *updates?)?$//'"  --  strip some variants of "updated changelog"
+        ,"-e 's/^ +\\[ci skip\\] *$//'"  --  strip [ci skip] lines
+        ,"-e 's/^ +$//'"                 --  replace lines containing only spaces with empty lines
+        -- ,"-e 's/\r//'"                   --  strip windows carriage returns (XXX \r untested. IDEA doesn't like a real ^M here)
+        ,"-e '/./,/^$/!d'"               --  replace consecutive newlines with one
+        ]
+
+      -- Things to exclude when doing git log for project-wide changelog.
+      -- git exclude pathspecs, https://git-scm.com/docs/gitglossary.html#gitglossary-aiddefpathspecapathspec
+      projectChangelogExcludeDirs = unwords [
+         ":!hledger-lib"
+        ,":!hledger"
+        ,":!hledger-ui"
+        ,":!hledger-web"
+        ,":!hledger-api"
+        ,":!tests"
+        ]
+
+    -- update all changelogs with latest commits
+    phony "changelogs" $ need changelogs
+
+    -- show the changelogs updates that would be written
+    -- phony "changelogs-dry" $ need changelogsdry
+
+    -- CHANGES.md */CHANGES.md CHANGES.md-dry */CHANGES.md-dry
+    -- Add commits to the specified changelog since the tag/commit in
+    -- the topmost heading, also removing that previous heading if it
+    -- was an interim heading (a commit hash). Or (the -dry variants)
+    -- just print the new changelog items to stdout without saving.
+    phonys (\out' -> if
+      | not $ out' `elem` (changelogs ++ changelogsdry) -> Nothing
+      | otherwise -> Just $ do
+        let (out, dryrun) | "-dry" `isSuffixOf` out' = (take (length out' - 4) out', True)
+                          | otherwise                = (out', False)
+        old <- liftIO $ lines <$> readFileStrictly out
+
+        let dir = takeDirectory out
+            pkg | dir=="."  = Nothing
+                | otherwise = Just dir
+            gitlogpaths = fromMaybe projectChangelogExcludeDirs pkg
+            isnotheading = not . ("#" `isPrefixOf`)
+            iscommithash s = length s > 6 && all isAlphaNum s
+            (preamble, oldheading:rest) = span isnotheading old
+            lastversion = words oldheading !! 1
+            lastrev | iscommithash lastversion = lastversion
+                    | otherwise                = fromMaybe "hledger" pkg ++ "-" ++ lastversion
+
+        headrev <- unwords . words . fromStdout <$>
+                   (cmd Shell gitlog "-1 --pretty=%h -- " gitlogpaths :: Action (Stdout String))
+
+        if headrev == lastrev
+        then liftIO $ putStrLn $ out ++ ": up to date"
+        else do
+          newitems <- fromStdout <$>
+                        (cmd Shell gitlog changelogGitFormat (lastrev++"..") "--" gitlogpaths
+                         "|" changelogCleanupCmd :: Action (Stdout String))
+          let newcontent = "# "++headrev++"\n\n" ++ newitems
+              newfile = unlines $ concat [
+                 preamble
+                ,[newcontent]
+                ,if iscommithash lastrev then [] else [oldheading]
+                ,rest
+                ]
+          liftIO $ if dryrun
+                   then putStr newcontent
+                   else writeFile out newfile
+        )
+
     -- MISC
 
     -- Generate the web manuals based on the current checkout and save
@@ -425,3 +516,6 @@ manualNameToManpageName s
 
 dropDirectory2 = dropDirectory1 . dropDirectory1
 
+readFileStrictly :: FilePath -> IO String
+readFileStrictly f = readFile f >>= \s -> C.evaluate (length s) >> return s
+