tools: new build system for man/web docs

Goal:

Generate man pages and web docs from one source.

Current plan:

The master docs for each package are now the pandoc-style
manpage-markdown files in the package directories -
hledger/hledger.1.md, hledger-lib/hledger_journal.5.md, etc.

Parts of these will be marked as web-only, and parts as man-only, using
divs recognisable by custom pandoc filters.

When generating man pages we strip the web-only parts, and all html
blocks, inline html and hyperlinks.

When generating web docs we strip the man-only parts and apply any other
tweaks needed for easy presentation, perhaps combining them into a
single web page similar to the old user manual.

Shake:

This was hard to do with GNU Make, and so I've introduced Shake, which
is working very well. Both coexist for now but it's probably time to
switch.

Makefile

@@ -1041,6 +1041,9 @@ haddock: \
 # in subsequent rules, allow automatic variables to be used in prerequisites (use $$)
 .SECONDEXPANSION:
 
+########################
+# man pages
+
 MANPAGES=\
 	hledger-lib/hledger_csv.5 \
 	hledger-lib/hledger_journal.5 \
@@ -1053,6 +1056,18 @@ MANPAGES=\
 manpages: $(MANPAGES) \
 		$(call def-help,manpages, generate man pages )
 
+# %.1 %.5: $$@.md doc/manpage.template
+# 	pandoc $< -t man -s --template doc/manpage.template -o $@ \
+# 		--filter tools/pandocCapitalizeHeaders.hs \
+# 		--filter tools/pandocRemoveNotes.hs \
+# 		--filter tools/pandocRemoveLinks.hs \
+# 		--filter tools/pandocRemoveHtmlBlocks.hs \
+# 		--filter tools/pandocRemoveHtmlInlines.hs \
+
+# much faster when compiled
+# but note, filters need to be compiled with a version of
+# pandoc-types compatible with the pandoc you run them with
+# (filters compiled for pandoc 1.15 broke with pandoc 1.16)
 %.1 %.5: $$@.md doc/manpage.template
 	pandoc $< -t man -s --template doc/manpage.template -o $@ \
 		--filter tools/pandocCapitalizeHeaders \
@@ -1060,26 +1075,45 @@ manpages: $(MANPAGES) \
 		--filter tools/pandocRemoveLinks \
 		--filter tools/pandocRemoveHtmlBlocks \
 		--filter tools/pandocRemoveHtmlInlines \
-# faster when compiled
-#		--filter tools/pandocCapitalizeHeaders.hs \
-#		--filter tools/pandocRemoveNotes.hs \
-#		--filter tools/pandocRemoveLinks.hs \
-#		--filter tools/pandocRemoveHtmlBlocks.hs \
-#		--filter tools/pandocRemoveHtmlInlines.hs \
 
 clean-manpages:
 	rm -f $(MANPAGES)
 
-site/manual2.md: site/manual-start.md site/manual-end.md $(MANPAGES) \
+########################
+# web manual 2, man page synced
+
+# how to generate the web-based user manual and man pages from one source ?
+
+# plan 1:
+#
+# core docs are maintained as multiple pandoc-style man-markdown files
+#  eg hledger/hledger.1.md, hledger-ui/hledger-ui.1.md, hledger-lib/hledger_journal.5.md etc.
+#
+# to generate a man page, apply filters:
+#  strip html blocks and inline html
+#
+# to generate web manual content,
+#  extract marked content (inside <div class="userguide">)
+#  and apply filters:
+#   demote headings 4 steps
+
+site/manual2-1.md: site/manual-start.md site/manual-end.md $(MANPAGES) \
 		$(call def-help,site/manual2.md, generate combined user manual )
-	cat site/manual-start.md >site/manual2.md && \
+	cat site/manual-start.md >$@ && \
 	pandoc \
 		--filter tools/pandocRemoveManpageBlocks \
-		hledger-ui/hledger-ui.1.md -w markdown >>site/manual2.md && \
-	cat site/manual-end.md >>site/manual2.md
-
+		hledger-ui/hledger-ui.1.md -w markdown >>$@ && \
+	cat site/manual-end.md >>$@
 #--template doc/userguide.template \
 
+# plan 2:
+#
+# split web manual into pages corresponding to man pages, at least initially
+
+# %.1.html %.5: $$@.md doc/webmanual.template
+# 	echo pandoc $< -t man -s --template doc/webmanual.template -o $@
+# too hard, see Shake.hs
+
 ###############################################################################
 $(call def-help-subsection,RELEASING:)
 #$(call def-help-subsection,see also developer guide -> how to -> do a release)

doc/manpage.html

@@ -0,0 +1,38 @@
+<!--
+"$title$" "$section$" "$date$" "$footer$" "hledger User Manuals"
+
+$for(header-includes)$
+$header-includes$
+$endfor$
+
+$for(include-before)$
+$include-before$
+$endfor$
+-->
+
+<!-- * toc -->
+
+$body$
+
+<!--
+$for(include-after)$
+$include-after$
+$endfor$
+
+$if(author)$
+$for(author)$$author$$sep$; $endfor$.
+$else$
+Simon Michael <simon@joyful.com>
+$endif$
+
+.SH COPYRIGHT
+
+Copyright (C) 2007-2015 Simon Michael.
+.br
+Released under GNU GPLv3+.
+
+.SH SEE ALSO
+hledger(1), hledger\-ui(1), hledger\-web(1), ledger(1)
+.br
+hledger_csv(5), hledger_journal(5), hledger_timelog(5)
+-->

shake.hs

@@ -0,0 +1,102 @@
+#!/usr/bin/env stack
+-- stack runghc --package shake
+
+import Development.Shake
+import Development.Shake.FilePath
+import Data.List
+import System.Directory as S (getDirectoryContents)
+
+manpages :: [String]
+manpages = [
+   "hledger_csv.5"
+  ,"hledger_journal.5"
+  ,"hledger_timedot.5"
+  ,"hledger_timelog.5"
+  ,"hledger.1"
+  ,"hledger-api.1"
+  ,"hledger-ui.1"
+  ,"hledger-web.1"
+  ]
+
+manpageDir :: String -> FilePath
+manpageDir p
+  | '_' `elem` p = "hledger-lib"
+  | otherwise    = dropExtension p
+
+buildDir :: FilePath
+buildDir = ".build"
+
+main :: IO ()
+main = do
+
+  pandocFilters <-
+    map ("tools" </>). nub . sort . map (-<.> "") . filter ("pandoc" `isPrefixOf`)
+    <$> S.getDirectoryContents "tools"
+
+  -- man pages, still markdown but with man-only sections removed.
+  -- (We let hakyll do the html rendering since it's good
+  -- at applying the site style, table of contents etc.)
+  let manpageFilteredMds = ["site" </> p <.>".md" | p <- manpages]
+
+  -- man pages, converted to man nroff with web-only sections removed
+  let manpageNroffs = [manpageDir p </> p | p <- manpages]
+
+  shakeArgs shakeOptions{shakeFiles=buildDir} $ do
+
+    want $ manpageNroffs ++ manpageFilteredMds
+
+    manpageNroffs |%> \out -> do
+      let
+        md = out <.> "md"
+        tmpl = "doc/manpage.nroff"
+      need $ md : tmpl : pandocFilters
+      cmd "pandoc" md "--to man -s --template" tmpl
+        -- XXX assume these are compiled
+        "--filter tools/pandocRemoveHtmlBlocks"
+        "--filter tools/pandocRemoveHtmlInlines"
+        "--filter tools/pandocRemoveLinks"
+        "--filter tools/pandocRemoveNotes"
+        "--filter tools/pandocCapitalizeHeaders"
+        "-o" out
+
+    manpageFilteredMds |%> \out -> do
+      let
+        p = dropExtension $ takeFileName out
+        md = manpageDir p </> p <.> "md"
+        tmpl = "doc/manpage.html"
+      need $ md : tmpl : pandocFilters
+      cmd "pandoc" md "--to markdown"
+        -- XXX assume this is compiled
+        "--filter tools/pandocRemoveManonlyBlocks"
+        "-o" out
+
+    pandocFilters |%> \out -> do
+      need [out <.> "hs"]
+      cmd "stack ghc" out
+
+    phony "clean" $ do
+      putNormal "Cleaning generated files"
+      removeFilesAfter "" manpageNroffs
+      removeFilesAfter "" manpageFilteredMds
+      putNormal "Cleaning object files"
+      removeFilesAfter "tools" ["*.o","*.p_o","*.hi"]
+      putNormal "Cleaning shake build files"
+      removeFilesAfter buildDir ["//*"]
+
+    -- manpageHtmls |%> \out -> do
+    --   let
+    --     p = dropExtension $ takeFileName out
+    --     md = manpageDir p </> p <.> "md"
+    --     tmpl = "doc/manpage.html"
+    --   need [md, tmpl]
+    --   cmd "pandoc" md "--to html --filter tools/pandocRemoveManpageBlocks.hs --template" tmpl "-o" out
+
+    -- "site/manual2.html" %> \out -> do
+    --   need ["site/manual2.md"]
+    --   cmd "pandoc site/manual2.md -o" out
+
+    -- "_build//*.o" %> \out -> do
+    --     let c = dropDirectory1 $ out -<.> "c"
+    --     let m = out -<.> "m"
+    --     () <- cmd "gcc -c" [c] "-o" [out] "-MMD -MF" [m]
+    --     needMakefileDependencies m

site/.gitignore

@@ -2,3 +2,11 @@ _cache
 _site
 files/downloads
 files/static
+hledger-api.1.md
+hledger-ui.1.md
+hledger-web.1.md
+hledger.1.md
+hledger_csv.5.md
+hledger_journal.5.md
+hledger_timedot.5.md
+hledger_timelog.5.md

site/manual2.md

@@ -0,0 +1,39 @@
+* toc
+
+# hledger man pages
+
+These are the main content of hledger's unix manual pages, cleaned
+and converted back to web format.
+The current plan is to reassemble them into something like the
+original [user manual](manual.html), but we're not there yet.
+
+<div class="container">
+<div class="row">
+<div class="col-md-4">
+
+## Tools
+
+### [hledger(1)](hledger.1.html)
+
+### [hledger-api(1)](hledger-api.1.html)
+
+### [hledger-ui(1)](hledger-ui.1.html)
+
+### [hledger-web(1)](hledger-web.1.html)
+
+</div>
+<div class="col-md-4">
+
+## File formats
+
+### [hledger_csv(5)](hledger_csv.5.html)
+
+### [hledger_journal(5)](hledger_journal.5.html)
+
+### [hledger_timelog(5)](hledger_timelog.5.html)
+
+### [hledger_timedot(5)](hledger_timedot.5.html)
+
+</div>
+</div>
+</div>

tools/.gitignore

@@ -0,0 +1,8 @@
+generatetimelog
+pandocCapitalizeHeaders
+pandocRemoveHtmlBlocks
+pandocRemoveHtmlInlines
+pandocRemoveLinks
+pandocRemoveManonlyBlocks
+pandocRemoveNotes
+pandocRemoveWebonlyBlocks

tools/pandocRemoveHtmlBlocks.hs

@@ -0,0 +1,11 @@
+#!/usr/bin/env stack
+-- stack runghc --package pandoc-types
+
+import Text.Pandoc.JSON
+
+main :: IO ()
+main = toJSONFilter removeHtmlBlocks
+
+removeHtmlBlocks :: Block -> Block
+removeHtmlBlocks (RawBlock (Format "html") _) = Plain []
+removeHtmlBlocks x = x

tools/pandocRemoveHtmlInlines.hs

@@ -0,0 +1,12 @@
+#!/usr/bin/env stack
+-- stack runghc --package pandoc-types
+
+import Text.Pandoc.JSON
+
+main :: IO ()
+main = toJSONFilter removeHtmlInlines
+
+removeHtmlInlines :: Inline -> Inline
+removeHtmlInlines (RawInline (Format "html") _) = Str ""
+removeHtmlInlines x = x
+

tools/pandocRemoveManonlyBlocks.hs

@@ -0,0 +1,12 @@
+#!/usr/bin/env stack
+-- stack runghc --package pandoc-types
+
+import Text.Pandoc.Builder
+import Text.Pandoc.JSON
+
+main :: IO ()
+main = toJSONFilter removeManonlyBlocks
+
+removeManonlyBlocks :: Block -> Block
+removeManonlyBlocks (Div ("",["manonly"],[]) _) = Plain []
+removeManonlyBlocks x = x

tools/pandocRemoveWebonlyBlocks.hs

@@ -0,0 +1,12 @@
+#!/usr/bin/env stack
+-- stack runghc --package pandoc-types
+
+import Text.Pandoc.Builder
+import Text.Pandoc.JSON
+
+main :: IO ()
+main = toJSONFilter removeWebonlyBlocks
+
+removeWebonlyBlocks :: Block -> Block
+removeWebonlyBlocks (Div ("",["webonly"],[]) _) = Plain []
+removeWebonlyBlocks x = x