From e1c9e51775e53f28f67eac01ea87d858ea7e7d25 Mon Sep 17 00:00:00 2001 From: Simon Michael Date: Sun, 21 Feb 2016 02:21:41 -0800 Subject: [PATCH] tools: new build system for man/web docs Goal: Generate man pages and web docs from one source. Current plan: The master docs for each package are now the pandoc-style manpage-markdown files in the package directories - hledger/hledger.1.md, hledger-lib/hledger_journal.5.md, etc. Parts of these will be marked as web-only, and parts as man-only, using divs recognisable by custom pandoc filters. When generating man pages we strip the web-only parts, and all html blocks, inline html and hyperlinks. When generating web docs we strip the man-only parts and apply any other tweaks needed for easy presentation, perhaps combining them into a single web page similar to the old user manual. Shake: This was hard to do with GNU Make, and so I've introduced Shake, which is working very well. Both coexist for now but it's probably time to switch. --- Makefile | 56 ++++++++++--- doc/manpage.html | 38 +++++++++ doc/{manpage.template => manpage.nroff} | 0 shake.hs | 102 ++++++++++++++++++++++++ site/.gitignore | 8 ++ site/manual2.md | 39 +++++++++ tools/.gitignore | 8 ++ tools/pandocRemoveHtmlBlocks.hs | 11 +++ tools/pandocRemoveHtmlInlines.hs | 12 +++ tools/pandocRemoveManonlyBlocks.hs | 12 +++ tools/pandocRemoveWebonlyBlocks.hs | 12 +++ 11 files changed, 287 insertions(+), 11 deletions(-) create mode 100644 doc/manpage.html rename doc/{manpage.template => manpage.nroff} (100%) create mode 100755 shake.hs create mode 100644 site/manual2.md create mode 100644 tools/.gitignore create mode 100755 tools/pandocRemoveHtmlBlocks.hs create mode 100755 tools/pandocRemoveHtmlInlines.hs create mode 100755 tools/pandocRemoveManonlyBlocks.hs create mode 100755 tools/pandocRemoveWebonlyBlocks.hs diff --git a/Makefile b/Makefile index 6f44c7c98..92bc739e2 100644 --- a/Makefile +++ b/Makefile @@ -1041,6 +1041,9 @@ haddock: \ # in subsequent rules, allow automatic variables to be used in prerequisites (use $$) .SECONDEXPANSION: +######################## +# man pages + MANPAGES=\ hledger-lib/hledger_csv.5 \ hledger-lib/hledger_journal.5 \ @@ -1053,6 +1056,18 @@ MANPAGES=\ manpages: $(MANPAGES) \ $(call def-help,manpages, generate man pages ) +# %.1 %.5: $$@.md doc/manpage.template +# pandoc $< -t man -s --template doc/manpage.template -o $@ \ +# --filter tools/pandocCapitalizeHeaders.hs \ +# --filter tools/pandocRemoveNotes.hs \ +# --filter tools/pandocRemoveLinks.hs \ +# --filter tools/pandocRemoveHtmlBlocks.hs \ +# --filter tools/pandocRemoveHtmlInlines.hs \ + +# much faster when compiled +# but note, filters need to be compiled with a version of +# pandoc-types compatible with the pandoc you run them with +# (filters compiled for pandoc 1.15 broke with pandoc 1.16) %.1 %.5: $$@.md doc/manpage.template pandoc $< -t man -s --template doc/manpage.template -o $@ \ --filter tools/pandocCapitalizeHeaders \ @@ -1060,26 +1075,45 @@ manpages: $(MANPAGES) \ --filter tools/pandocRemoveLinks \ --filter tools/pandocRemoveHtmlBlocks \ --filter tools/pandocRemoveHtmlInlines \ -# faster when compiled -# --filter tools/pandocCapitalizeHeaders.hs \ -# --filter tools/pandocRemoveNotes.hs \ -# --filter tools/pandocRemoveLinks.hs \ -# --filter tools/pandocRemoveHtmlBlocks.hs \ -# --filter tools/pandocRemoveHtmlInlines.hs \ clean-manpages: rm -f $(MANPAGES) -site/manual2.md: site/manual-start.md site/manual-end.md $(MANPAGES) \ +######################## +# web manual 2, man page synced + +# how to generate the web-based user manual and man pages from one source ? + +# plan 1: +# +# core docs are maintained as multiple pandoc-style man-markdown files +# eg hledger/hledger.1.md, hledger-ui/hledger-ui.1.md, hledger-lib/hledger_journal.5.md etc. +# +# to generate a man page, apply filters: +# strip html blocks and inline html +# +# to generate web manual content, +# extract marked content (inside
) +# and apply filters: +# demote headings 4 steps + +site/manual2-1.md: site/manual-start.md site/manual-end.md $(MANPAGES) \ $(call def-help,site/manual2.md, generate combined user manual ) - cat site/manual-start.md >site/manual2.md && \ + cat site/manual-start.md >$@ && \ pandoc \ --filter tools/pandocRemoveManpageBlocks \ - hledger-ui/hledger-ui.1.md -w markdown >>site/manual2.md && \ - cat site/manual-end.md >>site/manual2.md - + hledger-ui/hledger-ui.1.md -w markdown >>$@ && \ + cat site/manual-end.md >>$@ #--template doc/userguide.template \ +# plan 2: +# +# split web manual into pages corresponding to man pages, at least initially + +# %.1.html %.5: $$@.md doc/webmanual.template +# echo pandoc $< -t man -s --template doc/webmanual.template -o $@ +# too hard, see Shake.hs + ############################################################################### $(call def-help-subsection,RELEASING:) #$(call def-help-subsection,see also developer guide -> how to -> do a release) diff --git a/doc/manpage.html b/doc/manpage.html new file mode 100644 index 000000000..722055536 --- /dev/null +++ b/doc/manpage.html @@ -0,0 +1,38 @@ + + + + +$body$ + + diff --git a/doc/manpage.template b/doc/manpage.nroff similarity index 100% rename from doc/manpage.template rename to doc/manpage.nroff diff --git a/shake.hs b/shake.hs new file mode 100755 index 000000000..6726bb1cf --- /dev/null +++ b/shake.hs @@ -0,0 +1,102 @@ +#!/usr/bin/env stack +-- stack runghc --package shake + +import Development.Shake +import Development.Shake.FilePath +import Data.List +import System.Directory as S (getDirectoryContents) + +manpages :: [String] +manpages = [ + "hledger_csv.5" + ,"hledger_journal.5" + ,"hledger_timedot.5" + ,"hledger_timelog.5" + ,"hledger.1" + ,"hledger-api.1" + ,"hledger-ui.1" + ,"hledger-web.1" + ] + +manpageDir :: String -> FilePath +manpageDir p + | '_' `elem` p = "hledger-lib" + | otherwise = dropExtension p + +buildDir :: FilePath +buildDir = ".build" + +main :: IO () +main = do + + pandocFilters <- + map ("tools" ). nub . sort . map (-<.> "") . filter ("pandoc" `isPrefixOf`) + <$> S.getDirectoryContents "tools" + + -- man pages, still markdown but with man-only sections removed. + -- (We let hakyll do the html rendering since it's good + -- at applying the site style, table of contents etc.) + let manpageFilteredMds = ["site" p <.>".md" | p <- manpages] + + -- man pages, converted to man nroff with web-only sections removed + let manpageNroffs = [manpageDir p p | p <- manpages] + + shakeArgs shakeOptions{shakeFiles=buildDir} $ do + + want $ manpageNroffs ++ manpageFilteredMds + + manpageNroffs |%> \out -> do + let + md = out <.> "md" + tmpl = "doc/manpage.nroff" + need $ md : tmpl : pandocFilters + cmd "pandoc" md "--to man -s --template" tmpl + -- XXX assume these are compiled + "--filter tools/pandocRemoveHtmlBlocks" + "--filter tools/pandocRemoveHtmlInlines" + "--filter tools/pandocRemoveLinks" + "--filter tools/pandocRemoveNotes" + "--filter tools/pandocCapitalizeHeaders" + "-o" out + + manpageFilteredMds |%> \out -> do + let + p = dropExtension $ takeFileName out + md = manpageDir p p <.> "md" + tmpl = "doc/manpage.html" + need $ md : tmpl : pandocFilters + cmd "pandoc" md "--to markdown" + -- XXX assume this is compiled + "--filter tools/pandocRemoveManonlyBlocks" + "-o" out + + pandocFilters |%> \out -> do + need [out <.> "hs"] + cmd "stack ghc" out + + phony "clean" $ do + putNormal "Cleaning generated files" + removeFilesAfter "" manpageNroffs + removeFilesAfter "" manpageFilteredMds + putNormal "Cleaning object files" + removeFilesAfter "tools" ["*.o","*.p_o","*.hi"] + putNormal "Cleaning shake build files" + removeFilesAfter buildDir ["//*"] + + -- manpageHtmls |%> \out -> do + -- let + -- p = dropExtension $ takeFileName out + -- md = manpageDir p p <.> "md" + -- tmpl = "doc/manpage.html" + -- need [md, tmpl] + -- cmd "pandoc" md "--to html --filter tools/pandocRemoveManpageBlocks.hs --template" tmpl "-o" out + + -- "site/manual2.html" %> \out -> do + -- need ["site/manual2.md"] + -- cmd "pandoc site/manual2.md -o" out + + -- "_build//*.o" %> \out -> do + -- let c = dropDirectory1 $ out -<.> "c" + -- let m = out -<.> "m" + -- () <- cmd "gcc -c" [c] "-o" [out] "-MMD -MF" [m] + -- needMakefileDependencies m diff --git a/site/.gitignore b/site/.gitignore index caa3b5d25..8f8b8ecf4 100644 --- a/site/.gitignore +++ b/site/.gitignore @@ -2,3 +2,11 @@ _cache _site files/downloads files/static +hledger-api.1.md +hledger-ui.1.md +hledger-web.1.md +hledger.1.md +hledger_csv.5.md +hledger_journal.5.md +hledger_timedot.5.md +hledger_timelog.5.md diff --git a/site/manual2.md b/site/manual2.md new file mode 100644 index 000000000..568adf492 --- /dev/null +++ b/site/manual2.md @@ -0,0 +1,39 @@ +* toc + +# hledger man pages + +These are the main content of hledger's unix manual pages, cleaned +and converted back to web format. +The current plan is to reassemble them into something like the +original [user manual](manual.html), but we're not there yet. + +
+
+
+ +## Tools + +### [hledger(1)](hledger.1.html) + +### [hledger-api(1)](hledger-api.1.html) + +### [hledger-ui(1)](hledger-ui.1.html) + +### [hledger-web(1)](hledger-web.1.html) + +
+
+ +## File formats + +### [hledger_csv(5)](hledger_csv.5.html) + +### [hledger_journal(5)](hledger_journal.5.html) + +### [hledger_timelog(5)](hledger_timelog.5.html) + +### [hledger_timedot(5)](hledger_timedot.5.html) + +
+
+
diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 000000000..a6540ad27 --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1,8 @@ +generatetimelog +pandocCapitalizeHeaders +pandocRemoveHtmlBlocks +pandocRemoveHtmlInlines +pandocRemoveLinks +pandocRemoveManonlyBlocks +pandocRemoveNotes +pandocRemoveWebonlyBlocks diff --git a/tools/pandocRemoveHtmlBlocks.hs b/tools/pandocRemoveHtmlBlocks.hs new file mode 100755 index 000000000..acc99052a --- /dev/null +++ b/tools/pandocRemoveHtmlBlocks.hs @@ -0,0 +1,11 @@ +#!/usr/bin/env stack +-- stack runghc --package pandoc-types + +import Text.Pandoc.JSON + +main :: IO () +main = toJSONFilter removeHtmlBlocks + +removeHtmlBlocks :: Block -> Block +removeHtmlBlocks (RawBlock (Format "html") _) = Plain [] +removeHtmlBlocks x = x diff --git a/tools/pandocRemoveHtmlInlines.hs b/tools/pandocRemoveHtmlInlines.hs new file mode 100755 index 000000000..63863bdea --- /dev/null +++ b/tools/pandocRemoveHtmlInlines.hs @@ -0,0 +1,12 @@ +#!/usr/bin/env stack +-- stack runghc --package pandoc-types + +import Text.Pandoc.JSON + +main :: IO () +main = toJSONFilter removeHtmlInlines + +removeHtmlInlines :: Inline -> Inline +removeHtmlInlines (RawInline (Format "html") _) = Str "" +removeHtmlInlines x = x + diff --git a/tools/pandocRemoveManonlyBlocks.hs b/tools/pandocRemoveManonlyBlocks.hs new file mode 100755 index 000000000..4f762860c --- /dev/null +++ b/tools/pandocRemoveManonlyBlocks.hs @@ -0,0 +1,12 @@ +#!/usr/bin/env stack +-- stack runghc --package pandoc-types + +import Text.Pandoc.Builder +import Text.Pandoc.JSON + +main :: IO () +main = toJSONFilter removeManonlyBlocks + +removeManonlyBlocks :: Block -> Block +removeManonlyBlocks (Div ("",["manonly"],[]) _) = Plain [] +removeManonlyBlocks x = x diff --git a/tools/pandocRemoveWebonlyBlocks.hs b/tools/pandocRemoveWebonlyBlocks.hs new file mode 100755 index 000000000..0326bd2b3 --- /dev/null +++ b/tools/pandocRemoveWebonlyBlocks.hs @@ -0,0 +1,12 @@ +#!/usr/bin/env stack +-- stack runghc --package pandoc-types + +import Text.Pandoc.Builder +import Text.Pandoc.JSON + +main :: IO () +main = toJSONFilter removeWebonlyBlocks + +removeWebonlyBlocks :: Block -> Block +removeWebonlyBlocks (Div ("",["webonly"],[]) _) = Plain [] +removeWebonlyBlocks x = x