git.iko.soy/typeable/pandoc
on 229b310 help sign in
typeable/pandoc/src/wrappers/html2markdown.in
157 lines 3.9 kB history raw
1#!/bin/sh -e
2# converts HTML from a URL, file, or stdin to markdown
3# uses an available program to fetch URL and tidy to normalize it first
5REQUIRED="tidy"
6SYNOPSIS="converts HTML from a URL, file, or STDIN to markdown-formatted text."
8### common.sh
10grab_url_with () {
11 url="${1:?internal error: grab_url_with: url required}"
13 shift
14 cmdline="$@"
16 prog=
17 prog_opts=
18 if [ -n "$cmdline" ]; then
19 eval "set -- $cmdline"
20 prog=$1
21 shift
22 prog_opts="$@"
23 fi
25 if [ -z "$prog" ]; then
26 # Locate a sensible web grabber (note the order).
27 for p in wget lynx w3m curl links w3c; do
28 if pathfind $p; then
29 prog=$p
30 break
31 fi
32 done
34 [ -n "$prog" ] || {
35 errn "$THIS: Couldn't find a program to fetch the file from URL "
36 err "(e.g. wget, w3m, lynx, w3c, or curl)."
37 return 1
38 }
39 else
40 pathfind "$prog" || {
41 err "$THIS: No such web grabber '$prog' found; aborting."
42 return 1
43 }
44 fi
46 # Setup proper base options for known grabbers.
47 base_opts=
48 case "$prog" in
49 wget) base_opts="-O-" ;;
50 lynx) base_opts="-source" ;;
51 w3m) base_opts="-dump_source" ;;
52 curl) base_opts="" ;;
53 links) base_opts="-source" ;;
54 w3c) base_opts="-n -get" ;;
55 *) err "$THIS: unhandled web grabber '$prog'; hope it succeeds."
56 esac
58 err "$THIS: invoking '$prog $base_opts $prog_opts $url'..."
59 eval "set -- $base_opts $prog_opts"
60 $prog "$@" "$url"
63# Parse command-line arguments
64parse_arguments () {
65 while [ $# -gt 0 ]; do
66 case "$1" in
67 --encoding=*)
68 wholeopt="$1"
69 # extract encoding from after =
70 encoding="${wholeopt#*=}" ;;
71 -e|--encoding|-encoding)
72 shift
73 encoding="$1" ;;
74 --grabber=*)
75 wholeopt="$1"
76 # extract encoding from after =
77 grabber="\"${wholeopt#*=}\"" ;;
78 -g|--grabber|-grabber)
79 shift
80 grabber="$1" ;;
81 *)
82 if [ -z "$argument" ]; then
83 argument="$1"
84 else
85 err "Warning: extra argument '$1' will be ignored."
86 fi ;;
87 esac
88 shift
89 done
92argument=
93encoding=
94grabber=
96oldifs="$IFS"
97IFS=$NEWLINE
98parse_arguments $ARGS
99IFS="$oldifs"
101inurl=
102if [ -n "$argument" ] && ! [ -f "$argument" ]; then
103 # Treat given argument as an URL.
104 inurl="$argument"
105fi
107if [ -n "$inurl" ]; then
108 err "Attempting to fetch file from '$inurl'..."
110 ### tempdir.sh
112 grabber_out=$THIS_TEMPDIR/grabber.out
113 grabber_log=$THIS_TEMPDIR/grabber.log
114 if ! grab_url_with "$inurl" "$grabber" 1>$grabber_out 2>$grabber_log; then
115 errn "grab_url_with failed"
116 if [ -f $grabber_log ]; then
117 err " with the following error log."
118 err
119 cat >&2 $grabber_log
120 else
121 err .
122 fi
123 exit 1
124 fi
126 argument="$grabber_out"
127fi
129if [ -z "$encoding" ] && [ "x$argument" != "x" ]; then
130 # Try to determine character encoding if not specified
131 # and input is not STDIN.
132 encoding=$(
133 head "$argument" |
134 LC_ALL=C tr 'A-Z' 'a-z' |
135 sed -ne '/<meta .*content-type.*charset=/ {
136 s/.*charset=["'\'']*\([-a-zA-Z0-9]*\).*["'\'']*/\1/p
137 }'
138 )
139fi
141if [ -n "$encoding" ] && pathfind iconv; then
142 alias to_utf8='iconv -f "$encoding" -t utf-8'
143else # assume UTF-8
144 alias to_utf8='cat'
145fi
147if [ -z "$argument" ]; then
148 tidy -asxhtml -utf8 2>/dev/null | pandoc --ignore-args -r html -w markdown "$@"
149else
150 if [ -f "$argument" ]; then
151 to_utf8 "$argument" |
152 tidy -asxhtml -utf8 2>/dev/null | pandoc --ignore-args -r html -w markdown "$@"
153 else
154 err "File '$argument' not found."
155 exit 1
156 fi
157fi