git.iko.soy/typeable/pandoc
on 1.3 help sign in
typeable/pandoc/html2markdown
221 lines 5.4 kB history raw
1#!/bin/sh -e
2# converts HTML from a URL, file, or stdin to markdown
3# uses an available program to fetch URL and tidy to normalize it first
5REQUIRED="tidy"
6SYNOPSIS="converts HTML from a URL, file, or STDIN to markdown-formatted text."
8THIS=${0##*/}
10NEWLINE='
13err () { echo "$*" | fold -s -w ${COLUMNS:-110} >&2; }
14errn () { printf "$*" | fold -s -w ${COLUMNS:-110} >&2; }
16usage () {
17 err "$1 - $2" # short description
18 err "See the $1(1) man page for usage."
21# Portable which(1).
22pathfind () {
23 oldifs="$IFS"; IFS=':'
24 for _p in $PATH; do
25 if [ -x "$_p/$*" ] && [ -f "$_p/$*" ]; then
26 IFS="$oldifs"
27 return 0
28 fi
29 done
30 IFS="$oldifs"
31 return 1
34for p in pandoc $REQUIRED; do
35 pathfind $p || {
36 err "You need '$p' to use this program!"
37 exit 1
38 }
39done
41CONF=$(pandoc --dump-args "$@" 2>&1) || {
42 errcode=$?
43 echo "$CONF" | sed -e '/^pandoc \[OPTIONS\] \[FILES\]/,$d' >&2
44 [ $errcode -eq 2 ] && usage "$THIS" "$SYNOPSIS"
45 exit $errcode
48OUTPUT=$(echo "$CONF" | sed -ne '1p')
49ARGS=$(echo "$CONF" | sed -e '1d')
52grab_url_with () {
53 url="${1:?internal error: grab_url_with: url required}"
55 shift
56 cmdline="$@"
58 prog=
59 prog_opts=
60 if [ -n "$cmdline" ]; then
61 eval "set -- $cmdline"
62 prog=$1
63 shift
64 prog_opts="$@"
65 fi
67 if [ -z "$prog" ]; then
68 # Locate a sensible web grabber (note the order).
69 for p in wget lynx w3m curl links w3c; do
70 if pathfind $p; then
71 prog=$p
72 break
73 fi
74 done
76 [ -n "$prog" ] || {
77 errn "$THIS: Couldn't find a program to fetch the file from URL "
78 err "(e.g. wget, w3m, lynx, w3c, or curl)."
79 return 1
80 }
81 else
82 pathfind "$prog" || {
83 err "$THIS: No such web grabber '$prog' found; aborting."
84 return 1
85 }
86 fi
88 # Setup proper base options for known grabbers.
89 base_opts=
90 case "$prog" in
91 wget) base_opts="-O-" ;;
92 lynx) base_opts="-source" ;;
93 w3m) base_opts="-dump_source" ;;
94 curl) base_opts="" ;;
95 links) base_opts="-source" ;;
96 w3c) base_opts="-n -get" ;;
97 *) err "$THIS: unhandled web grabber '$prog'; hope it succeeds."
98 esac
100 err "$THIS: invoking '$prog $base_opts $prog_opts $url'..."
101 eval "set -- $base_opts $prog_opts"
102 $prog "$@" "$url"
105# Parse command-line arguments
106parse_arguments () {
107 while [ $# -gt 0 ]; do
108 case "$1" in
109 --encoding=*)
110 wholeopt="$1"
111 # extract encoding from after =
112 encoding="${wholeopt#*=}" ;;
113 -e|--encoding|-encoding)
114 shift
115 encoding="$1" ;;
116 --grabber=*)
117 wholeopt="$1"
118 # extract encoding from after =
119 grabber="\"${wholeopt#*=}\"" ;;
120 -g|--grabber|-grabber)
121 shift
122 grabber="$1" ;;
123 *)
124 if [ -z "$argument" ]; then
125 argument="$1"
126 else
127 err "Warning: extra argument '$1' will be ignored."
128 fi ;;
129 esac
130 shift
131 done
134argument=
135encoding=
136grabber=
138oldifs="$IFS"
139IFS=$NEWLINE
140parse_arguments $ARGS
141IFS="$oldifs"
143inurl=
144if [ -n "$argument" ] && ! [ -f "$argument" ]; then
145 # Treat given argument as an URL.
146 inurl="$argument"
147fi
149# As a security measure refuse to proceed if mktemp is not available.
150pathfind mktemp || { err "Couldn't find 'mktemp'; aborting."; exit 1; }
152# Avoid issues with /tmp directory on Windows/Cygwin
153cygwin=
154cygwin=$(uname | sed -ne '/^CYGWIN/p')
155if [ -n "$cygwin" ]; then
156 TMPDIR=.
157 export TMPDIR
158fi
160THIS_TEMPDIR=
161THIS_TEMPDIR="$(mktemp -d -t $THIS.XXXXXXXX)" || exit 1
162readonly THIS_TEMPDIR
164trap 'exitcode=$?
165 [ -z "$THIS_TEMPDIR" ] || rm -rf "$THIS_TEMPDIR"
166 exit $exitcode' 0 1 2 3 13 15
168if [ -n "$inurl" ]; then
169 err "Attempting to fetch file from '$inurl'..."
171 grabber_out=$THIS_TEMPDIR/grabber.out
172 grabber_log=$THIS_TEMPDIR/grabber.log
173 if ! grab_url_with "$inurl" "$grabber" 1>$grabber_out 2>$grabber_log; then
174 errn "grab_url_with failed"
175 if [ -f $grabber_log ]; then
176 err " with the following error log."
177 err
178 cat >&2 $grabber_log
179 else
180 err .
181 fi
182 exit 1
183 fi
185 argument="$grabber_out"
186fi
188if [ -z "$encoding" ] && [ "x$argument" != "x" ]; then
189 # Try to determine character encoding if not specified
190 # and input is not STDIN.
191 encoding=$(
192 head "$argument" |
193 LC_ALL=C tr 'A-Z' 'a-z' |
194 sed -ne '/<meta .*content-type.*charset=/ {
195 s/.*charset=["'\'']*\([-a-zA-Z0-9]*\).*["'\'']*/\1/p
196 }'
197 )
198fi
200if [ -n "$encoding" ] && pathfind iconv; then
201 alias to_utf8='iconv -f "$encoding" -t utf-8'
202else # assume UTF-8
203 alias to_utf8='cat'
204fi
206htmlinput=$THIS_TEMPDIR/htmlinput
208if [ -z "$argument" ]; then
209 to_utf8 > $htmlinput # read from STDIN
210elif [ -f "$argument" ]; then
211 to_utf8 "$argument" > $htmlinput # read from file
212else
213 err "File '$argument' not found."
214 exit 1
215fi
217if ! cat $htmlinput | pandoc --ignore-args -r html -w markdown "$@" ; then
218 err "Failed to parse HTML. Trying again with tidy..."
219 tidy -q -asxhtml -utf8 $htmlinput | \
220 pandoc --ignore-args -r html -w markdown "$@"
221fi