aecbf81 · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

Remove Text.Pandoc.BCP47 module.

aecbf81John MacFarlane6 years ago

pandoc.cabal−1

compare with master

unifiedsplit

@@ -601,7 +601,6 @@ library

601 Text.Pandoc.Asciify,

602 Text.Pandoc.Emoji,

603 Text.Pandoc.ImageSize,

604− Text.Pandoc.BCP47,

605 Text.Pandoc.Class,

604 Text.Pandoc.Class,

606 Text.Pandoc.Citeproc

605 Text.Pandoc.Citeproc

607 other-modules: Text.Pandoc.App.CommandLineOptions,

606 other-modules: Text.Pandoc.App.CommandLineOptions,

src/Text/Pandoc/App.hs+3−3

@@ -55,7 +55,7 @@ import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), defaultOpts,

55import Text.Pandoc.App.CommandLineOptions (parseOptions, parseOptionsFromArgs,

56 options)

57import Text.Pandoc.App.OutputSettings (OutputSettings (..), optToOutputSettings)

58−import Text.Pandoc.BCP47 (Lang (..), parseBCP47)

58+import UnicodeCollation.Lang (Lang (..), parseLang)

59import Text.Pandoc.Filter (Filter (JSONFilter, LuaFilter), applyFilters)

60import Text.Pandoc.PDF (makePDF)

61import Text.Pandoc.SelfContained (makeSelfContained)

@@ -200,8 +200,8 @@ convertWithOpts opts = do

200 Just f -> readFileStrict f

201

202 case lookupMetaString "lang" (optMetadata opts) of

203− "" -> setTranslations $ Lang "en" "" "US" []

203+ "" -> setTranslations $ Lang "en" Nothing (Just "US") [] [] []

204− l -> case parseBCP47 l of

204+ l -> case parseLang l of

205 Left _ -> report $ InvalidLang l

206 Right l' -> setTranslations l'

207

src/Text/Pandoc/BCP47.hsdeleted−99

@@ -1,99 +0,0 @@

1−{-# LANGUAGE OverloadedStrings #-}

2−{- |

3− Module : Text.Pandoc.BCP47

5− License : GNU GPL, version 2 or above

6−

7− Maintainer : John MacFarlane <jgm@berkeley.edu>

8− Stability : alpha

9− Portability : portable

10−

11−Functions for parsing and rendering BCP47 language identifiers.

12−-}

13−module Text.Pandoc.BCP47 (

14− getLang

15− , parseBCP47

16− , Lang(..)

17− , renderLang

18− )

19−where

20−import Control.Monad (guard)

21−import Data.Char (isAlphaNum, isAscii, isLetter, isLower, isUpper)

22−import Text.Pandoc.Definition

23−import Text.Pandoc.Options

24−import Text.DocTemplates (FromContext(..))

25−import qualified Data.Text as T

26−import qualified Text.Parsec as P

27−

28−-- | Represents BCP 47 language/country code.

29−data Lang = Lang{ langLanguage :: T.Text

30− , langScript :: T.Text

31− , langRegion :: T.Text

32− , langVariants :: [T.Text] }

33− deriving (Eq, Ord, Show)

34−

35−-- | Render a Lang as BCP 47.

36−renderLang :: Lang -> T.Text

37−renderLang lang = T.intercalate "-" (langLanguage lang : filter (not . T.null)

38− ([langScript lang, langRegion lang] ++ langVariants lang))

39−

40−-- | Parse a BCP 47 string as a Lang. Currently we parse

41−-- extensions and private-use fields as "variants," even

42−-- though officially they aren't.

43−parseBCP47 :: T.Text -> Either T.Text Lang

44−parseBCP47 lang =

45− case P.parse bcp47 "lang" lang of

46− Right r -> Right r

47− Left e -> Left $ T.pack $ show e

48− where bcp47 = do

49− language <- pLanguage

50− script <- P.option "" pScript

51− region <- P.option "" pRegion

52− variants <- P.many (pVariant P.<|> pExtension P.<|> pPrivateUse)

53− P.eof

54− return Lang{ langLanguage = language

55− , langScript = script

56− , langRegion = region

57− , langVariants = variants }

58− asciiLetter = P.satisfy (\c -> isAscii c && isLetter c)

59− pLanguage = do

60− cs <- P.many1 asciiLetter

61− let lcs = length cs

62− guard $ lcs == 2 || lcs == 3

63− return $ T.toLower $ T.pack cs

64− pScript = P.try $ do

65− P.char '-'

66− x <- P.satisfy (\c -> isAscii c && isLetter c && isUpper c)

67− xs <- P.count 3

68− (P.satisfy (\c -> isAscii c && isLetter c && isLower c))

69− return $ T.toLower $ T.pack (x:xs)

70− pRegion = P.try $ do

71− P.char '-'

72− cs <- P.many1 asciiLetter

73− let lcs = length cs

74− guard $ lcs == 2 || lcs == 3

75− return $ T.toUpper $ T.pack cs

76− pVariant = P.try $ do

77− P.char '-'

78− ds <- P.option "" (P.count 1 P.digit)

79− cs <- P.many1 asciiLetter

80− let var = ds ++ cs

81− lv = length var

82− guard $ if null ds

83− then lv >= 5 && lv <= 8

84− else lv == 4

85− return $ T.toLower $ T.pack var

86− pExtension = P.try $ do

87− P.char '-'

88− cs <- P.many1 $ P.satisfy (\c -> isAscii c && isAlphaNum c)

89− let lcs = length cs

90− guard $ lcs >= 2 && lcs <= 8

91− return $ T.toLower $ T.pack cs

92− pPrivateUse = P.try $ do

93− P.char '-'

94− P.char 'x'

95− P.char '-'

96− cs <- P.many1 $ P.satisfy (\c -> isAscii c && isAlphaNum c)

97− guard $ not (null cs) && length cs <= 8

98− let var = "x-" ++ cs

99− return $ T.toLower $ T.pack var

src/Text/Pandoc/Citeproc.hs+2−8

@@ -18,7 +18,6 @@ import Text.Pandoc.Citeproc.CslJson (cslJsonToReferences)

18import Text.Pandoc.Citeproc.BibTeX (readBibtexString, Variant(..))

19import Text.Pandoc.Citeproc.MetaValue (metaValueToReference, metaValueToText)

20import Text.Pandoc.Readers.Markdown (yamlToRefs)

21−import qualified Text.Pandoc.BCP47 as BCP47

22import Text.Pandoc.Builder (Inlines, Many(..), deleteMeta, setMeta)

21import Text.Pandoc.Builder (Inlines, Many(..), deleteMeta, setMeta)

23import qualified Text.Pandoc.Builder as B

22import qualified Text.Pandoc.Builder as B

24import Text.Pandoc.Definition as Pandoc

23import Text.Pandoc.Definition as Pandoc

@@ -630,13 +629,8 @@ removeFinalPeriod ils =

630

629

631bcp47LangToIETF :: PandocMonad m => Text -> m (Maybe Lang)

630bcp47LangToIETF :: PandocMonad m => Text -> m (Maybe Lang)

632bcp47LangToIETF bcplang =

631bcp47LangToIETF bcplang =

633− case BCP47.parseBCP47 bcplang of

632+ case parseLang bcplang of

634 Left _ -> do

633 Left _ -> do

635 report $ InvalidLang bcplang

634 report $ InvalidLang bcplang

636 return Nothing

635 return Nothing

637− Right lang ->

638− return $ Just

636+ Right lang -> return $ Just lang

639− $ Lang (BCP47.langLanguage lang)

640− (if T.null (BCP47.langRegion lang)

641− then Nothing

642− else Just (BCP47.langRegion lang))

src/Text/Pandoc/Citeproc/BibTeX.hs+11−9

@@ -205,10 +205,13 @@ writeBibtexString opts variant mblang ref =

205 [ (", " <>) <$> nameGiven name,

206 nameDroppingParticle name ]

207

208− mblang' = (parseLang <$> getVariableAsText "language") <|> mblang

208+ mblang' = case getVariableAsText "language" of

209+ Just l -> either (const Nothing) Just $ parseLang l

210+ Nothing -> mblang

209

211

210 titlecase = case mblang' of

212 titlecase = case mblang' of

213+ Just lang | langLanguage lang == "en"

211− Just (Lang "en" _) -> titlecase'

214+ -> titlecase'

212 Nothing -> titlecase'

215 Nothing -> titlecase'

213 _ ->

216 _ ->

214 case variant of

217 case variant of

@@ -331,7 +334,7 @@ writeBibtexString opts variant mblang ref =

331 renderFields = mconcat . intersperse ("," <> cr) . mapMaybe renderField

334 renderFields = mconcat . intersperse ("," <> cr) . mapMaybe renderField

332

335

333defaultLang :: Lang

336defaultLang :: Lang

334−defaultLang = Lang "en" (Just "US")

337+defaultLang = Lang "en" Nothing (Just "US") [] [] []

335

338

336-- a map of bibtex "string" macros

339-- a map of bibtex "string" macros

337type StringMap = Map.Map Text Text

340type StringMap = Map.Map Text Text

@@ -351,9 +354,7 @@ itemToReference locale variant item = do

351 bib item $ do

354 bib item $ do

352 let lang = fromMaybe defaultLang $ localeLanguage locale

355 let lang = fromMaybe defaultLang $ localeLanguage locale

353 modify $ \st -> st{ localeLang = lang,

356 modify $ \st -> st{ localeLang = lang,

354− untitlecase = case lang of

357+ untitlecase = langLanguage lang == "en" }

355− (Lang "en" _) -> True

356− _ -> False }

357

358

358 id' <- asks identifier

359 id' <- asks identifier

359 otherIds <- (Just <$> getRawField "ids")

360 otherIds <- (Just <$> getRawField "ids")

@@ -711,7 +712,7 @@ itemToReference locale variant item = do

711

712

713

713bib :: Item -> Bib a -> BibParser a

714bib :: Item -> Bib a -> BibParser a

714−bib entry m = fst <$> evalRWST m entry (BibState True (Lang "en" (Just "US")))

715+bib entry m = fst <$> evalRWST m entry (BibState True defaultLang)

715

716

716resolveCrossRefs :: Variant -> [Item] -> [Item]

717resolveCrossRefs :: Variant -> [Item] -> [Item]

717resolveCrossRefs variant entries =

718resolveCrossRefs variant entries =

@@ -1456,8 +1457,9 @@ resolveKey lang ils = Walk.walk go ils

1456 go x = x

1457 go x = x

1457

1458

1458resolveKey' :: Lang -> Text -> Text

1459resolveKey' :: Lang -> Text -> Text

1459−resolveKey' lang@(Lang l _) k =

1460+resolveKey' lang k =

1460− case Map.lookup l biblatexStringMap >>= Map.lookup (T.toLower k) of

1461+ case Map.lookup (langLanguage lang) biblatexStringMap >>=

1462+ Map.lookup (T.toLower k) of

1461 Nothing -> k

1463 Nothing -> k

1462 Just (x, _) -> either (const k) stringify $ parseLaTeX lang x

1464 Just (x, _) -> either (const k) stringify $ parseLaTeX lang x

1463

1465

src/Text/Pandoc/Citeproc/Data.hs+6−6

@@ -21,12 +21,12 @@ biblatexStringMap :: M.Map Text (M.Map Text (Text, Text))

21biblatexStringMap = foldr go mempty biblatexLocalizations

22 where

23 go (fp, bs) =

24− let Lang lang _ _ _ _ _ = parseLang

25− (toIETF $ T.takeWhile (/= '.') $ T.pack fp)

26− ls = T.lines $ TE.decodeUtf8 bs

24+ let ls = T.lines $ TE.decodeUtf8 bs

25+ in case parseLang (toIETF $ T.takeWhile (/= '.') $ T.pack fp) of

27− in if length ls > 4

26+ Right lang | length ls > 4

27+ -> M.insert (langLanguage lang)

28− then M.insert lang (toStringMap $ map (T.splitOn "|") ls)

28+ (toStringMap $ map (T.splitOn "|") ls)

29− else id

29+ _ -> id

30 toStringMap = foldr go' mempty

31 go' [term, x, y] = M.insert term (x, y)

32 go' _ = id

src/Text/Pandoc/Class/CommonState.hs+1−1

@@ -19,7 +19,7 @@ where

20import Data.Default (Default (def))

21import Data.Text (Text)

22−import Text.Pandoc.BCP47 (Lang)

22+import UnicodeCollation.Lang (Lang)

23import Text.Pandoc.MediaBag (MediaBag)

24import Text.Pandoc.Logging (LogMessage, Verbosity (WARNING))

25import Text.Pandoc.Translations (Translations)

src/Text/Pandoc/Class/PandocMonad.hs+2−2

@@ -70,7 +70,7 @@ import Network.URI ( escapeURIString, nonStrictRelativeTo,

70import System.FilePath ((</>), (<.>), takeExtension, dropExtension,

71 isRelative, splitDirectories)

72import System.Random (StdGen)

73−import Text.Pandoc.BCP47 (Lang(..), parseBCP47, renderLang)

73+import UnicodeCollation.Lang (Lang(..), parseLang, renderLang)

74import Text.Pandoc.Class.CommonState (CommonState (..))

75import Text.Pandoc.Definition

76import Text.Pandoc.Error

@@ -285,7 +285,7 @@ readFileFromDirs (d:ds) f = catchError

285toLang :: PandocMonad m => Maybe T.Text -> m (Maybe Lang)

286toLang Nothing = return Nothing

287toLang (Just s) =

288− case parseBCP47 s of

288+ case parseLang s of

289 Left _ -> do

290 report $ InvalidLang s

291 return Nothing

src/Text/Pandoc/Readers/BibTeX.hs+6−3

@@ -48,11 +48,14 @@ readBibLaTeX = readBibTeX' BibTeX.Biblatex

49readBibTeX' :: PandocMonad m => Variant -> ReaderOptions -> Text -> m Pandoc

50readBibTeX' variant _opts t = do

51− lang <- maybe (Lang "en" (Just "US")) parseLang

51+ mblangEnv <- lookupEnv "LANG"

52− <$> lookupEnv "LANG"

52+ let defaultLang = Lang "en" Nothing (Just "US") [] [] []

53+ let lang = case mblangEnv of

54+ Nothing -> defaultLang

55+ Just l -> either (const defaultLang) id $ parseLang l

53 locale <- case getLocale lang of

56 locale <- case getLocale lang of

54 Left e ->

57 Left e ->

55− case getLocale (Lang "en" (Just "US")) of

58+ case getLocale (Lang "en" Nothing (Just "US") [] [] []) of

56 Right l -> return l

59 Right l -> return l

57 Left _ -> throwError $ PandocCiteprocError e

60 Left _ -> throwError $ PandocCiteprocError e

58 Right l -> return l

61 Right l -> return l

src/Text/Pandoc/Readers/LaTeX.hs+1−1

@@ -33,7 +33,7 @@ import qualified Data.Set as Set

33import Data.Text (Text)

34import qualified Data.Text as T

35import System.FilePath (addExtension, replaceExtension, takeExtension)

36−import Text.Pandoc.BCP47 (renderLang)

36+import UnicodeCollation.Lang (renderLang)

37import Text.Pandoc.Builder as B

38import Text.Pandoc.Class.PandocPure (PandocPure)

39import Text.Pandoc.Class.PandocMonad (PandocMonad (..), getResourcePath,

src/Text/Pandoc/Readers/LaTeX/Lang.hs+122−119

@@ -23,7 +23,7 @@ import qualified Data.Map as M

23import Data.Text (Text)

24import qualified Data.Text as T

25import Text.Pandoc.Shared (extractSpaces)

26−import Text.Pandoc.BCP47 (Lang(..), renderLang)

26+import UnicodeCollation.Lang (Lang(..), renderLang)

27import Text.Pandoc.Class (PandocMonad(..), setTranslations)

28import Text.Pandoc.Readers.LaTeX.Parsing

29import Text.Pandoc.Parsing (updateState, option, getState, QuoteContext(..),

@@ -99,133 +99,136 @@ setDefaultLanguage = do

99polyglossiaLangToBCP47 :: M.Map T.Text (T.Text -> Lang)

100polyglossiaLangToBCP47 = M.fromList

101 [ ("arabic", \o -> case T.filter (/=' ') o of

102− "locale=algeria" -> Lang "ar" "" "DZ" []

102+ "locale=algeria" -> Lang "ar" Nothing (Just "DZ") [] [] []

103− "locale=mashriq" -> Lang "ar" "" "SY" []

103+ "locale=mashriq" -> Lang "ar" Nothing (Just "SY") [] [] []

104− "locale=libya" -> Lang "ar" "" "LY" []

104+ "locale=libya" -> Lang "ar" Nothing (Just "LY") [] [] []

105− "locale=morocco" -> Lang "ar" "" "MA" []

105+ "locale=morocco" -> Lang "ar" Nothing (Just "MA") [] [] []

106− "locale=mauritania" -> Lang "ar" "" "MR" []

106+ "locale=mauritania" -> Lang "ar" Nothing (Just "MR") [] [] []

107− "locale=tunisia" -> Lang "ar" "" "TN" []

107+ "locale=tunisia" -> Lang "ar" Nothing (Just "TN") [] [] []

108− _ -> Lang "ar" "" "" [])

108+ _ -> Lang "ar" Nothing (Just "") [] [] [])

109 , ("german", \o -> case T.filter (/=' ') o of

110− "spelling=old" -> Lang "de" "" "DE" ["1901"]

110+ "spelling=old" -> Lang "de" Nothing (Just "DE") ["1901"] [] []

111 "variant=austrian,spelling=old"

112− -> Lang "de" "" "AT" ["1901"]

112+ -> Lang "de" Nothing (Just "AT") ["1901"] [] []

113− "variant=austrian" -> Lang "de" "" "AT" []

113+ "variant=austrian" -> Lang "de" Nothing (Just "AT") [] [] []

114 "variant=swiss,spelling=old"

115− -> Lang "de" "" "CH" ["1901"]

115+ -> Lang "de" Nothing (Just "CH") ["1901"] [] []

116− "variant=swiss" -> Lang "de" "" "CH" []

116+ "variant=swiss" -> Lang "de" Nothing (Just "CH") [] [] []

117− _ -> Lang "de" "" "" [])

117+ _ -> Lang "de" Nothing Nothing [] [] [])

118− , ("lsorbian", \_ -> Lang "dsb" "" "" [])

118+ , ("lsorbian", \_ -> Lang "dsb" Nothing Nothing [] [] [])

119 , ("greek", \o -> case T.filter (/=' ') o of

120− "variant=poly" -> Lang "el" "" "polyton" []

120+ "variant=poly" -> Lang "el" Nothing (Just "polyton") [] [] []

121− "variant=ancient" -> Lang "grc" "" "" []

121+ "variant=ancient" -> Lang "grc" Nothing Nothing [] [] []

122− _ -> Lang "el" "" "" [])

122+ _ -> Lang "el" Nothing Nothing [] [] [])

123 , ("english", \o -> case T.filter (/=' ') o of

124− "variant=australian" -> Lang "en" "" "AU" []

124+ "variant=australian" -> Lang "en" Nothing (Just "AU") [] [] []

125− "variant=canadian" -> Lang "en" "" "CA" []

125+ "variant=canadian" -> Lang "en" Nothing (Just "CA") [] [] []

126− "variant=british" -> Lang "en" "" "GB" []

126+ "variant=british" -> Lang "en" Nothing (Just "GB") [] [] []

127− "variant=newzealand" -> Lang "en" "" "NZ" []

127+ "variant=newzealand" -> Lang "en" Nothing (Just "NZ") [] [] []

128− "variant=american" -> Lang "en" "" "US" []

128+ "variant=american" -> Lang "en" Nothing (Just "US") [] [] []

129− _ -> Lang "en" "" "" [])

129+ _ -> Lang "en" Nothing (Just "") [] [] [])

130− , ("usorbian", \_ -> Lang "hsb" "" "" [])

130+ , ("usorbian", \_ -> Lang "hsb" Nothing Nothing [] [] [])

131 , ("latin", \o -> case T.filter (/=' ') o of

132− "variant=classic" -> Lang "la" "" "" ["x-classic"]

132+ "variant=classic" -> Lang "la" Nothing Nothing ["x-classic"] [] []

133− _ -> Lang "la" "" "" [])

133+ _ -> Lang "la" Nothing Nothing [] [] [])

134− , ("slovenian", \_ -> Lang "sl" "" "" [])

134+ , ("slovenian", \_ -> Lang "sl" Nothing Nothing [] [] [])

135− , ("serbianc", \_ -> Lang "sr" "cyrl" "" [])

135+ , ("serbianc", \_ -> Lang "sr" (Just "Cyrl") Nothing [] [] [])

136− , ("pinyin", \_ -> Lang "zh" "Latn" "" ["pinyin"])

136+ , ("pinyin", \_ -> Lang "zh" (Just "Latn") Nothing ["pinyin"] [] [])

137− , ("afrikaans", \_ -> Lang "af" "" "" [])

137+ , ("afrikaans", \_ -> simpleLang "af")

138− , ("amharic", \_ -> Lang "am" "" "" [])

138+ , ("amharic", \_ -> simpleLang "am")

139− , ("assamese", \_ -> Lang "as" "" "" [])

139+ , ("assamese", \_ -> simpleLang "as")

140− , ("asturian", \_ -> Lang "ast" "" "" [])

140+ , ("asturian", \_ -> simpleLang "ast")

141− , ("bulgarian", \_ -> Lang "bg" "" "" [])

141+ , ("bulgarian", \_ -> simpleLang "bg")

142− , ("bengali", \_ -> Lang "bn" "" "" [])

142+ , ("bengali", \_ -> simpleLang "bn")

143− , ("tibetan", \_ -> Lang "bo" "" "" [])

143+ , ("tibetan", \_ -> simpleLang "bo")

144− , ("breton", \_ -> Lang "br" "" "" [])

144+ , ("breton", \_ -> simpleLang "br")

145− , ("catalan", \_ -> Lang "ca" "" "" [])

145+ , ("catalan", \_ -> simpleLang "ca")

146− , ("welsh", \_ -> Lang "cy" "" "" [])

146+ , ("welsh", \_ -> simpleLang "cy")

147− , ("czech", \_ -> Lang "cs" "" "" [])

147+ , ("czech", \_ -> simpleLang "cs")

148− , ("coptic", \_ -> Lang "cop" "" "" [])

148+ , ("coptic", \_ -> simpleLang "cop")

149− , ("danish", \_ -> Lang "da" "" "" [])

149+ , ("danish", \_ -> simpleLang "da")

150− , ("divehi", \_ -> Lang "dv" "" "" [])

150+ , ("divehi", \_ -> simpleLang "dv")

151− , ("esperanto", \_ -> Lang "eo" "" "" [])

151+ , ("esperanto", \_ -> simpleLang "eo")

152− , ("spanish", \_ -> Lang "es" "" "" [])

152+ , ("spanish", \_ -> simpleLang "es")

153− , ("estonian", \_ -> Lang "et" "" "" [])

153+ , ("estonian", \_ -> simpleLang "et")

154− , ("basque", \_ -> Lang "eu" "" "" [])

154+ , ("basque", \_ -> simpleLang "eu")

155− , ("farsi", \_ -> Lang "fa" "" "" [])

155+ , ("farsi", \_ -> simpleLang "fa")

156− , ("finnish", \_ -> Lang "fi" "" "" [])

156+ , ("finnish", \_ -> simpleLang "fi")

157− , ("french", \_ -> Lang "fr" "" "" [])

157+ , ("french", \_ -> simpleLang "fr")

158− , ("friulan", \_ -> Lang "fur" "" "" [])

158+ , ("friulan", \_ -> simpleLang "fur")

159− , ("irish", \_ -> Lang "ga" "" "" [])

159+ , ("irish", \_ -> simpleLang "ga")

160− , ("scottish", \_ -> Lang "gd" "" "" [])

160+ , ("scottish", \_ -> simpleLang "gd")

161− , ("ethiopic", \_ -> Lang "gez" "" "" [])

161+ , ("ethiopic", \_ -> simpleLang "gez")

162− , ("galician", \_ -> Lang "gl" "" "" [])

162+ , ("galician", \_ -> simpleLang "gl")

163− , ("hebrew", \_ -> Lang "he" "" "" [])

163+ , ("hebrew", \_ -> simpleLang "he")

164− , ("hindi", \_ -> Lang "hi" "" "" [])

164+ , ("hindi", \_ -> simpleLang "hi")

165− , ("croatian", \_ -> Lang "hr" "" "" [])

165+ , ("croatian", \_ -> simpleLang "hr")

166− , ("magyar", \_ -> Lang "hu" "" "" [])

166+ , ("magyar", \_ -> simpleLang "hu")

167− , ("armenian", \_ -> Lang "hy" "" "" [])

167+ , ("armenian", \_ -> simpleLang "hy")

168− , ("interlingua", \_ -> Lang "ia" "" "" [])

168+ , ("interlingua", \_ -> simpleLang "ia")

169− , ("indonesian", \_ -> Lang "id" "" "" [])

169+ , ("indonesian", \_ -> simpleLang "id")

170− , ("icelandic", \_ -> Lang "is" "" "" [])

170+ , ("icelandic", \_ -> simpleLang "is")

171− , ("italian", \_ -> Lang "it" "" "" [])

171+ , ("italian", \_ -> simpleLang "it")

172− , ("japanese", \_ -> Lang "jp" "" "" [])

172+ , ("japanese", \_ -> simpleLang "jp")

173− , ("khmer", \_ -> Lang "km" "" "" [])

173+ , ("khmer", \_ -> simpleLang "km")

174− , ("kurmanji", \_ -> Lang "kmr" "" "" [])

174+ , ("kurmanji", \_ -> simpleLang "kmr")

175− , ("kannada", \_ -> Lang "kn" "" "" [])

175+ , ("kannada", \_ -> simpleLang "kn")

176− , ("korean", \_ -> Lang "ko" "" "" [])

176+ , ("korean", \_ -> simpleLang "ko")

177− , ("lao", \_ -> Lang "lo" "" "" [])

177+ , ("lao", \_ -> simpleLang "lo")

178− , ("lithuanian", \_ -> Lang "lt" "" "" [])

178+ , ("lithuanian", \_ -> simpleLang "lt")

179− , ("latvian", \_ -> Lang "lv" "" "" [])

179+ , ("latvian", \_ -> simpleLang "lv")

180− , ("malayalam", \_ -> Lang "ml" "" "" [])

180+ , ("malayalam", \_ -> simpleLang "ml")

181− , ("mongolian", \_ -> Lang "mn" "" "" [])

181+ , ("mongolian", \_ -> simpleLang "mn")

182− , ("marathi", \_ -> Lang "mr" "" "" [])

182+ , ("marathi", \_ -> simpleLang "mr")

183− , ("dutch", \_ -> Lang "nl" "" "" [])

183+ , ("dutch", \_ -> simpleLang "nl")

184− , ("nynorsk", \_ -> Lang "nn" "" "" [])

184+ , ("nynorsk", \_ -> simpleLang "nn")

185− , ("norsk", \_ -> Lang "no" "" "" [])

185+ , ("norsk", \_ -> simpleLang "no")

186− , ("nko", \_ -> Lang "nqo" "" "" [])

186+ , ("nko", \_ -> simpleLang "nqo")

187− , ("occitan", \_ -> Lang "oc" "" "" [])

187+ , ("occitan", \_ -> simpleLang "oc")

188− , ("panjabi", \_ -> Lang "pa" "" "" [])

188+ , ("panjabi", \_ -> simpleLang "pa")

189− , ("polish", \_ -> Lang "pl" "" "" [])

189+ , ("polish", \_ -> simpleLang "pl")

190− , ("piedmontese", \_ -> Lang "pms" "" "" [])

190+ , ("piedmontese", \_ -> simpleLang "pms")

191− , ("portuguese", \_ -> Lang "pt" "" "" [])

191+ , ("portuguese", \_ -> simpleLang "pt")

192− , ("romansh", \_ -> Lang "rm" "" "" [])

192+ , ("romansh", \_ -> simpleLang "rm")

193− , ("romanian", \_ -> Lang "ro" "" "" [])

193+ , ("romanian", \_ -> simpleLang "ro")

194− , ("russian", \_ -> Lang "ru" "" "" [])

194+ , ("russian", \_ -> simpleLang "ru")

195− , ("sanskrit", \_ -> Lang "sa" "" "" [])

195+ , ("sanskrit", \_ -> simpleLang "sa")

196− , ("samin", \_ -> Lang "se" "" "" [])

196+ , ("samin", \_ -> simpleLang "se")

197− , ("slovak", \_ -> Lang "sk" "" "" [])

197+ , ("slovak", \_ -> simpleLang "sk")

198− , ("albanian", \_ -> Lang "sq" "" "" [])

198+ , ("albanian", \_ -> simpleLang "sq")

199− , ("serbian", \_ -> Lang "sr" "" "" [])

199+ , ("serbian", \_ -> simpleLang "sr")

200− , ("swedish", \_ -> Lang "sv" "" "" [])

200+ , ("swedish", \_ -> simpleLang "sv")

201− , ("syriac", \_ -> Lang "syr" "" "" [])

201+ , ("syriac", \_ -> simpleLang "syr")

202− , ("tamil", \_ -> Lang "ta" "" "" [])

202+ , ("tamil", \_ -> simpleLang "ta")

203− , ("telugu", \_ -> Lang "te" "" "" [])

203+ , ("telugu", \_ -> simpleLang "te")

204− , ("thai", \_ -> Lang "th" "" "" [])

204+ , ("thai", \_ -> simpleLang "th")

205− , ("turkmen", \_ -> Lang "tk" "" "" [])

205+ , ("turkmen", \_ -> simpleLang "tk")

206− , ("turkish", \_ -> Lang "tr" "" "" [])

206+ , ("turkish", \_ -> simpleLang "tr")

207− , ("ukrainian", \_ -> Lang "uk" "" "" [])

207+ , ("ukrainian", \_ -> simpleLang "uk")

208− , ("urdu", \_ -> Lang "ur" "" "" [])

208+ , ("urdu", \_ -> simpleLang "ur")

209− , ("vietnamese", \_ -> Lang "vi" "" "" [])

209+ , ("vietnamese", \_ -> simpleLang "vi")

210 ]

211

212+simpleLang :: Text -> Lang

213+simpleLang l = Lang l Nothing Nothing [] [] []

214+

212babelLangToBCP47 :: T.Text -> Maybe Lang

215babelLangToBCP47 :: T.Text -> Maybe Lang

213babelLangToBCP47 s =

216babelLangToBCP47 s =

214 case s of

217 case s of

215− "austrian" -> Just $ Lang "de" "" "AT" ["1901"]

218+ "austrian" -> Just $ Lang "de" Nothing (Just "AT") ["1901"] [] []

216− "naustrian" -> Just $ Lang "de" "" "AT" []

219+ "naustrian" -> Just $ Lang "de" Nothing (Just "AT") [] [] []

217− "swissgerman" -> Just $ Lang "de" "" "CH" ["1901"]

220+ "swissgerman" -> Just $ Lang "de" Nothing (Just "CH") ["1901"] [] []

218− "nswissgerman" -> Just $ Lang "de" "" "CH" []

221+ "nswissgerman" -> Just $ Lang "de" Nothing (Just "CH") [] [] []

219− "german" -> Just $ Lang "de" "" "DE" ["1901"]

222+ "german" -> Just $ Lang "de" Nothing (Just "DE") ["1901"] [] []

220− "ngerman" -> Just $ Lang "de" "" "DE" []

223+ "ngerman" -> Just $ Lang "de" Nothing (Just "DE") [] [] []

221− "lowersorbian" -> Just $ Lang "dsb" "" "" []

224+ "lowersorbian" -> Just $ Lang "dsb" Nothing Nothing [] [] []

222− "uppersorbian" -> Just $ Lang "hsb" "" "" []

225+ "uppersorbian" -> Just $ Lang "hsb" Nothing Nothing [] [] []

223− "polutonikogreek" -> Just $ Lang "el" "" "" ["polyton"]

226+ "polutonikogreek" -> Just $ Lang "el" Nothing Nothing ["polyton"] [] []

224− "slovene" -> Just $ Lang "sl" "" "" []

227+ "slovene" -> Just $ simpleLang "sl"

225− "australian" -> Just $ Lang "en" "" "AU" []

228+ "australian" -> Just $ Lang "en" Nothing (Just "AU") [] [] []

226− "canadian" -> Just $ Lang "en" "" "CA" []

229+ "canadian" -> Just $ Lang "en" Nothing (Just "CA") [] [] []

227− "british" -> Just $ Lang "en" "" "GB" []

230+ "british" -> Just $ Lang "en" Nothing (Just "GB") [] [] []

228− "newzealand" -> Just $ Lang "en" "" "NZ" []

231+ "newzealand" -> Just $ Lang "en" Nothing (Just "NZ") [] [] []

229− "american" -> Just $ Lang "en" "" "US" []

232+ "american" -> Just $ Lang "en" Nothing (Just "US") [] [] []

230− "classiclatin" -> Just $ Lang "la" "" "" ["x-classic"]

233+ "classiclatin" -> Just $ Lang "la" Nothing Nothing ["x-classic"] [] []

231 _ -> ($ "") <$> M.lookup s polyglossiaLangToBCP47

234 _ -> ($ "") <$> M.lookup s polyglossiaLangToBCP47

src/Text/Pandoc/Writers/BibTeX.hs+1−1

@@ -43,7 +43,7 @@ writeBibTeX' :: PandocMonad m => Variant -> WriterOptions -> Pandoc -> m Text

43writeBibTeX' variant opts (Pandoc meta _) = do

44 let mblang = case lookupMetaString "lang" meta of

45 "" -> Nothing

46− t -> Just $ parseLang t

46+ t -> either (const Nothing) Just $ parseLang t

47 let refs = case lookupMeta "references" meta of

48 Just (MetaList xs) -> mapMaybe metaValueToReference xs

49 _ -> []

src/Text/Pandoc/Writers/ConTeXt.hs+24−24

@@ -21,7 +21,7 @@ import Data.Maybe (mapMaybe)

21import Data.Text (Text)

22import qualified Data.Text as T

23import Network.URI (unEscapeString)

24−import Text.Pandoc.BCP47

24+import UnicodeCollation.Lang (Lang(..))

25import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

26import Text.Pandoc.Definition

27import Text.Pandoc.ImageSize

@@ -555,26 +555,26 @@ fromBCP47 mbs = fromBCP47' <$> toLang mbs

555-- https://tools.ietf.org/html/bcp47#section-2.1

556-- http://wiki.contextgarden.net/Language_Codes

557fromBCP47' :: Maybe Lang -> Maybe Text

558−fromBCP47' (Just (Lang "ar" _ "SY" _) ) = Just "ar-sy"

558+fromBCP47' (Just (Lang "ar" _ (Just "SY") _ _ _)) = Just "ar-sy"

559−fromBCP47' (Just (Lang "ar" _ "IQ" _) ) = Just "ar-iq"

559+fromBCP47' (Just (Lang "ar" _ (Just "IQ") _ _ _)) = Just "ar-iq"

560−fromBCP47' (Just (Lang "ar" _ "JO" _) ) = Just "ar-jo"

560+fromBCP47' (Just (Lang "ar" _ (Just "JO") _ _ _)) = Just "ar-jo"

561−fromBCP47' (Just (Lang "ar" _ "LB" _) ) = Just "ar-lb"

561+fromBCP47' (Just (Lang "ar" _ (Just "LB") _ _ _)) = Just "ar-lb"

562−fromBCP47' (Just (Lang "ar" _ "DZ" _) ) = Just "ar-dz"

562+fromBCP47' (Just (Lang "ar" _ (Just "DZ") _ _ _)) = Just "ar-dz"

563−fromBCP47' (Just (Lang "ar" _ "MA" _) ) = Just "ar-ma"

563+fromBCP47' (Just (Lang "ar" _ (Just "MA") _ _ _)) = Just "ar-ma"

564−fromBCP47' (Just (Lang "de" _ _ ["1901"]) ) = Just "deo"

564+fromBCP47' (Just (Lang "de" _ _ ["1901"] _ _)) = Just "deo"

565−fromBCP47' (Just (Lang "de" _ "DE" _) ) = Just "de-de"

565+fromBCP47' (Just (Lang "de" _ (Just "DE") _ _ _)) = Just "de-de"

566−fromBCP47' (Just (Lang "de" _ "AT" _) ) = Just "de-at"

566+fromBCP47' (Just (Lang "de" _ (Just "AT") _ _ _)) = Just "de-at"

567−fromBCP47' (Just (Lang "de" _ "CH" _) ) = Just "de-ch"

567+fromBCP47' (Just (Lang "de" _ (Just "CH") _ _ _)) = Just "de-ch"

568−fromBCP47' (Just (Lang "el" _ _ ["poly"]) ) = Just "agr"

568+fromBCP47' (Just (Lang "el" _ _ ["poly"] _ _)) = Just "agr"

569−fromBCP47' (Just (Lang "en" _ "US" _) ) = Just "en-us"

569+fromBCP47' (Just (Lang "en" _ (Just "US") _ _ _)) = Just "en-us"

570−fromBCP47' (Just (Lang "en" _ "GB" _) ) = Just "en-gb"

570+fromBCP47' (Just (Lang "en" _ (Just "GB") _ _ _)) = Just "en-gb"

571−fromBCP47' (Just (Lang "grc"_ _ _) ) = Just "agr"

571+fromBCP47' (Just (Lang "grc"_ _ _ _ _)) = Just "agr"

572−fromBCP47' (Just (Lang "el" _ _ _) ) = Just "gr"

572+fromBCP47' (Just (Lang "el" _ _ _ _ _)) = Just "gr"

573−fromBCP47' (Just (Lang "eu" _ _ _) ) = Just "ba"

573+fromBCP47' (Just (Lang "eu" _ _ _ _ _)) = Just "ba"

574−fromBCP47' (Just (Lang "he" _ _ _) ) = Just "il"

574+fromBCP47' (Just (Lang "he" _ _ _ _ _)) = Just "il"

575−fromBCP47' (Just (Lang "jp" _ _ _) ) = Just "ja"

575+fromBCP47' (Just (Lang "jp" _ _ _ _ _)) = Just "ja"

576−fromBCP47' (Just (Lang "uk" _ _ _) ) = Just "ua"

576+fromBCP47' (Just (Lang "uk" _ _ _ _ _)) = Just "ua"

577−fromBCP47' (Just (Lang "vi" _ _ _) ) = Just "vn"

577+fromBCP47' (Just (Lang "vi" _ _ _ _ _)) = Just "vn"

578−fromBCP47' (Just (Lang "zh" _ _ _) ) = Just "cn"

578+fromBCP47' (Just (Lang "zh" _ _ _ _ _)) = Just "cn"

579−fromBCP47' (Just (Lang l _ _ _) ) = Just l

579+fromBCP47' (Just (Lang l _ _ _ _ _)) = Just l

580−fromBCP47' Nothing = Nothing

580+fromBCP47' Nothing = Nothing

src/Text/Pandoc/Writers/CslJson.hs+4−3

@@ -34,15 +34,16 @@ import Control.Monad.Identity

34import Citeproc.Locale (getLocale)

35import Citeproc.CslJson

36import Text.Pandoc.Options (WriterOptions)

37−import Data.Maybe (mapMaybe)

37+import Data.Maybe (mapMaybe, fromMaybe)

38import Data.Aeson.Encode.Pretty (Config (..), Indent (Spaces),

39 NumberFormat (Generic),

40 defConfig, encodePretty')

42writeCslJson :: PandocMonad m => WriterOptions -> Pandoc -> m Text

43writeCslJson _opts (Pandoc meta _) = do

44− let lang = maybe (Lang "en" (Just "US")) parseLang

44+ let lang = fromMaybe (Lang "en" Nothing (Just "US") [] [] [])

45− (lookupMeta "lang" meta >>= metaValueToText)

45+ (lookupMeta "lang" meta >>= metaValueToText >>=

46+ either (const Nothing) Just . parseLang)

46 locale <- case getLocale lang of

47 locale <- case getLocale lang of

47 Left e -> throwError $ PandocCiteprocError e

48 Left e -> throwError $ PandocCiteprocError e

48 Right l -> return l

49 Right l -> return l

src/Text/Pandoc/Writers/Docx.hs+1−1

@@ -36,7 +36,7 @@ import qualified Data.Text.Lazy as TL

36import Data.Time.Clock.POSIX

37import Data.Digest.Pure.SHA (sha1, showDigest)

38import Skylighting

39−import Text.Pandoc.BCP47 (getLang, renderLang)

39+import UnicodeCollation.Lang (renderLang)

40import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

41import qualified Text.Pandoc.Class.PandocMonad as P

42import Data.Time

src/Text/Pandoc/Writers/LaTeX.hs+1−1

@@ -29,7 +29,7 @@ import qualified Data.Text as T

29import Network.URI (unEscapeString)

30import Text.DocTemplates (FromContext(lookupContext), renderTemplate,

31 Val(..), Context(..))

32−import Text.Pandoc.BCP47 (Lang (..), getLang, renderLang)

32+import UnicodeCollation.Lang (Lang (..), renderLang)

33import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

34import Text.Pandoc.Definition

35import Text.Pandoc.Highlighting (formatLaTeXBlock, formatLaTeXInline, highlight,

src/Text/Pandoc/Writers/LaTeX/Lang.hs+4−4

@@ -46,7 +46,7 @@ toPolyglossia (Lang "de" _ (Just "AT") vars _ _)

46toPolyglossia (Lang "de" _ (Just "AT") _ _ _) = ("german", "variant=austrian")

47toPolyglossia (Lang "de" _ (Just "CH") vars _ _)

48 | "1901" `elem` vars = ("german", "variant=swiss, spelling=old")

49−toPolyglossia (Lang "de" _ (Just "CH") _ _ _ _) = ("german", "variant=swiss")

49+toPolyglossia (Lang "de" _ (Just "CH") _ _ _) = ("german", "variant=swiss")

50toPolyglossia (Lang "de" _ _ _ _ _) = ("german", "")

51toPolyglossia (Lang "dsb" _ _ _ _ _) = ("lsorbian", "")

52toPolyglossia (Lang "el" _ _ vars _ _)

@@ -61,9 +61,9 @@ toPolyglossia (Lang "grc" _ _ _ _ _) = ("greek", "variant=ancient")

61toPolyglossia (Lang "hsb" _ _ _ _ _) = ("usorbian", "")

62toPolyglossia (Lang "la" _ _ vars _ _)

63 | "x-classic" `elem` vars = ("latin", "variant=classic")

64−toPolyglossia (Lang "pt" _ "BR" _ _ _) = ("portuguese", "variant=brazilian")

64+toPolyglossia (Lang "pt" _ (Just "BR") _ _ _) = ("portuguese", "variant=brazilian")

65toPolyglossia (Lang "sl" _ _ _ _ _) = ("slovenian", "")

66−toPolyglossia x = (commonFromBcp47 x, "")

66+toPolyglossia x = (commonFromBcp47 x, "")

68-- Takes a list of the constituents of a BCP47 language code and

69-- converts it to a Babel language string.

@@ -81,7 +81,7 @@ toBabel (Lang "de" _ _ vars _ _)

81 | "1901" `elem` vars = "german"

82 | otherwise = "ngerman"

83toBabel (Lang "dsb" _ _ _ _ _) = "lowersorbian"

84−toBabel (Lang "el" _ _ vars)

84+toBabel (Lang "el" _ _ vars _ _)

85 | "polyton" `elem` vars = "polutonikogreek"

86toBabel (Lang "en" _ (Just "AU") _ _ _) = "australian"

87toBabel (Lang "en" _ (Just "CA") _ _ _) = "canadian"

src/Text/Pandoc/Writers/ODT.hs+4−3

@@ -16,6 +16,7 @@ import Codec.Archive.Zip

16import Control.Monad.Except (catchError, throwError)

17import Control.Monad.State.Strict

18import qualified Data.ByteString.Lazy as B

19+import Data.Maybe (fromMaybe)

19import Data.Generics (everywhere', mkT)

20import Data.Generics (everywhere', mkT)

20import Data.List (isPrefixOf)

21import Data.List (isPrefixOf)

21import qualified Data.Map as Map

22import qualified Data.Map as Map

@@ -23,7 +24,7 @@ import qualified Data.Text as T

23import qualified Data.Text.Lazy as TL

24import qualified Data.Text.Lazy as TL

24import Data.Time

25import Data.Time

25import System.FilePath (takeDirectory, takeExtension, (<.>))

26import System.FilePath (takeDirectory, takeExtension, (<.>))

26−import Text.Pandoc.BCP47 (Lang (..), getLang, renderLang)

27+import UnicodeCollation.Lang (Lang (..), renderLang)

27import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

28import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

28import qualified Text.Pandoc.Class.PandocMonad as P

29import qualified Text.Pandoc.Class.PandocMonad as P

29import Text.Pandoc.Definition

30import Text.Pandoc.Definition

@@ -35,7 +36,7 @@ import Text.Pandoc.Options (WrapOption (..), WriterOptions (..))

35import Text.DocLayout

36import Text.DocLayout

36import Text.Pandoc.Shared (stringify, pandocVersion, tshow)

37import Text.Pandoc.Shared (stringify, pandocVersion, tshow)

37import Text.Pandoc.Writers.Shared (lookupMetaString, lookupMetaBlocks,

38import Text.Pandoc.Writers.Shared (lookupMetaString, lookupMetaBlocks,

38− fixDisplayMath)

39+ fixDisplayMath, getLang)

39import Text.Pandoc.UTF8 (fromStringLazy, fromTextLazy, toTextLazy)

40import Text.Pandoc.UTF8 (fromStringLazy, fromTextLazy, toTextLazy)

40import Text.Pandoc.Walk

41import Text.Pandoc.Walk

41import Text.Pandoc.Writers.OpenDocument (writeOpenDocument)

42import Text.Pandoc.Writers.OpenDocument (writeOpenDocument)

@@ -194,7 +195,7 @@ addLang lang = everywhere' (mkT updateLangAttr)

194 where updateLangAttr (Attr n@(QName "language" _ (Just "fo")) _)

195 where updateLangAttr (Attr n@(QName "language" _ (Just "fo")) _)

195 = Attr n (langLanguage lang)

196 = Attr n (langLanguage lang)

196 updateLangAttr (Attr n@(QName "country" _ (Just "fo")) _)

197 updateLangAttr (Attr n@(QName "country" _ (Just "fo")) _)

197− = Attr n (langRegion lang)

198+ = Attr n (fromMaybe "" $ langRegion lang)

198 updateLangAttr x = x

199 updateLangAttr x = x

199

200

200-- | transform both Image and Math elements

201-- | transform both Image and Math elements

src/Text/Pandoc/Writers/OpenDocument.hs+4−4

@@ -25,7 +25,7 @@ import Data.Ord (comparing)

25import qualified Data.Set as Set

26import Data.Text (Text)

27import qualified Data.Text as T

28−import Text.Pandoc.BCP47 (Lang (..), parseBCP47)

28+import UnicodeCollation.Lang (Lang (..), parseLang)

29import Text.Pandoc.Class.PandocMonad (PandocMonad, report, translateTerm,

30 setTranslations, toLang)

31import Text.Pandoc.Definition

@@ -236,7 +236,7 @@ handleSpaces s = case T.uncons s of

236-- | Convert Pandoc document to string in OpenDocument format.

237writeOpenDocument :: PandocMonad m => WriterOptions -> Pandoc -> m Text

238writeOpenDocument opts (Pandoc meta blocks) = do

239− let defLang = Lang "en" "US" "" []

239+ let defLang = Lang "en" (Just "US") Nothing [] [] []

240 lang <- case lookupMetaString "lang" meta of

241 "" -> pure defLang

242 s -> fromMaybe defLang <$> toLang (Just s)

@@ -893,7 +893,7 @@ textStyleAttr m s

893 Map.insert "style:font-name-complex" "Courier New" $ m

894 | Language lang <- s

895 = Map.insert "fo:language" (langLanguage lang) .

896− Map.insert "fo:country" (langRegion lang) $ m

896+ maybe id (Map.insert "fo:country") (langRegion lang) $ m

897 | otherwise = m

898

899withLangFromAttr :: PandocMonad m => Attr -> OD m a -> OD m a

@@ -901,7 +901,7 @@ withLangFromAttr (_,_,kvs) action =

901 case lookup "lang" kvs of

902 Nothing -> action

903 Just l ->

904− case parseBCP47 l of

904+ case parseLang l of

905 Right lang -> withTextStyle (Language lang) action

906 Left _ -> do

907 report $ InvalidLang l

src/Text/Pandoc/Writers/Shared.hs+1−1

@@ -149,7 +149,7 @@ defField field val (Context m) =

149 f _newval oldval = oldval

150

151-- | Get the contents of the `lang` metadata field or variable.

152−getLang :: WriterOptions -> Meta -> Maybe Text

152+getLang :: WriterOptions -> Meta -> Maybe T.Text

153getLang opts meta =

154 case lookupContext "lang" (writerVariables opts) of

155 Just s -> Just s