aecbf81 · typeable/pandoc

git.iko.soy/typeable/pandoc

Remove Text.Pandoc.BCP47 module.

[API change] Use Lang from UnicodeCollation.Lang instead. This is a richer implementation of BCP 47.

commitaecbf8156eb7c36c4b41de27797e262c23728db5 authorJohn MacFarlane <jgm@berkeley.edu> dateSun 11 Apr 2021, 21:28 (6 years ago) parent7ba8c0d

20 files changed+198−294

compare with master

unifiedsplit

pandoc.cabal−1

@@ -601,7 +601,6 @@ library

601601 Text.Pandoc.Asciify,

602602 Text.Pandoc.Emoji,

603603 Text.Pandoc.ImageSize,

604− Text.Pandoc.BCP47,

605604 Text.Pandoc.Class,

606605 Text.Pandoc.Citeproc

607606 other-modules: Text.Pandoc.App.CommandLineOptions,

src/Text/Pandoc/App.hs+3−3

@@ -55,7 +55,7 @@ import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), defaultOpts,

5555import Text.Pandoc.App.CommandLineOptions (parseOptions, parseOptionsFromArgs,

5656 options)

5757import Text.Pandoc.App.OutputSettings (OutputSettings (..), optToOutputSettings)

58−import Text.Pandoc.BCP47 (Lang (..), parseBCP47)

58+import UnicodeCollation.Lang (Lang (..), parseLang)

5959import Text.Pandoc.Filter (Filter (JSONFilter, LuaFilter), applyFilters)

6060import Text.Pandoc.PDF (makePDF)

6161import Text.Pandoc.SelfContained (makeSelfContained)

@@ -200,8 +200,8 @@ convertWithOpts opts = do

200200 Just f -> readFileStrict f

201201

202202 case lookupMetaString "lang" (optMetadata opts) of

203− "" -> setTranslations $ Lang "en" "" "US" []

204− l -> case parseBCP47 l of

203+ "" -> setTranslations $ Lang "en" Nothing (Just "US") [] [] []

204+ l -> case parseLang l of

205205 Left _ -> report $ InvalidLang l

206206 Right l' -> setTranslations l'

207207

src/Text/Pandoc/BCP47.hsdeleted−99

@@ -1,99 +0,0 @@

1−{-# LANGUAGE OverloadedStrings #-}

2−{- |

3− Module : Text.Pandoc.BCP47

5− License : GNU GPL, version 2 or above

6−

7− Maintainer : John MacFarlane <jgm@berkeley.edu>

8− Stability : alpha

9− Portability : portable

10−

11−Functions for parsing and rendering BCP47 language identifiers.

12−-}

13−module Text.Pandoc.BCP47 (

14− getLang

15− , parseBCP47

16− , Lang(..)

17− , renderLang

18− )

19−where

20−import Control.Monad (guard)

21−import Data.Char (isAlphaNum, isAscii, isLetter, isLower, isUpper)

22−import Text.Pandoc.Definition

23−import Text.Pandoc.Options

24−import Text.DocTemplates (FromContext(..))

25−import qualified Data.Text as T

26−import qualified Text.Parsec as P

27−

28−-- | Represents BCP 47 language/country code.

29−data Lang = Lang{ langLanguage :: T.Text

30− , langScript :: T.Text

31− , langRegion :: T.Text

32− , langVariants :: [T.Text] }

33− deriving (Eq, Ord, Show)

34−

35−-- | Render a Lang as BCP 47.

36−renderLang :: Lang -> T.Text

37−renderLang lang = T.intercalate "-" (langLanguage lang : filter (not . T.null)

38− ([langScript lang, langRegion lang] ++ langVariants lang))

39−

40−-- | Parse a BCP 47 string as a Lang. Currently we parse

41−-- extensions and private-use fields as "variants," even

42−-- though officially they aren't.

43−parseBCP47 :: T.Text -> Either T.Text Lang

44−parseBCP47 lang =

45− case P.parse bcp47 "lang" lang of

46− Right r -> Right r

47− Left e -> Left $ T.pack $ show e

48− where bcp47 = do

49− language <- pLanguage

50− script <- P.option "" pScript

51− region <- P.option "" pRegion

52− variants <- P.many (pVariant P.<|> pExtension P.<|> pPrivateUse)

53− P.eof

54− return Lang{ langLanguage = language

55− , langScript = script

56− , langRegion = region

57− , langVariants = variants }

58− asciiLetter = P.satisfy (\c -> isAscii c && isLetter c)

59− pLanguage = do

60− cs <- P.many1 asciiLetter

61− let lcs = length cs

62− guard $ lcs == 2 || lcs == 3

63− return $ T.toLower $ T.pack cs

64− pScript = P.try $ do

65− P.char '-'

66− x <- P.satisfy (\c -> isAscii c && isLetter c && isUpper c)

67− xs <- P.count 3

68− (P.satisfy (\c -> isAscii c && isLetter c && isLower c))

69− return $ T.toLower $ T.pack (x:xs)

70− pRegion = P.try $ do

71− P.char '-'

72− cs <- P.many1 asciiLetter

73− let lcs = length cs

74− guard $ lcs == 2 || lcs == 3

75− return $ T.toUpper $ T.pack cs

76− pVariant = P.try $ do

77− P.char '-'

78− ds <- P.option "" (P.count 1 P.digit)

79− cs <- P.many1 asciiLetter

80− let var = ds ++ cs

81− lv = length var

82− guard $ if null ds

83− then lv >= 5 && lv <= 8

84− else lv == 4

85− return $ T.toLower $ T.pack var

86− pExtension = P.try $ do

87− P.char '-'

88− cs <- P.many1 $ P.satisfy (\c -> isAscii c && isAlphaNum c)

89− let lcs = length cs

90− guard $ lcs >= 2 && lcs <= 8

91− return $ T.toLower $ T.pack cs

92− pPrivateUse = P.try $ do

93− P.char '-'

94− P.char 'x'

95− P.char '-'

96− cs <- P.many1 $ P.satisfy (\c -> isAscii c && isAlphaNum c)

97− guard $ not (null cs) && length cs <= 8

98− let var = "x-" ++ cs

99− return $ T.toLower $ T.pack var

src/Text/Pandoc/Citeproc.hs+2−8

@@ -18,7 +18,6 @@ import Text.Pandoc.Citeproc.CslJson (cslJsonToReferences)

1818import Text.Pandoc.Citeproc.BibTeX (readBibtexString, Variant(..))

1919import Text.Pandoc.Citeproc.MetaValue (metaValueToReference, metaValueToText)

2020import Text.Pandoc.Readers.Markdown (yamlToRefs)

21−import qualified Text.Pandoc.BCP47 as BCP47

2221import Text.Pandoc.Builder (Inlines, Many(..), deleteMeta, setMeta)

2322import qualified Text.Pandoc.Builder as B

2423import Text.Pandoc.Definition as Pandoc

@@ -630,13 +629,8 @@ removeFinalPeriod ils =

630629

631630bcp47LangToIETF :: PandocMonad m => Text -> m (Maybe Lang)

632631bcp47LangToIETF bcplang =

633− case BCP47.parseBCP47 bcplang of

632+ case parseLang bcplang of

634633 Left _ -> do

635634 report $ InvalidLang bcplang

636635 return Nothing

637− Right lang ->

638− return $ Just

639− $ Lang (BCP47.langLanguage lang)

640− (if T.null (BCP47.langRegion lang)

641− then Nothing

642− else Just (BCP47.langRegion lang))

636+ Right lang -> return $ Just lang

src/Text/Pandoc/Citeproc/BibTeX.hs+11−9

@@ -205,10 +205,13 @@ writeBibtexString opts variant mblang ref =

205205 [ (", " <>) <$> nameGiven name,

206206 nameDroppingParticle name ]

207207

208− mblang' = (parseLang <$> getVariableAsText "language") <|> mblang

208+ mblang' = case getVariableAsText "language" of

209+ Just l -> either (const Nothing) Just $ parseLang l

210+ Nothing -> mblang

209211

210212 titlecase = case mblang' of

211− Just (Lang "en" _) -> titlecase'

213+ Just lang | langLanguage lang == "en"

214+ -> titlecase'

212215 Nothing -> titlecase'

213216 _ ->

214217 case variant of

@@ -331,7 +334,7 @@ writeBibtexString opts variant mblang ref =

331334 renderFields = mconcat . intersperse ("," <> cr) . mapMaybe renderField

332335

333336defaultLang :: Lang

334−defaultLang = Lang "en" (Just "US")

337+defaultLang = Lang "en" Nothing (Just "US") [] [] []

335338

336339-- a map of bibtex "string" macros

337340type StringMap = Map.Map Text Text

@@ -351,9 +354,7 @@ itemToReference locale variant item = do

351354 bib item $ do

352355 let lang = fromMaybe defaultLang $ localeLanguage locale

353356 modify $ \st -> st{ localeLang = lang,

354− untitlecase = case lang of

355− (Lang "en" _) -> True

356− _ -> False }

357+ untitlecase = langLanguage lang == "en" }

357358

358359 id' <- asks identifier

359360 otherIds <- (Just <$> getRawField "ids")

@@ -711,7 +712,7 @@ itemToReference locale variant item = do

711712

712713

713714bib :: Item -> Bib a -> BibParser a

714−bib entry m = fst <$> evalRWST m entry (BibState True (Lang "en" (Just "US")))

715+bib entry m = fst <$> evalRWST m entry (BibState True defaultLang)

715716

716717resolveCrossRefs :: Variant -> [Item] -> [Item]

717718resolveCrossRefs variant entries =

@@ -1456,8 +1457,9 @@ resolveKey lang ils = Walk.walk go ils

14561457 go x = x

14571458

14581459resolveKey' :: Lang -> Text -> Text

1459−resolveKey' lang@(Lang l _) k =

1460− case Map.lookup l biblatexStringMap >>= Map.lookup (T.toLower k) of

1460+resolveKey' lang k =

1461+ case Map.lookup (langLanguage lang) biblatexStringMap >>=

1462+ Map.lookup (T.toLower k) of

14611463 Nothing -> k

14621464 Just (x, _) -> either (const k) stringify $ parseLaTeX lang x

14631465

src/Text/Pandoc/Citeproc/Data.hs+6−6

@@ -21,12 +21,12 @@ biblatexStringMap :: M.Map Text (M.Map Text (Text, Text))

2121biblatexStringMap = foldr go mempty biblatexLocalizations

2222 where

2323 go (fp, bs) =

24− let Lang lang _ _ _ _ _ = parseLang

25− (toIETF $ T.takeWhile (/= '.') $ T.pack fp)

26− ls = T.lines $ TE.decodeUtf8 bs

27− in if length ls > 4

28− then M.insert lang (toStringMap $ map (T.splitOn "|") ls)

29− else id

24+ let ls = T.lines $ TE.decodeUtf8 bs

25+ in case parseLang (toIETF $ T.takeWhile (/= '.') $ T.pack fp) of

26+ Right lang | length ls > 4

27+ -> M.insert (langLanguage lang)

28+ (toStringMap $ map (T.splitOn "|") ls)

29+ _ -> id

3030 toStringMap = foldr go' mempty

3131 go' [term, x, y] = M.insert term (x, y)

3232 go' _ = id

src/Text/Pandoc/Class/CommonState.hs+1−1

@@ -19,7 +19,7 @@ where

1919

2020import Data.Default (Default (def))

2121import Data.Text (Text)

22−import Text.Pandoc.BCP47 (Lang)

22+import UnicodeCollation.Lang (Lang)

2323import Text.Pandoc.MediaBag (MediaBag)

2424import Text.Pandoc.Logging (LogMessage, Verbosity (WARNING))

2525import Text.Pandoc.Translations (Translations)

src/Text/Pandoc/Class/PandocMonad.hs+2−2

@@ -70,7 +70,7 @@ import Network.URI ( escapeURIString, nonStrictRelativeTo,

7070import System.FilePath ((</>), (<.>), takeExtension, dropExtension,

7171 isRelative, splitDirectories)

7272import System.Random (StdGen)

73−import Text.Pandoc.BCP47 (Lang(..), parseBCP47, renderLang)

73+import UnicodeCollation.Lang (Lang(..), parseLang, renderLang)

7474import Text.Pandoc.Class.CommonState (CommonState (..))

7575import Text.Pandoc.Definition

7676import Text.Pandoc.Error

@@ -285,7 +285,7 @@ readFileFromDirs (d:ds) f = catchError

285285toLang :: PandocMonad m => Maybe T.Text -> m (Maybe Lang)

286286toLang Nothing = return Nothing

287287toLang (Just s) =

288− case parseBCP47 s of

288+ case parseLang s of

289289 Left _ -> do

290290 report $ InvalidLang s

291291 return Nothing

src/Text/Pandoc/Readers/BibTeX.hs+6−3

@@ -48,11 +48,14 @@ readBibLaTeX = readBibTeX' BibTeX.Biblatex

4848

4949readBibTeX' :: PandocMonad m => Variant -> ReaderOptions -> Text -> m Pandoc

5050readBibTeX' variant _opts t = do

51− lang <- maybe (Lang "en" (Just "US")) parseLang

52− <$> lookupEnv "LANG"

51+ mblangEnv <- lookupEnv "LANG"

52+ let defaultLang = Lang "en" Nothing (Just "US") [] [] []

53+ let lang = case mblangEnv of

54+ Nothing -> defaultLang

55+ Just l -> either (const defaultLang) id $ parseLang l

5356 locale <- case getLocale lang of

5457 Left e ->

55− case getLocale (Lang "en" (Just "US")) of

58+ case getLocale (Lang "en" Nothing (Just "US") [] [] []) of

5659 Right l -> return l

5760 Left _ -> throwError $ PandocCiteprocError e

5861 Right l -> return l

src/Text/Pandoc/Readers/LaTeX.hs+1−1

@@ -33,7 +33,7 @@ import qualified Data.Set as Set

3333import Data.Text (Text)

3434import qualified Data.Text as T

3535import System.FilePath (addExtension, replaceExtension, takeExtension)

36−import Text.Pandoc.BCP47 (renderLang)

36+import UnicodeCollation.Lang (renderLang)

3737import Text.Pandoc.Builder as B

3838import Text.Pandoc.Class.PandocPure (PandocPure)

3939import Text.Pandoc.Class.PandocMonad (PandocMonad (..), getResourcePath,

src/Text/Pandoc/Readers/LaTeX/Lang.hs+122−119

@@ -23,7 +23,7 @@ import qualified Data.Map as M

2323import Data.Text (Text)

2424import qualified Data.Text as T

2525import Text.Pandoc.Shared (extractSpaces)

26−import Text.Pandoc.BCP47 (Lang(..), renderLang)

26+import UnicodeCollation.Lang (Lang(..), renderLang)

2727import Text.Pandoc.Class (PandocMonad(..), setTranslations)

2828import Text.Pandoc.Readers.LaTeX.Parsing

2929import Text.Pandoc.Parsing (updateState, option, getState, QuoteContext(..),

@@ -99,133 +99,136 @@ setDefaultLanguage = do

9999polyglossiaLangToBCP47 :: M.Map T.Text (T.Text -> Lang)

100100polyglossiaLangToBCP47 = M.fromList

101101 [ ("arabic", \o -> case T.filter (/=' ') o of

102− "locale=algeria" -> Lang "ar" "" "DZ" []

103− "locale=mashriq" -> Lang "ar" "" "SY" []

104− "locale=libya" -> Lang "ar" "" "LY" []

105− "locale=morocco" -> Lang "ar" "" "MA" []

106− "locale=mauritania" -> Lang "ar" "" "MR" []

107− "locale=tunisia" -> Lang "ar" "" "TN" []

108− _ -> Lang "ar" "" "" [])

102+ "locale=algeria" -> Lang "ar" Nothing (Just "DZ") [] [] []

103+ "locale=mashriq" -> Lang "ar" Nothing (Just "SY") [] [] []

104+ "locale=libya" -> Lang "ar" Nothing (Just "LY") [] [] []

105+ "locale=morocco" -> Lang "ar" Nothing (Just "MA") [] [] []

106+ "locale=mauritania" -> Lang "ar" Nothing (Just "MR") [] [] []

107+ "locale=tunisia" -> Lang "ar" Nothing (Just "TN") [] [] []

108+ _ -> Lang "ar" Nothing (Just "") [] [] [])

109109 , ("german", \o -> case T.filter (/=' ') o of

110− "spelling=old" -> Lang "de" "" "DE" ["1901"]

110+ "spelling=old" -> Lang "de" Nothing (Just "DE") ["1901"] [] []

111111 "variant=austrian,spelling=old"

112− -> Lang "de" "" "AT" ["1901"]

113− "variant=austrian" -> Lang "de" "" "AT" []

112+ -> Lang "de" Nothing (Just "AT") ["1901"] [] []

113+ "variant=austrian" -> Lang "de" Nothing (Just "AT") [] [] []

114114 "variant=swiss,spelling=old"

115− -> Lang "de" "" "CH" ["1901"]

116− "variant=swiss" -> Lang "de" "" "CH" []

117− _ -> Lang "de" "" "" [])

118− , ("lsorbian", \_ -> Lang "dsb" "" "" [])

115+ -> Lang "de" Nothing (Just "CH") ["1901"] [] []

116+ "variant=swiss" -> Lang "de" Nothing (Just "CH") [] [] []

117+ _ -> Lang "de" Nothing Nothing [] [] [])

118+ , ("lsorbian", \_ -> Lang "dsb" Nothing Nothing [] [] [])

119119 , ("greek", \o -> case T.filter (/=' ') o of

120− "variant=poly" -> Lang "el" "" "polyton" []

121− "variant=ancient" -> Lang "grc" "" "" []

122− _ -> Lang "el" "" "" [])

120+ "variant=poly" -> Lang "el" Nothing (Just "polyton") [] [] []

121+ "variant=ancient" -> Lang "grc" Nothing Nothing [] [] []

122+ _ -> Lang "el" Nothing Nothing [] [] [])

123123 , ("english", \o -> case T.filter (/=' ') o of

124− "variant=australian" -> Lang "en" "" "AU" []

125− "variant=canadian" -> Lang "en" "" "CA" []

126− "variant=british" -> Lang "en" "" "GB" []

127− "variant=newzealand" -> Lang "en" "" "NZ" []

128− "variant=american" -> Lang "en" "" "US" []

129− _ -> Lang "en" "" "" [])

130− , ("usorbian", \_ -> Lang "hsb" "" "" [])

124+ "variant=australian" -> Lang "en" Nothing (Just "AU") [] [] []

125+ "variant=canadian" -> Lang "en" Nothing (Just "CA") [] [] []

126+ "variant=british" -> Lang "en" Nothing (Just "GB") [] [] []

127+ "variant=newzealand" -> Lang "en" Nothing (Just "NZ") [] [] []

128+ "variant=american" -> Lang "en" Nothing (Just "US") [] [] []

129+ _ -> Lang "en" Nothing (Just "") [] [] [])

130+ , ("usorbian", \_ -> Lang "hsb" Nothing Nothing [] [] [])

131131 , ("latin", \o -> case T.filter (/=' ') o of

132− "variant=classic" -> Lang "la" "" "" ["x-classic"]

133− _ -> Lang "la" "" "" [])

134− , ("slovenian", \_ -> Lang "sl" "" "" [])

135− , ("serbianc", \_ -> Lang "sr" "cyrl" "" [])

136− , ("pinyin", \_ -> Lang "zh" "Latn" "" ["pinyin"])

137− , ("afrikaans", \_ -> Lang "af" "" "" [])

138− , ("amharic", \_ -> Lang "am" "" "" [])

139− , ("assamese", \_ -> Lang "as" "" "" [])

140− , ("asturian", \_ -> Lang "ast" "" "" [])

141− , ("bulgarian", \_ -> Lang "bg" "" "" [])

142− , ("bengali", \_ -> Lang "bn" "" "" [])

143− , ("tibetan", \_ -> Lang "bo" "" "" [])

144− , ("breton", \_ -> Lang "br" "" "" [])

145− , ("catalan", \_ -> Lang "ca" "" "" [])

146− , ("welsh", \_ -> Lang "cy" "" "" [])

147− , ("czech", \_ -> Lang "cs" "" "" [])

148− , ("coptic", \_ -> Lang "cop" "" "" [])

149− , ("danish", \_ -> Lang "da" "" "" [])

150− , ("divehi", \_ -> Lang "dv" "" "" [])

151− , ("esperanto", \_ -> Lang "eo" "" "" [])

152− , ("spanish", \_ -> Lang "es" "" "" [])

153− , ("estonian", \_ -> Lang "et" "" "" [])

154− , ("basque", \_ -> Lang "eu" "" "" [])

155− , ("farsi", \_ -> Lang "fa" "" "" [])

156− , ("finnish", \_ -> Lang "fi" "" "" [])

157− , ("french", \_ -> Lang "fr" "" "" [])

158− , ("friulan", \_ -> Lang "fur" "" "" [])

159− , ("irish", \_ -> Lang "ga" "" "" [])

160− , ("scottish", \_ -> Lang "gd" "" "" [])

161− , ("ethiopic", \_ -> Lang "gez" "" "" [])

162− , ("galician", \_ -> Lang "gl" "" "" [])

163− , ("hebrew", \_ -> Lang "he" "" "" [])

164− , ("hindi", \_ -> Lang "hi" "" "" [])

165− , ("croatian", \_ -> Lang "hr" "" "" [])

166− , ("magyar", \_ -> Lang "hu" "" "" [])

167− , ("armenian", \_ -> Lang "hy" "" "" [])

168− , ("interlingua", \_ -> Lang "ia" "" "" [])

169− , ("indonesian", \_ -> Lang "id" "" "" [])

170− , ("icelandic", \_ -> Lang "is" "" "" [])

171− , ("italian", \_ -> Lang "it" "" "" [])

172− , ("japanese", \_ -> Lang "jp" "" "" [])

173− , ("khmer", \_ -> Lang "km" "" "" [])

174− , ("kurmanji", \_ -> Lang "kmr" "" "" [])

175− , ("kannada", \_ -> Lang "kn" "" "" [])

176− , ("korean", \_ -> Lang "ko" "" "" [])

177− , ("lao", \_ -> Lang "lo" "" "" [])

178− , ("lithuanian", \_ -> Lang "lt" "" "" [])

179− , ("latvian", \_ -> Lang "lv" "" "" [])

180− , ("malayalam", \_ -> Lang "ml" "" "" [])

181− , ("mongolian", \_ -> Lang "mn" "" "" [])

182− , ("marathi", \_ -> Lang "mr" "" "" [])

183− , ("dutch", \_ -> Lang "nl" "" "" [])

184− , ("nynorsk", \_ -> Lang "nn" "" "" [])

185− , ("norsk", \_ -> Lang "no" "" "" [])

186− , ("nko", \_ -> Lang "nqo" "" "" [])

187− , ("occitan", \_ -> Lang "oc" "" "" [])

188− , ("panjabi", \_ -> Lang "pa" "" "" [])

189− , ("polish", \_ -> Lang "pl" "" "" [])

190− , ("piedmontese", \_ -> Lang "pms" "" "" [])

191− , ("portuguese", \_ -> Lang "pt" "" "" [])

192− , ("romansh", \_ -> Lang "rm" "" "" [])

193− , ("romanian", \_ -> Lang "ro" "" "" [])

194− , ("russian", \_ -> Lang "ru" "" "" [])

195− , ("sanskrit", \_ -> Lang "sa" "" "" [])

196− , ("samin", \_ -> Lang "se" "" "" [])

197− , ("slovak", \_ -> Lang "sk" "" "" [])

198− , ("albanian", \_ -> Lang "sq" "" "" [])

199− , ("serbian", \_ -> Lang "sr" "" "" [])

200− , ("swedish", \_ -> Lang "sv" "" "" [])

201− , ("syriac", \_ -> Lang "syr" "" "" [])

202− , ("tamil", \_ -> Lang "ta" "" "" [])

203− , ("telugu", \_ -> Lang "te" "" "" [])

204− , ("thai", \_ -> Lang "th" "" "" [])

205− , ("turkmen", \_ -> Lang "tk" "" "" [])

206− , ("turkish", \_ -> Lang "tr" "" "" [])

207− , ("ukrainian", \_ -> Lang "uk" "" "" [])

208− , ("urdu", \_ -> Lang "ur" "" "" [])

209− , ("vietnamese", \_ -> Lang "vi" "" "" [])

132+ "variant=classic" -> Lang "la" Nothing Nothing ["x-classic"] [] []

133+ _ -> Lang "la" Nothing Nothing [] [] [])

134+ , ("slovenian", \_ -> Lang "sl" Nothing Nothing [] [] [])

135+ , ("serbianc", \_ -> Lang "sr" (Just "Cyrl") Nothing [] [] [])

136+ , ("pinyin", \_ -> Lang "zh" (Just "Latn") Nothing ["pinyin"] [] [])

137+ , ("afrikaans", \_ -> simpleLang "af")

138+ , ("amharic", \_ -> simpleLang "am")

139+ , ("assamese", \_ -> simpleLang "as")

140+ , ("asturian", \_ -> simpleLang "ast")

141+ , ("bulgarian", \_ -> simpleLang "bg")

142+ , ("bengali", \_ -> simpleLang "bn")

143+ , ("tibetan", \_ -> simpleLang "bo")

144+ , ("breton", \_ -> simpleLang "br")

145+ , ("catalan", \_ -> simpleLang "ca")

146+ , ("welsh", \_ -> simpleLang "cy")

147+ , ("czech", \_ -> simpleLang "cs")

148+ , ("coptic", \_ -> simpleLang "cop")

149+ , ("danish", \_ -> simpleLang "da")

150+ , ("divehi", \_ -> simpleLang "dv")

151+ , ("esperanto", \_ -> simpleLang "eo")

152+ , ("spanish", \_ -> simpleLang "es")

153+ , ("estonian", \_ -> simpleLang "et")

154+ , ("basque", \_ -> simpleLang "eu")

155+ , ("farsi", \_ -> simpleLang "fa")

156+ , ("finnish", \_ -> simpleLang "fi")

157+ , ("french", \_ -> simpleLang "fr")

158+ , ("friulan", \_ -> simpleLang "fur")

159+ , ("irish", \_ -> simpleLang "ga")

160+ , ("scottish", \_ -> simpleLang "gd")

161+ , ("ethiopic", \_ -> simpleLang "gez")

162+ , ("galician", \_ -> simpleLang "gl")

163+ , ("hebrew", \_ -> simpleLang "he")

164+ , ("hindi", \_ -> simpleLang "hi")

165+ , ("croatian", \_ -> simpleLang "hr")

166+ , ("magyar", \_ -> simpleLang "hu")

167+ , ("armenian", \_ -> simpleLang "hy")

168+ , ("interlingua", \_ -> simpleLang "ia")

169+ , ("indonesian", \_ -> simpleLang "id")

170+ , ("icelandic", \_ -> simpleLang "is")

171+ , ("italian", \_ -> simpleLang "it")

172+ , ("japanese", \_ -> simpleLang "jp")

173+ , ("khmer", \_ -> simpleLang "km")

174+ , ("kurmanji", \_ -> simpleLang "kmr")

175+ , ("kannada", \_ -> simpleLang "kn")

176+ , ("korean", \_ -> simpleLang "ko")

177+ , ("lao", \_ -> simpleLang "lo")

178+ , ("lithuanian", \_ -> simpleLang "lt")

179+ , ("latvian", \_ -> simpleLang "lv")

180+ , ("malayalam", \_ -> simpleLang "ml")

181+ , ("mongolian", \_ -> simpleLang "mn")

182+ , ("marathi", \_ -> simpleLang "mr")

183+ , ("dutch", \_ -> simpleLang "nl")

184+ , ("nynorsk", \_ -> simpleLang "nn")

185+ , ("norsk", \_ -> simpleLang "no")

186+ , ("nko", \_ -> simpleLang "nqo")

187+ , ("occitan", \_ -> simpleLang "oc")

188+ , ("panjabi", \_ -> simpleLang "pa")

189+ , ("polish", \_ -> simpleLang "pl")

190+ , ("piedmontese", \_ -> simpleLang "pms")

191+ , ("portuguese", \_ -> simpleLang "pt")

192+ , ("romansh", \_ -> simpleLang "rm")

193+ , ("romanian", \_ -> simpleLang "ro")

194+ , ("russian", \_ -> simpleLang "ru")

195+ , ("sanskrit", \_ -> simpleLang "sa")

196+ , ("samin", \_ -> simpleLang "se")

197+ , ("slovak", \_ -> simpleLang "sk")

198+ , ("albanian", \_ -> simpleLang "sq")

199+ , ("serbian", \_ -> simpleLang "sr")

200+ , ("swedish", \_ -> simpleLang "sv")

201+ , ("syriac", \_ -> simpleLang "syr")

202+ , ("tamil", \_ -> simpleLang "ta")

203+ , ("telugu", \_ -> simpleLang "te")

204+ , ("thai", \_ -> simpleLang "th")

205+ , ("turkmen", \_ -> simpleLang "tk")

206+ , ("turkish", \_ -> simpleLang "tr")

207+ , ("ukrainian", \_ -> simpleLang "uk")

208+ , ("urdu", \_ -> simpleLang "ur")

209+ , ("vietnamese", \_ -> simpleLang "vi")

210210 ]

211211

212+simpleLang :: Text -> Lang

213+simpleLang l = Lang l Nothing Nothing [] [] []

214+

212215babelLangToBCP47 :: T.Text -> Maybe Lang

213216babelLangToBCP47 s =

214217 case s of

215− "austrian" -> Just $ Lang "de" "" "AT" ["1901"]

216− "naustrian" -> Just $ Lang "de" "" "AT" []

217− "swissgerman" -> Just $ Lang "de" "" "CH" ["1901"]

218− "nswissgerman" -> Just $ Lang "de" "" "CH" []

219− "german" -> Just $ Lang "de" "" "DE" ["1901"]

220− "ngerman" -> Just $ Lang "de" "" "DE" []

221− "lowersorbian" -> Just $ Lang "dsb" "" "" []

222− "uppersorbian" -> Just $ Lang "hsb" "" "" []

223− "polutonikogreek" -> Just $ Lang "el" "" "" ["polyton"]

224− "slovene" -> Just $ Lang "sl" "" "" []

225− "australian" -> Just $ Lang "en" "" "AU" []

226− "canadian" -> Just $ Lang "en" "" "CA" []

227− "british" -> Just $ Lang "en" "" "GB" []

228− "newzealand" -> Just $ Lang "en" "" "NZ" []

229− "american" -> Just $ Lang "en" "" "US" []

230− "classiclatin" -> Just $ Lang "la" "" "" ["x-classic"]

218+ "austrian" -> Just $ Lang "de" Nothing (Just "AT") ["1901"] [] []

219+ "naustrian" -> Just $ Lang "de" Nothing (Just "AT") [] [] []

220+ "swissgerman" -> Just $ Lang "de" Nothing (Just "CH") ["1901"] [] []

221+ "nswissgerman" -> Just $ Lang "de" Nothing (Just "CH") [] [] []

222+ "german" -> Just $ Lang "de" Nothing (Just "DE") ["1901"] [] []

223+ "ngerman" -> Just $ Lang "de" Nothing (Just "DE") [] [] []

224+ "lowersorbian" -> Just $ Lang "dsb" Nothing Nothing [] [] []

225+ "uppersorbian" -> Just $ Lang "hsb" Nothing Nothing [] [] []

226+ "polutonikogreek" -> Just $ Lang "el" Nothing Nothing ["polyton"] [] []

227+ "slovene" -> Just $ simpleLang "sl"

228+ "australian" -> Just $ Lang "en" Nothing (Just "AU") [] [] []

229+ "canadian" -> Just $ Lang "en" Nothing (Just "CA") [] [] []

230+ "british" -> Just $ Lang "en" Nothing (Just "GB") [] [] []

231+ "newzealand" -> Just $ Lang "en" Nothing (Just "NZ") [] [] []

232+ "american" -> Just $ Lang "en" Nothing (Just "US") [] [] []

233+ "classiclatin" -> Just $ Lang "la" Nothing Nothing ["x-classic"] [] []

231234 _ -> ($ "") <$> M.lookup s polyglossiaLangToBCP47

src/Text/Pandoc/Writers/BibTeX.hs+1−1

@@ -43,7 +43,7 @@ writeBibTeX' :: PandocMonad m => Variant -> WriterOptions -> Pandoc -> m Text

4343writeBibTeX' variant opts (Pandoc meta _) = do

4444 let mblang = case lookupMetaString "lang" meta of

4545 "" -> Nothing

46− t -> Just $ parseLang t

46+ t -> either (const Nothing) Just $ parseLang t

4747 let refs = case lookupMeta "references" meta of

4848 Just (MetaList xs) -> mapMaybe metaValueToReference xs

4949 _ -> []

src/Text/Pandoc/Writers/ConTeXt.hs+24−24

@@ -21,7 +21,7 @@ import Data.Maybe (mapMaybe)

2121import Data.Text (Text)

2222import qualified Data.Text as T

2323import Network.URI (unEscapeString)

24−import Text.Pandoc.BCP47

24+import UnicodeCollation.Lang (Lang(..))

2525import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

2626import Text.Pandoc.Definition

2727import Text.Pandoc.ImageSize

@@ -555,26 +555,26 @@ fromBCP47 mbs = fromBCP47' <$> toLang mbs

555555-- https://tools.ietf.org/html/bcp47#section-2.1

556556-- http://wiki.contextgarden.net/Language_Codes

557557fromBCP47' :: Maybe Lang -> Maybe Text

558−fromBCP47' (Just (Lang "ar" _ "SY" _) ) = Just "ar-sy"

559−fromBCP47' (Just (Lang "ar" _ "IQ" _) ) = Just "ar-iq"

560−fromBCP47' (Just (Lang "ar" _ "JO" _) ) = Just "ar-jo"

561−fromBCP47' (Just (Lang "ar" _ "LB" _) ) = Just "ar-lb"

562−fromBCP47' (Just (Lang "ar" _ "DZ" _) ) = Just "ar-dz"

563−fromBCP47' (Just (Lang "ar" _ "MA" _) ) = Just "ar-ma"

564−fromBCP47' (Just (Lang "de" _ _ ["1901"]) ) = Just "deo"

565−fromBCP47' (Just (Lang "de" _ "DE" _) ) = Just "de-de"

566−fromBCP47' (Just (Lang "de" _ "AT" _) ) = Just "de-at"

567−fromBCP47' (Just (Lang "de" _ "CH" _) ) = Just "de-ch"

568−fromBCP47' (Just (Lang "el" _ _ ["poly"]) ) = Just "agr"

569−fromBCP47' (Just (Lang "en" _ "US" _) ) = Just "en-us"

570−fromBCP47' (Just (Lang "en" _ "GB" _) ) = Just "en-gb"

571−fromBCP47' (Just (Lang "grc"_ _ _) ) = Just "agr"

572−fromBCP47' (Just (Lang "el" _ _ _) ) = Just "gr"

573−fromBCP47' (Just (Lang "eu" _ _ _) ) = Just "ba"

574−fromBCP47' (Just (Lang "he" _ _ _) ) = Just "il"

575−fromBCP47' (Just (Lang "jp" _ _ _) ) = Just "ja"

576−fromBCP47' (Just (Lang "uk" _ _ _) ) = Just "ua"

577−fromBCP47' (Just (Lang "vi" _ _ _) ) = Just "vn"

578−fromBCP47' (Just (Lang "zh" _ _ _) ) = Just "cn"

579−fromBCP47' (Just (Lang l _ _ _) ) = Just l

580−fromBCP47' Nothing = Nothing

558+fromBCP47' (Just (Lang "ar" _ (Just "SY") _ _ _)) = Just "ar-sy"

559+fromBCP47' (Just (Lang "ar" _ (Just "IQ") _ _ _)) = Just "ar-iq"

560+fromBCP47' (Just (Lang "ar" _ (Just "JO") _ _ _)) = Just "ar-jo"

561+fromBCP47' (Just (Lang "ar" _ (Just "LB") _ _ _)) = Just "ar-lb"

562+fromBCP47' (Just (Lang "ar" _ (Just "DZ") _ _ _)) = Just "ar-dz"

563+fromBCP47' (Just (Lang "ar" _ (Just "MA") _ _ _)) = Just "ar-ma"

564+fromBCP47' (Just (Lang "de" _ _ ["1901"] _ _)) = Just "deo"

565+fromBCP47' (Just (Lang "de" _ (Just "DE") _ _ _)) = Just "de-de"

566+fromBCP47' (Just (Lang "de" _ (Just "AT") _ _ _)) = Just "de-at"

567+fromBCP47' (Just (Lang "de" _ (Just "CH") _ _ _)) = Just "de-ch"

568+fromBCP47' (Just (Lang "el" _ _ ["poly"] _ _)) = Just "agr"

569+fromBCP47' (Just (Lang "en" _ (Just "US") _ _ _)) = Just "en-us"

570+fromBCP47' (Just (Lang "en" _ (Just "GB") _ _ _)) = Just "en-gb"

571+fromBCP47' (Just (Lang "grc"_ _ _ _ _)) = Just "agr"

572+fromBCP47' (Just (Lang "el" _ _ _ _ _)) = Just "gr"

573+fromBCP47' (Just (Lang "eu" _ _ _ _ _)) = Just "ba"

574+fromBCP47' (Just (Lang "he" _ _ _ _ _)) = Just "il"

575+fromBCP47' (Just (Lang "jp" _ _ _ _ _)) = Just "ja"

576+fromBCP47' (Just (Lang "uk" _ _ _ _ _)) = Just "ua"

577+fromBCP47' (Just (Lang "vi" _ _ _ _ _)) = Just "vn"

578+fromBCP47' (Just (Lang "zh" _ _ _ _ _)) = Just "cn"

579+fromBCP47' (Just (Lang l _ _ _ _ _)) = Just l

580+fromBCP47' Nothing = Nothing

src/Text/Pandoc/Writers/CslJson.hs+4−3

@@ -34,15 +34,16 @@ import Control.Monad.Identity

3434import Citeproc.Locale (getLocale)

3535import Citeproc.CslJson

3636import Text.Pandoc.Options (WriterOptions)

37−import Data.Maybe (mapMaybe)

37+import Data.Maybe (mapMaybe, fromMaybe)

3838import Data.Aeson.Encode.Pretty (Config (..), Indent (Spaces),

3939 NumberFormat (Generic),

4040 defConfig, encodePretty')

4141

4242writeCslJson :: PandocMonad m => WriterOptions -> Pandoc -> m Text

4343writeCslJson _opts (Pandoc meta _) = do

44− let lang = maybe (Lang "en" (Just "US")) parseLang

45− (lookupMeta "lang" meta >>= metaValueToText)

44+ let lang = fromMaybe (Lang "en" Nothing (Just "US") [] [] [])

45+ (lookupMeta "lang" meta >>= metaValueToText >>=

46+ either (const Nothing) Just . parseLang)

4647 locale <- case getLocale lang of

4748 Left e -> throwError $ PandocCiteprocError e

4849 Right l -> return l

src/Text/Pandoc/Writers/Docx.hs+1−1

@@ -36,7 +36,7 @@ import qualified Data.Text.Lazy as TL

3636import Data.Time.Clock.POSIX

3737import Data.Digest.Pure.SHA (sha1, showDigest)

3838import Skylighting

39−import Text.Pandoc.BCP47 (getLang, renderLang)

39+import UnicodeCollation.Lang (renderLang)

4040import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

4141import qualified Text.Pandoc.Class.PandocMonad as P

4242import Data.Time

src/Text/Pandoc/Writers/LaTeX.hs+1−1

@@ -29,7 +29,7 @@ import qualified Data.Text as T

2929import Network.URI (unEscapeString)

3030import Text.DocTemplates (FromContext(lookupContext), renderTemplate,

3131 Val(..), Context(..))

32−import Text.Pandoc.BCP47 (Lang (..), getLang, renderLang)

32+import UnicodeCollation.Lang (Lang (..), renderLang)

3333import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

3434import Text.Pandoc.Definition

3535import Text.Pandoc.Highlighting (formatLaTeXBlock, formatLaTeXInline, highlight,

src/Text/Pandoc/Writers/LaTeX/Lang.hs+4−4

@@ -46,7 +46,7 @@ toPolyglossia (Lang "de" _ (Just "AT") vars _ _)

4646toPolyglossia (Lang "de" _ (Just "AT") _ _ _) = ("german", "variant=austrian")

4747toPolyglossia (Lang "de" _ (Just "CH") vars _ _)

4848 | "1901" `elem` vars = ("german", "variant=swiss, spelling=old")

49−toPolyglossia (Lang "de" _ (Just "CH") _ _ _ _) = ("german", "variant=swiss")

49+toPolyglossia (Lang "de" _ (Just "CH") _ _ _) = ("german", "variant=swiss")

5050toPolyglossia (Lang "de" _ _ _ _ _) = ("german", "")

5151toPolyglossia (Lang "dsb" _ _ _ _ _) = ("lsorbian", "")

5252toPolyglossia (Lang "el" _ _ vars _ _)

@@ -61,9 +61,9 @@ toPolyglossia (Lang "grc" _ _ _ _ _) = ("greek", "variant=ancient")

6161toPolyglossia (Lang "hsb" _ _ _ _ _) = ("usorbian", "")

6262toPolyglossia (Lang "la" _ _ vars _ _)

6363 | "x-classic" `elem` vars = ("latin", "variant=classic")

64−toPolyglossia (Lang "pt" _ "BR" _ _ _) = ("portuguese", "variant=brazilian")

64+toPolyglossia (Lang "pt" _ (Just "BR") _ _ _) = ("portuguese", "variant=brazilian")

6565toPolyglossia (Lang "sl" _ _ _ _ _) = ("slovenian", "")

66−toPolyglossia x = (commonFromBcp47 x, "")

66+toPolyglossia x = (commonFromBcp47 x, "")

6767

6868-- Takes a list of the constituents of a BCP47 language code and

6969-- converts it to a Babel language string.

@@ -81,7 +81,7 @@ toBabel (Lang "de" _ _ vars _ _)

8181 | "1901" `elem` vars = "german"

8282 | otherwise = "ngerman"

8383toBabel (Lang "dsb" _ _ _ _ _) = "lowersorbian"

84−toBabel (Lang "el" _ _ vars)

84+toBabel (Lang "el" _ _ vars _ _)

8585 | "polyton" `elem` vars = "polutonikogreek"

8686toBabel (Lang "en" _ (Just "AU") _ _ _) = "australian"

8787toBabel (Lang "en" _ (Just "CA") _ _ _) = "canadian"

src/Text/Pandoc/Writers/ODT.hs+4−3

@@ -16,6 +16,7 @@ import Codec.Archive.Zip

1616import Control.Monad.Except (catchError, throwError)

1717import Control.Monad.State.Strict

1818import qualified Data.ByteString.Lazy as B

19+import Data.Maybe (fromMaybe)

1920import Data.Generics (everywhere', mkT)

2021import Data.List (isPrefixOf)

2122import qualified Data.Map as Map

@@ -23,7 +24,7 @@ import qualified Data.Text as T

2324import qualified Data.Text.Lazy as TL

2425import Data.Time

2526import System.FilePath (takeDirectory, takeExtension, (<.>))

26−import Text.Pandoc.BCP47 (Lang (..), getLang, renderLang)

27+import UnicodeCollation.Lang (Lang (..), renderLang)

2728import Text.Pandoc.Class.PandocMonad (PandocMonad, report, toLang)

2829import qualified Text.Pandoc.Class.PandocMonad as P

2930import Text.Pandoc.Definition

@@ -35,7 +36,7 @@ import Text.Pandoc.Options (WrapOption (..), WriterOptions (..))

3536import Text.DocLayout

3637import Text.Pandoc.Shared (stringify, pandocVersion, tshow)

3738import Text.Pandoc.Writers.Shared (lookupMetaString, lookupMetaBlocks,

38− fixDisplayMath)

39+ fixDisplayMath, getLang)

3940import Text.Pandoc.UTF8 (fromStringLazy, fromTextLazy, toTextLazy)

4041import Text.Pandoc.Walk

4142import Text.Pandoc.Writers.OpenDocument (writeOpenDocument)

@@ -194,7 +195,7 @@ addLang lang = everywhere' (mkT updateLangAttr)

194195 where updateLangAttr (Attr n@(QName "language" _ (Just "fo")) _)

195196 = Attr n (langLanguage lang)

196197 updateLangAttr (Attr n@(QName "country" _ (Just "fo")) _)

197− = Attr n (langRegion lang)

198+ = Attr n (fromMaybe "" $ langRegion lang)

198199 updateLangAttr x = x

199200

200201-- | transform both Image and Math elements

src/Text/Pandoc/Writers/OpenDocument.hs+4−4

@@ -25,7 +25,7 @@ import Data.Ord (comparing)

2525import qualified Data.Set as Set

2626import Data.Text (Text)

2727import qualified Data.Text as T

28−import Text.Pandoc.BCP47 (Lang (..), parseBCP47)

28+import UnicodeCollation.Lang (Lang (..), parseLang)

2929import Text.Pandoc.Class.PandocMonad (PandocMonad, report, translateTerm,

3030 setTranslations, toLang)

3131import Text.Pandoc.Definition

@@ -236,7 +236,7 @@ handleSpaces s = case T.uncons s of

236236-- | Convert Pandoc document to string in OpenDocument format.

237237writeOpenDocument :: PandocMonad m => WriterOptions -> Pandoc -> m Text

238238writeOpenDocument opts (Pandoc meta blocks) = do

239− let defLang = Lang "en" "US" "" []

239+ let defLang = Lang "en" (Just "US") Nothing [] [] []

240240 lang <- case lookupMetaString "lang" meta of

241241 "" -> pure defLang

242242 s -> fromMaybe defLang <$> toLang (Just s)

@@ -893,7 +893,7 @@ textStyleAttr m s

893893 Map.insert "style:font-name-complex" "Courier New" $ m

894894 | Language lang <- s

895895 = Map.insert "fo:language" (langLanguage lang) .

896− Map.insert "fo:country" (langRegion lang) $ m

896+ maybe id (Map.insert "fo:country") (langRegion lang) $ m

897897 | otherwise = m

898898

899899withLangFromAttr :: PandocMonad m => Attr -> OD m a -> OD m a

@@ -901,7 +901,7 @@ withLangFromAttr (_,_,kvs) action =

901901 case lookup "lang" kvs of

902902 Nothing -> action

903903 Just l ->

904− case parseBCP47 l of

904+ case parseLang l of

905905 Right lang -> withTextStyle (Language lang) action

906906 Left _ -> do

907907 report $ InvalidLang l

src/Text/Pandoc/Writers/Shared.hs+1−1

@@ -149,7 +149,7 @@ defField field val (Context m) =

149149 f _newval oldval = oldval

150150

151151-- | Get the contents of the `lang` metadata field or variable.

152−getLang :: WriterOptions -> Meta -> Maybe Text

152+getLang :: WriterOptions -> Meta -> Maybe T.Text

153153getLang opts meta =

154154 case lookupContext "lang" (writerVariables opts) of

155155 Just s -> Just s