From 79078121840587c6f34080b6e4cdfc939f73e7d0 Mon Sep 17 00:00:00 2001 From: "leandro.guisandez@pgconocimiento.com" Date: Thu, 8 Apr 2021 14:27:17 -0700 Subject: [PATCH] Initialise Catalan language with Numeral Summary: Adds Catalan language and Numeral rules for it Reviewed By: haoxuany Differential Revision: D26518604 Pulled By: chessai fbshipit-source-id: e6b4b0ceb9b7931d086c732dd03fb5cbbe062d5b --- Duckling/Dimensions.hs | 2 + Duckling/Dimensions/CA.hs | 17 ++ Duckling/Locale.hs | 6 +- Duckling/Numeral/CA/Corpus.hs | 162 ++++++++++++ Duckling/Numeral/CA/Rules.hs | 300 +++++++++++++++++++++++ Duckling/Ranking/Classifiers.hs | 2 + Duckling/Ranking/Classifiers/CA_XX.hs | 22 ++ Duckling/Rules.hs | 4 + Duckling/Rules/CA.hs | 46 ++++ Duckling/Rules/EN.hs | 5 +- duckling.cabal | 6 + exe/Duckling/Ranking/Generate.hs | 111 +++++---- tests/Duckling/AmountOfMoney/EN/Tests.hs | 4 +- tests/Duckling/Numeral/CA/Tests.hs | 21 ++ tests/Duckling/Numeral/Tests.hs | 2 + tests/Duckling/Time/EN/Tests.hs | 4 +- 16 files changed, 652 insertions(+), 62 deletions(-) create mode 100644 Duckling/Dimensions/CA.hs create mode 100644 Duckling/Numeral/CA/Corpus.hs create mode 100644 Duckling/Numeral/CA/Rules.hs create mode 100644 Duckling/Ranking/Classifiers/CA_XX.hs create mode 100644 Duckling/Rules/CA.hs create mode 100644 tests/Duckling/Numeral/CA/Tests.hs diff --git a/Duckling/Dimensions.hs b/Duckling/Dimensions.hs index 2cdb6fe8..a1dbaa83 100644 --- a/Duckling/Dimensions.hs +++ b/Duckling/Dimensions.hs @@ -25,6 +25,7 @@ import qualified Duckling.Dimensions.AF as AFDimensions import qualified Duckling.Dimensions.AR as ARDimensions import qualified Duckling.Dimensions.BG as BGDimensions import qualified Duckling.Dimensions.BN as BNDimensions +import qualified Duckling.Dimensions.CA as CADimensions import qualified Duckling.Dimensions.CS as CSDimensions import qualified Duckling.Dimensions.DA as DADimensions import qualified Duckling.Dimensions.DE as DEDimensions @@ -105,6 +106,7 @@ langDimensions AF = AFDimensions.allDimensions langDimensions AR = ARDimensions.allDimensions langDimensions BG = BGDimensions.allDimensions langDimensions BN = BNDimensions.allDimensions +langDimensions CA = CADimensions.allDimensions langDimensions CS = CSDimensions.allDimensions langDimensions DA = DADimensions.allDimensions langDimensions DE = DEDimensions.allDimensions diff --git a/Duckling/Dimensions/CA.hs b/Duckling/Dimensions/CA.hs new file mode 100644 index 00000000..5ed26781 --- /dev/null +++ b/Duckling/Dimensions/CA.hs @@ -0,0 +1,17 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +module Duckling.Dimensions.CA + ( allDimensions + ) where + +import Duckling.Dimensions.Types + +allDimensions :: [Seal Dimension] +allDimensions = + [ Seal Numeral + ] diff --git a/Duckling/Locale.hs b/Duckling/Locale.hs index 751a7c1b..7ce111e4 100644 --- a/Duckling/Locale.hs +++ b/Duckling/Locale.hs @@ -15,7 +15,6 @@ module Duckling.Locale ( AU , BE , BZ - , CA , CL , CN , CO @@ -52,12 +51,14 @@ import qualified TextShow as TS import Duckling.Region hiding ( AR + , CA , ES , NL ) import qualified Duckling.Region as R ( Region ( AR + , CA , ES , NL ) @@ -70,6 +71,7 @@ data Lang | AR | BG | BN + | CA | CS | DA | DE @@ -141,7 +143,7 @@ allLocales :: HashMap Lang (HashSet Region) allLocales = HashMap.fromList [ (AR, HashSet.fromList [EG]) - , (EN, HashSet.fromList [AU, BZ, CA, GB, IN, IE, JM, NZ, PH, ZA, TT, US]) + , (EN, HashSet.fromList [AU, BZ, R.CA, GB, IN, IE, JM, NZ, PH, ZA, TT, US]) , (ES, HashSet.fromList [R.AR, CL, CO, R.ES, MX, PE, VE]) , (NL, HashSet.fromList [BE, R.NL]) , (ZH, HashSet.fromList [CN, HK, MO, TW]) diff --git a/Duckling/Numeral/CA/Corpus.hs b/Duckling/Numeral/CA/Corpus.hs new file mode 100644 index 00000000..95470cf7 --- /dev/null +++ b/Duckling/Numeral/CA/Corpus.hs @@ -0,0 +1,162 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +{-# LANGUAGE OverloadedStrings #-} + +module Duckling.Numeral.CA.Corpus (corpus) where + +import Data.String +import Prelude + +import Duckling.Locale +import Duckling.Numeral.Types +import Duckling.Resolve +import Duckling.Testing.Types + +corpus :: Corpus +corpus = + (testContext { locale = makeLocale CA Nothing }, testOptions, allExamples) + +allExamples :: [Example] +allExamples = + concat + [ examples (NumeralValue 1) + [ "1" + , "u" + , "un" + , "una" + ] + , examples (NumeralValue 11) + [ "onze" + ] + , examples (NumeralValue 12) + [ "dotze" + ] + , examples (NumeralValue 13) + [ "tretze" + ] + , examples (NumeralValue 14) + [ "catorze" + ] + , examples (NumeralValue 15) + [ "quinze" + ] + , examples (NumeralValue 16) + [ "setze" + ] + , examples (NumeralValue 17) + [ "disset" + , "dèsset" + ] + , examples (NumeralValue 18) + [ "divuit" + , "dihuit" + , "devuit" + ] + , examples (NumeralValue 19) + [ "dinou" + , "dènou" + , "denou" + ] + , examples (NumeralValue 20) + [ "vint" + ] + , examples (NumeralValue 21) + [ "vint-i-un" + , "vint i un" + ] + , examples (NumeralValue 22) + [ "vint-i-dos" + , "vint i dos" + ] + , examples (NumeralValue 23) + [ "vint-i-tres" + , "vint i tres" + ] + , examples (NumeralValue 37) + [ "trenta-set" + ] + , examples (NumeralValue 40) + [ "quaranta" + ] + , examples (NumeralValue 70) + [ "setanta" + ] + , examples (NumeralValue 78) + [ "Setanta-vuit" + ] + , examples (NumeralValue 80) + [ "vuitanta" + ] + , examples (NumeralValue 33) + [ "33" + , "trenta-tres" + , "trenta-3" + ] + , examples (NumeralValue 100000) + [ "100000" + , "100K" + , "100k" + ] + , examples (NumeralValue 300) + [ "tres-cents" + ] + , examples (NumeralValue 243) + [ "243" + ] + , examples (NumeralValue 85) + [ "vuitanta-cinc" + ] + , examples (NumeralValue 3000000) + [ "3M" + , "3000K" + , "3000000" + ] + , examples (NumeralValue 1200000) + [ "1200000" + , "1200K" + ] + , examples (NumeralValue (-1200000)) + [ "-1200000" + , "-1200K" + ] + , examples (NumeralValue 1.5) + [ "1 coma cinc" + , "una coma cinc" + , "u coma cinc" + ] + , examples (NumeralValue 1) + [ "zero u" + , "zero un" + ] + , examples (NumeralValue 2) + [ "zero dos" + ] + , examples (NumeralValue 3) + [ "zero tres" + ] + , examples (NumeralValue 4) + [ "zero quatre" + ] + , examples (NumeralValue 5) + [ "zero cinc" + ] + , examples (NumeralValue 6) + [ "zero sis" + ] + , examples (NumeralValue 7) + [ "zero set" + ] + , examples (NumeralValue 8) + [ "zero vuit" + ] + , examples (NumeralValue 9) + [ "zero nou" + ] + ] + +-- Ull, revisar la xifra amb decimals diff --git a/Duckling/Numeral/CA/Rules.hs b/Duckling/Numeral/CA/Rules.hs new file mode 100644 index 00000000..db6e6104 --- /dev/null +++ b/Duckling/Numeral/CA/Rules.hs @@ -0,0 +1,300 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +{-# LANGUAGE GADTs #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} + +module Duckling.Numeral.CA.Rules (rules) where + +import Data.HashMap.Strict (HashMap) +import qualified Data.HashMap.Strict as HashMap +import Data.Maybe +import Data.String +import Data.Text (Text) +import qualified Data.Text as Text +import Prelude + +import Duckling.Dimensions.Types +import Duckling.Numeral.Helpers +import Duckling.Numeral.Types (NumeralData(..)) +import qualified Duckling.Numeral.Types as TNumeral +import Duckling.Regex.Types +import Duckling.Types + +zeroToFifteenMap :: HashMap Text Integer +zeroToFifteenMap = + HashMap.fromList + [ ("zero", 0) + , ("u", 1) + , ("un", 1) + , ("una", 1) + , ("dos", 2) + , ("dues", 2) + , ("tres", 3) + , ("quatre", 4) + , ("cinc", 5) + , ("sis", 6) + , ("set", 7) + , ("vuit", 8) + , ("nou", 9) + , ("deu", 10) + , ("onze", 11) + , ("dotze", 12) + , ("tretze", 13) + , ("catorze", 14) + , ("quinze", 15) + ] + +ruleZeroToFifteen :: Rule +ruleZeroToFifteen = Rule + { name = "number (0..15)" + , pattern = + [ regex + "(zero|u(na|n)?|d(o|ue)s|tres|quatre|cinc|sis|set|vuit|nou|deu|onze|dotze|tretze|catorze|quinze)" + ] + , prod = \case + (Token RegexMatch (GroupMatch (match:_)):_) -> + HashMap.lookup (Text.toLower match) zeroToFifteenMap >>= integer + _ -> Nothing + } + +ruleNumeralsPrefixWithNegativeOrMinus :: Rule +ruleNumeralsPrefixWithNegativeOrMinus = Rule + { name = "numbers prefix with -, negative or minus" + , pattern = [regex "-|menys", Predicate isPositive] + , prod = \case + (_ : Token Numeral NumeralData { TNumeral.value = v } : _) -> + double $ negate v + _ -> Nothing + } + +tensMap :: HashMap Text Integer +tensMap = + HashMap.fromList + [ ("vint", 20) + , ("trenta", 30) + , ("quaranta", 40) + , ("cinquanta", 50) + , ("seixanta", 60) + , ("setanta", 70) + , ("vuitanta", 80) + , ("noranta", 90) + ] + +ruleTens :: Rule +ruleTens = Rule + { name = "number (20..90)" + , pattern = + [ regex + "(vint|(tre|quara|cinqua|seixa|seta|vuita|nora)nta)" + ] + , prod = \case + (Token RegexMatch (GroupMatch (match:_)) : _) -> + HashMap.lookup (Text.toLower match) tensMap >>= integer + _ -> Nothing + } + +sixteenToTwentyNineMap :: HashMap Text Integer +sixteenToTwentyNineMap = + HashMap.fromList + [ ("setze", 16) + , ("disset", 17) + , ("dèsset", 17) + , ("devuit", 18) + , ("divuit", 18) + , ("dihuit", 18) + , ("dinou", 19) + , ("dènou", 19) + , ("denou", 19) + , ("vint-i-u", 21) + , ("vint-i-una", 21) + , ("vint-i-dos", 22) + , ("vint-i-tres", 23) + , ("vint-i-quatre", 24) + , ("vint-i-cinc", 25) + , ("vint-i-sis", 26) + , ("vint-i-set", 27) + , ("vint-i-vuit", 28) + , ("vint-i-nou", 29) + ] + +ruleLowerTensWithOnes :: Rule +ruleLowerTensWithOnes = Rule + { name = "number (16..19 21..29)" + , pattern = + [ regex + "(setze|d(i|e|è)sset|d(e|i)(v|h)uit|d(i|e|è)nou|vint-i-u(na)?|vint-i-dos|vint-i-tres|vint-i-quatre|vint-i-cinc|vint-i-sis|vint-i-set|vint-i-vuit|vint-i-nou)" + ] + , prod = \case + (Token RegexMatch (GroupMatch (match:_)):_) -> + HashMap.lookup (Text.toLower match) sixteenToTwentyNineMap >>= integer + _ -> Nothing + } + +ruleHigherTensWithOnes :: Rule +ruleHigherTensWithOnes = Rule + { name = "number (31..39 41..49 51..59 61..69 71..79 81..89 91..99)" + , pattern = + [oneOf [30, 40, 50, 60, 70, 80, 90], regex "-", numberBetween 1 9] + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v1}: + _: + Token Numeral NumeralData{TNumeral.value = v2}: + _) -> double $ v1 + v2 + _ -> Nothing + } + +ruleNumeralsSuffixesKMG :: Rule +ruleNumeralsSuffixesKMG = Rule + { name = "numbers suffixes (K, M, G)" + , pattern = [dimension Numeral, regex "([kmg])(?=[\\W\\$€]|$)"] + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v}: + Token RegexMatch (GroupMatch (match:_)): + _) -> + case Text.toLower match of + "k" -> double $ v * 1e3 + "m" -> double $ v * 1e6 + "g" -> double $ v * 1e9 + _ -> Nothing + _ -> Nothing + } + +oneHundredToThousandMap :: HashMap Text Integer +oneHundredToThousandMap = + HashMap.fromList + [ ("cent", 100) + , ("cents", 100) + , ("dos-cents", 200) + , ("tres-cents", 300) + , ("quatre-cents", 400) + , ("cinc-cents", 500) + , ("sis-cents", 600) + , ("set-cents", 700) + , ("vuit-cents", 800) + , ("nou-cents", 900) + , ("mil", 1000) + ] + +ruleTwenties :: Rule +ruleTwenties = Rule + { name = "number (21..29)" + , pattern = + [oneOf [20], regex "(-i-| i )", numberBetween 1 10] + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v1}: + _: + Token Numeral NumeralData{TNumeral.value = v2}: + _) -> double $ v1 + v2 + _ -> Nothing + } + +ruleHundreds :: Rule +ruleHundreds = Rule + { name = "number 100..1000 " + , pattern = + [ regex + "(cent(s)?|dos-cents|tres-cents|quatre-cents|cinc-cents|sis-cents|set-cents|vuit-cents|nou-cents|mil)" + ] + , prod = \case + (Token RegexMatch (GroupMatch (match:_)):_) -> + HashMap.lookup (Text.toLower match) oneHundredToThousandMap >>= integer + _ -> Nothing + } + +-- Afegeixo regex "-" perque les centenes s'escriuen dos-cent, tres-cent +ruleNumerals :: Rule +ruleNumerals = Rule + { name = "numbers 200..999" + , pattern = + [ numberBetween 2 10 + , regex "-" + , numberWith TNumeral.value (== 100) + , numberBetween 0 100 + ] + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v1}: + _: + Token Numeral NumeralData{TNumeral.value = v2}: + _) -> double $ 100 * v1 + v2 + _ -> Nothing + } + +ruleNumeralDotNumeral :: Rule +ruleNumeralDotNumeral = Rule + { name = "number dot number" + , pattern = [dimension Numeral, regex "coma", Predicate $ not . hasGrain] + , prod = \case + (Token Numeral NumeralData{TNumeral.value = v1}: + _: + Token Numeral NumeralData{TNumeral.value = v2}: + _) -> double $ v1 + decimalsToDouble v2 + _ -> Nothing + } + +ruleBelowTenWithTwoDigits :: Rule +ruleBelowTenWithTwoDigits = Rule + { name = "integer (0-9) with two digits" + , pattern = + [ regex "zero|0" + , numberBetween 1 10 + ] + , prod = \case + (_:Token Numeral NumeralData{TNumeral.value = v}:_) -> double v + _ -> Nothing + } + +ruleDecimalWithThousandsSeparator :: Rule +ruleDecimalWithThousandsSeparator = Rule + { name = "decimal with thousands separator ." + , pattern = [regex "(\\d+(\\.\\d\\d\\d)+,\\d+)"] + , prod = \case + (Token RegexMatch (GroupMatch (match:_)):_) -> + let fmt = Text.replace "," "." . Text.replace "." Text.empty $ match + in parseDouble fmt >>= double + _ -> Nothing + } + +ruleDecimalNumeral :: Rule +ruleDecimalNumeral = Rule + { name = "decimal number ," + , pattern = [regex "(\\d*,\\d+)"] + , prod = \case + (Token RegexMatch (GroupMatch (match:_)):_) -> + parseDecimal False match + _ -> Nothing + } + +ruleIntegerWithThousandsSeparator :: Rule +ruleIntegerWithThousandsSeparator = Rule + { name = "integer with thousands separator ." + , pattern = [regex "(\\d{1,3}(\\.\\d\\d\\d){1,5})"] + , prod = \case + (Token RegexMatch (GroupMatch (match:_)):_) -> + parseDouble (Text.replace "." Text.empty match) >>= double + _ -> Nothing + } + +rules :: [Rule] +rules = + [ ruleBelowTenWithTwoDigits + , ruleZeroToFifteen + , ruleTens + , ruleTwenties + , ruleLowerTensWithOnes + , ruleHigherTensWithOnes + , ruleHundreds + + , ruleNumeralDotNumeral + , ruleNumerals + , ruleNumeralsPrefixWithNegativeOrMinus + , ruleNumeralsSuffixesKMG + , ruleDecimalNumeral + , ruleDecimalWithThousandsSeparator + , ruleIntegerWithThousandsSeparator + ] diff --git a/Duckling/Ranking/Classifiers.hs b/Duckling/Ranking/Classifiers.hs index ab1b4bc4..d4969f7b 100644 --- a/Duckling/Ranking/Classifiers.hs +++ b/Duckling/Ranking/Classifiers.hs @@ -17,6 +17,7 @@ import qualified Duckling.Ranking.Classifiers.AF_XX as AF_XXClassifiers import qualified Duckling.Ranking.Classifiers.AR_XX as AR_XXClassifiers import qualified Duckling.Ranking.Classifiers.BG_XX as BG_XXClassifiers import qualified Duckling.Ranking.Classifiers.BN_XX as BN_XXClassifiers +import qualified Duckling.Ranking.Classifiers.CA_XX as CA_XXClassifiers import qualified Duckling.Ranking.Classifiers.CS_XX as CS_XXClassifiers import qualified Duckling.Ranking.Classifiers.DA_XX as DA_XXClassifiers import qualified Duckling.Ranking.Classifiers.DE_XX as DE_XXClassifiers @@ -69,6 +70,7 @@ classifiers (Locale AF _) = AF_XXClassifiers.classifiers classifiers (Locale AR _) = AR_XXClassifiers.classifiers classifiers (Locale BG _) = BG_XXClassifiers.classifiers classifiers (Locale BN _) = BN_XXClassifiers.classifiers +classifiers (Locale CA _) = CA_XXClassifiers.classifiers classifiers (Locale CS _) = CS_XXClassifiers.classifiers classifiers (Locale DA _) = DA_XXClassifiers.classifiers classifiers (Locale DE _) = DE_XXClassifiers.classifiers diff --git a/Duckling/Ranking/Classifiers/CA_XX.hs b/Duckling/Ranking/Classifiers/CA_XX.hs new file mode 100644 index 00000000..d44a095b --- /dev/null +++ b/Duckling/Ranking/Classifiers/CA_XX.hs @@ -0,0 +1,22 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. An additional grant +-- of patent rights can be found in the PATENTS file in the same directory. + +----------------------------------------------------------------- +-- Auto-generated by regenClassifiers +-- +-- DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +-- @generated +----------------------------------------------------------------- +{-# LANGUAGE OverloadedStrings #-} +module Duckling.Ranking.Classifiers.CA_XX (classifiers) where +import Data.String +import Prelude +import qualified Data.HashMap.Strict as HashMap +import Duckling.Ranking.Types + +classifiers :: Classifiers +classifiers = HashMap.fromList [] \ No newline at end of file diff --git a/Duckling/Rules.hs b/Duckling/Rules.hs index 3093e7aa..47767c97 100644 --- a/Duckling/Rules.hs +++ b/Duckling/Rules.hs @@ -26,6 +26,7 @@ import qualified Duckling.Rules.AR as ARRules import qualified Duckling.Rules.Common as CommonRules import qualified Duckling.Rules.BG as BGRules import qualified Duckling.Rules.BN as BNRules +import qualified Duckling.Rules.CA as CARules import qualified Duckling.Rules.CS as CSRules import qualified Duckling.Rules.DA as DARules import qualified Duckling.Rules.DE as DERules @@ -96,6 +97,7 @@ defaultRules AF = AFRules.defaultRules defaultRules AR = ARRules.defaultRules defaultRules BG = BGRules.defaultRules defaultRules BN = BNRules.defaultRules +defaultRules CA = CARules.defaultRules defaultRules CS = CSRules.defaultRules defaultRules DA = DARules.defaultRules defaultRules DE = DERules.defaultRules @@ -146,6 +148,7 @@ localeRules AF = AFRules.localeRules localeRules AR = ARRules.localeRules localeRules BG = BGRules.localeRules localeRules BN = BNRules.localeRules +localeRules CA = CARules.localeRules localeRules CS = CSRules.localeRules localeRules DA = DARules.localeRules localeRules DE = DERules.localeRules @@ -196,6 +199,7 @@ langRules AF = AFRules.langRules langRules AR = ARRules.langRules langRules BG = BGRules.langRules langRules BN = BNRules.langRules +langRules CA = CARules.langRules langRules CS = CSRules.langRules langRules DA = DARules.langRules langRules DE = DERules.langRules diff --git a/Duckling/Rules/CA.hs b/Duckling/Rules/CA.hs new file mode 100644 index 00000000..4efdc5f2 --- /dev/null +++ b/Duckling/Rules/CA.hs @@ -0,0 +1,46 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +{-# LANGUAGE GADTs #-} + + +module Duckling.Rules.CA + ( defaultRules + , langRules + , localeRules + ) where + + +import Duckling.Dimensions.Types +import Duckling.Locale +import qualified Duckling.Numeral.CA.Rules as Numeral +import Duckling.Types + +defaultRules :: Seal Dimension -> [Rule] +defaultRules = langRules + +localeRules :: Region -> Seal Dimension -> [Rule] +localeRules region (Seal (CustomDimension dim)) = dimLocaleRules region dim +localeRules _ _ = [] + +langRules :: Seal Dimension -> [Rule] +langRules (Seal AmountOfMoney) = [] +langRules (Seal CreditCardNumber) = [] +langRules (Seal Distance) = [] +langRules (Seal Duration) = [] +langRules (Seal Email) = [] +langRules (Seal Numeral) = Numeral.rules +langRules (Seal Ordinal) = [] +langRules (Seal PhoneNumber) = [] +langRules (Seal Quantity) = [] +langRules (Seal RegexMatch) = [] +langRules (Seal Temperature) = [] +langRules (Seal Time) = [] +langRules (Seal TimeGrain) = [] +langRules (Seal Url) = [] +langRules (Seal Volume) = [] +langRules (Seal (CustomDimension _)) = [] diff --git a/Duckling/Rules/EN.hs b/Duckling/Rules/EN.hs index 3af87fdc..3ff2eb02 100644 --- a/Duckling/Rules/EN.hs +++ b/Duckling/Rules/EN.hs @@ -38,6 +38,7 @@ import qualified Duckling.Email.EN.Rules as Email import qualified Duckling.Numeral.EN.Rules as Numeral import qualified Duckling.Ordinal.EN.Rules as Ordinal import qualified Duckling.Quantity.EN.Rules as Quantity +import qualified Duckling.Region as R import qualified Duckling.Temperature.EN.Rules as Temperature import qualified Duckling.Time.EN.Rules as Time import qualified Duckling.Time.EN.AU.Rules as TimeAU @@ -62,7 +63,7 @@ defaultRules dim = langRules dim localeRules :: Region -> Seal Dimension -> [Rule] localeRules AU (Seal AmountOfMoney) = AmountOfMoneyAU.rules localeRules BZ (Seal AmountOfMoney) = AmountOfMoneyBZ.rules -localeRules CA (Seal AmountOfMoney) = AmountOfMoneyCA.rules +localeRules R.CA (Seal AmountOfMoney) = AmountOfMoneyCA.rules localeRules GB (Seal AmountOfMoney) = AmountOfMoneyGB.rules localeRules IE (Seal AmountOfMoney) = AmountOfMoneyIE.rules localeRules IN (Seal AmountOfMoney) = AmountOfMoneyIN.rules @@ -74,7 +75,7 @@ localeRules US (Seal AmountOfMoney) = AmountOfMoneyUS.rules localeRules ZA (Seal AmountOfMoney) = AmountOfMoneyZA.rules localeRules AU (Seal Time) = TimeAU.rules localeRules BZ (Seal Time) = TimeBZ.rules -localeRules CA (Seal Time) = TimeCA.rules +localeRules R.CA (Seal Time) = TimeCA.rules localeRules GB (Seal Time) = TimeGB.rules localeRules IE (Seal Time) = TimeIE.rules localeRules IN (Seal Time) = TimeIN.rules diff --git a/duckling.cabal b/duckling.cabal index 1bd8fb5a..a192ab49 100644 --- a/duckling.cabal +++ b/duckling.cabal @@ -51,6 +51,7 @@ library , Duckling.Rules.AR , Duckling.Rules.BG , Duckling.Rules.BN + , Duckling.Rules.CA , Duckling.Rules.CS , Duckling.Rules.DA , Duckling.Rules.DE @@ -107,6 +108,7 @@ library , Duckling.Ranking.Classifiers.AR_XX , Duckling.Ranking.Classifiers.BG_XX , Duckling.Ranking.Classifiers.BN_XX + , Duckling.Ranking.Classifiers.CA_XX , Duckling.Ranking.Classifiers.CS_XX , Duckling.Ranking.Classifiers.DA_XX , Duckling.Ranking.Classifiers.DE_XX @@ -170,6 +172,7 @@ library , Duckling.Dimensions.AR , Duckling.Dimensions.BG , Duckling.Dimensions.BN + , Duckling.Dimensions.CA , Duckling.Dimensions.CS , Duckling.Dimensions.DA , Duckling.Dimensions.DE @@ -414,6 +417,8 @@ library , Duckling.Numeral.BG.Rules , Duckling.Numeral.BN.Corpus , Duckling.Numeral.BN.Rules + , Duckling.Numeral.CA.Corpus + , Duckling.Numeral.CA.Rules , Duckling.Numeral.CS.Corpus , Duckling.Numeral.CS.Rules , Duckling.Numeral.DA.Corpus @@ -982,6 +987,7 @@ test-suite duckling-test , Duckling.Numeral.AR.Tests , Duckling.Numeral.BG.Tests , Duckling.Numeral.BN.Tests + , Duckling.Numeral.CA.Tests , Duckling.Numeral.CS.Tests , Duckling.Numeral.DA.Tests , Duckling.Numeral.DE.Tests diff --git a/exe/Duckling/Ranking/Generate.hs b/exe/Duckling/Ranking/Generate.hs index 88c91efb..5898c910 100644 --- a/exe/Duckling/Ranking/Generate.hs +++ b/exe/Duckling/Ranking/Generate.hs @@ -4,7 +4,7 @@ -- This source code is licensed under the BSD-style license found in the -- LICENSE file in the root directory of this source tree. - +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE NoRebindableSyntax #-} {-# LANGUAGE RecordWildCards #-} @@ -27,6 +27,7 @@ import Duckling.Ranking.Train import Duckling.Ranking.Types import Duckling.Rules import Duckling.Testing.Types +import qualified Duckling.Region as R import qualified Duckling.Time.AR.Corpus as ARTime import qualified Duckling.Time.BG.Corpus as BGTime import qualified Duckling.Time.DA.Corpus as DATime @@ -52,7 +53,6 @@ import qualified Duckling.Time.PT.Corpus as PTTime import qualified Duckling.Time.RO.Corpus as ROTime import qualified Duckling.Time.RU.Corpus as RUTime import qualified Duckling.Time.SV.Corpus as SVTime -import qualified Duckling.Time.TR.Corpus as TRTime import qualified Duckling.Time.UK.Corpus as UKTime import qualified Duckling.Time.VI.Corpus as VITime import qualified Duckling.Time.ZH.Corpus as ZHTime @@ -167,62 +167,65 @@ getCorpus locale@(Locale lang (Just region)) = -- | For backward compatibility. getDefaultCorpusForLang :: Lang -> Corpus -getDefaultCorpusForLang EN = ENTime.defaultCorpus -getDefaultCorpusForLang NL = NLTime.defaultCorpus -getDefaultCorpusForLang lang = getCorpusForLang lang +getDefaultCorpusForLang = \case + EN -> ENTime.defaultCorpus + NL -> NLTime.defaultCorpus + lang -> getCorpusForLang lang getCorpusForLang :: Lang -> Corpus -getCorpusForLang AF = (testContext, testOptions, []) -getCorpusForLang AR = ARTime.corpus -getCorpusForLang BG = BGTime.corpus -getCorpusForLang BN = (testContext, testOptions, []) -getCorpusForLang CS = (testContext, testOptions, []) -getCorpusForLang DA = DATime.corpus -getCorpusForLang DE = DETime.corpus -getCorpusForLang EL = ELTime.corpus -getCorpusForLang EN = ENTime.corpus -getCorpusForLang ES = ESTime.corpus -getCorpusForLang ET = (testContext, testOptions, []) -getCorpusForLang FI = (testContext, testOptions, []) -getCorpusForLang FA = (testContext, testOptions, []) -getCorpusForLang FR = FRTime.corpus -getCorpusForLang GA = GATime.corpus -getCorpusForLang HR = HRTime.corpus -getCorpusForLang HE = HETime.corpus -getCorpusForLang HU = HUTime.corpus -getCorpusForLang HI = (testContext, testOptions, []) -getCorpusForLang ID = (testContext, testOptions, []) -getCorpusForLang IS = (testContext, testOptions, []) -getCorpusForLang IT = ITTime.corpus -getCorpusForLang JA = (testContext, testOptions, []) -getCorpusForLang KA = (testContext, testOptions, []) -getCorpusForLang KM = (testContext, testOptions, []) -getCorpusForLang KN = (testContext, testOptions, []) -getCorpusForLang KO = KOTime.corpus -getCorpusForLang LO = (testContext, testOptions, []) -getCorpusForLang ML = (testContext, testOptions, []) -getCorpusForLang MN = (testContext, testOptions, []) -getCorpusForLang MY = (testContext, testOptions, []) -getCorpusForLang NB = NBTime.corpus -getCorpusForLang NE = (testContext, testOptions, []) -getCorpusForLang NL = NLTime.corpus -getCorpusForLang PL = PLTime.corpus -getCorpusForLang PT = PTTime.corpus -getCorpusForLang RO = ROTime.corpus -getCorpusForLang RU = RUTime.corpus -getCorpusForLang SK = (testContext, testOptions, []) -getCorpusForLang SV = SVTime.corpus -getCorpusForLang SW = (testContext, testOptions, []) -getCorpusForLang TA = (testContext, testOptions, []) -getCorpusForLang TE = (testContext, testOptions, []) -getCorpusForLang TH = (testContext, testOptions, []) -getCorpusForLang TR = TRTime.corpus -getCorpusForLang UK = UKTime.corpus -getCorpusForLang VI = VITime.corpus -getCorpusForLang ZH = ZHTime.corpus +getCorpusForLang = \case + AF -> (testContext, testOptions, []) + AR -> ARTime.corpus + BG -> BGTime.corpus + BN -> (testContext, testOptions, []) + CA -> (testContext, testOptions, []) + CS -> (testContext, testOptions, []) + DA -> DATime.corpus + DE -> DETime.corpus + EL -> ELTime.corpus + EN -> ENTime.corpus + ES -> ESTime.corpus + ET -> (testContext, testOptions, []) + FI -> (testContext, testOptions, []) + FA -> (testContext, testOptions, []) + FR -> FRTime.corpus + GA -> GATime.corpus + HR -> HRTime.corpus + HE -> HETime.corpus + HU -> HUTime.corpus + HI -> (testContext, testOptions, []) + ID -> (testContext, testOptions, []) + IS -> (testContext, testOptions, []) + IT -> ITTime.corpus + JA -> (testContext, testOptions, []) + KA -> (testContext, testOptions, []) + KM -> (testContext, testOptions, []) + KN -> (testContext, testOptions, []) + KO -> KOTime.corpus + LO -> (testContext, testOptions, []) + ML -> (testContext, testOptions, []) + MN -> (testContext, testOptions, []) + MY -> (testContext, testOptions, []) + NB -> NBTime.corpus + NE -> (testContext, testOptions, []) + NL -> NLTime.corpus + PL -> PLTime.corpus + PT -> PTTime.corpus + RO -> ROTime.corpus + RU -> RUTime.corpus + SK -> (testContext, testOptions, []) + SV -> SVTime.corpus + SW -> (testContext, testOptions, []) + TA -> (testContext, testOptions, []) + TE -> (testContext, testOptions, []) + TH -> (testContext, testOptions, []) + TR -> (testContext, testOptions, []) + UK -> UKTime.corpus + VI -> VITime.corpus + ZH -> ZHTime.corpus getExamplesForLocale :: Lang -> Region -> [Example] -getExamplesForLocale EN CA = EN_CATime.allExamples +getExamplesForLocale EN R.CA = EN_CATime.allExamples getExamplesForLocale EN GB = EN_GBTime.allExamples getExamplesForLocale EN US = EN_USTime.allExamples getExamplesForLocale NL BE = NL_BETime.allExamples diff --git a/tests/Duckling/AmountOfMoney/EN/Tests.hs b/tests/Duckling/AmountOfMoney/EN/Tests.hs index 02308332..a692ca4a 100644 --- a/tests/Duckling/AmountOfMoney/EN/Tests.hs +++ b/tests/Duckling/AmountOfMoney/EN/Tests.hs @@ -24,7 +24,6 @@ import qualified Duckling.AmountOfMoney.EN.AU.Corpus as AU import qualified Duckling.AmountOfMoney.EN.BZ.Corpus as BZ import qualified Duckling.AmountOfMoney.EN.CA.Corpus as CA import qualified Duckling.AmountOfMoney.EN.GB.Corpus as GB -import qualified Duckling.AmountOfMoney.EN.IE.Corpus as GIE import qualified Duckling.AmountOfMoney.EN.IN.Corpus as IN import qualified Duckling.AmountOfMoney.EN.IE.Corpus as IE import qualified Duckling.AmountOfMoney.EN.JM.Corpus as JM @@ -33,6 +32,7 @@ import qualified Duckling.AmountOfMoney.EN.PH.Corpus as PH import qualified Duckling.AmountOfMoney.EN.TT.Corpus as TT import qualified Duckling.AmountOfMoney.EN.US.Corpus as US import qualified Duckling.AmountOfMoney.EN.ZA.Corpus as ZA +import qualified Duckling.Region as R tests :: TestTree tests = testGroup "EN Tests" @@ -122,7 +122,7 @@ localeTests = testGroup "Locale Tests" where localeAU = makeLocale EN $ Just AU localeBZ = makeLocale EN $ Just BZ - localeCA = makeLocale EN $ Just CA + localeCA = makeLocale EN $ Just R.CA localeGB = makeLocale EN $ Just GB localeIE = makeLocale EN $ Just IE localeIN = makeLocale EN $ Just IN diff --git a/tests/Duckling/Numeral/CA/Tests.hs b/tests/Duckling/Numeral/CA/Tests.hs new file mode 100644 index 00000000..5cd91b16 --- /dev/null +++ b/tests/Duckling/Numeral/CA/Tests.hs @@ -0,0 +1,21 @@ +-- Copyright (c) 2016-present, Facebook, Inc. +-- All rights reserved. +-- +-- This source code is licensed under the BSD-style license found in the +-- LICENSE file in the root directory of this source tree. + + +module Duckling.Numeral.CA.Tests + ( tests ) where + +import Data.String +import Test.Tasty + +import Duckling.Dimensions.Types +import Duckling.Numeral.CA.Corpus +import Duckling.Testing.Asserts + +tests :: TestTree +tests = testGroup "CA Tests" + [ makeCorpusTest [Seal Numeral] corpus + ] diff --git a/tests/Duckling/Numeral/Tests.hs b/tests/Duckling/Numeral/Tests.hs index 8cd83ca4..bec0ff5e 100644 --- a/tests/Duckling/Numeral/Tests.hs +++ b/tests/Duckling/Numeral/Tests.hs @@ -15,6 +15,7 @@ import qualified Duckling.Numeral.AF.Tests as AF import qualified Duckling.Numeral.AR.Tests as AR import qualified Duckling.Numeral.BG.Tests as BG import qualified Duckling.Numeral.BN.Tests as BN +import qualified Duckling.Numeral.CA.Tests as CA import qualified Duckling.Numeral.CS.Tests as CS import qualified Duckling.Numeral.DA.Tests as DA import qualified Duckling.Numeral.DE.Tests as DE @@ -66,6 +67,7 @@ tests = testGroup "Numeral Tests" , AR.tests , BG.tests , BN.tests + , CA.tests , CS.tests , DA.tests , DE.tests diff --git a/tests/Duckling/Time/EN/Tests.hs b/tests/Duckling/Time/EN/Tests.hs index 8f751c86..a70b4d1e 100644 --- a/tests/Duckling/Time/EN/Tests.hs +++ b/tests/Duckling/Time/EN/Tests.hs @@ -15,7 +15,6 @@ module Duckling.Time.EN.Tests import Data.Aeson import Data.Aeson.Types ((.:), parseMaybe, withObject) import Data.String -import Data.Text (Text) import Prelude import Test.Tasty import Test.Tasty.HUnit @@ -29,6 +28,7 @@ import Duckling.Time.Corpus import Duckling.Time.EN.Corpus import Duckling.TimeGrain.Types (Grain(..)) import Duckling.Types (Range(..)) +import qualified Duckling.Region as R import qualified Duckling.Time.EN.AU.Corpus as AU import qualified Duckling.Time.EN.BZ.Corpus as BZ import qualified Duckling.Time.EN.CA.Corpus as CA @@ -109,7 +109,7 @@ localeTests = testGroup "Locale Tests" where localeAU = makeLocale EN $ Just AU localeBZ = makeLocale EN $ Just BZ - localeCA = makeLocale EN $ Just CA + localeCA = makeLocale EN $ Just R.CA localeGB = makeLocale EN $ Just GB localeIE = makeLocale EN $ Just IE localeIN = makeLocale EN $ Just IN