Update ref and ua databases

Dockerfile

@@ -41,6 +41,8 @@ RUN npm run deploy --prefix ./assets && \
     npm run deploy --prefix ./tracker && \
     mix phx.digest priv/static && \
     mix download_country_database && \
+    mix ref_inspector.download -f && \
+    mix ua_inspector.download -f && \
 # https://hexdocs.pm/sentry/Sentry.Sources.html#module-source-code-storage
     mix sentry_recompile
 

mix.exs

@@ -68,7 +68,7 @@ defmodule Plausible.MixProject do
       {:poison, "~> 3.1"},
       {:ref_inspector, "~> 1.3"},
       {:timex, "~> 3.6"},
-      {:ua_inspector, "~> 0.18"},
+      {:ua_inspector, "~> 2.2"},
       {:bamboo, "~> 2.0"},
       {:hackney, "~> 1.8"},
       {:bamboo_phoenix, "~> 1.0.0"},

mix.lock

@@ -86,7 +86,7 @@
   "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"},
   "timex": {:hex, :timex, "3.6.3", "58ce6c9eda8ed47fc80c24dde09d481465838d3bcfc230949287fc1b0b0041c1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "6d69f4f95fcf5684102a9cb3cf92c5ba6545bd60ed8d8a6a93cd2a4a4fb0d9ec"},
   "tzdata": {:hex, :tzdata, "1.0.5", "69f1ee029a49afa04ad77801febaf69385f3d3e3d1e4b56b9469025677b89a28", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "55519aa2a99e5d2095c1e61cc74c9be69688f8ab75c27da724eb8279ff402a5a"},
-  "ua_inspector": {:hex, :ua_inspector, "0.20.0", "01939baf5706f7d6c2dc0affbbd7f5e14309ba43ebf8967aa6479ee2204f23bc", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.0", [hex: :poolboy, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "30e8623b9f55e7d58be12fc2afd50be8792ec14192c289701d3cc93ad6027f26"},
+  "ua_inspector": {:hex, :ua_inspector, "2.2.0", "7b737d7668a10c6a380c949e3d93acc992df51f77cc2659a6352dafc8b6fb09c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "1701dcfd81c00d5ec1322a0f609b93260f5de85317ed1afb2e11917a52748958"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
   "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"},
   "yamerl": {:hex, :yamerl, "0.8.1", "07da13ffa1d8e13948943789665c62ccd679dfa7b324a4a2ed3149df17f453a4", [:rebar3], [], "hexpm", "96cb30f9d64344fed0ef8a92e9f16f207de6c04dfff4f366752ca79f5bceb23f"},

priv/ref_inspector/ref_inspector.readme.md

@@ -0,0 +1,5 @@
+# RefInspector Parser Database
+
+The file in this directory is taken from the
+[snowplow-referer-parser](https://github.com/snowplow-referer-parser/referer-parser)
+project. See there for detailed license information about the data contained.

priv/ref_inspector/referers.yml

@@ -51,7 +51,15 @@ unknown:
       - omg.yahoo.com
       - match.yahoo.net
 
-
+  Yandex Maps:
+    parameters:
+      - text
+    domains:
+      - maps.yandex.ru
+      - maps.yandex.ua
+      - maps.yandex.com
+      - maps.yandex.by
+      - n.maps.yandex.ru
 
 # #######################################################################################################
 #
@@ -79,6 +87,10 @@ email:
     domains:
       - mail.aol.com
 
+  Beeline:
+    domains:
+      - post.ru
+
   Bigpond:
     domains:
       - webmail.bigpond.com
@@ -99,6 +111,10 @@ email:
     domains:
       - webmail.dodo.com.au
 
+  E1.ru:
+    domains:
+      - mail.e1.ru
+
   Freenet:
     domains:
       - webmail.freenet.de
@@ -121,6 +137,16 @@ email:
     domains:
       - webmail.iprimus.com.au
 
+  Mail.ru:
+    domains:
+      - e.mail.ru
+      - touch.mail.ru
+
+  Mastermail:
+    domains:
+      - mastermail.ru
+      - m.mastermail.ru
+
   Mynet Mail:
     domains:
       - mail.mynet.com
@@ -147,14 +173,31 @@ email:
       - mail.live.com
       - outlook.live.com
 
+  QIP:
+    domains:
+      - mail.qip.ru
+
   QQ Mail:
     domains:
-      - mail.qq.com  
+      - mail.qq.com
+      - exmail.qq.com
+
+  Rambler:
+    domains:
+      - mail.rambler.ru 
 
   Seznam Mail:
     domains:
       - email.seznam.cz
 
+  Sibmail:
+    domains:
+      - sibmail.com
+
+  Ukr.net:
+    domains:
+      - mail.ukr.net
+
   Virgin:
     domains:
       - webmail.virginbroadband.com.au
@@ -167,6 +210,14 @@ email:
     domains:
       - webmail.westnet.com.au
 
+  Yandex:
+    domains:
+      - mail.yandex.ru
+      - mail.yandex.com
+      - mail.yandex.kz
+      - mail.yandex.ua
+      - mail.yandex.by
+
   Yahoo! Mail:
     domains:
       - mail.yahoo.net
@@ -208,6 +259,7 @@ social:
   Instagram:
     domains:
       - instagram.com
+      - l.instagram.com
 
   Youtube:
     domains:
@@ -237,7 +289,9 @@ social:
 
   Vkontakte:
     domains:
+      - m.vk.com
       - vk.com
+      - away.vk.com
       - vkontakte.ru
 
   Tagged:
@@ -300,6 +354,7 @@ social:
   Odnoklassniki:
     domains:
       - odnoklassniki.ru
+      - ok.ru
 
   Viadeo:
     domains:
@@ -470,7 +525,7 @@ social:
   Pocket:
     domains:
       - getpocket.com
-      
+
   ITU Sozluk:
     domains:
       - itusozluk.com
@@ -501,7 +556,7 @@ social:
 
   Donanimhaber:
     domains:
-      - donanimhaber.com 
+      - donanimhaber.com
 
   Disqus:
     domains:
@@ -513,9 +568,17 @@ social:
     domains:
       - quora.com
 
-    Whirlpool:
+  Skype:
-      domains:
+    domains:
-        - forums.whirlpool.net.au
+      - web.skype.com
+
+  WhatsApp:
+    domains:
+      - web.whatsapp.com
+
+  Whirlpool:
+    domains:
+      - forums.whirlpool.net.au
 
 # #######################################################################################################
 #
@@ -543,6 +606,13 @@ search:
     domains:
       - search.1und1.de
 
+  2gis:
+    domains:
+      - 2gis.ru
+      - www.2gis.ru
+      - link.2gis.ru
+      - www.link.2gis.ru
+
   360.cn:
     parameters:
       - q
@@ -2969,6 +3039,12 @@ search:
       - www.toile.com
       - web.toile.com
 
+  Liveinternet:
+    parameters:
+      - q
+    domains:
+      - liveinternet.ru
+
   Looksmart:
     parameters:
       - key
@@ -2999,6 +3075,8 @@ search:
     parameters:
       - q
     domains:
+      - mail.ru
+      - m.mail.ru
       - go.mail.ru
 
   Mamma:
@@ -3085,6 +3163,7 @@ search:
       - searchfor
       - searchFor
     domains:
+      - mysearch.com
       - www.mysearch.com
       - ms114.mysearch.com
       - ms146.mysearch.com
@@ -3167,7 +3246,7 @@ search:
     domains:
       - busca.orange.es
       - search.orange.co.uk
-      - lemoteur.orange.fr   
+      - lemoteur.orange.fr
 
   Paperball:
     parameters:
@@ -3220,6 +3299,13 @@ search:
       - www.qualigo.de
       - www.qualigo.nl
 
+  Qwant:
+    parameters:
+      - q
+    domains:
+      - www.qwant.com
+      - lite.qwant.com
+
   Rakuten:
     parameters:
       - qt
@@ -3338,6 +3424,18 @@ search:
       - sosodesktop.com
       - search.sosodesktop.com
 
+  Shenma:
+    parameters:
+      - q
+    domains:
+      - so.m.sm.cn
+      - yz.m.sm.cn
+      - m.sm.cn
+      - quark.sm.cn
+      - m.sp.sm.cn
+      - m.yz2.sm.cn
+      - m.yz.sm.cn
+
   Snapdo:
     parameters:
       - q
@@ -3598,6 +3696,8 @@ search:
       - au.yahoo.com
       - br.search.yahoo.com
       - br.yahoo.com
+      - ca.search.yahoo.com
+      - ca.yahoo.com
       - cade.searchde.yahoo.com
       - cade.yahoo.com
       - chinese.searchinese.yahoo.com
@@ -3614,8 +3714,12 @@ search:
       - espanol.yahoo.com
       - fr.search.yahoo.com
       - fr.yahoo.com
+      - hk.search.yahoo.com
+      - hk.yahoo.com
       - ie.search.yahoo.com
       - ie.yahoo.com
+      - in.search.yahoo.com
+      - in.yahoo.com
       - it.search.yahoo.com
       - it.yahoo.com
       - kr.search.yahoo.com
@@ -3630,11 +3734,17 @@ search:
       - one.searchn.yahoo.com
       - qc.search.yahoo.com
       - qc.yahoo.com
+      - ru.search.yahoo.com
+      - ru.yahoo.com
       - se.search.yahoo.com
       - se.yahoo.com
       - search.searcharch.yahoo.com
+      - tw.search.yahoo.com
+      - tw.yahoo.com
       - uk.search.yahoo.com
       - uk.yahoo.com
+      - us.search.yahoo.com
+      - us.yahoo.com
       - www.yahoo.co.jp
       - search.yahoo.co.jp
       # powered by Yahoo
@@ -3668,6 +3778,10 @@ search:
       - www.yandex.ua
       - www.yandex.com
       - www.yandex.by
+      - clck.yandex.ru
+      - clck.yandex.ua
+      - clck.yandex.com
+      - clck.yandex.by
 
   Yandex Images:
     parameters:
@@ -3676,6 +3790,7 @@ search:
       - images.yandex.ru
       - images.yandex.ua
       - images.yandex.com
+      - images.yandex.by
 
   Yasni:
     parameters:
@@ -3759,10 +3874,31 @@ paid:
     domains:
       - adform.net
 
+  ADFOX:
+    domains:
+      - adfox.ru
+      - www.adfox.ru
+      - ads.adfox.ru
+      - www.ads.adfox.ru
+
+  Adition:
+    domains:
+      - adition.com
+
+  AdNET:
+    domains:
+      - adnet.de
+
   AdRoll:
     domains:
       - adroll.com
 
+  AdSpirit:
+    domains:
+      - adspirit.de
+      - rtbcity.com
+      - plusperformance.com
+
   AppNexus:
     domains:
       - ib.adnxs.com
@@ -3821,6 +3957,12 @@ paid:
     domains:
       - lfstmedia.com
 
+  LowerMyBills:
+    parameters:
+      - leadid
+    domains:
+      - lowermybills.com
+
   Jivox:
     domains:
       - jivox.com
@@ -3861,6 +4003,11 @@ paid:
     domains:
       - farm.plista.com
 
+  Price.ru:
+    domains:
+      - price.ru
+      - v.price.ru
+
   PubMatic:
     domains:
       - sshowads.pubmatic.com
@@ -3900,6 +4047,10 @@ paid:
       - api.taboola.com
       - taboola.com
 
+  Torg.Mail.ru:
+    domains:
+      - torg.mail.ru
+
   Tribal Fusion:
     domains:
       - cdnx.tribalfusion.com
@@ -3909,6 +4060,19 @@ paid:
       - www.whitepages.com.au
       - mobile.whitepages.com.au
 
+  Yandex.Direct:
+    domains:
+      - an.yandex.ru
+      - yabs.yandex.ru
+      - yabs.yandex.ua
+      - yabs.yandex.com
+      - yabs.yandex.by
+
+  Yandex.Market:
+    domains:
+      - market.yandex.ru
+      - m.market.yandex.ru
+
   Yieldmo:
     domains:
       - yieldmo.com

priv/ua_inspector/device.notebooks.yml

@@ -0,0 +1,114 @@
+###############
+# Device Detector - The Universal Device Detection library for parsing User Agents
+#
+# @link https://matomo.org
+# @license http://www.gnu.org/licenses/lgpl.html LGPL v3 or later
+###############
+
+Acer:
+  regex: 'FBMD/(?:Aspire E5-421G|Z5WAL|One S1003);'
+  device: 'desktop'
+  models:
+    - regex: 'FBMD/Aspire E5-421G;'
+      model: 'Aspire E5-421G'
+    - regex: 'FBMD/Z5WAL;'
+      model: 'Aspire E5-511'
+    - regex: 'FBMD/One S1003;'
+      model: 'One 10'
+
+Asus:
+  regex: 'FBMD/(?:K50IN|K54L|T100HAN|T103HAF|UX360CAK|X550LB|X553MA|X555LN|X556UQK);'
+  device: 'desktop'
+  models:
+    - regex: 'FBMD/K50IN;'
+      model: 'K50IN'
+    - regex: 'FBMD/K54L;'
+      model: 'K54L'
+    - regex: 'FBMD/T100HAN;'
+      model: 'Transformer Book'
+    - regex: 'FBMD/T103HAF;'
+      model: 'Transformer Mini'
+    - regex: 'FBMD/UX360CAK;'
+      model: 'ZenBook Flip'
+    - regex: 'FBMD/X550LB;'
+      model: 'X550LB'
+    - regex: 'FBMD/X553MA;'
+      model: 'X553MA'
+    - regex: 'FBMD/X555LN;'
+      model: 'X555LN'
+    - regex: 'FBMD/X556UQK;'
+      model: 'X556UQK'
+
+Dell:
+  regex: 'FBMD/(?:Latitude E4300|Inspiron 3541|XPS 15 95[35]0);'
+  device: 'desktop'
+  models:
+    - regex: 'Latitude E4300'
+      model: 'Latitude E4300'
+    - regex: 'Inspiron 3541'
+      model: 'Inspiron 3541'
+    - regex: 'XPS 15 9530'
+      model: 'XPS 15 9530'
+    - regex: 'XPS 15 9550'
+      model: 'XPS 15 9550'
+
+HP:
+  regex: 'FBMD/((?:Compaq|HP) |23-f364)'
+  device: 'desktop'
+  models:
+    - regex: 'Compaq Presario CQ61 Notebook PC'
+      model: 'Compaq Presario CQ61'
+    - regex: 'HP Pavilion x2 Detachable'
+      model: 'Pavilion x2'
+    - regex: 'HP Laptop 15-bs0xx'
+      model: '15 Laptop PC'
+    - regex: 'HP ENVY x360 Convertible 15-bp0xx'
+      model: 'ENVY x360 Convertible PC'
+    - regex: 'HP EliteBook (25[67]0p)'
+      model: 'EliteBook $1'
+    - regex: 'HP ProBook (440 G5|6[35]60b)'
+      model: 'ProBook $1'
+    - regex: 'HP Pavilion dv6 Notebook PC'
+      model: 'Pavilion dv6'
+    - regex: 'HP Pavilion Notebook'
+      model: 'Pavilion'
+    - regex: 'HP Spectre x360 Convertible'
+      model: 'Spectre x360'
+    - regex: 'HP Pavilion All-in-One 24-r0xx'
+      model: 'Pavilion 24-r0xx All-in-One Desktop PC'
+      device: 'desktop'
+    - regex: '23-f364'
+      model: 'Pavilion TouchSmart 23-f364 All-in-One Desktop PC'
+      device: 'desktop'
+
+Lenovo:
+  regex: 'FBMD/(?:37021C5|80E5|80SM|80VR);'
+  device: 'desktop'
+  models:
+    - regex: 'FBMD/37021C5;'
+      model: 'ThinkPad Helix 3702'
+    - regex: 'FBMD/80E5;'
+      model: 'G50-80'
+    - regex: 'FBMD/80SM;'
+      model: 'Ideapad 310-15ISK'
+    - regex: 'FBMD/80VR;'
+      model: 'Legion Y720'
+
+Schneider:
+  regex: 'FBMD/SCL141CTP;'
+  device: 'desktop'
+  model: 'Notebook 14" Cherry Trail'
+
+Thomson:
+  regex: 'FBMD/TH360R12.32CTW;'
+  device: 'desktop'
+  model: 'Prestige TH-360R12.32CTW'
+
+Toshiba:
+  regex: 'FBMD/Satellite '
+  device: 'desktop'
+  models:
+    - regex: 'Satellite (A[25]00|C650|C855|L650|S855)'
+      model: 'Satellite $1'
+    - regex: 'Satellite ([^;\)]+);'
+      model: 'Satellite $1'

priv/ua_inspector/os.oss.yml

@@ -98,6 +98,10 @@
 ##########
 # Android
 ##########
+- regex: 'Android; (\d+[\.\d]*); Mobile;'
+  name: 'Android'
+  version: '$1'
+
 - regex: '[ ]([\d\.]+)\) AppleWebKit.*ROBLOX Android App'
   name: 'Android'
   version: '$1'

priv/ua_inspector/short_codes.browser_families.yml

@@ -0,0 +1,144 @@
+- "Android Browser":
+  - "AN"
+  - "MU"
+- "BlackBerry Browser":
+  - "BB"
+- "Baidu":
+  - "BD"
+  - "BS"
+- "Amiga":
+  - "AV"
+  - "AW"
+- "Chrome":
+  - "CH"
+  - "BA"
+  - "BR"
+  - "CC"
+  - "CD"
+  - "CM"
+  - "CI"
+  - "CF"
+  - "CN"
+  - "CR"
+  - "CP"
+  - "DD"
+  - "IR"
+  - "RM"
+  - "AO"
+  - "TS"
+  - "VI"
+  - "PT"
+  - "AS"
+  - "TB"
+  - "AD"
+  - "SB"
+  - "WP"
+  - "I3"
+  - "CV"
+  - "WH"
+  - "SZ"
+  - "QW"
+  - "LF"
+  - "KW"
+  - "2B"
+  - "CE"
+  - "EC"
+  - "MT"
+  - "MS"
+  - "HA"
+  - "OC"
+  - "MZ"
+  - "BM"
+  - "KN"
+  - "SW"
+  - "M1"
+  - "FA"
+  - "TA"
+  - "AH"
+  - "CL"
+  - "SU"
+  - "EU"
+  - "UB"
+  - "LO"
+  - "VG"
+  - "TV"
+  - "A0"
+  - "1B"
+  - "S4"
+  - "EE"
+  - "AE"
+  - "VM"
+  - "O0"
+  - "TG"
+  - "GB"
+  - "SY"
+  - "HH"
+  - "YJ"
+  - "LL"
+  - "TU"
+  - "XV"
+  - "C2"
+  - "QU"
+  - "YN"
+- "Firefox":
+  - "FF"
+  - "FE"
+  - "FM"
+  - "SX"
+  - "FB"
+  - "PX"
+  - "MB"
+  - "EI"
+  - "WF"
+  - "CU"
+  - "TF"
+  - "QM"
+  - "FR"
+  - "I4"
+  - "GZ"
+  - "MO"
+  - "F1"
+  - "BI"
+  - "MN"
+  - "BH"
+  - "TO"
+  - "OS"
+  - "MY"
+  - "FY"
+  - "AX"
+  - "C0"
+  - "LH"
+  - "S5"
+  - "ZV"
+  - "IW"
+- "Internet Explorer":
+  - "IE"
+  - "IM"
+  - "PS"
+- "Konqueror":
+  - "KO"
+- "NetFront":
+  - "NF"
+- "NetSurf":
+  - "NE"
+- "Nokia Browser":
+  - "NB"
+  - "NO"
+  - "NV"
+  - "DO"
+- "Opera":
+  - "OP"
+  - "OM"
+  - "OI"
+  - "ON"
+  - "OO"
+  - "OG"
+  - "OH"
+  - "O1"
+  - "OX"
+- "Safari":
+  - "SF"
+  - "MF"
+  - "SO"
+- "Sailfish Browser":
+  - "SA"

priv/ua_inspector/short_codes.device_brands.yml

@@ -1,8 +1,10 @@
 - "36": "360"
+- "88": "8848"
 - "3Q": "3Q"
 - "4G": "4Good"
 - "AE": "Ace"
 - "AA": "AllCall"
+- "3A": "AllDocube"
 - "AC": "Acer"
 - "00": "Accent"
 - "A9": "Advan"
@@ -15,8 +17,8 @@
 - "AK": "Akai"
 - "1A": "Alba"
 - "AL": "Alcatel"
+- "20": "Alcor"
 - "4A": "Aligator"
-- "3A": "AllDocube"
 - "A2": "Allview"
 - "A7": "Allwinner"
 - "A1": "Altech UEC"
@@ -193,6 +195,7 @@
 - "FL": "Fly"
 - "F1": "FinePower"
 - "FT": "Freetel"
+- "F7": "Fero"
 - "FR": "Forstar"
 - "FO": "Foxconn"
 - "F2": "FORME"
@@ -248,6 +251,7 @@
 - "IJ": "i-Joy"
 - "IC": "iDroid"
 - "IY": "iBerry"
+- "7I": "iBrit"
 - "IH": "iHunt"
 - "IK": "iKoMo"
 - "IE": "iView"
@@ -458,6 +462,7 @@
 - "OU": "OUYA"
 - "OO": "Opsson"
 - "OV": "Overmax"
+- "30": "Ovvi"
 - "OY": "Oysters"
 - "O6": "Oyyu"
 - "OW": "öwn"
@@ -591,6 +596,7 @@
 - "TA": "Tesla"
 - "TK": "Takara"
 - "4T": "Tronsmart"
+- "5R": "Transpeed"
 - "T5": "TB Touch"
 - "TC": "TCL"
 - "T7": "Teclast"
@@ -667,6 +673,7 @@
 - "VL": "Verykool"
 - "V6": "VGO TEL"
 - "VV": "Vivo"
+- "3V": "VVETIME"
 - "VX": "Vertex"
 - "V3": "Vinsoc"
 - "V2": "Vonino"