diff --git a/Dockerfile b/Dockerfile index c1c8fa720..bfd6f7eda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,8 @@ RUN npm run deploy --prefix ./assets && \ npm run deploy --prefix ./tracker && \ mix phx.digest priv/static && \ mix download_country_database && \ + mix ref_inspector.download -f && \ + mix ua_inspector.download -f && \ # https://hexdocs.pm/sentry/Sentry.Sources.html#module-source-code-storage mix sentry_recompile diff --git a/mix.exs b/mix.exs index 6ba5e37fe..825a53b93 100644 --- a/mix.exs +++ b/mix.exs @@ -68,7 +68,7 @@ defmodule Plausible.MixProject do {:poison, "~> 3.1"}, {:ref_inspector, "~> 1.3"}, {:timex, "~> 3.6"}, - {:ua_inspector, "~> 0.18"}, + {:ua_inspector, "~> 2.2"}, {:bamboo, "~> 2.0"}, {:hackney, "~> 1.8"}, {:bamboo_phoenix, "~> 1.0.0"}, diff --git a/mix.lock b/mix.lock index a98f902aa..7aef3546a 100644 --- a/mix.lock +++ b/mix.lock @@ -86,7 +86,7 @@ "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "timex": {:hex, :timex, "3.6.3", "58ce6c9eda8ed47fc80c24dde09d481465838d3bcfc230949287fc1b0b0041c1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "6d69f4f95fcf5684102a9cb3cf92c5ba6545bd60ed8d8a6a93cd2a4a4fb0d9ec"}, "tzdata": {:hex, :tzdata, "1.0.5", "69f1ee029a49afa04ad77801febaf69385f3d3e3d1e4b56b9469025677b89a28", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "55519aa2a99e5d2095c1e61cc74c9be69688f8ab75c27da724eb8279ff402a5a"}, - "ua_inspector": {:hex, :ua_inspector, "0.20.0", "01939baf5706f7d6c2dc0affbbd7f5e14309ba43ebf8967aa6479ee2204f23bc", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.0", [hex: :poolboy, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "30e8623b9f55e7d58be12fc2afd50be8792ec14192c289701d3cc93ad6027f26"}, + "ua_inspector": {:hex, :ua_inspector, "2.2.0", "7b737d7668a10c6a380c949e3d93acc992df51f77cc2659a6352dafc8b6fb09c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "1701dcfd81c00d5ec1322a0f609b93260f5de85317ed1afb2e11917a52748958"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, "yamerl": {:hex, :yamerl, "0.8.1", "07da13ffa1d8e13948943789665c62ccd679dfa7b324a4a2ed3149df17f453a4", [:rebar3], [], "hexpm", "96cb30f9d64344fed0ef8a92e9f16f207de6c04dfff4f366752ca79f5bceb23f"}, diff --git a/priv/ref_inspector/ref_inspector.readme.md b/priv/ref_inspector/ref_inspector.readme.md index e69de29bb..0f0dbee3c 100644 --- a/priv/ref_inspector/ref_inspector.readme.md +++ b/priv/ref_inspector/ref_inspector.readme.md @@ -0,0 +1,5 @@ +# RefInspector Parser Database + +The file in this directory is taken from the +[snowplow-referer-parser](https://github.com/snowplow-referer-parser/referer-parser) +project. See there for detailed license information about the data contained. diff --git a/priv/ref_inspector/referers.yml b/priv/ref_inspector/referers.yml index 35577b65d..a6f88f3c7 100644 --- a/priv/ref_inspector/referers.yml +++ b/priv/ref_inspector/referers.yml @@ -51,7 +51,15 @@ unknown: - omg.yahoo.com - match.yahoo.net - + Yandex Maps: + parameters: + - text + domains: + - maps.yandex.ru + - maps.yandex.ua + - maps.yandex.com + - maps.yandex.by + - n.maps.yandex.ru # ####################################################################################################### # @@ -79,6 +87,10 @@ email: domains: - mail.aol.com + Beeline: + domains: + - post.ru + Bigpond: domains: - webmail.bigpond.com @@ -99,6 +111,10 @@ email: domains: - webmail.dodo.com.au + E1.ru: + domains: + - mail.e1.ru + Freenet: domains: - webmail.freenet.de @@ -121,6 +137,16 @@ email: domains: - webmail.iprimus.com.au + Mail.ru: + domains: + - e.mail.ru + - touch.mail.ru + + Mastermail: + domains: + - mastermail.ru + - m.mastermail.ru + Mynet Mail: domains: - mail.mynet.com @@ -147,14 +173,31 @@ email: - mail.live.com - outlook.live.com + QIP: + domains: + - mail.qip.ru + QQ Mail: domains: - - mail.qq.com + - mail.qq.com + - exmail.qq.com + + Rambler: + domains: + - mail.rambler.ru Seznam Mail: domains: - email.seznam.cz + Sibmail: + domains: + - sibmail.com + + Ukr.net: + domains: + - mail.ukr.net + Virgin: domains: - webmail.virginbroadband.com.au @@ -167,6 +210,14 @@ email: domains: - webmail.westnet.com.au + Yandex: + domains: + - mail.yandex.ru + - mail.yandex.com + - mail.yandex.kz + - mail.yandex.ua + - mail.yandex.by + Yahoo! Mail: domains: - mail.yahoo.net @@ -208,6 +259,7 @@ social: Instagram: domains: - instagram.com + - l.instagram.com Youtube: domains: @@ -237,7 +289,9 @@ social: Vkontakte: domains: + - m.vk.com - vk.com + - away.vk.com - vkontakte.ru Tagged: @@ -300,6 +354,7 @@ social: Odnoklassniki: domains: - odnoklassniki.ru + - ok.ru Viadeo: domains: @@ -470,7 +525,7 @@ social: Pocket: domains: - getpocket.com - + ITU Sozluk: domains: - itusozluk.com @@ -501,7 +556,7 @@ social: Donanimhaber: domains: - - donanimhaber.com + - donanimhaber.com Disqus: domains: @@ -513,9 +568,17 @@ social: domains: - quora.com - Whirlpool: - domains: - - forums.whirlpool.net.au + Skype: + domains: + - web.skype.com + + WhatsApp: + domains: + - web.whatsapp.com + + Whirlpool: + domains: + - forums.whirlpool.net.au # ####################################################################################################### # @@ -543,6 +606,13 @@ search: domains: - search.1und1.de + 2gis: + domains: + - 2gis.ru + - www.2gis.ru + - link.2gis.ru + - www.link.2gis.ru + 360.cn: parameters: - q @@ -2969,6 +3039,12 @@ search: - www.toile.com - web.toile.com + Liveinternet: + parameters: + - q + domains: + - liveinternet.ru + Looksmart: parameters: - key @@ -2999,6 +3075,8 @@ search: parameters: - q domains: + - mail.ru + - m.mail.ru - go.mail.ru Mamma: @@ -3085,6 +3163,7 @@ search: - searchfor - searchFor domains: + - mysearch.com - www.mysearch.com - ms114.mysearch.com - ms146.mysearch.com @@ -3167,7 +3246,7 @@ search: domains: - busca.orange.es - search.orange.co.uk - - lemoteur.orange.fr + - lemoteur.orange.fr Paperball: parameters: @@ -3220,6 +3299,13 @@ search: - www.qualigo.de - www.qualigo.nl + Qwant: + parameters: + - q + domains: + - www.qwant.com + - lite.qwant.com + Rakuten: parameters: - qt @@ -3338,6 +3424,18 @@ search: - sosodesktop.com - search.sosodesktop.com + Shenma: + parameters: + - q + domains: + - so.m.sm.cn + - yz.m.sm.cn + - m.sm.cn + - quark.sm.cn + - m.sp.sm.cn + - m.yz2.sm.cn + - m.yz.sm.cn + Snapdo: parameters: - q @@ -3598,6 +3696,8 @@ search: - au.yahoo.com - br.search.yahoo.com - br.yahoo.com + - ca.search.yahoo.com + - ca.yahoo.com - cade.searchde.yahoo.com - cade.yahoo.com - chinese.searchinese.yahoo.com @@ -3614,8 +3714,12 @@ search: - espanol.yahoo.com - fr.search.yahoo.com - fr.yahoo.com + - hk.search.yahoo.com + - hk.yahoo.com - ie.search.yahoo.com - ie.yahoo.com + - in.search.yahoo.com + - in.yahoo.com - it.search.yahoo.com - it.yahoo.com - kr.search.yahoo.com @@ -3630,11 +3734,17 @@ search: - one.searchn.yahoo.com - qc.search.yahoo.com - qc.yahoo.com + - ru.search.yahoo.com + - ru.yahoo.com - se.search.yahoo.com - se.yahoo.com - search.searcharch.yahoo.com + - tw.search.yahoo.com + - tw.yahoo.com - uk.search.yahoo.com - uk.yahoo.com + - us.search.yahoo.com + - us.yahoo.com - www.yahoo.co.jp - search.yahoo.co.jp # powered by Yahoo @@ -3668,6 +3778,10 @@ search: - www.yandex.ua - www.yandex.com - www.yandex.by + - clck.yandex.ru + - clck.yandex.ua + - clck.yandex.com + - clck.yandex.by Yandex Images: parameters: @@ -3676,6 +3790,7 @@ search: - images.yandex.ru - images.yandex.ua - images.yandex.com + - images.yandex.by Yasni: parameters: @@ -3759,10 +3874,31 @@ paid: domains: - adform.net + ADFOX: + domains: + - adfox.ru + - www.adfox.ru + - ads.adfox.ru + - www.ads.adfox.ru + + Adition: + domains: + - adition.com + + AdNET: + domains: + - adnet.de + AdRoll: domains: - adroll.com + AdSpirit: + domains: + - adspirit.de + - rtbcity.com + - plusperformance.com + AppNexus: domains: - ib.adnxs.com @@ -3821,6 +3957,12 @@ paid: domains: - lfstmedia.com + LowerMyBills: + parameters: + - leadid + domains: + - lowermybills.com + Jivox: domains: - jivox.com @@ -3861,6 +4003,11 @@ paid: domains: - farm.plista.com + Price.ru: + domains: + - price.ru + - v.price.ru + PubMatic: domains: - sshowads.pubmatic.com @@ -3900,6 +4047,10 @@ paid: - api.taboola.com - taboola.com + Torg.Mail.ru: + domains: + - torg.mail.ru + Tribal Fusion: domains: - cdnx.tribalfusion.com @@ -3909,6 +4060,19 @@ paid: - www.whitepages.com.au - mobile.whitepages.com.au + Yandex.Direct: + domains: + - an.yandex.ru + - yabs.yandex.ru + - yabs.yandex.ua + - yabs.yandex.com + - yabs.yandex.by + + Yandex.Market: + domains: + - market.yandex.ru + - m.market.yandex.ru + Yieldmo: domains: - yieldmo.com diff --git a/priv/ua_inspector/device.mobiles.yml b/priv/ua_inspector/device.mobiles.yml index f8d1939b1..a0a964e4b 100644 --- a/priv/ua_inspector/device.mobiles.yml +++ b/priv/ua_inspector/device.mobiles.yml @@ -16,7 +16,7 @@ Ace: model: 'Buzz 1 Plus' 'Tunisie Telecom': - regex: 'StarTrail TT[);/ ]' + regex: 'StarTrail TT(?:[);/ ]|$)' device: 'smartphone' model: 'StarTrail by TT' @@ -25,37 +25,37 @@ SFR: regex: 'StarShine|StarTrail|STARADDICT|StarText|StarNaute|StarXtrem|StarTab' device: 'smartphone' models: - - regex: 'StarXtrem[);/ ]' + - regex: 'StarXtrem(?:[);/ ]|$)' model: 'StarXtrem' # ZTE - - regex: 'StarTrail ?4[);/ ]' + - regex: 'StarTrail ?4(?:[);/ ]|$)' model: 'StarTrail 4' # ZTE Blade Q - - regex: 'StarTrail III[);/ ]' + - regex: 'StarTrail III(?:[);/ ]|$)' model: 'StarTrail 3' # ZTE - - regex: 'StarTrail II[);/ ]' + - regex: 'StarTrail II(?:[);/ ]|$)' model: 'StarTrail 2' # Huawei Ascend Y200 - - regex: 'StarTrail[);/ ]' + - regex: 'StarTrail(?:[);/ ]|$)' model: 'StarTrail' # ZTE - - regex: 'StarShine II[);/ ]' + - regex: 'StarShine II(?:[);/ ]|$)' model: 'StarShine 2' # ZTE - - regex: 'StarShine[);/ ]' + - regex: 'StarShine(?:[);/ ]|$)' model: 'StarShine' # Huawei U8180 - - regex: 'STARADDICT 4[);/ ]' + - regex: 'STARADDICT 4(?:[);/ ]|$)' model: 'Staraddict 4' - - regex: 'STARADDICT III[);/ ]' + - regex: 'STARADDICT III(?:[);/ ]|$)' model: 'Staraddict 3' # Coolpad 8861U - - regex: 'STARADDICT II Plus[);/ ]' + - regex: 'STARADDICT II Plus(?:[);/ ]|$)' model: 'Staraddict 2 Plus' # ZTE Grand X - - regex: 'STARADDICT II[);/ ]' + - regex: 'STARADDICT II(?:[);/ ]|$)' model: 'Staraddict 2' # Alcatel One Touch 995 - - regex: 'STARADDICT[);/ ]' + - regex: 'STARADDICT(?:[);/ ]|$)' model: 'Staraddict' # ZTE - - regex: 'StarText II[);/ ]' + - regex: 'StarText II(?:[);/ ]|$)' model: 'StarText 2' # ZTE - - regex: 'StarText[);/ ]' + - regex: 'StarText(?:[);/ ]|$)' model: 'StarText' # ZTE - - regex: 'StarNaute II[);/ ]' + - regex: 'StarNaute II(?:[);/ ]|$)' model: 'StarNaute 2' # ZTE - - regex: 'StarNaute[);/ ]' + - regex: 'StarNaute(?:[);/ ]|$)' model: 'StarNaute' # ZTE Amigo - regex: 'StarTab' model: 'StarTab' @@ -66,33 +66,35 @@ SFR: # HTC HTC: - regex: 'HTC|Sprint (?:APA|ATP)|ADR(?!910L)[a-z0-9]+|NexusHD2|Amaze[ _]4G[);/ ]|(Desire|Sensation|Evo ?3D|IncredibleS|Wildfire|Butterfly)[ _]?([^;/)]+)(?: Build|\))|(Amaze[ _]4G|One ?[XELSV\+]+)[);/ ]|SPV E6[05]0|(One M8|X525a|PG86100|PC36100|XV6975|PJ83100|2PYB2|2PZC5|0PJA10|0PJA2|HTV33|0PM92|0PCV1|601HT|0PAJ5|HTL2[23]|HTV32|2PQ93)[);/ ]' + regex: 'HTC|Sprint (?:APA|ATP)|ADR(?!910L)[a-z0-9]+|NexusHD2|Amaze[ _]4G(?:[);/ ]|$)|(Desire|Sensation|Evo ?3D|IncredibleS|Wildfire|Butterfly)[ _]?([^;/)]+)(?: Build|\))|(Amaze[ _]4G|One ?[XELSV\+]+)(?:[);/ ]|$)|SPV E6[05]0|(One M8|X525a|PG86100|PC36100|XV6975|PJ83100|2PYB2|2PZC5|0PJA10|0PJA2|HTV33|0PM92|0PCV1|601HT|0PAJ5|HTL2[23]|HTV32|2PQ93|X2-HT)(?:[);/ ]|$)' device: 'smartphone' models: # explicit smartphone models - - regex: '2PZC5[);/ ]' + - regex: '2PZC5(?:[);/ ]|$)' model: 'U11' - - regex: 'XV6975[);/ ]' + - regex: 'XV6975(?:[);/ ]|$)' model: 'Imagio' - - regex: 'PG86100[);/ ]' + - regex: 'PG86100(?:[);/ ]|$)' model: 'Evo 3G' - - regex: 'PC36100[);/ ]' + - regex: 'PC36100(?:[);/ ]|$)' model: 'Evo 4G' - - regex: '(?:HTV33|601HT)[);/ ]' + - regex: '(?:HTV33|601HT)(?:[);/ ]|$)' model: 'U11' - regex: '(?:801e|802[wdt])' model: 'One' - - regex: 'PJ83100[);/ ]' + - regex: 'PJ83100(?:[);/ ]|$)' model: 'One X' - - regex: 'HTL22[);/ ]' + - regex: 'X2-HT(?:[);/ ]|$)' + model: 'One X2' + - regex: 'HTL22(?:[);/ ]|$)' model: 'J One' - - regex: 'HTL23[);/ ]' + - regex: 'HTL23(?:[);/ ]|$)' model: 'J Butterfly' - - regex: '(?:0PJA2|0PJA10)[);/ ]' + - regex: '(?:0PJA2|0PJA10)(?:[);/ ]|$)' model: 'One M9' - - regex: 'X525a[);/ ]' + - regex: 'X525a(?:[);/ ]|$)' model: 'One X+' - - regex: '2PQ93[);/ ]' + - regex: '2PQ93(?:[);/ ]|$)' model: 'One A9' - regex: 'ADR6300' model: 'Droid Incredible' @@ -106,11 +108,11 @@ HTC: model: 'Vox' - regex: '2PYB2' model: 'Bolt' - - regex: '0PM92[);/ ]' + - regex: '0PM92(?:[);/ ]|$)' model: 'Desire 626S' - - regex: '0PCV1[);/ ]' + - regex: '0PCV1(?:[);/ ]|$)' model: 'Desire 510' - - regex: '608t[);/ ]' + - regex: '608t(?:[);/ ]|$)' model: 'Desire 608t' - regex: 'NexusHD2' # custom rom for hd2 @@ -124,7 +126,7 @@ HTC: - regex: 'HTC[ _\-]Flyer[ _\-]([\w]{1,5})' device: 'tablet' model: 'Flyer $1' - - regex: 'HTC[ _\-]One[ _\-]max[);/ ]' + - regex: 'HTC[ _\-]One[ _\-]max(?:[);/ ]|$)' device: 'phablet' model: 'One max' - regex: 'HTC[ _]([^/;]+) [0-9]+(?:\.[0-9]+)+ Build' @@ -154,9 +156,9 @@ HTC: model: '$1' - regex: '(One M8)' model: '$1' - - regex: '0PAJ5[);/ ]' + - regex: '0PAJ5(?:[);/ ]|$)' model: 'One E8' - - regex: 'HTV32[);/ ]' + - regex: 'HTV32(?:[);/ ]|$)' model: '10' # microsoft @@ -203,7 +205,7 @@ Microsoft: - regex: 'Lumia 1530' model: 'Lumia 1530' - - regex: 'Microsoft; Lumia ([^;/)]+)[;/)]' + - regex: 'Microsoft; Lumia ([^;/)]+)(?:[;/)]|$)' model: 'Lumia $1' # NOKIA @@ -660,14 +662,14 @@ Apple: # micromax MicroMax: - regex: '(?:MicroMax[ \-\_]?[a-z0-9]+|Q327)|4560MMX|P70221 Build|(?:YU(?:5040|5530)|AQ5001 Canvas Power)[);/ ]' + regex: '(?:MicroMax[ \-\_]?[a-z0-9]+|Q327)|4560MMX|P70221 Build|(?:YU(?:5040|5530)|AQ5001 Canvas Power)(?:[);/ ]|$)' device: 'smartphone' models: - regex: 'AQ5001 Canvas Power' model: 'Canvas Power' - regex: 'AQ5001' model: 'Canvas Juice 2' - - regex: '4560MMX[);/ ]' + - regex: '4560MMX(?:[);/ ]|$)' model: '4560MMX' - regex: 'E481' model: 'Canvas 5' @@ -680,9 +682,9 @@ MicroMax: model: 'Canvas Mega' - regex: 'Q402\+' model: 'Bolt Pace' - - regex: 'YU5040[);/ ]' + - regex: 'YU5040(?:[);/ ]|$)' model: 'YU Yureka YU5040' - - regex: 'YU5530[);/ ]' + - regex: 'YU5530(?:[);/ ]|$)' model: 'YU Yureka S YU5530' - regex: 'Q424' model: 'Bolt Selfie' @@ -744,7 +746,7 @@ Alba: # AllCall AllCall: - regex: 'AllCall_|Heat[34]|Hot ?[125][^;/]+ Build' + regex: 'AllCall_|Heat[34]|Rio_X(?:[);/ ]|$)|Hot ?[125][^;/]+ Build' device: 'smartphone' models: - regex: 'AllCall_(Alpha|Atom|Bro|Madrid)' @@ -761,6 +763,8 @@ AllCall: model: 'Rio' - regex: 'Heat([34])' model: 'Heat $1' + - regex: 'Rio_X(?:[);/ ]|$)' + model: 'Rio X' - regex: 'Hot ?([125][^;/]+) Build' model: 'Hot $1' @@ -838,17 +842,17 @@ Bravis: # Acer Acer: - regex: 'acer|(?