Fetch and display tweets (#27)

assets/css/app.css

@@ -222,3 +222,21 @@ a {
 .table-striped tbody tr:nth-child(odd) {
   background-color: #f1f5f8;
 }
+
+.twitter-icon {
+  width: 1.25em;
+  height: 1.25em;
+  display: inline-block;
+  background-repeat: no-repeat;
+  background-size: contain;
+  vertical-align: text-bottom;
+  background-image: url(data:image/svg+xml;charset=utf-8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%2072%2072%22%3E%3Cpath%20fill%3D%22none%22%20d%3D%22M0%200h72v72H0z%22%2F%3E%3Cpath%20class%3D%22icon%22%20fill%3D%22%231da1f2%22%20d%3D%22M68.812%2015.14c-2.348%201.04-4.87%201.744-7.52%202.06%202.704-1.62%204.78-4.186%205.757-7.243-2.53%201.5-5.33%202.592-8.314%203.176C56.35%2010.59%2052.948%209%2049.182%209c-7.23%200-13.092%205.86-13.092%2013.093%200%201.026.118%202.02.338%202.98C25.543%2024.527%2015.9%2019.318%209.44%2011.396c-1.125%201.936-1.77%204.184-1.77%206.58%200%204.543%202.312%208.552%205.824%2010.9-2.146-.07-4.165-.658-5.93-1.64-.002.056-.002.11-.002.163%200%206.345%204.513%2011.638%2010.504%2012.84-1.1.298-2.256.457-3.45.457-.845%200-1.666-.078-2.464-.23%201.667%205.2%206.5%208.985%2012.23%209.09-4.482%203.51-10.13%205.605-16.26%205.605-1.055%200-2.096-.06-3.122-.184%205.794%203.717%2012.676%205.882%2020.067%205.882%2024.083%200%2037.25-19.95%2037.25-37.25%200-.565-.013-1.133-.038-1.693%202.558-1.847%204.778-4.15%206.532-6.774z%22%2F%3E%3C%2Fsvg%3E);
+}
+
+.tweet-text a {
+  @apply text-blue;
+}
+
+.tweet-text a:hover {
+  text-decoration: underline;
+}

assets/js/dashboard/stats/modals/referrer-drilldown.js

@@ -1,5 +1,6 @@
 import React from "react";
 import { Link, withRouter } from 'react-router-dom'
+import TweetEmbed from 'react-tweet-embed'
 
 import Modal from './modal'
 import * as api from '../../api'
@@ -34,18 +35,69 @@ class ReferrerDrilldownModal extends React.Component {
     }
   }
 
-  renderReferrer(referrer) {
+  renderReferrerName(name) {
+    if (name) {
+      return <a className="hover:underline" target="_blank" href={'//' + name}>{name}</a>
+    } else {
+      return '(no referrer)'
+    }
+  }
+
+  renderTweet(tweet, index) {
+    const authorUrl = `https://twitter.com/${tweet.author_handle}`
+    const tweetUrl = `${authorUrl}/status/${tweet.tweet_id}`
+    const border = index === 0 ? '' : ' pt-4 border-t border-grey-light'
+
     return (
-      <tr className="text-sm" key={referrer.name}>
-        <td className="p-2 truncate">
-          <a className="hover:underline" target="_blank" href={'//' + referrer.name}>{ referrer.name }</a>
-        </td>
-        <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.count)}</td>
-        {this.showBounceRate() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(referrer)}</td> }
-      </tr>
+      <div key={tweet.tweet_id}>
+        <div className={"flex items-center my-4" + border} >
+          <a className="flex items-center group" href={authorUrl} target="_blank">
+            <img className="rounded-full w-6" src={tweet.author_image} />
+            <div className="font-bold ml-2 group-hover:text-blue">{tweet.author_name}</div>
+            <div className="ml-2 text-xs text-grey-dark">@{tweet.author_handle}</div>
+          </a>
+          <a className="ml-auto twitter-icon" href={tweetUrl} target="_blank"></a>
+        </div>
+        <div className="my-2 cursor-text tweet-text" dangerouslySetInnerHTML={{__html: tweet.text}}>
+        </div>
+        <div className="text-xs text-grey-darker font-medium">
+          {tweet.created}
+        </div>
+      </div>
     )
   }
 
+  renderReferrer(referrer) {
+    if (false && referrer.tweets) {
+      return (
+        <tr className="text-sm" key={referrer.name}>
+          <td className="p-2">
+            { this.renderReferrerName(referrer.name) }
+            <span className="text-grey-dark ml-2 text-xs">
+              appears in {referrer.tweets.length} tweets
+              <svg className="feather ml-1"><use xlinkHref="#feather-chevron-down" /></svg>
+            </span>
+            <div className="my-4 ml-4">
+              { referrer.tweets.map(this.renderTweet) }
+            </div>
+          </td>
+          <td className="p-2 w-32 font-medium" align="right" valign="top">{numberFormatter(referrer.count)}</td>
+          {this.showBounceRate() && <td className="p-2 w-32 font-medium" align="right" valign="top">{this.formatBounceRate(referrer)}</td> }
+        </tr>
+      )
+    } else {
+      return (
+        <tr className="text-sm" key={referrer.name}>
+          <td className="p-2 truncate">
+            { this.renderReferrerName(referrer.name) }
+          </td>
+          <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.count)}</td>
+          {this.showBounceRate() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(referrer)}</td> }
+        </tr>
+      )
+    }
+  }
+
   renderGoalText() {
     if (this.state.query.filters.goal) {
       return (

assets/package-lock.json

@@ -8033,6 +8033,11 @@
         "tiny-warning": "^1.0.0"
       }
     },
+    "react-tweet-embed": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/react-tweet-embed/-/react-tweet-embed-1.2.2.tgz",
+      "integrity": "sha512-Y932BlSaJsDUsKDucC2opzzd+uhc0YNhrlTa/4Beb2be1od+AjLGo6Fhuo2wPT0D+fF4VTXOyoZyA8Yc88RdYA=="
+    },
     "read-file-stdin": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/read-file-stdin/-/read-file-stdin-0.2.1.tgz",

assets/package.json

@@ -13,6 +13,7 @@
     "react": "^16.11.0",
     "react-dom": "^16.11.0",
     "react-router-dom": "^5.1.2",
+    "react-tweet-embed": "^1.2.2",
     "url-search-params-polyfill": "^7.0.0"
   },
   "devDependencies": {

lib/mix/tasks/fetch_tweets.ex

@@ -0,0 +1,62 @@
+defmodule Mix.Tasks.FetchTweets do
+  use Plausible.Repo
+  alias Plausible.Twitter.Tweet
+	@oauth_credentials Application.get_env(:plausible, :twitter, %{}) |> OAuther.credentials()
+
+  def run(_args) do
+    Application.ensure_all_started(:plausible)
+    execute()
+  end
+
+  def execute() do
+    new_links = Repo.all(
+      from e in Plausible.Event,
+      where: e.timestamp > fragment("(now() - '6 days'::interval)") and e.timestamp < fragment("(now() - '5 days'::interval)"),
+      or_where: e.timestamp > fragment("(now() - '1 days'::interval)"),
+      where: e.referrer_source == "Twitter",
+      where: e.referrer not in ["t.co", "t.co/"],
+      distinct: true,
+      select: e.referrer
+    )
+
+    for link <- new_links do
+      results = search(link)
+
+      for tweet <- results do
+        {:ok, created} = Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}")
+
+        Tweet.changeset(%Tweet{}, %{
+          link: link,
+          tweet_id: tweet["id_str"],
+          author_handle: tweet["user"]["screen_name"],
+          author_name: tweet["user"]["name"],
+          author_image: tweet["user"]["profile_image_url"],
+          text: html_body(tweet),
+          created: created
+        }) |> Repo.insert!(on_conflict: :nothing)
+      end
+    end
+  end
+
+  def html_body(tweet) do
+    body = Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text ->
+      html = "<a href=\"#{url["url"]}\" target=\"_blank\">#{url["display_url"]}</a>"
+      String.replace(text, url["url"], html)
+    end)
+
+    Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text ->
+      link = "https://twitter.com/#{mention["screen_name"]}"
+      html = "<a href=\"#{link}\" target=\"_blank\">@#{mention["screen_name"]}</a>"
+      String.replace(text, "@" <> mention["screen_name"], html)
+    end)
+  end
+
+  defp search(link) do
+    params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]
+    params = OAuther.sign("get", "https://api.twitter.com/1.1/search/tweets.json", params, @oauth_credentials)
+		uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)
+    response = HTTPoison.get!(uri)
+    Jason.decode!(response.body)
+    |> Map.get("statuses")
+  end
+end

lib/plausible/stats/stats.ex

@@ -228,7 +228,8 @@ defmodule Plausible.Stats do
   end
 
   def referrer_drilldown(site, query, referrer, include \\ []) do
-    referring_urls = Repo.all(from e in base_query(site, query),
+    referring_urls = Repo.all(
+      from e in base_query(site, query),
       select: %{name: e.referrer, count: count(e.user_id, :distinct)},
       group_by: e.referrer,
       where: e.referrer_source == ^referrer,
@@ -236,7 +237,7 @@ defmodule Plausible.Stats do
       limit: 100
     )
 
-    if "bounce_rate" in include do
+    referring_urls = if "bounce_rate" in include do
       bounce_rates = bounce_rates_by_referring_url(site, query, Enum.map(referring_urls, fn ref -> ref[:name] end))
 
       Enum.map(referring_urls, fn url ->
@@ -245,6 +246,24 @@ defmodule Plausible.Stats do
     else
       referring_urls
     end
+
+    if referrer == "Twitter" do
+      urls = Enum.map(referring_urls, &(&1[:name]))
+
+      tweets = Repo.all(
+        from t in Plausible.Twitter.Tweet,
+        where: t.link in ^urls
+      ) |> Enum.reduce(%{}, fn tweet, acc ->
+        Map.update(acc, tweet.link, [tweet], &([tweet | &1]))
+      end)
+      |> IO.inspect
+
+      Enum.map(referring_urls, fn url ->
+        Map.put(url, :tweets, tweets[url[:name]])
+      end)
+    else
+      referring_urls
+    end
   end
 
   defp bounce_rates_by_referring_url(site, query, referring_urls) do

lib/plausible/twitter/tweet.ex

@@ -0,0 +1,26 @@
+defmodule Plausible.Twitter.Tweet do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  @required_fields [:link, :tweet_id, :author_handle, :author_name, :author_image, :text, :created]
+
+  @derive {Jason.Encoder, only: @required_fields}
+  schema "tweets" do
+    field :link, :string
+
+    field :tweet_id, :string
+    field :author_handle, :string
+    field :author_name, :string
+    field :author_image, :string
+    field :text, :string
+    field :created, :naive_datetime, null: false
+
+    timestamps()
+  end
+
+  def changeset(tweet, attrs) do
+    tweet
+    |> cast(attrs, @required_fields)
+    |> validate_required(@required_fields)
+  end
+end

mix.exs

@@ -59,7 +59,8 @@ defmodule Plausible.MixProject do
       {:excoveralls, "~> 0.10", only: :test},
       {:joken, "~> 2.0"},
       {:php_serializer, "~> 0.9.0"},
-      {:csv, "~> 2.3"}
+      {:csv, "~> 2.3"},
+      {:oauther, "~> 1.1"}
     ]
   end
 

mix.lock

@@ -20,6 +20,7 @@
   "elixir_uuid": {:hex, :elixir_uuid, "1.2.0", "ff26e938f95830b1db152cb6e594d711c10c02c6391236900ddd070a6b01271d", [:mix], [], "hexpm"},
   "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"},
   "excoveralls": {:hex, :excoveralls, "0.12.0", "50e17a1b116fdb7facc2fe127a94db246169f38d7627b391376a0bc418413ce1", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
+  "extwitter": {:hex, :extwitter, "0.11.0", "9472e19f1711bc60bc7efa594353164532475d7c47ea9f1bb66d4faa889b079e", [:mix], [{:oauther, "~> 1.1", [hex: :oauther, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
   "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"},
   "gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm"},
   "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
@@ -31,6 +32,7 @@
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
   "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"},
+  "oauther": {:hex, :oauther, "1.1.1", "7d8b16167bb587ecbcddd3f8792beb9ec3e7b65c1f8ebd86b8dd25318d535752", [:mix], [], "hexpm"},
   "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
   "phoenix": {:hex, :phoenix, "1.4.0", "56fe9a809e0e735f3e3b9b31c1b749d4b436e466d8da627b8d82f90eaae714d2", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},

priv/repo/migrations/20200114131538_add_tweets.exs

@@ -0,0 +1,20 @@
+defmodule Plausible.Repo.Migrations.AddTweets do
+  use Ecto.Migration
+
+  def change do
+    create table(:tweets) do
+      add :tweet_id, :text, null: false
+      add :text, :text, null: false
+      add :author_handle, :text, null: false
+      add :author_name, :text, null: false
+      add :author_image, :text, null: false
+      add :created, :naive_datetime, null: false
+      add :link, :string, null: false
+
+      timestamps()
+    end
+
+    create index(:tweets, :link)
+    create unique_index(:tweets, [:link, :tweet_id])
+  end
+end

test/mix/tasks/fetch_tweets_test.exs

@@ -0,0 +1,40 @@
+defmodule Mix.Tasks.FetchTweetsTest do
+  use Plausible.DataCase
+  alias Mix.Tasks.FetchTweets
+
+  describe "processing tweet entities" do
+    test "inlines links to the body" do
+      tweet = %{
+        "full_text" => "asd https://t.co/somelink",
+        "entities" => %{
+          "user_mentions" => [],
+          "urls" => [%{
+            "display_url" => "plausible.io",
+            "indices" => [4, 17],
+            "url" => "https://t.co/somelink"
+          }]
+        }
+      }
+      body = FetchTweets.html_body(tweet)
+
+      assert body == "asd <a href=\"https://t.co/somelink\" target=\"_blank\">plausible.io</a>"
+    end
+
+    test "inlines user mentions to the body" do
+      tweet = %{
+        "full_text" => "asd @hello",
+        "entities" => %{
+          "user_mentions" => [%{
+            "screen_name" => "hello",
+            "id_str" => "123123"
+          }],
+          "urls" => []
+        }
+      }
+      body = FetchTweets.html_body(tweet)
+
+      assert body == "asd <a href=\"https://twitter.com/hello\" target=\"_blank\">@hello</a>"
+    end
+  end
+
+end