Initial tinybird setup

ref https://linear.app/tryghost/issue/ANAL-27/setup-tinybird-project-and-cicd

- Tinybird has a system for managing it's configuration as code, with full ci/cd support
- The tinybird CLI tool uses python, so we'll run that using docker, via `yarn tb`
- Some of the files tinybird adds should not be in source control, so we've added those to git ignore
- Everything in /ghost/tinybird is tinybird's init config

.gitignore

@@ -167,3 +167,8 @@ tsconfig.tsbuildinfo
 /apps/admin-x-settings/test-results/
 /apps/admin-x-settings/playwright-report/
 /apps/admin-x-settings/playwright/.cache/
+
+# Tinybird
+.tinyb
+.venv
+.diff_tmp

ghost/tinybird/.tinyenv

@@ -0,0 +1,31 @@
+
+# VERSION format is major.minor.patch-post where major, minor, patch and post are integer numbers
+# bump post to deploy to the current live Release, rollback to previous post version is not available
+# bump patch or minor to deploy a new Release and auto-promote it to live. Add TB_AUTO_PROMOTE=0 to create the Release in preview status
+# bump major to deploy a new Release in preview status
+VERSION=0.0.0
+
+
+
+##########
+# OPTIONAL env vars
+# Deploy a new Release in preview status (default is 1)
+# TB_AUTO_PROMOTE=0
+
+# Check if deploy requires backfilling on preview (default is 1)
+# TB_CHECK_BACKFILL_REQUIRED=0
+
+# Force old Releases deletion on promote (default is 0)
+# Setting it to 1 will remove oldest rollback Releases even when some resource is still in use
+# TB_FORCE_REMOVE_OLDEST_ROLLBACK=0
+
+# Don't print CLI version warning message if there's a new available version
+# TB_VERSION_WARNING=0
+
+# Skip regression tests
+# TB_SKIP_REGRESSION=0
+
+# Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
+# OBFUSCATE_REGEX_PATTERN="https://(www\.)?[^/]+||^Follow these instructions =>"
+# OBFUSCATE_PATTERN_SEPARATOR=||
+##########

ghost/tinybird/README.md

@@ -0,0 +1,18 @@
+# Tinybird
+
+This folder contains configuration for Tinybird, so that the stats feature can be used.
+
+We sync this configuration with Tinybird via the Tinybird CLI.
+
+## Tinybird CLI
+
+The Tinybird CLI is used via Docker.
+
+```bash
+yarn tb
+```
+
+Documentation for the Tinybird CLI: https://docs.tinybird.co/v/0.22.0/cli/overview
+Note: you can use python if you prefer, but we use Docker for consistency.
+
+How to work with version control: https://docs.tinybird.co/v/0.22.0/guides/version-control

ghost/tinybird/requirements.txt

@@ -0,0 +1 @@
+tinybird-cli>=4,<5

ghost/tinybird/scripts/append_fixtures.sh

@@ -0,0 +1,21 @@
+
+#!/usr/bin/env bash
+set -euxo pipefail
+
+directory="datasources/fixtures"
+extensions=("csv" "ndjson")
+
+absolute_directory=$(realpath "$directory")
+
+for extension in "${extensions[@]}"; do
+  file_list=$(find "$absolute_directory" -type f -name "*.$extension")
+
+  for file_path in $file_list; do
+    file_name=$(basename "$file_path")
+    file_name_without_extension="${file_name%.*}"
+
+    command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name"
+    echo $command
+    $command
+  done
+done

ghost/tinybird/scripts/exec_test.sh

@@ -0,0 +1,58 @@
+
+#!/usr/bin/env bash
+set -euxo pipefail
+
+export TB_VERSION_WARNING=0
+
+run_test() {
+    t=$1
+    echo "** Running $t **"
+    echo "** $(cat $t)"
+    tmpfile=$(mktemp)
+    retries=0
+    TOTAL_RETRIES=3
+
+    # When appending fixtures, we need to retry in case of the data is not replicated in time
+    while [ $retries -lt $TOTAL_RETRIES ]; do
+        # Run the test and store the output in a temporary file
+        bash $t $2 >$tmpfile
+        exit_code=$?
+        if [ "$exit_code" -eq 0 ]; then
+            # If the test passed, break the loop
+            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+                break
+            # If the test failed, increment the retries counter and try again
+            else
+                retries=$((retries+1))
+            fi
+        # If the bash command failed, print an error message and break the loop
+        else
+            break
+        fi
+    done
+
+    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+        echo "✅ Test $t passed"
+        rm $tmpfile
+        return 0
+    elif [ $retries -eq $TOTAL_RETRIES ]; then
+        echo "🚨 ERROR: Test $t failed, diff:";
+        diff -B ${t}.result $tmpfile
+        rm $tmpfile
+        return 1
+    else
+        echo "🚨 ERROR: Test $t failed with bash command exit code $?"
+        cat $tmpfile
+        rm $tmpfile
+        return 1
+    fi
+    echo ""
+}
+export -f run_test
+
+fail=0
+find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1
+
+if [ $fail == 1 ]; then
+  exit -1;
+fi

package.json

@@ -40,7 +40,8 @@
     "main": "yarn main:monorepo && yarn main:submodules",
     "main:monorepo": "git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn",
     "main:submodules": "git submodule sync && git submodule update && git submodule foreach \"git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn\"",
-    "prepare": "husky install .github/hooks"
+    "prepare": "husky install .github/hooks",
+    "tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker"
   },
   "resolutions": {
     "@tryghost/errors": "1.3.5",