graphql-engine/scripts/data-sources-util.sh
Abby Sassel a091110364 server/tests: ephemeral BigQuery projects for CI test jobs
_Problem_
We currently run teardown/`DELETE` statements on the same, shared `hasura_test` dataset. This is not ideal as parallel test runs can and do affect each other, resulting in nondeterministic CI failures.

Closes https://github.com/hasura/graphql-engine-mono/issues/2521

_Solution and design_
This PR introduces ephemeral, isolated projects for each test run _in CI only_. Projects are created within [the Google Cloud Platform `data-sources-test-bigquery` directory](https://console.cloud.google.com/iam-admin/settings?folder=704256416468&orgonly=true&supportedpurview=organizationId) on each test run, and destroyed afterwards.

I've only introduced this change in CI for the time being:
  1. this isn't as much of an issue locally because we're less likely to run bigquery tests in parallel.
  2. to more quickly unblock https://github.com/hasura/graphql-engine/issues/7929.
  3. to limit the number of new projects created until we have a better idea of our usage vs GCP quota/limits.

Also updated the [internal wiki here](https://github.com/hasura/graphql-engine-mono/wiki/Testing-BigQuery) with this info.

_To verify_
- CI: [this job](https://buildkite.com/hasura/graphql-engine-mono/builds/3770#89e5bac6-16fe-447e-bcda-85cd47ea1b77) successfully runs all tests on a temporary project & dataset
- local: follow [these steps](https://github.com/hasura/graphql-engine-mono/wiki/Testing-BigQuery#ci--optional-dedicated-gcp-project-for-tests) to try the same setup locally

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/3240
GitOrigin-RevId: d88d9cb7922266bfa962cfcb481e0272b8929a5d
2022-01-25 08:28:06 +00:00

129 lines
4.1 KiB
Bash

#!/usr/bin/env bash
# functions common to dev.sh (local testing) and test-server.sh (CI tests)
function add_sources() {
hasura_graphql_server_port=${1:-8181}
# always add a Postgres source for metadata, at least
add_postgres_source "$hasura_graphql_server_port" "$PG_DB_URL"
case "$BACKEND" in
citus)
add_citus_source "$hasura_graphql_server_port" "$CITUS_DB_URL"
;;
mssql)
add_mssql_source "$hasura_graphql_server_port" "$MSSQL_CONN_STR"
;;
mysql)
add_mysql_source "$hasura_graphql_server_port"
;;
bigquery)
add_bigquery_source "$hasura_graphql_server_port"
;;
esac
echo ""
}
function add_postgres_source() {
hasura_graphql_server_port=${1}
db_url=${2}
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
echo ""
echo "Adding Postgres source"
curl --fail "$metadata_url" \
--data-raw '{"type":"pg_add_source","args":{"name":"default","configuration":{"connection_info":{"database_url":"'"$db_url"'"}}}}'
}
function add_citus_source() {
hasura_graphql_server_port=${1}
db_url=${2}
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
echo ""
echo "Adding Citus source"
curl --fail "$metadata_url" \
--data-raw '{"type":"citus_add_source","args":{"name":"citus","configuration":{"connection_info":{"database_url":"'"$db_url"'"}}}}'
}
function add_mssql_source() {
hasura_graphql_server_port=${1}
connection_string=${2}
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
echo ""
echo "Adding SQL Server source"
curl --fail "$metadata_url" \
--data-raw '{"type":"mssql_add_source","args":{"name":"mssql","configuration":{"connection_info":{"connection_string":"'"$connection_string"'"}}}}'
}
function add_mysql_source() {
hasura_graphql_server_port=${1}
# connection_string currently unused as mysql_add_source not yet supported
# connection_string=${2}
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
echo ""
echo "Adding MySQL source"
curl --fail "$metadata_url" \
--data-raw '{"type":"replace_metadata","args":{"version":3,"sources":[{"name":"mysql","kind":"mysql","tables":[],"configuration":{"database":"hasura","user":"'"$MYSQL_USER"'","password":"'"$MYSQL_PASSWORD"'","host":"127.0.0.1","port":'"$MYSQL_PORT"',"pool_settings":{}}}]}}'
}
function add_bigquery_source() {
hasura_graphql_server_port=${1}
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
echo ""
echo "Adding BigQuery sources"
curl --fail "$metadata_url" \
--data-raw '
{
"type": "replace_metadata",
"args": {
"metadata": {
"version": 3,
"sources": [
{
"name": "bigquery",
"kind": "bigquery",
"tables": [],
"configuration": {
"service_account": {
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
},
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
"datasets": ["hasura_test"]
}
},
{
"name": "bigquery2",
"kind": "bigquery",
"tables": [],
"configuration": {
"service_account": {
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
},
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
"datasets": ["hasura_test"]
}
},
{
"name": "hasura_global_limited",
"kind": "bigquery",
"tables": [],
"configuration": {
"global_select_limit": 1,
"service_account": {
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
},
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
"datasets": ["hasura_test"]
}
}
]
}
}
}
'
}