mirror of
https://github.com/hasura/graphql-engine.git
synced 2024-12-14 17:02:49 +03:00
a091110364
_Problem_ We currently run teardown/`DELETE` statements on the same, shared `hasura_test` dataset. This is not ideal as parallel test runs can and do affect each other, resulting in nondeterministic CI failures. Closes https://github.com/hasura/graphql-engine-mono/issues/2521 _Solution and design_ This PR introduces ephemeral, isolated projects for each test run _in CI only_. Projects are created within [the Google Cloud Platform `data-sources-test-bigquery` directory](https://console.cloud.google.com/iam-admin/settings?folder=704256416468&orgonly=true&supportedpurview=organizationId) on each test run, and destroyed afterwards. I've only introduced this change in CI for the time being: 1. this isn't as much of an issue locally because we're less likely to run bigquery tests in parallel. 2. to more quickly unblock https://github.com/hasura/graphql-engine/issues/7929. 3. to limit the number of new projects created until we have a better idea of our usage vs GCP quota/limits. Also updated the [internal wiki here](https://github.com/hasura/graphql-engine-mono/wiki/Testing-BigQuery) with this info. _To verify_ - CI: [this job](https://buildkite.com/hasura/graphql-engine-mono/builds/3770#89e5bac6-16fe-447e-bcda-85cd47ea1b77) successfully runs all tests on a temporary project & dataset - local: follow [these steps](https://github.com/hasura/graphql-engine-mono/wiki/Testing-BigQuery#ci--optional-dedicated-gcp-project-for-tests) to try the same setup locally PR-URL: https://github.com/hasura/graphql-engine-mono/pull/3240 GitOrigin-RevId: d88d9cb7922266bfa962cfcb481e0272b8929a5d
129 lines
4.1 KiB
Bash
129 lines
4.1 KiB
Bash
#!/usr/bin/env bash
|
|
# functions common to dev.sh (local testing) and test-server.sh (CI tests)
|
|
function add_sources() {
|
|
hasura_graphql_server_port=${1:-8181}
|
|
|
|
# always add a Postgres source for metadata, at least
|
|
add_postgres_source "$hasura_graphql_server_port" "$PG_DB_URL"
|
|
|
|
case "$BACKEND" in
|
|
citus)
|
|
add_citus_source "$hasura_graphql_server_port" "$CITUS_DB_URL"
|
|
;;
|
|
mssql)
|
|
add_mssql_source "$hasura_graphql_server_port" "$MSSQL_CONN_STR"
|
|
;;
|
|
mysql)
|
|
add_mysql_source "$hasura_graphql_server_port"
|
|
;;
|
|
bigquery)
|
|
add_bigquery_source "$hasura_graphql_server_port"
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
}
|
|
|
|
function add_postgres_source() {
|
|
hasura_graphql_server_port=${1}
|
|
db_url=${2}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding Postgres source"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '{"type":"pg_add_source","args":{"name":"default","configuration":{"connection_info":{"database_url":"'"$db_url"'"}}}}'
|
|
}
|
|
|
|
function add_citus_source() {
|
|
hasura_graphql_server_port=${1}
|
|
db_url=${2}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding Citus source"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '{"type":"citus_add_source","args":{"name":"citus","configuration":{"connection_info":{"database_url":"'"$db_url"'"}}}}'
|
|
}
|
|
|
|
function add_mssql_source() {
|
|
hasura_graphql_server_port=${1}
|
|
connection_string=${2}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding SQL Server source"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '{"type":"mssql_add_source","args":{"name":"mssql","configuration":{"connection_info":{"connection_string":"'"$connection_string"'"}}}}'
|
|
}
|
|
|
|
function add_mysql_source() {
|
|
hasura_graphql_server_port=${1}
|
|
# connection_string currently unused as mysql_add_source not yet supported
|
|
# connection_string=${2}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding MySQL source"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '{"type":"replace_metadata","args":{"version":3,"sources":[{"name":"mysql","kind":"mysql","tables":[],"configuration":{"database":"hasura","user":"'"$MYSQL_USER"'","password":"'"$MYSQL_PASSWORD"'","host":"127.0.0.1","port":'"$MYSQL_PORT"',"pool_settings":{}}}]}}'
|
|
}
|
|
|
|
function add_bigquery_source() {
|
|
hasura_graphql_server_port=${1}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding BigQuery sources"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '
|
|
{
|
|
"type": "replace_metadata",
|
|
"args": {
|
|
"metadata": {
|
|
"version": 3,
|
|
"sources": [
|
|
{
|
|
"name": "bigquery",
|
|
"kind": "bigquery",
|
|
"tables": [],
|
|
"configuration": {
|
|
"service_account": {
|
|
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
|
|
},
|
|
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
|
|
"datasets": ["hasura_test"]
|
|
}
|
|
},
|
|
{
|
|
"name": "bigquery2",
|
|
"kind": "bigquery",
|
|
"tables": [],
|
|
"configuration": {
|
|
"service_account": {
|
|
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
|
|
},
|
|
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
|
|
"datasets": ["hasura_test"]
|
|
}
|
|
},
|
|
{
|
|
"name": "hasura_global_limited",
|
|
"kind": "bigquery",
|
|
"tables": [],
|
|
"configuration": {
|
|
"global_select_limit": 1,
|
|
"service_account": {
|
|
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
|
|
},
|
|
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
|
|
"datasets": ["hasura_test"]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
'
|
|
}
|