mirror of
https://github.com/hasura/graphql-engine.git
synced 2024-12-15 01:12:56 +03:00
fc0352a995
This PR addresses a couple of recent issues with BigQuery tests in CI, plus some refactoring. TL;DR: most issues were fixed by being [explicit about failures](http://redsymbol.net/articles/unofficial-bash-strict-mode/) and variables in scope. Thanks to @qrilka for addressing a couple of these in your last PR already, sharing my notes about them here too for posterity as I found the troubleshooting exercise useful. _commands listed in execution order_ `generate_test_project` * issue: this rarely fails AFAICT, but a failure in any of the `gcloud` commands will result in unclear failures further along the call stack * fix: fail faster with `set -e` `create_bigquery_dataset` * error: `BigQuery error in mk operation: Not found: Project`. examples [[1]](https://github.com/hasura/graphql-engine-mono/issues/3600)[[2]](https://buildkite.com/hasura/graphql-engine-mono/builds/4857#f70990eb-3721-45c6-b11a-af7e4ebd9fe3) * issue: a failure on a missing project, which I could verify was created successfully * cause: missing `HASURA_BIGQUERY_PROJECT_ID` , most likely caused by the above failure and dodgy subshell scoping * fix: pass `project_id` as an argument. This might not have been necessary since `generate_test_project` fails faster, but I thought it was clearer to pass explicitly. `ensure_bigquery_db` * issue: this could fail, but return a 0 exit code. [example](https://buildkite.com/hasura/graphql-engine-mono/builds/4836#e3dfbc1e-4034-40bb-beb1-181fb5d9489f) * fix: add a `--fail` flag to `curl` call. Kirill addressed in https://github.com/hasura/graphql-engine-mono/pull/3435/files#diff-0525000dbb36f436dcc17570541378de51471200d294b468c4b288e0292441b6R94 `delete_temp_bigquery_project` * error: `(gcloud.projects.delete) value for field [projectId] in collection [cloudresourcemanager.projects] is required but was not provided`. [example](https://buildkite.com/hasura/graphql-engine-mono/builds/4836#e3dfbc1e-4034-40bb-beb1-181fb5d9489f) * issue: attempting to delete a tmp file that didn't exist due to earlier failures * fix: Kirill addressed in https://github.com/hasura/graphql-engine-mono/pull/3435/files#diff-0525000dbb36f436dcc17570541378de51471200d294b468c4b288e0292441b6R106 to verify: check [bigquery](https://buildkite.com/hasura/graphql-engine-mono/builds/5115#41d059eb-329c-46fb-a745-b2b97fffd328) and [hspec](https://buildkite.com/hasura/graphql-engine-mono/builds/5115#5623a53d-c18d-478e-bf44-446e1287453b) jobs in CI PR-URL: https://github.com/hasura/graphql-engine-mono/pull/3616 Co-authored-by: Divi <32202683+imperfect-fourth@users.noreply.github.com> GitOrigin-RevId: b85420ef57036b4dbb05202b73ee9e954113bf9d
129 lines
4.1 KiB
Bash
129 lines
4.1 KiB
Bash
#!/usr/bin/env bash
|
|
# functions common to dev.sh (local testing) and test-server.sh (CI tests)
|
|
function add_sources() {
|
|
hasura_graphql_server_port=${1:-8181}
|
|
|
|
# always add a Postgres source for metadata, at least
|
|
add_postgres_source "$hasura_graphql_server_port" "$PG_DB_URL"
|
|
|
|
case "$BACKEND" in
|
|
citus)
|
|
add_citus_source "$hasura_graphql_server_port" "$CITUS_DB_URL"
|
|
;;
|
|
mssql)
|
|
add_mssql_source "$hasura_graphql_server_port" "$MSSQL_CONN_STR"
|
|
;;
|
|
mysql)
|
|
add_mysql_source "$hasura_graphql_server_port"
|
|
;;
|
|
bigquery)
|
|
add_bigquery_source "$hasura_graphql_server_port"
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
}
|
|
|
|
function add_postgres_source() {
|
|
hasura_graphql_server_port=${1}
|
|
db_url=${2}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding Postgres source"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '{"type":"pg_add_source","args":{"name":"default","configuration":{"connection_info":{"database_url":"'"$db_url"'"}}}}'
|
|
}
|
|
|
|
function add_citus_source() {
|
|
hasura_graphql_server_port=${1}
|
|
db_url=${2}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding Citus source"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '{"type":"citus_add_source","args":{"name":"citus","configuration":{"connection_info":{"database_url":"'"$db_url"'"}}}}'
|
|
}
|
|
|
|
function add_mssql_source() {
|
|
hasura_graphql_server_port=${1}
|
|
connection_string=${2}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding SQL Server source"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '{"type":"mssql_add_source","args":{"name":"mssql","configuration":{"connection_info":{"connection_string":"'"$connection_string"'"}}}}'
|
|
}
|
|
|
|
function add_mysql_source() {
|
|
hasura_graphql_server_port=${1}
|
|
# connection_string currently unused as mysql_add_source not yet supported
|
|
# connection_string=${2}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding MySQL source"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '{"type":"replace_metadata","args":{"version":3,"sources":[{"name":"mysql","kind":"mysql","tables":[],"configuration":{"database":"hasura","user":"'"$MYSQL_USER"'","password":"'"$MYSQL_PASSWORD"'","host":"127.0.0.1","port":'"$MYSQL_PORT"',"pool_settings":{}}}]}}'
|
|
}
|
|
|
|
function add_bigquery_source() {
|
|
hasura_graphql_server_port=${1}
|
|
metadata_url=http://127.0.0.1:$hasura_graphql_server_port/v1/metadata
|
|
|
|
echo ""
|
|
echo "Adding BigQuery sources to project $HASURA_BIGQUERY_PROJECT_ID"
|
|
curl --fail "$metadata_url" \
|
|
--data-raw '
|
|
{
|
|
"type": "replace_metadata",
|
|
"args": {
|
|
"metadata": {
|
|
"version": 3,
|
|
"sources": [
|
|
{
|
|
"name": "bigquery",
|
|
"kind": "bigquery",
|
|
"tables": [],
|
|
"configuration": {
|
|
"service_account": {
|
|
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
|
|
},
|
|
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
|
|
"datasets": ["hasura"]
|
|
}
|
|
},
|
|
{
|
|
"name": "bigquery2",
|
|
"kind": "bigquery",
|
|
"tables": [],
|
|
"configuration": {
|
|
"service_account": {
|
|
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
|
|
},
|
|
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
|
|
"datasets": ["hasura"]
|
|
}
|
|
},
|
|
{
|
|
"name": "hasura_global_limited",
|
|
"kind": "bigquery",
|
|
"tables": [],
|
|
"configuration": {
|
|
"global_select_limit": 1,
|
|
"service_account": {
|
|
"from_env": "HASURA_BIGQUERY_SERVICE_ACCOUNT"
|
|
},
|
|
"project_id": { "from_env": "HASURA_BIGQUERY_PROJECT_ID" },
|
|
"datasets": ["hasura"]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
'
|
|
}
|