From 1e7ab8665d8bebeabf02cd13f821ad6dfc68c3b7 Mon Sep 17 00:00:00 2001 From: Abby Sassel <3883855+sassela@users.noreply.github.com> Date: Thu, 26 Aug 2021 19:11:12 +0100 Subject: [PATCH] server/bigquery: fix data source references & pytest filter in CI https://github.com/hasura/graphql-engine-mono/pull/2169 GitOrigin-RevId: a0a7190a404ee558a965e06fe1a022953f6f5a10 --- .circleci/test-server.sh | 2 +- ...et_table_customization_and_test_query.yaml | 3 +- .../bigquery/setup_bigquery.yaml | 286 ++++++++---------- .../bigquery/teardown_bigquery.yaml | 70 +---- ...ble_with_customization_and_test_query.yaml | 21 +- server/tests-py/test_graphql_queries.py | 33 +- 6 files changed, 136 insertions(+), 279 deletions(-) diff --git a/.circleci/test-server.sh b/.circleci/test-server.sh index ac42185a916..3f3cbd296b3 100755 --- a/.circleci/test-server.sh +++ b/.circleci/test-server.sh @@ -1207,7 +1207,7 @@ admin_users = postgres' > pgbouncer/pgbouncer.ini wait_for_port 8080 # See note [Specifying Pytests with -k flag] - pytest -n 1 --hge-urls "$HGE_URL" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" --backend bigquery -k "BigQuery or Common" + pytest -n 1 --hge-urls "$HGE_URL" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" --backend bigquery -k "Bigquery or Common" kill_hge_servers ;; diff --git a/server/tests-py/queries/graphql_query/bigquery/set_table_customization_and_test_query.yaml b/server/tests-py/queries/graphql_query/bigquery/set_table_customization_and_test_query.yaml index 6751bcfe8a4..b01664f73e7 100644 --- a/server/tests-py/queries/graphql_query/bigquery/set_table_customization_and_test_query.yaml +++ b/server/tests-py/queries/graphql_query/bigquery/set_table_customization_and_test_query.yaml @@ -6,7 +6,7 @@ query: type: bigquery_set_table_customization args: - source: bigquery + source: hasura table: dataset: hasura name: article @@ -49,4 +49,3 @@ } } } - diff --git a/server/tests-py/queries/graphql_query/bigquery/setup_bigquery.yaml b/server/tests-py/queries/graphql_query/bigquery/setup_bigquery.yaml index 53723ec1235..dc71a7a7576 100644 --- a/server/tests-py/queries/graphql_query/bigquery/setup_bigquery.yaml +++ b/server/tests-py/queries/graphql_query/bigquery/setup_bigquery.yaml @@ -1,175 +1,125 @@ type: bulk args: - - -- type: bigquery_track_table +- type: replace_metadata args: - source: hasura_global_limited - table: - dataset: hasura - name: LimitedAlbum - -- type: bigquery_track_table - args: - source: hasura_global_limited - table: - dataset: hasura - name: LimitedArtist - -- type: bigquery_create_object_relationship - args: - source: hasura_global_limited - table: - dataset: hasura - name: LimitedAlbum - name: artist - using: - manual_configuration: - remote_table: - dataset: hasura - name: LimitedArtist - column_mapping: - artist_other_id: artist_self_id - -- type: bigquery_create_array_relationship - args: - source: hasura_global_limited - table: - dataset: hasura - name: LimitedArtist - name: albums - using: - manual_configuration: - remote_table: - dataset: hasura + version: 3 + sources: + - name: hasura_global_limited + kind: bigquery + configuration: + service_account: {from_env: HASURA_BIGQUERY_SERVICE_ACCOUNT} + project_id: {from_env: HASURA_BIGQUERY_PROJECT_ID} + datasets: + - hasura + global_select_limit: 1 + tables: + - table: name: LimitedAlbum - column_mapping: - artist_self_id: artist_other_id - - -- type: bigquery_track_table - args: - source: bigquery - table: - dataset: hasura - name: author - -- type: bigquery_track_table - args: - source: bigquery - table: - dataset: hasura - name: article - -- type: bigquery_create_array_relationship - args: - source: bigquery - table: - dataset: hasura - name: author - name: articles - using: - manual_configuration: - remote_table: dataset: hasura - name: article - column_mapping: - id: author_id - -- type: bigquery_create_object_relationship - args: - source: bigquery - table: - dataset: hasura - name: article - name: author - using: - manual_configuration: - remote_table: + object_relationships: + - name: artist + using: + manual_configuration: + remote_table: + name: LimitedArtist + dataset: hasura + column_mapping: + artist_other_id: artist_self_id + - table: + name: LimitedArtist + dataset: hasura + array_relationships: + - name: albums + using: + manual_configuration: + remote_table: + name: LimitedAlbum + dataset: hasura + column_mapping: + artist_self_id: artist_other_id + - name: hasura + kind: bigquery + configuration: + service_account: {from_env: HASURA_BIGQUERY_SERVICE_ACCOUNT} + project_id: {from_env: HASURA_BIGQUERY_PROJECT_ID} + datasets: + - hasura + global_select_limit: 10 + tables: + - table: + name: Album + dataset: hasura + object_relationships: + - name: artist + using: + manual_configuration: + remote_table: + name: Artist + dataset: hasura + column_mapping: + artist_other_id: artist_self_id + - table: + name: Artist + dataset: hasura + array_relationships: + - name: albums + using: + manual_configuration: + remote_table: + name: Album + dataset: hasura + column_mapping: + artist_self_id: artist_other_id + - table: dataset: hasura name: author - column_mapping: - author_id: id - -- type: bigquery_create_select_permission - args: - role: published_only - source: bigquery - table: - dataset: hasura - name: article - permission: - columns: - - is_published - - id - - title - filter: - is_published: true - -- type: bigquery_create_select_permission - args: - role: exact_article_id - source: bigquery - table: - dataset: hasura - name: article - permission: - columns: - - id - - title - - content - filter: - id: {'_eq': 2} - -- type: bigquery_create_select_permission - args: - role: empty_perms - source: bigquery - table: - dataset: hasura - name: article - permission: - columns: [] - filter: {} - -- type: bigquery_create_select_permission - args: - role: user - source: bigquery - table: - dataset: hasura - name: article - permission: - columns: - - id - - title - - content - - author_id - - is_published - - published_on - - created_at - filter: {} - -- type: bigquery_create_select_permission - args: - role: test_timestamp_perm - source: bigquery - table: - dataset: hasura - name: article - permission: - columns: - - id - - title - - content - filter: - created_at: {'_eq': '2008-12-25T07:30:02'} - - -- type: bigquery_track_table - args: - source: bigquery - table: - dataset: hasura - name: all_types - + array_relationships: + - using: + manual_configuration: + remote_table: + name: article + dataset: hasura + column_mapping: + id: author_id + name: articles + - table: + dataset: hasura + name: article + object_relationships: + - using: + manual_configuration: + remote_table: + name: author + dataset: hasura + column_mapping: + author_id: id + name: author + select_permissions: + - role: published_only + permission: + columns: [is_published, id, title] + filter: + is_published: true + - role: exact_article_id + permission: + columns: [id, title, content] + filter: + id: + '_eq': 2 + - role: empty_perms + permission: + columns: [] + filter: {} + - role: user + permission: + columns: [id, title, content, author_id, is_published, published_on, created_at] + filter: {} + - role: test_timestamp_perm + permission: + columns: [id, title, content] + filter: + created_at: + '_eq': '2008-12-25T07:30:02' + - table: + dataset: hasura + name: all_types diff --git a/server/tests-py/queries/graphql_query/bigquery/teardown_bigquery.yaml b/server/tests-py/queries/graphql_query/bigquery/teardown_bigquery.yaml index 1d6785b2d99..54fdb9f6898 100644 --- a/server/tests-py/queries/graphql_query/bigquery/teardown_bigquery.yaml +++ b/server/tests-py/queries/graphql_query/bigquery/teardown_bigquery.yaml @@ -1,70 +1,2 @@ type: bulk -args: - -- type: bigquery_untrack_table - args: - source: bigquery - table: - dataset: hasura - name: all_types - -- type: bigquery_drop_relationship - args: - source: bigquery - table: - dataset: hasura - name: article - relationship: author - -- type: bigquery_drop_relationship - args: - source: bigquery - table: - dataset: hasura - name: author - relationship: articles - -- type: bigquery_untrack_table - args: - source: bigquery - table: - dataset: hasura - name: article - -- type: bigquery_untrack_table - args: - source: bigquery - table: - dataset: hasura - name: author - -- type: bigquery_drop_relationship - args: - source: hasura_global_limited - table: - dataset: hasura - name: LimitedArtist - relationship: albums - -- type: bigquery_drop_relationship - args: - source: hasura_global_limited - table: - dataset: hasura - name: LimitedAlbum - relationship: artist - -- type: bigquery_untrack_table - args: - source: hasura_global_limited - table: - dataset: hasura - name: LimitedArtist - -- type: bigquery_untrack_table - args: - source: hasura_global_limited - table: - dataset: hasura - name: LimitedAlbum - +args: [] diff --git a/server/tests-py/queries/graphql_query/bigquery/track_table_with_customization_and_test_query.yaml b/server/tests-py/queries/graphql_query/bigquery/track_table_with_customization_and_test_query.yaml index bdca833fd6b..c32177c87a0 100644 --- a/server/tests-py/queries/graphql_query/bigquery/track_table_with_customization_and_test_query.yaml +++ b/server/tests-py/queries/graphql_query/bigquery/track_table_with_customization_and_test_query.yaml @@ -19,14 +19,14 @@ # explicitly drop non-cascading dependency - type: bigquery_drop_relationship args: - source: bigquery + source: hasura table: dataset: hasura name: author relationship: articles - type: bigquery_untrack_table args: - source: bigquery + source: hasura table: dataset: hasura name: article @@ -34,7 +34,7 @@ # track table with customizations - type: bigquery_track_table args: - source: bigquery + source: hasura table: dataset: hasura name: article @@ -51,7 +51,7 @@ # restore previously (explicitly and implicitly) dropped dependencies - type: bigquery_create_array_relationship args: - source: bigquery + source: hasura table: dataset: hasura name: author @@ -66,7 +66,7 @@ - type: bigquery_create_object_relationship args: - source: bigquery + source: hasura table: dataset: hasura name: article @@ -82,7 +82,7 @@ - type: bigquery_create_select_permission args: role: published_only - source: bigquery + source: hasura table: dataset: hasura name: article @@ -97,7 +97,7 @@ - type: bigquery_create_select_permission args: role: exact_article_id - source: bigquery + source: hasura table: dataset: hasura name: article @@ -112,7 +112,7 @@ - type: bigquery_create_select_permission args: role: empty_perms - source: bigquery + source: hasura table: dataset: hasura name: article @@ -123,7 +123,7 @@ - type: bigquery_create_select_permission args: role: user - source: bigquery + source: hasura table: dataset: hasura name: article @@ -141,7 +141,7 @@ - type: bigquery_create_select_permission args: role: test_timestamp_perm - source: bigquery + source: hasura table: dataset: hasura name: article @@ -183,4 +183,3 @@ } } } - diff --git a/server/tests-py/test_graphql_queries.py b/server/tests-py/test_graphql_queries.py index 2127d6e8135..572e76dc22c 100644 --- a/server/tests-py/test_graphql_queries.py +++ b/server/tests-py/test_graphql_queries.py @@ -49,13 +49,7 @@ class TestGraphQLQueryBasicMySQL: @pytest.mark.parametrize("transport", ['http', 'websocket']) @pytest.mark.parametrize("backend", ['bigquery']) @usefixtures('per_class_tests_db_state') -class TestGraphQLQueryTableCustomizationsWithReplaceMetadataBigquery: - - def test_replace_metadata_with_customization(self, hge_ctx, transport): - if transport == 'http': - check_query_f(hge_ctx, self.dir() + '/replace_metadata_with_customization_and_test_query.yaml') - # This also sets the global_select_limit for hasura_global_limited to 1 - # so, having that test (and related) right after here is convenient. +class TestGraphQLQueryTableCustomizationsBigquery: def test_global_limit(self, hge_ctx, transport): check_query_f(hge_ctx, self.dir() + "/global_limit.yaml", transport) @@ -63,39 +57,22 @@ class TestGraphQLQueryTableCustomizationsWithReplaceMetadataBigquery: def test_offset_regression(self, hge_ctx, transport): check_query_f(hge_ctx, self.dir() + "/offset_regression.yaml", transport) - @classmethod - def dir(cls): - return 'queries/graphql_query/bigquery' - - -@pytest.mark.parametrize("transport", ['http', 'websocket']) -@pytest.mark.parametrize("backend", ['bigquery']) -@usefixtures('per_class_tests_db_state') -class TestGraphQLQueryTableCustomizationsWithTrackTableBigquery: - def test_track_table_with_customization(self, hge_ctx, transport): if transport == 'http': check_query_f(hge_ctx, self.dir() + '/track_table_with_customization_and_test_query.yaml') - @classmethod - def dir(cls): - return 'queries/graphql_query/bigquery' - - -@pytest.mark.parametrize("transport", ['http', 'websocket']) -@pytest.mark.parametrize("backend", ['bigquery']) -@usefixtures('per_class_tests_db_state') -class TestGraphQLQueryCustomizationsWithSetTableCustomizationBigquery: - def test_set_table_customization(self, hge_ctx, transport): if transport == 'http': check_query_f(hge_ctx, self.dir() + '/set_table_customization_and_test_query.yaml') + def test_replace_metadata_with_customization(self, hge_ctx, transport): + if transport == 'http': + check_query_f(hge_ctx, self.dir() + '/replace_metadata_with_customization_and_test_query.yaml') + @classmethod def dir(cls): return 'queries/graphql_query/bigquery' - @pytest.mark.parametrize("transport", ['http', 'websocket']) @pytest.mark.parametrize("backend", ['bigquery']) @usefixtures('per_class_tests_db_state')