server/bigquery: fix data source references & pytest filter in CI

https://github.com/hasura/graphql-engine-mono/pull/2169

GitOrigin-RevId: a0a7190a404ee558a965e06fe1a022953f6f5a10

.circleci/test-server.sh

@@ -1207,7 +1207,7 @@ admin_users = postgres' > pgbouncer/pgbouncer.ini
     wait_for_port 8080
 
     # See note [Specifying Pytests with -k flag]
-    pytest -n 1 --hge-urls "$HGE_URL" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" --backend bigquery -k "BigQuery or Common"
+    pytest -n 1 --hge-urls "$HGE_URL" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" --backend bigquery -k "Bigquery or Common"
 
     kill_hge_servers
     ;;

server/tests-py/queries/graphql_query/bigquery/set_table_customization_and_test_query.yaml

@@ -6,7 +6,7 @@
   query:
     type: bigquery_set_table_customization
     args:
-      source: bigquery
+      source: hasura
       table:
         dataset: hasura
         name: article
@@ -49,4 +49,3 @@
           }
         }
       }
-

server/tests-py/queries/graphql_query/bigquery/setup_bigquery.yaml

@@ -1,175 +1,125 @@
 type: bulk
 args:
-
-
-- type: bigquery_track_table
+- type: replace_metadata
   args:
-    source: hasura_global_limited
-    table:
-      dataset: hasura
-      name: LimitedAlbum
-
-- type: bigquery_track_table
-  args:
-    source: hasura_global_limited
-    table:
-      dataset: hasura
-      name: LimitedArtist
-
-- type: bigquery_create_object_relationship
-  args:
-    source: hasura_global_limited
-    table:
-      dataset: hasura
-      name: LimitedAlbum
-    name: artist
-    using:
-      manual_configuration:
-        remote_table:
-          dataset: hasura
-          name: LimitedArtist
-        column_mapping:
-          artist_other_id: artist_self_id
-
-- type: bigquery_create_array_relationship
-  args:
-    source: hasura_global_limited
-    table:
-      dataset: hasura
-      name: LimitedArtist
-    name: albums
-    using:
-      manual_configuration:
-        remote_table:
-          dataset: hasura
+    version: 3
+    sources:
+    - name: hasura_global_limited
+      kind: bigquery
+      configuration:
+        service_account: {from_env: HASURA_BIGQUERY_SERVICE_ACCOUNT}
+        project_id: {from_env: HASURA_BIGQUERY_PROJECT_ID}
+        datasets:
+        - hasura
+        global_select_limit: 1
+      tables:
+      - table:
           name: LimitedAlbum
-        column_mapping:
-          artist_self_id: artist_other_id
-
-
-- type: bigquery_track_table
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: author
-
-- type: bigquery_track_table
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-
-- type: bigquery_create_array_relationship
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: author
-    name: articles
-    using:
-      manual_configuration:
-        remote_table:
           dataset: hasura
-          name: article
-        column_mapping:
-          id: author_id
-
-- type: bigquery_create_object_relationship
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-    name: author
-    using:
-      manual_configuration:
-        remote_table:
+        object_relationships:
+        - name: artist
+          using:
+            manual_configuration:
+              remote_table:
+                name: LimitedArtist
+                dataset: hasura
+              column_mapping:
+                artist_other_id: artist_self_id
+      - table:
+          name: LimitedArtist
+          dataset: hasura
+        array_relationships:
+        - name: albums
+          using:
+            manual_configuration:
+              remote_table:
+                name: LimitedAlbum
+                dataset: hasura
+              column_mapping:
+                artist_self_id: artist_other_id
+    - name: hasura
+      kind: bigquery
+      configuration:
+        service_account: {from_env: HASURA_BIGQUERY_SERVICE_ACCOUNT}
+        project_id: {from_env: HASURA_BIGQUERY_PROJECT_ID}
+        datasets:
+        - hasura
+        global_select_limit: 10
+      tables:
+      - table:
+          name: Album
+          dataset: hasura
+        object_relationships:
+        - name: artist
+          using:
+            manual_configuration:
+              remote_table:
+                name: Artist
+                dataset: hasura
+              column_mapping:
+                artist_other_id: artist_self_id
+      - table:
+          name: Artist
+          dataset: hasura
+        array_relationships:
+        - name: albums
+          using:
+            manual_configuration:
+              remote_table:
+                name: Album
+                dataset: hasura
+              column_mapping:
+                artist_self_id: artist_other_id
+      - table:
           dataset: hasura
           name: author
-        column_mapping:
-          author_id: id
-
-- type: bigquery_create_select_permission
-  args:
-    role: published_only
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-    permission:
-      columns:
-        - is_published
-        - id
-        - title
-      filter:
-        is_published: true
-
-- type: bigquery_create_select_permission
-  args:
-    role: exact_article_id
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-    permission:
-      columns:
-        - id
-        - title
-        - content
-      filter:
-        id: {'_eq': 2}
-
-- type: bigquery_create_select_permission
-  args:
-    role: empty_perms
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-    permission:
-      columns: []
-      filter: {}
-
-- type: bigquery_create_select_permission
-  args:
-    role: user
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-    permission:
-      columns:
-        - id
-        - title
-        - content
-        - author_id
-        - is_published
-        - published_on
-        - created_at
-      filter: {}
-
-- type: bigquery_create_select_permission
-  args:
-    role: test_timestamp_perm
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-    permission:
-      columns:
-        - id
-        - title
-        - content
-      filter:
-        created_at: {'_eq': '2008-12-25T07:30:02'}
-
-
-- type: bigquery_track_table
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: all_types
-
+        array_relationships:
+        - using:
+            manual_configuration:
+              remote_table:
+                name: article
+                dataset: hasura
+              column_mapping:
+                id: author_id
+          name: articles
+      - table:
+          dataset: hasura
+          name: article
+        object_relationships:
+        - using:
+            manual_configuration:
+              remote_table:
+                name: author
+                dataset: hasura
+              column_mapping:
+                author_id: id
+          name: author
+        select_permissions:
+        - role: published_only
+          permission:
+            columns: [is_published, id, title]
+            filter:
+              is_published: true
+        - role: exact_article_id
+          permission:
+            columns: [id, title, content]
+            filter:
+              id:
+                '_eq': 2
+        - role: empty_perms
+          permission:
+            columns: []
+            filter: {}
+        - role: user
+          permission:
+            columns: [id, title, content, author_id, is_published, published_on, created_at]
+            filter: {}
+        - role: test_timestamp_perm
+          permission:
+            columns: [id, title, content]
+            filter:
+              created_at:
+                '_eq': '2008-12-25T07:30:02'
+      - table:
+          dataset: hasura
+          name: all_types

server/tests-py/queries/graphql_query/bigquery/teardown_bigquery.yaml

@@ -1,70 +1,2 @@
 type: bulk
-args:
-
-- type: bigquery_untrack_table
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: all_types
-
-- type: bigquery_drop_relationship
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-    relationship: author
-
-- type: bigquery_drop_relationship
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: author
-    relationship: articles
-
-- type: bigquery_untrack_table
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: article
-
-- type: bigquery_untrack_table
-  args:
-    source: bigquery
-    table:
-      dataset: hasura
-      name: author
-
-- type: bigquery_drop_relationship
-  args:
-    source: hasura_global_limited
-    table:
-      dataset: hasura
-      name: LimitedArtist
-    relationship: albums
-
-- type: bigquery_drop_relationship
-  args:
-    source: hasura_global_limited
-    table:
-      dataset: hasura
-      name: LimitedAlbum
-    relationship: artist
-
-- type: bigquery_untrack_table
-  args:
-    source: hasura_global_limited
-    table:
-      dataset: hasura
-      name: LimitedArtist
-
-- type: bigquery_untrack_table
-  args:
-    source: hasura_global_limited
-    table:
-      dataset: hasura
-      name: LimitedAlbum
-
+args: []

server/tests-py/queries/graphql_query/bigquery/track_table_with_customization_and_test_query.yaml

@@ -19,14 +19,14 @@
     # explicitly drop non-cascading dependency
     - type: bigquery_drop_relationship
       args:
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: author
         relationship: articles
     - type: bigquery_untrack_table
       args:
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: article
@@ -34,7 +34,7 @@
     # track table with customizations
     - type: bigquery_track_table
       args:
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: article
@@ -51,7 +51,7 @@
     # restore previously (explicitly and implicitly) dropped dependencies
     - type: bigquery_create_array_relationship
       args:
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: author
@@ -66,7 +66,7 @@
 
     - type: bigquery_create_object_relationship
       args:
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: article
@@ -82,7 +82,7 @@
     - type: bigquery_create_select_permission
       args:
         role: published_only
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: article
@@ -97,7 +97,7 @@
     - type: bigquery_create_select_permission
       args:
         role: exact_article_id
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: article
@@ -112,7 +112,7 @@
     - type: bigquery_create_select_permission
       args:
         role: empty_perms
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: article
@@ -123,7 +123,7 @@
     - type: bigquery_create_select_permission
       args:
         role: user
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: article
@@ -141,7 +141,7 @@
     - type: bigquery_create_select_permission
       args:
         role: test_timestamp_perm
-        source: bigquery
+        source: hasura
         table:
           dataset: hasura
           name: article
@@ -183,4 +183,3 @@
           }
         }
       }
-

server/tests-py/test_graphql_queries.py

@@ -49,13 +49,7 @@ class TestGraphQLQueryBasicMySQL:
 @pytest.mark.parametrize("transport", ['http', 'websocket'])
 @pytest.mark.parametrize("backend", ['bigquery'])
 @usefixtures('per_class_tests_db_state')
-class TestGraphQLQueryTableCustomizationsWithReplaceMetadataBigquery:
-
-    def test_replace_metadata_with_customization(self, hge_ctx, transport):
-        if transport == 'http':
-            check_query_f(hge_ctx, self.dir() + '/replace_metadata_with_customization_and_test_query.yaml')
-            # This also sets the global_select_limit for hasura_global_limited to 1
-            # so, having that test (and related) right after here is convenient.
+class TestGraphQLQueryTableCustomizationsBigquery:
 
     def test_global_limit(self, hge_ctx, transport):
         check_query_f(hge_ctx, self.dir() + "/global_limit.yaml", transport)
@@ -63,39 +57,22 @@ class TestGraphQLQueryTableCustomizationsWithReplaceMetadataBigquery:
     def test_offset_regression(self, hge_ctx, transport):
         check_query_f(hge_ctx, self.dir() + "/offset_regression.yaml", transport)
 
-    @classmethod
-    def dir(cls):
-        return 'queries/graphql_query/bigquery'
-
-
-@pytest.mark.parametrize("transport", ['http', 'websocket'])
-@pytest.mark.parametrize("backend", ['bigquery'])
-@usefixtures('per_class_tests_db_state')
-class TestGraphQLQueryTableCustomizationsWithTrackTableBigquery:
-
     def test_track_table_with_customization(self, hge_ctx, transport):
         if transport == 'http':
             check_query_f(hge_ctx, self.dir() + '/track_table_with_customization_and_test_query.yaml')
 
-    @classmethod
-    def dir(cls):
-        return 'queries/graphql_query/bigquery'
-
-
-@pytest.mark.parametrize("transport", ['http', 'websocket'])
-@pytest.mark.parametrize("backend", ['bigquery'])
-@usefixtures('per_class_tests_db_state')
-class TestGraphQLQueryCustomizationsWithSetTableCustomizationBigquery:
-
     def test_set_table_customization(self, hge_ctx, transport):
         if transport == 'http':
             check_query_f(hge_ctx, self.dir() + '/set_table_customization_and_test_query.yaml')
 
+    def test_replace_metadata_with_customization(self, hge_ctx, transport):
+        if transport == 'http':
+            check_query_f(hge_ctx, self.dir() + '/replace_metadata_with_customization_and_test_query.yaml')
+
     @classmethod
     def dir(cls):
         return 'queries/graphql_query/bigquery'
 
-
 @pytest.mark.parametrize("transport", ['http', 'websocket'])
 @pytest.mark.parametrize("backend", ['bigquery'])
 @usefixtures('per_class_tests_db_state')