#!/usr/bin/env python3 # This module is for tests that validate our tests or test framework, make sure # tests are running correctly, or test our python test helpers. import itertools from typing import Any, Generic, Optional, TypeVar import pytest from ruamel.yaml.comments import CommentedMap from validate import check_query_f, collapse_order_not_selset, validate_http_anyq_with_allowed_responses usefixtures = pytest.mark.usefixtures @usefixtures('postgis', 'per_class_tests_db_state') class TestTests1: """ Test various things about our test framework code. Validate that tests work as we expect. """ # NOTE: We don't care about this for now, but should adapt this to test # that xfail detection in code that handles `--accept` works correctly. @pytest.mark.xfail(reason="expected") def test_tests_xfail(self, request): try: marker = request.node.get_closest_marker("xfail") print(marker) if marker.name != 'xfail': print("FAIL!") return True # Force a test failure when xfail strict except: print("FAIL!") return True # Force a test failure when xfail strict assert 0, "Expected failure is expected" # Adapted arbitrarily from # `TestGraphQLQueryBasic.test_select_query_author_pk()` using original yaml # test case file that we later fixed. @pytest.mark.xfail(reason="expected, validating test code") def test_tests_detect_bad_ordering(self, hge_ctx): """We can detect bad ordering of selection set""" check_query_f(hge_ctx, 'test_tests/select_query_author_by_pkey_bad_ordering.yaml', 'http') # # E AssertionError: # E expected: # E data: # E author_by_pk: # E name: Author 1 # E id: 1 # E diff: (results differ only in their order of keys) # E response: # E data: # E author_by_pk: # E id: 1 # E name: Author 1 # Re-use setup and teardown from where we adapted this test case: @classmethod def dir(cls): return 'queries/graphql_query/basic' @usefixtures('postgis', 'per_class_tests_db_state') class TestTests2: """ Test various things about our test framework code. Validate that tests work as we expect. """ # Test another bad ordering scenario, while we're here: @pytest.mark.xfail(reason="expected, validating test code") def test_tests_detect_bad_ordering(self, hge_ctx): """We can detect bad ordering of selection set""" check_query_f(hge_ctx, 'test_tests/user_can_query_jsonb_values_filter_bad_order.yaml', 'http') # # E AssertionError: # E expected: # E data: # E jsonb_table: # E - jsonb_col: # E name: Hasura # E age: 7 # E id: 1 # E response: # E data: # E jsonb_table: # E - id: 1 # E jsonb_col: # E age: 7 # E name: Hasura # E diff: (results differ only in their order of keys) # Unit test for good measure, to validate above and check our assumptions # wrt comparisons of trees of ordered and unordered dicts and arrays: def test_tests_dict_ordering_assumptions_and_helpers(self): # fragment of yaml test file: example_query = {"query": """ query { thing1 jsonb_table{ id jsonb_col } thing2 } """ } # We want to collapse any ordering we don't care about here # (CommentedMap is ruamel.yaml's OrderedMap that also preserves # format): fully_ordered_result = \ CommentedMap([('data', CommentedMap([ ('thing1', "thing1"), ('jsonb_table', [ CommentedMap([ ('id', 1), ('jsonb_col', CommentedMap([('age', 7), ('name', 'Hasura')]))]), CommentedMap([ ('id', 2), ('jsonb_col', CommentedMap([('age', 8), ('name', 'Rawkz')]))]), ]), ('thing2', CommentedMap([("a",1), ("b",2), ("c",3)])), ]))]) relevant_ordered_result = collapse_order_not_selset(fully_ordered_result, example_query) # We expect to have discarded ordering of leaves not in selset: relevant_ordered_result_expected = \ dict([('data', CommentedMap([ ('thing1', "thing1"), ('jsonb_table', [ CommentedMap([ ('id', 1), ('jsonb_col', dict([('age', 7), ('name', 'Hasura')]))]), CommentedMap([ ('id', 2), ('jsonb_col', dict([('age', 8), ('name', 'Rawkz')]))]), ]), ('thing2', dict([("a",1), ("b",2), ("c",3)])), ]))]) # NOTE: use str() to actually do a stong equality comparison, comparing # types. Only works because str() on dict seems to have a canonical # ordering. assert str(relevant_ordered_result) == str(relevant_ordered_result_expected) # Demonstrate equality on different mixes of trees of ordered and unordered dicts: assert CommentedMap([("a", "a"), ("b", "b")]) == dict([("b", "b"), ("a", "a")]) assert CommentedMap([("a", "a"), ("b", "b")]) != CommentedMap([("b", "b"), ("a", "a")]) assert dict([ ("x", CommentedMap([("a", "a"), ("b", CommentedMap([("b1", "b1"), ("b2", "b2")]))])), ("y","y"),]) == \ CommentedMap([("y","y"), ("x", dict([("a", "a"), ("b", CommentedMap([("b1", "b1"), ("b2", "b2")]))])), ]) # type: ignore def test_tests_ordering_differences_correctly_ignored(self, hge_ctx): """ We don't care about ordering of stuff outside the selection set e.g. JSON fields. """ check_query_f(hge_ctx, 'test_tests/user_can_query_jsonb_values_filter_okay_orders.yaml', 'http') # Re-use setup and teardown from where we adapted this test case: @classmethod def dir(cls): return 'queries/graphql_query/permissions' @usefixtures("per_class_tests_db_state") class TestTests3: """ This test case is about testing validate_http_anyq_with_allowed_responses with an empty list of allowed responses wherein it should throw an exception """ @pytest.mark.xfail(reason="expected, validating function working") def test_tests_validate_http_anyq_with_allowed_responses_with_empty_list( self, hge_ctx ): # These values are being set as they are in address_not_null_constraint_error.yaml url = "/v1/graphql" query = { "query": 'mutation {\n insert_address(objects: [{street: "koramangala"}]){\n returning{\n id\n street\n }\n affected_rows\n }\n} \n' } try: resp, pass_test = validate_http_anyq_with_allowed_responses( hge_ctx, url, query, {}, 200, [] ) except: print("FAIL!") assert 0, "Test failure, occurs as expected" # It reaches here only if the exception wasn't caught, in which case # this current test should fail """ This test case is about testing validate_http_anyq_with_allowed_responses with a list of allowed responses which are incorrect wherein it should fail the test """ @pytest.mark.xfail(reason="expected, validating test code") def test_tests_validate_http_anyq_with_allowed_responses_with_no_correct_response( self, hge_ctx ): # These values are being set as they are in address_not_null_constraint_error.yaml url = "/v1/graphql" query = { "query": 'mutation {\n insert_address(objects: [{street: "koramangala"}]){\n returning{\n id\n street\n }\n affected_rows\n }\n} \n' } allowed_response_1 = { "response": { "errors": [ { "extensions": { "code": "constraint-violation", "path": "$.selectionSet.insert_address.args.objects", }, "message": 'Not-NULL. null value in column "door_no" violates not-null constraint', } ] } } allowed_response_2 = { "response": { "errors": [ { "extensions": { "code": "constraint-violation", "path": "$.selectionSet.insert_address.args.objects", }, "message": 'Not-NULL violation. null value in column "door_no" of relation "address" not-null constraint', } ] } } allowed_responses = [allowed_response_1, allowed_response_2] try: resp, err = validate_http_anyq_with_allowed_responses( hge_ctx, url, query, {}, 200, allowed_responses ) except: print("FAIL!") assert 0, "Test failed, as expected" # Re-use setup and teardown from where we adapted this test case: @classmethod def dir(cls): return "queries/graphql_mutation/insert/constraints" class TestParameterizedFixtures: ''' The following tests were an attempt to figure out how to convey information between fixtures when one of them is parameterized, and we have "soft" dependencies between fixtures—that is, B doesn't explicitly depend on A, but rather, A mutates a value provided by C, that B then uses. Unfortunately, there doesn't seem to be a way to do this reliably (see the `xfail` markers below). Preserving this as documentation. ''' # We use `itertools.count` as a simple way to create mutation that's shared across tests. # This allows us to test the parameterization is working. def test_simple_parameterized_fixture(self, simple_parameterized_fixture: int, count=itertools.count()): assert simple_parameterized_fixture == next(count) def test_parameterized_fixture_with_mark(self, request: pytest.FixtureRequest, parameterized_fixture_with_mark: int): marker = request.node.get_closest_marker('value') assert marker is not None assert parameterized_fixture_with_mark == marker.args[0] # Does not actually work; see below for `xfail` markers. def test_parameterized_fixture_mutating_value_used_in_another_fixture(self, parameterized_fixture_that_mutates: str, fixture_using_mutable_value: str): assert parameterized_fixture_that_mutates == fixture_using_mutable_value @pytest.mark.xfail(reason='The marker never makes it to the fixture.') @pytest.mark.parametrize( 'value', [ pytest.param(0, marks=pytest.mark.value(0)), pytest.param(1, marks=pytest.mark.value(1)), pytest.param(2, marks=pytest.mark.value(2)), ] ) def test_parameterized_test_using_marks_with_a_fixture(self, value: int, fixture_returning_mark: int): assert value == fixture_returning_mark @pytest.mark.parametrize( 'fixture_returning_indirect_param', [ pytest.param(0), pytest.param(1), pytest.param(2), ], indirect=True, ) def test_parameterized_test_using_indirect_fixture(self, fixture_returning_indirect_param: int, count=itertools.count()): assert fixture_returning_indirect_param == next(count) @pytest.mark.usefixtures('fixture_returning_indirect_param') def test_using_indirect_fixture_without_param(self, fixture_returning_indirect_param: Any): assert fixture_returning_indirect_param is None @pytest.mark.xfail(reason='`fixture_using_mutable_value` is only instantiated once.') @pytest.mark.parametrize( 'fixture_mutating_with_indirect_param', [ pytest.param(9), pytest.param(8), pytest.param(7), ], indirect=True, ) def test_parameterized_test_using_indirect_fixture_and_mutation(self, fixture_mutating_with_indirect_param: int, fixture_using_mutable_value: int): assert fixture_mutating_with_indirect_param == fixture_using_mutable_value T = TypeVar('T') class MutableValue(Generic[T]): value: Optional[T] = None def get(self) -> T: if self.value is None: raise IndexError('Value is None') return self.value def set(self, new_value: T): self.value = new_value @pytest.fixture(scope='class') def mutable_value() -> MutableValue[str]: return MutableValue() @pytest.fixture(scope='class', params=[0, 1, 2]) def simple_parameterized_fixture(request: pytest.FixtureRequest) -> int: return request.param @pytest.fixture(scope='class', params=[ pytest.param(0, marks=pytest.mark.value(0)), pytest.param(1, marks=pytest.mark.value(1)), pytest.param(2, marks=pytest.mark.value(2)), ]) def parameterized_fixture_with_mark(request: pytest.FixtureRequest) -> int: return request.param @pytest.fixture(scope='class', params=[ 'A', pytest.param('B', marks=pytest.mark.xfail(reason='`fixture_using_mutable_value` is only instantiated once.')), pytest.param('C', marks=pytest.mark.xfail(reason='`fixture_using_mutable_value` is only instantiated once.')), ]) def parameterized_fixture_that_mutates(request: pytest.FixtureRequest, mutable_value: MutableValue[str]) -> str: mutable_value.set(request.param) return request.param @pytest.fixture(scope='class') def fixture_using_mutable_value(mutable_value: MutableValue[T]) -> T: return mutable_value.get() @pytest.fixture(scope='class') def fixture_returning_mark(request: pytest.FixtureRequest) -> Any: marker = request.node.get_closest_marker('value') assert marker is not None return marker.args[0] @pytest.fixture(scope='class') def fixture_returning_indirect_param(request: pytest.FixtureRequest) -> Any: if 'param' not in dir(request): return None return request.param @pytest.fixture(scope='class') def fixture_mutating_with_indirect_param(request: pytest.FixtureRequest, mutable_value: MutableValue[Any]) -> Any: if 'param' not in dir(request): return mutable_value.set(request.param) return request.param