graphql-engine/server/benchmarks/benchmark_sets/chinook/config.query.yaml

# This tells graphql-bench that it's testing a hasura instance and should
# collect some additional metrics:
extended_hasura_checks: true
headers:
  X-Hasura-Admin-Secret: my-secret
  X-Hasura-Role: employee
  X-Hasura-User-Id: 4
  # On cloud about 95% of requests incoming Accept gzip (which as of writing
  # means we unconditionally compress, which has significant performance
  # impact), so it's important that we include it in all benchmarks unless we
  # have an intentional reason not to.
  Accept-Encoding: gzip

# Anchors to help us DRY below; settings here may be overridden selectively
constants:
  scalars:
    # We'll measure at just two consistent load levels, which makes comparing
    # benchmarks within the same run useful.
    #
    # NOTE: a load of 500 may cause hasura to fall over on a laptop. On our
    # beefy CI benchmark runner we cannot sustain 1,000 RPS for the
    # "large_result" queries.
    - &low_load 20
    - &high_load 500

  k6_custom: &k6_custom
    tools: [k6]
    execution_strategy: CUSTOM

  settings: &settings
    # This is equivalent to wrk2's approach:
    executor: 'constant-arrival-rate'
    timeUnit: '1s'
    maxVUs: 500 # NOTE: required, else defaults to `preAllocatedVUs`
    # NOTE: ideally we'd test all of the queries with the same *number of requests*
    # but that would mean running the "low_load" queries for much longer than
    # is acceptable.
    duration: '60s'

queries:
  ############################################################################
  # single-table query, small result size; makes use of permissions for
  # filtering; low RPS
  #
  # NOTE: According to data from cloud this is a pretty average response (we see
  # P50 compressed response body size of 123 bytes, and P75 290 bytes):
  #  
  #   uncompressed body size: ~600 bytes
  #   compressed body size:   ~200 bytes   (as of time of writing)
  - name: simple_query_low_load
    <<: *k6_custom
    options:
      k6:
        # NOTE: setting this to true will ignore graphql-layer errors, which
        # still return a 200 HTTP status code.  Unfortunately we need to set to
        # `true` so that we aren't measuring e.g. decompression on the k6 side
        # (see: https://github.com/grafana/k6/issues/2685)
        discardResponseBodies: true
        scenarios:
          main:
            <<: *settings
            rate: *low_load
            # tune this so it's just high enough that we can expect to not need
            # to allocate during the test:
            preAllocatedVUs: 10
    query: &simple_query |
      query MyQuery {
        Customer {
          Email
        }
      }      
  # ...above, but at high RPS
  - name: simple_query_high_load
    <<: *k6_custom
    options:
      k6:
        # See note on Chinook.simple_query_low_load
        discardResponseBodies: true
        scenarios:
          main:
            <<: *settings
            # NOTE: 500 RPS is easy-peasy for this query, but we want to be
            # able to compare to e.g. complex_query_high_load_large_result at
            # the same RPS
            rate: *high_load
            preAllocatedVUs: 50
    query: *simple_query

  ############################################################################
  # A more complex query, with some conditions and joins (exercising
  # bread-and-butter SQL query generation), with variables. We test the same
  # query on both low and high load, and with a small and large response size.


  ######## Small result size
  # 
  #   uncompressed body size: ~650 bytes
  #   compressed body size:   ~200 bytes

  - name: complex_query_low_load_small_result
    <<: *k6_custom
    options:
      k6:
        # See note on Chinook.simple_query_low_load
        discardResponseBodies: true
        scenarios:
          main:
            <<: *settings
            rate: *low_load
            preAllocatedVUs: 10
    variables:
      # Two playlists with comedy tracks; return one row each
      genre: Comedy
      track_lim: 1
    query: &complex_query |
      query MyQuery($genre: String!, $track_lim: Int = 1000) {
        playlist_containing_genre: Playlist(order_by: {Name: asc}, where: {PlaylistTracks: {Track: {Genre: {Name: {_eq: $genre}}}}}) {
          Name
          tracks_of_genre: PlaylistTracks(where: {Track: {Genre: {Name: {_eq: $genre}}}}, limit: $track_lim) {
            Track {
              Name
              Album {
                Title
                Artist {
                  Name
                }
              }
              MediaType {
                Name
              }
            }
          }
        }
      }      

  - name: complex_query_high_load_small_result
    <<: *k6_custom
    options:
      k6:
        # See note on Chinook.simple_query_low_load
        discardResponseBodies: true
        scenarios:
          main:
            <<: *settings
            rate: *high_load
            preAllocatedVUs: 100
    variables:
      # Two playlists with comedy tracks; return one row each
      genre: Comedy
      track_lim: 1
    query: *complex_query


  ######## Large result size
  #
  # NOTE: According to data from cloud, this is somewhere between 
  #       P90 (2 kB compressed) and P95 (4 kB compressed):
  #
  #   uncompressed body size: ~33 kB
  #   compressed body size:    ~3 kB   (as of time of writing)

  - name: complex_query_low_load_large_result
    <<: *k6_custom
    options:
      k6:
        # See note on Chinook.simple_query_low_load
        discardResponseBodies: true
        scenarios:
          main:
            <<: *settings
            rate: *low_load
            preAllocatedVUs: 10
    variables:
      # This yields ~30Kb response body; ~100x the size of simple_query
      #  (FYI "Rock" is around 300Kb) 
      genre: Jazz
    query: *complex_query 

  - name: complex_query_high_load_large_result
    <<: *k6_custom
    options:
      k6:
        # See note on Chinook.simple_query_low_load
        discardResponseBodies: true
        scenarios:
          main:
            <<: *settings
            # NOTE: this will fall start to fall over on a laptop
            rate: *high_load
            preAllocatedVUs: 100
    variables:
      # This yields ~30Kb response body; ~100x the size of simple_query
      genre: Jazz
    query: *complex_query 

  ############################################################################
  # The standard introspection query from server/src-rsr/introspection.json
  # We don't expect a server to be hammered with these, but they are crucial
  # for clients (like graphiql):
  #
  # NOTE: According to data from cloud, this is somewhere between 
  # P95 (4 kB compressed) and P99 (40 kB compressed):
  #
  #   uncompressed body size: ~190 kB
  #   compressed body size:    ~13 kB   (as of time of writing)

  - name: full_introspection
    <<: *k6_custom
    options:
      k6:
        # See note on Chinook.simple_query_low_load
        discardResponseBodies: true
        scenarios:
          main:
            <<: *settings
            rate: *low_load
            preAllocatedVUs: 10
    query: |
      query IntrospectionQuery {
        __schema {
          queryType {
            name
          }
          mutationType {
            name
          }
          subscriptionType {
            name
          }
          types {
            ...FullType
          }
          directives {
            name
            description
            locations
            args {
              ...InputValue
            }
          }
        }
      }

      fragment FullType on __Type {
        kind
        name
        description
        fields(includeDeprecated: true) {
          name
          description
          args {
            ...InputValue
          }
          type {
            ...TypeRef
          }
          isDeprecated
          deprecationReason
        }
        inputFields {
          ...InputValue
        }
        interfaces {
          ...TypeRef
        }
        enumValues(includeDeprecated: true) {
          name
          description
          isDeprecated
          deprecationReason
        }
        possibleTypes {
          ...TypeRef
        }
      }

      fragment InputValue on __InputValue {
        name
        description
        type {
          ...TypeRef
        }
        defaultValue
      }

      fragment TypeRef on __Type {
        kind
        name
        ofType {
          kind
          name
          ofType {
            kind
            name
            ofType {
              kind
              name
              ofType {
                kind
                name
                ofType {
                  kind
                  name
                  ofType {
                    kind
                    name
                    ofType {
                      kind
                      name
                    }
                  }
                }
              }
            }
          }
        }
      }