Improvements to query usage analytics data shape (#715)

This PR introduces the following changes to query usage analytics data
shape:
- The `name` field in `RelationshipUsage` is just `RelationshipName`
without `Qualified` wrapper. The `source` is already qualified, and the
same qualification applies to `name`.
- The `used` for both field and input field is a list. A field can use
multiple opendd objects at a time.
- Example: A root field can use `Model` and `Permission` (with both
filter and argument presets).
- The permission usage now revamped to express available permissions in
the opendd
  - Filter predicate - provides lists of fields and relationships
  - Field presets - provides a list of fields involved
  - Argument presets - provides a list of arguments involved
- The `GqlFieldArgument` is dropped in favor of `GqlInputField`.
- Opendd object usage is not specified for `GqlFieldArgument`. An input
argument with object type can have field presets permission. It is
replaced with `GqlInputField` to allow specifying the permission usage.

This PR also includes JSON schema for the data shape with a golden test
to verify.
V3_GIT_ORIGIN_REV_ID: f0bf9ba201471af367ef5027bc2c8b9f915994ac
This commit is contained in:
Rakesh Emmadi 2024-06-25 13:26:31 +05:30 committed by hasura-bot
parent d96bb22844
commit e8ec700d70
7 changed files with 644 additions and 60 deletions

3
v3/Cargo.lock generated
View File

@ -1899,6 +1899,7 @@ dependencies = [
"open-dds",
"ref-cast",
"reqwest",
"schemars",
"serde",
"serde_json",
"similar-asserts",
@ -2522,8 +2523,10 @@ dependencies = [
name = "query-usage-analytics"
version = "0.1.0"
dependencies = [
"goldenfile",
"metadata-resolve",
"open-dds",
"schemars",
"serde",
"serde_json",
]

View File

@ -19,6 +19,7 @@ ndc-models = { workspace = true }
nonempty = { workspace = true }
ref-cast = { workspace = true }
reqwest = { workspace = true, features = ["json", "multipart"] }
schemars = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }

View File

@ -12,6 +12,7 @@ use open_dds::{
models::{ModelGraphQlDefinition, ModelName, OrderableField},
types::{CustomTypeName, FieldName},
};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
@ -54,6 +55,7 @@ pub struct Model {
#[derive(
Serialize,
Deserialize,
JsonSchema,
Clone,
Debug,
PartialEq,

View File

@ -2,10 +2,13 @@ use std::fmt::Display;
use std::{collections::BTreeMap, fmt::Write};
use open_dds::types::{BaseType, CustomTypeName, InbuiltType, TypeName, TypeReference};
use schemars::JsonSchema;
use serde::{de::DeserializeOwned, ser::SerializeMap, Deserialize, Serialize};
use serde_json;
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Hash, Eq, PartialOrd, Ord)]
#[derive(
Serialize, Deserialize, JsonSchema, Clone, Debug, PartialEq, Hash, Eq, PartialOrd, Ord,
)]
pub struct Qualified<T: Display> {
pub subgraph: String,
pub name: T,

View File

@ -10,9 +10,15 @@ bench = false
[dependencies]
open-dds = { path = "../open-dds" }
metadata-resolve = { path = "../metadata-resolve" }
schemars = { version = "0.8.20", features = ["preserve_order"] }
serde = { workspace = true }
serde_json = { workspace = true }
[dev-dependencies]
goldenfile = "^1.7.1"
schemars = { version = "0.8.20", features = ["preserve_order"] }
[lints]
workspace = true

View File

@ -0,0 +1,517 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "GqlOperation",
"description": "This is the data to emit (serlialized) for analytics, when a GraphQL operation is executed.",
"oneOf": [
{
"type": "object",
"required": [
"query"
],
"properties": {
"query": {
"type": "object",
"required": [
"fields",
"operation_name"
],
"properties": {
"operation_name": {
"type": "string"
},
"fields": {
"type": "array",
"items": {
"$ref": "#/definitions/GqlField"
}
}
}
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"mutation"
],
"properties": {
"mutation": {
"type": "object",
"required": [
"fields",
"operation_name"
],
"properties": {
"operation_name": {
"type": "string"
},
"fields": {
"type": "array",
"items": {
"$ref": "#/definitions/GqlField"
}
}
}
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"subscription"
],
"properties": {
"subscription": {
"type": "object",
"required": [
"fields",
"operation_name"
],
"properties": {
"operation_name": {
"type": "string"
},
"fields": {
"type": "array",
"items": {
"$ref": "#/definitions/GqlField"
}
}
}
}
},
"additionalProperties": false
}
],
"definitions": {
"GqlField": {
"description": "A GraphQL field appearing in the query",
"type": "object",
"required": [
"alias",
"arguments",
"fields",
"name",
"used"
],
"properties": {
"name": {
"description": "Name of the GraphQL field",
"type": "string"
},
"alias": {
"description": "Alias of this field used in the query",
"type": "string"
},
"arguments": {
"description": "Arguments of this field",
"type": "array",
"items": {
"$ref": "#/definitions/GqlInputField"
}
},
"fields": {
"description": "Fields in its selection set",
"type": "array",
"items": {
"$ref": "#/definitions/GqlField"
}
},
"used": {
"description": "Which OpenDD objects it is using",
"type": "array",
"items": {
"$ref": "#/definitions/OpenddObject"
}
}
}
},
"GqlInputField": {
"description": "A GraphQL input field",
"type": "object",
"required": [
"fields",
"name",
"used"
],
"properties": {
"name": {
"description": "Name of the input field",
"type": "string"
},
"fields": {
"description": "Fields of this input field",
"type": "array",
"items": {
"$ref": "#/definitions/GqlInputField"
}
},
"used": {
"description": "Which OpenDD objects it is using",
"type": "array",
"items": {
"$ref": "#/definitions/OpenddObject"
}
}
}
},
"OpenddObject": {
"description": "All kinds of OpenDD objects that could be used in a GraphQL operation",
"oneOf": [
{
"type": "object",
"required": [
"model"
],
"properties": {
"model": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"$ref": "#/definitions/Qualified_for_ModelName"
}
}
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"command"
],
"properties": {
"command": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"$ref": "#/definitions/Qualified_for_CommandName"
}
}
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"field"
],
"properties": {
"field": {
"$ref": "#/definitions/FieldUsage"
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"permission"
],
"properties": {
"permission": {
"$ref": "#/definitions/PermissionUsage"
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"relationship"
],
"properties": {
"relationship": {
"$ref": "#/definitions/RelationshipUsage"
}
},
"additionalProperties": false
}
]
},
"Qualified_for_ModelName": {
"type": "object",
"required": [
"name",
"subgraph"
],
"properties": {
"subgraph": {
"type": "string"
},
"name": {
"$ref": "#/definitions/ModelName"
}
}
},
"ModelName": {
"$id": "https://hasura.io/jsonschemas/metadata/ModelName",
"title": "ModelName",
"description": "The name of data model.",
"type": "string",
"pattern": "^[_a-zA-Z][_a-zA-Z0-9]*$"
},
"Qualified_for_CommandName": {
"type": "object",
"required": [
"name",
"subgraph"
],
"properties": {
"subgraph": {
"type": "string"
},
"name": {
"$ref": "#/definitions/CommandName"
}
}
},
"CommandName": {
"$id": "https://hasura.io/jsonschemas/metadata/CommandName",
"title": "CommandName",
"description": "The name of a command.",
"type": "string",
"pattern": "^[_a-zA-Z][_a-zA-Z0-9]*$"
},
"FieldUsage": {
"type": "object",
"required": [
"name",
"opendd_type"
],
"properties": {
"name": {
"$ref": "#/definitions/FieldName"
},
"opendd_type": {
"$ref": "#/definitions/Qualified_for_CustomTypeName"
}
}
},
"FieldName": {
"$id": "https://hasura.io/jsonschemas/metadata/FieldName",
"title": "FieldName",
"description": "The name of a field in a user-defined object type.",
"type": "string",
"pattern": "^[_a-zA-Z][_a-zA-Z0-9]*$"
},
"Qualified_for_CustomTypeName": {
"type": "object",
"required": [
"name",
"subgraph"
],
"properties": {
"subgraph": {
"type": "string"
},
"name": {
"$ref": "#/definitions/CustomTypeName"
}
}
},
"CustomTypeName": {
"$id": "https://hasura.io/jsonschemas/metadata/CustomTypeName",
"title": "CustomTypeName",
"description": "The name of a user-defined type.",
"type": "string",
"pattern": "^[_a-zA-Z][_a-zA-Z0-9]*$"
},
"PermissionUsage": {
"oneOf": [
{
"type": "object",
"required": [
"field_presets"
],
"properties": {
"field_presets": {
"$ref": "#/definitions/FieldPresetsUsage"
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"filter_predicate"
],
"properties": {
"filter_predicate": {
"$ref": "#/definitions/FilterPredicateUsage"
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"argument_presets"
],
"properties": {
"argument_presets": {
"$ref": "#/definitions/ArgumentPresetsUsage"
}
},
"additionalProperties": false
}
]
},
"FieldPresetsUsage": {
"type": "object",
"required": [
"fields"
],
"properties": {
"fields": {
"type": "array",
"items": {
"$ref": "#/definitions/FieldUsage"
}
}
}
},
"FilterPredicateUsage": {
"type": "object",
"required": [
"fields",
"relationships"
],
"properties": {
"fields": {
"type": "array",
"items": {
"$ref": "#/definitions/FieldUsage"
}
},
"relationships": {
"type": "array",
"items": {
"$ref": "#/definitions/RelationshipUsage"
}
}
}
},
"RelationshipUsage": {
"type": "object",
"required": [
"name",
"source",
"target"
],
"properties": {
"name": {
"$ref": "#/definitions/RelationshipName"
},
"source": {
"$ref": "#/definitions/Qualified_for_CustomTypeName"
},
"target": {
"$ref": "#/definitions/RelationshipTarget"
}
}
},
"RelationshipName": {
"$id": "https://hasura.io/jsonschemas/metadata/RelationshipName",
"title": "RelationshipName",
"description": "The name of the GraphQL relationship field.",
"type": "string",
"pattern": "^[_a-zA-Z][_a-zA-Z0-9]*$"
},
"RelationshipTarget": {
"oneOf": [
{
"type": "object",
"required": [
"model"
],
"properties": {
"model": {
"type": "object",
"required": [
"model_name",
"relationship_type"
],
"properties": {
"model_name": {
"$ref": "#/definitions/Qualified_for_ModelName"
},
"relationship_type": {
"$ref": "#/definitions/RelationshipType"
}
}
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"command"
],
"properties": {
"command": {
"type": "object",
"required": [
"command_name"
],
"properties": {
"command_name": {
"$ref": "#/definitions/Qualified_for_CommandName"
}
}
}
},
"additionalProperties": false
}
]
},
"RelationshipType": {
"title": "RelationshipType",
"description": "Type of the relationship.",
"oneOf": [
{
"description": "Select one related object from the target.",
"type": "string",
"enum": [
"Object"
]
},
{
"description": "Select multiple related objects from the target.",
"type": "string",
"enum": [
"Array"
]
}
]
},
"ArgumentPresetsUsage": {
"type": "object",
"required": [
"arguments"
],
"properties": {
"arguments": {
"type": "array",
"items": {
"$ref": "#/definitions/ConnectorArgumentName"
}
}
}
},
"ConnectorArgumentName": {
"description": "The name of an argument as defined by a data connector",
"type": "string"
}
}
}

View File

@ -1,17 +1,18 @@
//! Usage analytics, like model, command, field usage analytics, from a GraphQL query
use metadata_resolve::Qualified;
use metadata_resolve::{ConnectorArgumentName, Qualified};
use open_dds::{
commands::CommandName,
models::ModelName,
relationships::{RelationshipName, RelationshipType},
types::{CustomTypeName, FieldName},
};
use schemars::JsonSchema;
use serde::Serialize;
/// This is the data to emit (serlialized) for analytics, when a GraphQL
/// operation is executed.
#[derive(Serialize)]
#[derive(Serialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum GqlOperation {
Query {
@ -22,71 +23,87 @@ pub enum GqlOperation {
operation_name: String,
fields: Vec<GqlField>,
},
Subscription {
operation_name: String,
fields: Vec<GqlField>,
},
}
/// A GraphQL field appearing in the query
#[derive(Serialize)]
#[derive(Serialize, JsonSchema)]
pub struct GqlField {
/// Name of the GraphQL field
pub name: String,
/// Alias of this field used in the query
pub alias: String,
/// Arguments of this field
pub arguments: Vec<GqlFieldArgument>,
pub arguments: Vec<GqlInputField>,
/// Fields in its selection set
pub fields: Vec<GqlField>,
/// Which OpenDD object it is using
pub used: OpenddObject,
/// Which OpenDD objects it is using
pub used: Vec<OpenddObject>,
}
#[derive(Serialize)]
#[derive(Serialize, JsonSchema)]
/// A GraphQL input field
pub struct GqlInputField {
/// Name of the input field
pub name: String,
/// Fields of this input field
pub fields: Vec<GqlInputField>,
/// Which OpenDD object it is using
pub used: Option<OpenddObject>,
}
/// Arguments of a GraphQL field
#[derive(Serialize)]
pub struct GqlFieldArgument {
pub name: String,
pub fields: Vec<GqlInputField>,
/// Which OpenDD objects it is using
pub used: Vec<OpenddObject>,
}
/// All kinds of OpenDD objects that could be used in a GraphQL operation
#[derive(Serialize, Clone)]
#[derive(Serialize, JsonSchema, Clone)]
#[serde(rename_all = "snake_case")]
pub enum OpenddObject {
Model { name: Qualified<ModelName> },
Command { name: Qualified<CommandName> },
Field(FieldUsage),
Permission(PermissionsUsage),
Permission(PermissionUsage),
Relationship(RelationshipUsage),
}
#[derive(Serialize, Clone)]
#[derive(Serialize, JsonSchema, Clone)]
pub struct FieldUsage {
pub name: FieldName,
pub opendd_type: Qualified<CustomTypeName>,
}
#[derive(Serialize, Clone)]
pub struct PermissionsUsage {
#[derive(Serialize, JsonSchema, Clone)]
#[serde(rename_all = "snake_case")]
pub enum PermissionUsage {
FieldPresets(FieldPresetsUsage),
FilterPredicate(FilterPredicateUsage),
ArgumentPresets(ArgumentPresetsUsage),
}
#[derive(Serialize, JsonSchema, Clone)]
pub struct FieldPresetsUsage {
pub fields: Vec<FieldUsage>,
}
#[derive(Serialize, Clone)]
#[derive(Serialize, JsonSchema, Clone)]
pub struct FilterPredicateUsage {
pub fields: Vec<FieldUsage>,
pub relationships: Vec<RelationshipUsage>,
}
#[derive(Serialize, JsonSchema, Clone)]
pub struct ArgumentPresetsUsage {
pub arguments: Vec<ConnectorArgumentName>,
}
#[derive(Serialize, JsonSchema, Clone)]
pub struct RelationshipUsage {
pub name: Qualified<RelationshipName>,
pub name: RelationshipName,
pub source: Qualified<CustomTypeName>,
pub target: RelationshipTarget,
}
#[derive(Serialize, Clone)]
#[derive(Serialize, JsonSchema, Clone)]
#[serde(rename_all = "snake_case")]
pub enum RelationshipTarget {
Model {
@ -95,15 +112,32 @@ pub enum RelationshipTarget {
},
Command {
command_name: Qualified<CommandName>,
relationship_type: RelationshipType,
},
}
#[cfg(test)]
mod tests {
use super::*;
use goldenfile::Mint;
use open_dds::identifier;
use open_dds::relationships::RelationshipType;
use schemars::schema_for;
use std::{io::Write, path::PathBuf};
#[test]
fn test_json_schema() {
let mut mint = Mint::new(PathBuf::from(env!("CARGO_MANIFEST_DIR")));
let mut expected = mint
.new_goldenfile("query_usage_analytics.jsonschema")
.unwrap();
let schema = schema_for!(super::GqlOperation);
write!(
expected,
"{}",
serde_json::to_string_pretty(&schema).unwrap()
)
.unwrap();
}
#[test]
// just a dummy serialize test for now to visualize the output
@ -128,7 +162,7 @@ mod tests {
}
*/
let product_relationship = OpenddObject::Relationship(RelationshipUsage {
name: Qualified::new("app".to_string(), RelationshipName(identifier!("product"))),
name: RelationshipName(identifier!("product")),
source: Qualified::new("app".to_string(), CustomTypeName(identifier!("Order"))),
target: RelationshipTarget::Model {
model_name: Qualified::new("app".to_string(), ModelName(identifier!("Products"))),
@ -139,16 +173,16 @@ mod tests {
// id: {_eq: 5}
let product_id_filter = GqlInputField {
name: "id".to_string(),
used: Some(OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("id")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Order")),
),
})),
})],
fields: vec![GqlInputField {
name: "_eq".to_string(),
used: None,
used: vec![],
fields: vec![],
}],
};
@ -157,82 +191,86 @@ mod tests {
name: "products".to_string(),
fields: vec![GqlInputField {
name: "price".to_string(),
used: Some(OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("price")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Product")),
),
})),
})],
fields: vec![GqlInputField {
name: "_gt".to_string(),
used: None,
used: vec![],
fields: vec![],
}],
}],
used: Some(product_relationship.clone()),
used: vec![product_relationship.clone()],
};
// where: {id: {_eq: 5}, products: {price: {_gt: 100}}}
let where_argument = GqlFieldArgument {
let where_argument = GqlInputField {
name: "where".to_string(),
fields: vec![product_id_filter, products_price_filter],
used: vec![],
};
// order_by: {product: {price: asc}}
let order_by_argument = GqlFieldArgument {
let order_by_argument = GqlInputField {
name: "order_by".to_string(),
fields: vec![GqlInputField {
name: "product".to_string(),
fields: vec![GqlInputField {
name: "price".to_string(),
fields: vec![],
used: Some(OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("price")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Product")),
),
})),
})],
}],
used: Some(product_relationship.clone()),
used: vec![product_relationship.clone()],
}],
used: vec![],
};
// quantity: {_gt: 2}
let products_quantity_filter = GqlInputField {
name: "quantity".to_string(),
used: Some(OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("quantity")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Product")),
),
})),
})],
fields: vec![GqlInputField {
name: "_gt".to_string(),
used: None,
used: vec![],
fields: vec![],
}],
};
// where: {quantity: {_gt: 2}}
let products_where_argument = GqlFieldArgument {
let products_where_argument = GqlInputField {
name: "where".to_string(),
fields: vec![products_quantity_filter],
used: vec![],
};
// order_by: {quantity: desc}
let products_order_by_argument = GqlFieldArgument {
let products_order_by_argument = GqlInputField {
name: "order_by".to_string(),
fields: vec![GqlInputField {
name: "quantity".to_string(),
fields: vec![],
used: Some(OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("quantity")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Product")),
),
})),
})],
}],
used: vec![],
};
let operation = GqlOperation::Query {
@ -241,20 +279,34 @@ mod tests {
name: "app_orders".to_string(),
alias: "orders".to_string(),
arguments: vec![where_argument, order_by_argument],
used: OpenddObject::Model {
name: Qualified::new("app".to_string(), ModelName(identifier!("Orders"))),
},
used: vec![
OpenddObject::Model {
name: Qualified::new("app".to_string(), ModelName(identifier!("Orders"))),
},
OpenddObject::Permission(PermissionUsage::FilterPredicate(
FilterPredicateUsage {
fields: vec![FieldUsage {
name: FieldName(identifier!("id")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Order")),
),
}],
relationships: vec![],
},
)),
],
fields: vec![
GqlField {
name: "date".to_string(),
alias: "date".to_string(),
used: OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("date")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Order")),
),
}),
})],
fields: vec![],
arguments: vec![],
},
@ -268,51 +320,51 @@ mod tests {
alias: "address_line_1".to_string(),
arguments: vec![],
fields: vec![],
used: OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("address_line_1")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Address")),
),
}),
})],
},
GqlField {
name: "address_line_2".to_string(),
alias: "address_line_2".to_string(),
arguments: vec![],
fields: vec![],
used: OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("address_line_2")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Address")),
),
}),
})],
},
],
used: OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("address")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Order")),
),
}),
})],
},
GqlField {
name: "product".to_string(),
alias: "product".to_string(),
arguments: vec![products_where_argument, products_order_by_argument],
used: product_relationship,
used: vec![product_relationship],
fields: vec![GqlField {
name: "name".to_string(),
alias: "name".to_string(),
used: OpenddObject::Field(FieldUsage {
used: vec![OpenddObject::Field(FieldUsage {
name: FieldName(identifier!("name")),
opendd_type: Qualified::new(
"app".to_string(),
CustomTypeName(identifier!("Product")),
),
}),
})],
arguments: vec![],
fields: vec![],
}],