From c726b528c4b5979318951f38545d49c2b28c3ed6 Mon Sep 17 00:00:00 2001 From: Daniel Chambers Date: Fri, 18 Aug 2023 14:27:49 +1000 Subject: [PATCH] Add table/function filtering and detail level to schema introspection for Data Connectors to improve performance PR-URL: https://github.com/hasura/graphql-engine-mono/pull/10074 GitOrigin-RevId: 12de7387746f456cbefd9f76e446f56e9e643dc9 --- dc-agents/DOCUMENTATION.md | 47 ++++++++++++++ dc-agents/dc-api-types/package.json | 2 +- dc-agents/dc-api-types/src/agent.openapi.json | 60 +++++++++++++++-- dc-agents/dc-api-types/src/index.ts | 3 + .../dc-api-types/src/models/DetailLevel.ts | 11 ++++ .../dc-api-types/src/models/FunctionInfo.ts | 6 +- .../dc-api-types/src/models/SchemaFilters.ts | 18 ++++++ .../dc-api-types/src/models/SchemaRequest.ts | 12 ++++ .../dc-api-types/src/models/TableInfo.ts | 2 +- dc-agents/package-lock.json | 10 +-- dc-agents/reference/package-lock.json | 4 +- dc-agents/reference/package.json | 2 +- dc-agents/reference/src/data/index.ts | 64 +++++++++++++++---- dc-agents/reference/src/index.ts | 14 +++- dc-agents/sqlite/package-lock.json | 4 +- dc-agents/sqlite/package.json | 2 +- dc-agents/sqlite/src/index.ts | 14 +++- dc-agents/sqlite/src/schema.ts | 55 +++++++++++----- server/lib/dc-api/dc-api.cabal | 2 +- .../src/Hasura/Backends/DataConnector/API.hs | 1 + .../Backends/DataConnector/API/V0/Function.hs | 10 +-- .../Backends/DataConnector/API/V0/Schema.hs | 60 ++++++++++++++++- .../Backends/DataConnector/API/V0/Table.hs | 2 +- server/lib/dc-api/test/Main.hs | 8 +-- server/lib/dc-api/test/Test/AgentAPI.hs | 8 ++- server/lib/dc-api/test/Test/Data.hs | 2 +- .../Specs/{UDFSpec.hs => FunctionsSpec.hs} | 40 ++++++++++-- .../lib/dc-api/test/Test/Specs/SchemaSpec.hs | 46 ++++++++++++- .../Backend/DataConnector/Mock/Server.hs | 4 +- .../DataConnector/Adapter/Metadata.hs | 37 ++++++++--- .../Backends/DataConnector/Agent/Client.hs | 6 +- .../DataConnector/API/V0/SchemaSpec.hs | 39 ++++++++++- 32 files changed, 501 insertions(+), 94 deletions(-) create mode 100644 dc-agents/dc-api-types/src/models/DetailLevel.ts create mode 100644 dc-agents/dc-api-types/src/models/SchemaFilters.ts create mode 100644 dc-agents/dc-api-types/src/models/SchemaRequest.ts rename server/lib/dc-api/test/Test/Specs/{UDFSpec.hs => FunctionsSpec.hs} (83%) diff --git a/dc-agents/DOCUMENTATION.md b/dc-agents/DOCUMENTATION.md index ac69e5aa066..1ed07c52f2f 100644 --- a/dc-agents/DOCUMENTATION.md +++ b/dc-agents/DOCUMENTATION.md @@ -415,6 +415,53 @@ Columns can have their value generated by the database, for example auto-increme If the agent declares a lack of mutability support in its capabilities, it should not declare tables/columns as mutable in its schema here. +#### Schema Request +The `/schema` endpoint may be sent a request body with optional filtering details designed to reduce the amount of schema data returned. Here's an example request body: + +```json +{ + "filters": { + "only_tables": [ + ["Artist"], + ["Album"] + ], + "only_functions": [ + ["SearchAlbums"] + ] + }, + "detail_level": "basic_info" +} +``` + +The `filters` property may contain an object with the following properties: +* `only_tables`: This is a list of table names, and the schema response must only contain the tables specified in this list, if it is specified. An empty list means return no tables. +* `only_functions`: This is a list of function names, and the schema response must only contain the functions specified in this list, if it is specified. An empty list means return no functions. + +The `detail_level` property controls what data needs to be returned about functions and tables. There are two values: `everything` and `basic_info` (the default, if omitted, is `everything`). `everything` requires the agent to return all properties described above. `basic_info` requires the agent to only return a reduced set of properties about tables and functions; specifically only table names and types, and function names and types should be returned. All other properties should be omitted. + +Here's an example response to the above request: + +```json +{ + "tables": [ + { + "name": ["Artist"], + "type": "table" + }, + { + "name": ["Album"], + "type": "table" + } + ], + "functions": [ + { + "name": ["SearchAlbums"], + "type": "read" + } + ] +} +``` + #### Type definitions The `SchemaResponse` TypeScript type from [the reference implementation](./reference/src/types/index.ts) describes the valid response body for the `GET /schema` endpoint. diff --git a/dc-agents/dc-api-types/package.json b/dc-agents/dc-api-types/package.json index 823b71c6216..4726ede40a1 100644 --- a/dc-agents/dc-api-types/package.json +++ b/dc-agents/dc-api-types/package.json @@ -1,6 +1,6 @@ { "name": "@hasura/dc-api-types", - "version": "0.40.0", + "version": "0.41.0", "description": "Hasura GraphQL Engine Data Connector Agent API types", "author": "Hasura (https://github.com/hasura/graphql-engine)", "license": "Apache-2.0", diff --git a/dc-agents/dc-api-types/src/agent.openapi.json b/dc-agents/dc-api-types/src/agent.openapi.json index 5abdef36ca8..b0f9755a843 100644 --- a/dc-agents/dc-api-types/src/agent.openapi.json +++ b/dc-agents/dc-api-types/src/agent.openapi.json @@ -63,6 +63,15 @@ } } ], + "requestBody": { + "content": { + "application/json;charset=utf-8": { + "schema": { + "$ref": "#/components/schemas/SchemaRequest" + } + } + } + }, "responses": { "200": { "content": { @@ -82,7 +91,7 @@ } } }, - "description": " or `X-Hasura-DataConnector-Config` or `X-Hasura-DataConnector-SourceName`" + "description": " or `body` or `X-Hasura-DataConnector-Config` or `X-Hasura-DataConnector-SourceName`" }, "500": { "content": { @@ -1286,6 +1295,7 @@ "TableInfo": { "properties": { "columns": { + "default": [], "description": "The columns of the table", "items": { "$ref": "#/components/schemas/ColumnInfo" @@ -1336,8 +1346,7 @@ } }, "required": [ - "name", - "columns" + "name" ], "type": "object" }, @@ -1428,6 +1437,7 @@ "FunctionInfo": { "properties": { "args": { + "default": [], "description": "argument info - name/types", "items": { "$ref": "#/components/schemas/FunctionInformationArgument" @@ -1448,6 +1458,7 @@ "one", "many" ], + "nullable": true, "type": "string" }, "returns": { @@ -1459,10 +1470,7 @@ }, "required": [ "name", - "type", - "returns", - "response_cardinality", - "args" + "type" ], "type": "object" }, @@ -1490,6 +1498,44 @@ ], "type": "object" }, + "SchemaRequest": { + "properties": { + "detail_level": { + "$ref": "#/components/schemas/DetailLevel" + }, + "filters": { + "$ref": "#/components/schemas/SchemaFilters" + } + }, + "type": "object" + }, + "SchemaFilters": { + "properties": { + "only_functions": { + "description": "Only get the schemas for these functions", + "items": { + "$ref": "#/components/schemas/FunctionName" + }, + "type": "array" + }, + "only_tables": { + "description": "Only get the schemas for these tables", + "items": { + "$ref": "#/components/schemas/TableName" + }, + "type": "array" + } + }, + "type": "object" + }, + "DetailLevel": { + "description": "How much information to return about the schema. Values:\n- 'everything': All information about the schema.\n- 'basic_info': For tables, only the table name and table type, for functions, only the function name and function type.\n", + "enum": [ + "everything", + "basic_info" + ], + "type": "string" + }, "QueryResponse": { "properties": { "aggregates": { diff --git a/dc-agents/dc-api-types/src/index.ts b/dc-agents/dc-api-types/src/index.ts index 4d1be956c89..06bc2152f7b 100644 --- a/dc-agents/dc-api-types/src/index.ts +++ b/dc-agents/dc-api-types/src/index.ts @@ -48,6 +48,7 @@ export type { DatasetTemplateName } from './models/DatasetTemplateName'; export type { DefaultValueGenerationStrategy } from './models/DefaultValueGenerationStrategy'; export type { DeleteCapabilities } from './models/DeleteCapabilities'; export type { DeleteMutationOperation } from './models/DeleteMutationOperation'; +export type { DetailLevel } from './models/DetailLevel'; export type { ErrorResponse } from './models/ErrorResponse'; export type { ErrorResponseType } from './models/ErrorResponseType'; export type { ExistsExpression } from './models/ExistsExpression'; @@ -132,6 +133,8 @@ export type { ScalarTypeCapabilities } from './models/ScalarTypeCapabilities'; export type { ScalarTypesCapabilities } from './models/ScalarTypesCapabilities'; export type { ScalarValue } from './models/ScalarValue'; export type { ScalarValueComparison } from './models/ScalarValueComparison'; +export type { SchemaFilters } from './models/SchemaFilters'; +export type { SchemaRequest } from './models/SchemaRequest'; export type { SchemaResponse } from './models/SchemaResponse'; export type { SetColumnRowUpdate } from './models/SetColumnRowUpdate'; export type { SingleColumnAggregate } from './models/SingleColumnAggregate'; diff --git a/dc-agents/dc-api-types/src/models/DetailLevel.ts b/dc-agents/dc-api-types/src/models/DetailLevel.ts new file mode 100644 index 00000000000..f58197a1f9f --- /dev/null +++ b/dc-agents/dc-api-types/src/models/DetailLevel.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * How much information to return about the schema. Values: + * - 'everything': All information about the schema. + * - 'basic_info': For tables, only the table name and table type, for functions, only the function name and function type. + * + */ +export type DetailLevel = 'everything' | 'basic_info'; diff --git a/dc-agents/dc-api-types/src/models/FunctionInfo.ts b/dc-agents/dc-api-types/src/models/FunctionInfo.ts index 323d5716cef..f6fffb64907 100644 --- a/dc-agents/dc-api-types/src/models/FunctionInfo.ts +++ b/dc-agents/dc-api-types/src/models/FunctionInfo.ts @@ -11,7 +11,7 @@ export type FunctionInfo = { /** * argument info - name/types */ - args: Array; + args?: Array; /** * Description of the table */ @@ -20,8 +20,8 @@ export type FunctionInfo = { /** * object response if false, rows if true */ - response_cardinality: 'one' | 'many'; - returns: FunctionReturnType; + response_cardinality?: 'one' | 'many' | null; + returns?: FunctionReturnType; type: FunctionType; }; diff --git a/dc-agents/dc-api-types/src/models/SchemaFilters.ts b/dc-agents/dc-api-types/src/models/SchemaFilters.ts new file mode 100644 index 00000000000..dbcfbf0993a --- /dev/null +++ b/dc-agents/dc-api-types/src/models/SchemaFilters.ts @@ -0,0 +1,18 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { FunctionName } from './FunctionName'; +import type { TableName } from './TableName'; + +export type SchemaFilters = { + /** + * Only get the schemas for these functions + */ + only_functions?: Array; + /** + * Only get the schemas for these tables + */ + only_tables?: Array; +}; + diff --git a/dc-agents/dc-api-types/src/models/SchemaRequest.ts b/dc-agents/dc-api-types/src/models/SchemaRequest.ts new file mode 100644 index 00000000000..5c424573fa3 --- /dev/null +++ b/dc-agents/dc-api-types/src/models/SchemaRequest.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { DetailLevel } from './DetailLevel'; +import type { SchemaFilters } from './SchemaFilters'; + +export type SchemaRequest = { + detail_level?: DetailLevel; + filters?: SchemaFilters; +}; + diff --git a/dc-agents/dc-api-types/src/models/TableInfo.ts b/dc-agents/dc-api-types/src/models/TableInfo.ts index 92f927794f8..9a8907b7e0b 100644 --- a/dc-agents/dc-api-types/src/models/TableInfo.ts +++ b/dc-agents/dc-api-types/src/models/TableInfo.ts @@ -11,7 +11,7 @@ export type TableInfo = { /** * The columns of the table */ - columns: Array; + columns?: Array; /** * Whether or not existing rows can be deleted in the table */ diff --git a/dc-agents/package-lock.json b/dc-agents/package-lock.json index 34b9119a2db..fda5b8cfb2e 100644 --- a/dc-agents/package-lock.json +++ b/dc-agents/package-lock.json @@ -24,7 +24,7 @@ }, "dc-api-types": { "name": "@hasura/dc-api-types", - "version": "0.40.0", + "version": "0.41.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", @@ -2227,7 +2227,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.40.0", + "@hasura/dc-api-types": "0.41.0", "fastify": "^4.13.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", @@ -2547,7 +2547,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.40.0", + "@hasura/dc-api-types": "0.41.0", "fastify": "^4.13.0", "fastify-metrics": "^9.2.1", "nanoid": "^3.3.4", @@ -2868,7 +2868,7 @@ "version": "file:reference", "requires": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.40.0", + "@hasura/dc-api-types": "0.41.0", "@tsconfig/node16": "^1.0.3", "@types/node": "^16.11.49", "@types/xml2js": "^0.4.11", @@ -3080,7 +3080,7 @@ "version": "file:sqlite", "requires": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.40.0", + "@hasura/dc-api-types": "0.41.0", "@tsconfig/node16": "^1.0.3", "@types/node": "^16.11.49", "@types/sqlite3": "^3.1.8", diff --git a/dc-agents/reference/package-lock.json b/dc-agents/reference/package-lock.json index a7fa518173b..6047f79430f 100644 --- a/dc-agents/reference/package-lock.json +++ b/dc-agents/reference/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.40.0", + "@hasura/dc-api-types": "0.41.0", "fastify": "^4.13.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", @@ -52,7 +52,7 @@ "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ==" }, "node_modules/@hasura/dc-api-types": { - "version": "0.40.0", + "version": "0.41.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", diff --git a/dc-agents/reference/package.json b/dc-agents/reference/package.json index a21683ecaa2..d6401601f5d 100644 --- a/dc-agents/reference/package.json +++ b/dc-agents/reference/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.40.0", + "@hasura/dc-api-types": "0.41.0", "fastify": "^4.13.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", diff --git a/dc-agents/reference/src/data/index.ts b/dc-agents/reference/src/data/index.ts index c3da62da4ea..37f18cb93a8 100644 --- a/dc-agents/reference/src/data/index.ts +++ b/dc-agents/reference/src/data/index.ts @@ -1,4 +1,4 @@ -import { FunctionInfo, SchemaResponse, TableName } from "@hasura/dc-api-types" +import { SchemaRequest, SchemaResponse, TableInfo, TableName } from "@hasura/dc-api-types" import { Casing, Config } from "../config"; import xml2js from "xml2js" import fs from "fs" @@ -26,7 +26,7 @@ const streamToBuffer = async (stream: stream.Readable): Promise => { // Only parse numeric columns as numbers, otherwise you get "number-like" columns like BillingPostCode // getting partially parsed as a number or a string depending on the individual postcode const parseNumbersInNumericColumns = (schema: SchemaResponse) => { - const numericColumns = new Set(schema.tables.flatMap(table => table.columns.filter(c => c.type === "number").map(c => c.name))); + const numericColumns = new Set(schema.tables.flatMap(table => (table.columns ?? []).filter(c => c.type === "number").map(c => c.name))); return (value: string, name: string): any => { return numericColumns.has(name) @@ -139,7 +139,7 @@ const applyCasing = (casing: Casing) => (str: string): string => { } } -export const getSchema = (store: Record, config: Config): SchemaResponse => { +export const getSchema = (store: Record, config: Config, request: SchemaRequest = {}): SchemaResponse => { const applyTableNameCasing = applyCasing(config.table_name_casing); const applyColumnNameCasing = applyCasing(config.column_name_casing); @@ -155,11 +155,19 @@ export const getSchema = (store: Record, config: Config): Sc throw new Error(`Couldn't find db store for ${dbName}`); } + const filterForOnlyTheseTables = request.filters?.only_tables + // If we're using a schema, only use those table names that belong to that schema + ?.filter(n => config.schema ? n.length === 2 && n[0] === config.schema : true) + // But the schema is fake, so just keep the actual table name + ?.map(n => n[n.length - 1]) + const filteredTables = schema.tables.filter(table => - config.tables === null ? true : config.tables.map(n => [n]).find(nameEquals(table.name)) !== undefined + config.tables || filterForOnlyTheseTables + ? (config.tables ?? []).concat(filterForOnlyTheseTables ?? []).map(n => [n]).find(nameEquals(table.name)) !== undefined + : true ); - const prefixedTables = filteredTables.map(table => ({ + const prefixedTables: TableInfo[] = filteredTables.map(table => ({ ...table, name: prefixSchemaToTableName(table.name.map(applyTableNameCasing)), primary_key: table.primary_key?.map(applyColumnNameCasing), @@ -170,17 +178,49 @@ export const getSchema = (store: Record, config: Config): Sc column_mapping: mapObject(constraint.column_mapping, ([outer, inner]) => [applyColumnNameCasing(outer), applyColumnNameCasing(inner)]) })) : table.foreign_keys, - columns: table.columns.map(column => ({ + columns: table.columns?.map(column => ({ ...column, name: applyColumnNameCasing(column.name), })) })); - const prefixedFunctions = (schema.functions ?? []); // TODO: Put some real prefixes here + const filterForOnlyTheseFunctions = request.filters?.only_functions + // If we're using a schema, only use those function names that belong to that schema + ?.filter(n => config.schema ? n.length === 2 && n[0] === config.schema : true) + // But the schema is fake, so just keep the actual function name + ?.map(n => n[n.length - 1]) + + const filteredFunctions = (schema.functions ?? []).filter(func => + filterForOnlyTheseFunctions + ? filterForOnlyTheseFunctions.map(n => [n]).find(nameEquals(func.name)) !== undefined + : true + ) + + const prefixedFunctions = filteredFunctions; // TODO: Put some real prefixes here + + const detailLevel = request?.detail_level ?? "everything"; + switch (detailLevel) { + case "everything": + return { + tables: prefixedTables, + functions: prefixedFunctions, + }; + + case "basic_info": + return { + tables: prefixedTables.map(table => ({ + name: table.name, + type: table.type + })), + functions: prefixedFunctions.map(func => ({ + name: func.name, + type: func.type, + })), + }; + + default: + return unreachable(detailLevel); + } + - return { - ...schema, - tables: prefixedTables, - functions: prefixedFunctions, - }; }; diff --git a/dc-agents/reference/src/index.ts b/dc-agents/reference/src/index.ts index 026f9752539..7866e0ceec6 100644 --- a/dc-agents/reference/src/index.ts +++ b/dc-agents/reference/src/index.ts @@ -4,7 +4,7 @@ import { filterAvailableTables, getSchema, getTable, loadStaticData, StaticData import { queryData } from './query'; import { getConfig } from './config'; import { capabilitiesResponse } from './capabilities'; -import { CapabilitiesResponse, SchemaResponse, QueryRequest, QueryResponse, DatasetGetTemplateResponse, DatasetCreateCloneRequest, DatasetCreateCloneResponse, DatasetDeleteCloneResponse } from '@hasura/dc-api-types'; +import { CapabilitiesResponse, SchemaResponse, QueryRequest, QueryResponse, DatasetGetTemplateResponse, DatasetCreateCloneRequest, DatasetCreateCloneResponse, DatasetDeleteCloneResponse, SchemaRequest } from '@hasura/dc-api-types'; import { cloneDataset, defaultDbStoreName, deleteDataset, getDataset, getDbStoreName } from './datasets'; const port = Number(process.env.PORT) || 8100; @@ -20,15 +20,23 @@ server.register(FastifyCors, { allowedHeaders: ["X-Hasura-DataConnector-Config", "X-Hasura-DataConnector-SourceName"] }); +// This is a hack to get Fastify to parse bodies on /schema GET requests +// We basically trick its code into thinking the request is actually a POST +// request so it doesn't skip parsing request bodies. +server.addHook("onRequest", async(request, reply) => { + if (request.routerPath === "/schema") + request.raw.method = "POST" +}) + server.get<{ Reply: CapabilitiesResponse }>("/capabilities", async (request, _response) => { server.log.info({ headers: request.headers, query: request.body, }, "capabilities.request"); return capabilitiesResponse; }); -server.get<{ Reply: SchemaResponse }>("/schema", async (request, _response) => { +server.get<{ Body: SchemaRequest | undefined, Reply: SchemaResponse }>("/schema", async (request, _response) => { server.log.info({ headers: request.headers, query: request.body, }, "schema.request"); const config = getConfig(request); - return getSchema(staticData, config); + return getSchema(staticData, config, request.body); }); server.post<{ Body: QueryRequest, Reply: QueryResponse }>("/query", async (request, _response) => { diff --git a/dc-agents/sqlite/package-lock.json b/dc-agents/sqlite/package-lock.json index e453029809d..a9fb7981c3d 100644 --- a/dc-agents/sqlite/package-lock.json +++ b/dc-agents/sqlite/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.40.0", + "@hasura/dc-api-types": "0.41.0", "fastify": "^4.13.0", "fastify-metrics": "^9.2.1", "nanoid": "^3.3.4", @@ -57,7 +57,7 @@ "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ==" }, "node_modules/@hasura/dc-api-types": { - "version": "0.40.0", + "version": "0.41.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", diff --git a/dc-agents/sqlite/package.json b/dc-agents/sqlite/package.json index 7dddee6b252..434485df5f6 100644 --- a/dc-agents/sqlite/package.json +++ b/dc-agents/sqlite/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.40.0", + "@hasura/dc-api-types": "0.41.0", "fastify-metrics": "^9.2.1", "fastify": "^4.13.0", "nanoid": "^3.3.4", diff --git a/dc-agents/sqlite/src/index.ts b/dc-agents/sqlite/src/index.ts index 9962e6e8d69..6cf38e41877 100644 --- a/dc-agents/sqlite/src/index.ts +++ b/dc-agents/sqlite/src/index.ts @@ -4,7 +4,7 @@ import { getSchema } from './schema'; import { explain, queryData } from './query'; import { getConfig, tryGetConfig } from './config'; import { capabilitiesResponse } from './capabilities'; -import { QueryResponse, SchemaResponse, QueryRequest, CapabilitiesResponse, ExplainResponse, RawRequest, RawResponse, ErrorResponse, MutationRequest, MutationResponse, DatasetTemplateName, DatasetGetTemplateResponse, DatasetCreateCloneRequest, DatasetCreateCloneResponse, DatasetDeleteCloneResponse } from '@hasura/dc-api-types'; +import { QueryResponse, SchemaResponse, QueryRequest, CapabilitiesResponse, ExplainResponse, RawRequest, RawResponse, ErrorResponse, MutationRequest, MutationResponse, DatasetTemplateName, DatasetGetTemplateResponse, DatasetCreateCloneRequest, DatasetCreateCloneResponse, DatasetDeleteCloneResponse, SchemaRequest } from '@hasura/dc-api-types'; import { defaultMode, withConnection } from './db'; import metrics from 'fastify-metrics'; import prometheus from 'prom-client'; @@ -97,6 +97,14 @@ if(PERMISSIVE_CORS) { }) })(); +// This is a hack to get Fastify to parse bodies on /schema GET requests +// We basically trick its code into thinking the request is actually a POST +// request so it doesn't skip parsing request bodies. +server.addHook("onRequest", async(request, reply) => { + if (request.routerPath === "/schema") + request.raw.method = "POST" +}) + // Serves as an example of a custom histogram // Not especially useful at present as this mirrors // http_request_duration_seconds_bucket but is less general @@ -118,10 +126,10 @@ server.get<{ Reply: CapabilitiesResponse }>("/capabilities", async (request, _re return capabilitiesResponse; }); -server.get<{ Reply: SchemaResponse }>("/schema", async (request, _response) => { +server.get<{ Body: SchemaRequest | undefined, Reply: SchemaResponse }>("/schema", async (request, _response) => { server.log.info({ headers: request.headers, query: request.body, }, "schema.request"); const config = getConfig(request); - return getSchema(config, sqlLogger); + return getSchema(config, sqlLogger, request.body); }); /** diff --git a/dc-agents/sqlite/src/schema.ts b/dc-agents/sqlite/src/schema.ts index 1ec204e6360..dbd1c06e248 100644 --- a/dc-agents/sqlite/src/schema.ts +++ b/dc-agents/sqlite/src/schema.ts @@ -1,8 +1,9 @@ -import { SchemaResponse, ColumnInfo, TableInfo, Constraint, ColumnValueGenerationStrategy } from "@hasura/dc-api-types" +import { SchemaResponse, ColumnInfo, TableInfo, Constraint, ColumnValueGenerationStrategy, SchemaRequest, DetailLevel, TableName } from "@hasura/dc-api-types" import { ScalarTypeKey } from "./capabilities"; import { Config } from "./config"; import { defaultMode, SqlLogger, withConnection } from './db'; import { MUTATIONS } from "./environment"; +import { unreachable } from "./util"; var sqliteParser = require('sqlite-parser'); @@ -62,18 +63,33 @@ function nullableCast(ds: any[]): boolean { return true; } -const formatTableInfo = (config: Config) => (info: TableInfoInternal): TableInfo => { +const formatTableInfo = (config: Config, detailLevel: DetailLevel): ((info: TableInfoInternal) => TableInfo) => { + switch (detailLevel) { + case "everything": return formatEverythingTableInfo(config); + case "basic_info": return formatBasicTableInfo(config); + default: return unreachable(detailLevel); + } +} + +const formatBasicTableInfo = (config: Config) => (info: TableInfoInternal): TableInfo => { const tableName = config.explicit_main_schema ? ["main", info.name] : [info.name]; + return { + name: tableName, + type: "table" + } +} + +const formatEverythingTableInfo = (config: Config) => (info: TableInfoInternal): TableInfo => { + const basicTableInfo = formatBasicTableInfo(config)(info); const ast = sqliteParser(info.sql); const columnsDdl = getColumnsDdl(ast); const primaryKeys = getPrimaryKeyNames(ast); - const foreignKeys = ddlFKs(config, tableName, ast); + const foreignKeys = ddlFKs(config, basicTableInfo.name, ast); const primaryKey = primaryKeys.length > 0 ? { primary_key: primaryKeys } : {}; const foreignKey = foreignKeys.length > 0 ? { foreign_keys: Object.fromEntries(foreignKeys) } : {}; return { - name: tableName, - type: "table", + ...basicTableInfo, ...primaryKey, ...foreignKey, description: info.sql, @@ -92,14 +108,21 @@ function isMeta(table : TableInfoInternal) { return table.type != 'table' || table.name === 'sqlite_sequence'; } -function includeTable(config: Config, table: TableInfoInternal): boolean { - if(config.tables === null) { - if(isMeta(table) && ! config.meta) { - return false; - } - return true; +const includeTable = (config: Config, only_tables?: TableName[]) => (table: TableInfoInternal): boolean => { + if (isMeta(table) && !config.meta) { + return false; + } + + const filterForOnlyTheseTables = only_tables + // If we're using an explicit main schema, only use those table names that belong to that schema + ?.filter(n => config.explicit_main_schema ? n.length === 2 && n[0] === "main" : true) + // Just keep the actual table name + ?.map(n => n[n.length - 1]) + + if (config.tables || only_tables) { + return (config.tables ?? []).concat(filterForOnlyTheseTables ?? []).indexOf(table.name) >= 0; } else { - return config.tables.indexOf(table.name) >= 0 + return true; } } @@ -227,12 +250,14 @@ function getPrimaryKeyNames(ddl: any): string[] { }) } -export async function getSchema(config: Config, sqlLogger: SqlLogger): Promise { +export async function getSchema(config: Config, sqlLogger: SqlLogger, schemaRequest: SchemaRequest = {}): Promise { return await withConnection(config, defaultMode, sqlLogger, async db => { + const detailLevel = schemaRequest.detail_level ?? "everything"; + const results = await db.query("SELECT * from sqlite_schema"); const resultsT: TableInfoInternal[] = results as TableInfoInternal[]; - const filtered: TableInfoInternal[] = resultsT.filter(table => includeTable(config,table)); - const result: TableInfo[] = filtered.map(formatTableInfo(config)); + const filtered: TableInfoInternal[] = resultsT.filter(includeTable(config, schemaRequest?.filters?.only_tables)); + const result: TableInfo[] = filtered.map(formatTableInfo(config, detailLevel)); return { tables: result diff --git a/server/lib/dc-api/dc-api.cabal b/server/lib/dc-api/dc-api.cabal index e4023d0699d..7f6f6749585 100644 --- a/server/lib/dc-api/dc-api.cabal +++ b/server/lib/dc-api/dc-api.cabal @@ -164,6 +164,7 @@ test-suite tests-dc-api Test.Specs.CapabilitiesSpec Test.Specs.ErrorSpec Test.Specs.ExplainSpec + Test.Specs.FunctionsSpec Test.Specs.HealthSpec Test.Specs.MetricsSpec Test.Specs.MutationSpec @@ -180,5 +181,4 @@ test-suite tests-dc-api Test.Specs.QuerySpec.RedactionSpec Test.Specs.QuerySpec.RelationshipsSpec Test.Specs.SchemaSpec - Test.Specs.UDFSpec Test.TestHelpers diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API.hs index 42646c195f2..dee1130f9c3 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API.hs @@ -84,6 +84,7 @@ type SchemaApi config = "schema" :> SourceNameHeader Required :> ConfigHeader config Required + :> ReqBody '[JSON] V0.SchemaRequest :> UVerb 'GET '[JSON] SchemaResponses -- | This function defines a central place to ensure that all cases are covered for query and error responses. diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Function.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Function.hs index 4a06ed4bd54..21ca31be9a6 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Function.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Function.hs @@ -194,8 +194,8 @@ data FunctionInfo = FunctionInfo { -- NOTE: Some fields from PG are omitted here due to initial implementation, or non-generality. _fiName :: FunctionName, _fiFunctionType :: FunctionType, - _fiReturns :: FunctionReturnType, -- Functions must currently return tables as per PG. - _fiResponseCardinality :: FunctionArity, + _fiReturns :: Maybe FunctionReturnType, -- Functions must currently return tables as per PG. + _fiResponseCardinality :: Maybe FunctionArity, _fiInputArgs :: [FunctionArg], -- Args info is listed grouped unlike PG. _fiDescription :: Maybe Text } @@ -209,9 +209,9 @@ instance HasCodec FunctionInfo where FunctionInfo <$> requiredField "name" "The name of the table" .= _fiName <*> requiredField "type" "read/write classification of the function" .= _fiFunctionType - <*> requiredField "returns" "table listed in schema that matches the return type of the function - to relax later" .= _fiReturns - <*> requiredField "response_cardinality" "object response if false, rows if true" .= _fiResponseCardinality - <*> requiredField "args" "argument info - name/types" .= _fiInputArgs + <*> optionalFieldOrNull "returns" "table listed in schema that matches the return type of the function - to relax later" .= _fiReturns + <*> optionalFieldOrNull "response_cardinality" "object response if false, rows if true" .= _fiResponseCardinality + <*> optionalFieldWithOmittedDefault "args" [] "argument info - name/types" .= _fiInputArgs <*> optionalFieldOrNull "description" "Description of the table" .= _fiDescription -------------------------------------------------------------------------------- diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Schema.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Schema.hs index 6178ed729b5..4ad2ffb0d48 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Schema.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Schema.hs @@ -1,7 +1,11 @@ {-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE OverloadedLists #-} module Hasura.Backends.DataConnector.API.V0.Schema - ( SchemaResponse (..), + ( SchemaRequest (..), + SchemaFilters (..), + DetailLevel (..), + SchemaResponse (..), ObjectTypeDefinition (..), ) where @@ -24,6 +28,60 @@ import Prelude -------------------------------------------------------------------------------- -- Schema Response +data SchemaRequest = SchemaRequest + { _srFilters :: SchemaFilters, + _srDetailLevel :: DetailLevel + } + deriving stock (Eq, Show, Generic) + deriving anyclass (NFData, Hashable) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec SchemaRequest + +instance HasCodec SchemaRequest where + codec = + object "SchemaRequest" $ + SchemaRequest + <$> optionalFieldWithOmittedDefault "filters" mempty "Optional schema filtering settings" .= _srFilters + <*> optionalFieldWithOmittedDefault "detail_level" Everything "Only return names for schema items" .= _srDetailLevel + +data SchemaFilters = SchemaFilters + { _sfOnlyTables :: Maybe [API.V0.TableName], + _sfOnlyFunctions :: Maybe [API.V0.FunctionName] + } + deriving stock (Eq, Show, Generic) + deriving anyclass (NFData, Hashable) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec SchemaFilters + +instance Semigroup SchemaFilters where + a <> b = + SchemaFilters + { _sfOnlyTables = _sfOnlyTables a <> _sfOnlyTables b, + _sfOnlyFunctions = _sfOnlyFunctions a <> _sfOnlyFunctions b + } + +instance Monoid SchemaFilters where + mempty = SchemaFilters Nothing Nothing + +instance HasCodec SchemaFilters where + codec = + object "SchemaFilters" $ + SchemaFilters + <$> optionalField "only_tables" "Only get the schemas for these tables" .= _sfOnlyTables + <*> optionalField "only_functions" "Only get the schemas for these functions" .= _sfOnlyFunctions + +data DetailLevel = Everything | BasicInfo + deriving stock (Eq, Show, Generic) + deriving anyclass (NFData, Hashable) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec DetailLevel + +instance HasCodec DetailLevel where + codec = + named "DetailLevel" $ + stringConstCodec [(Everything, "everything"), (BasicInfo, "basic_info")] + [ "How much information to return about the schema. Values:", + "- 'everything': All information about the schema.", + "- 'basic_info': For tables, only the table name and table type, for functions, only the function name and function type." + ] + -- | The Schema Response provides the schemas for tracked tables and -- 'Capabilities' supported by the service. data SchemaResponse = SchemaResponse diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Table.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Table.hs index 358561d2fd7..2308fd164b7 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Table.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Table.hs @@ -90,7 +90,7 @@ instance HasCodec TableInfo where TableInfo <$> requiredField "name" "The name of the table" .= _tiName <*> optionalFieldWithDefault "type" Table "The type of table" .= _tiType - <*> requiredField "columns" "The columns of the table" .= _tiColumns + <*> optionalFieldWithDefault "columns" [] "The columns of the table" .= _tiColumns <*> dimapMaybeNonEmpty (optionalFieldWithOmittedDefault "primary_key" [] "The primary key of the table") .= _tiPrimaryKey <*> optionalFieldWithOmittedDefault "foreign_keys" (ForeignKeys mempty) "Foreign key constraints" .= _tiForeignKeys <*> optionalFieldOrNull "description" "Description of the table" .= _tiDescription diff --git a/server/lib/dc-api/test/Main.hs b/server/lib/dc-api/test/Main.hs index 39e3b2b1584..b487af7c529 100644 --- a/server/lib/dc-api/test/Main.hs +++ b/server/lib/dc-api/test/Main.hs @@ -24,12 +24,12 @@ import Test.Sandwich.Options qualified as Sandwich import Test.Specs.CapabilitiesSpec qualified import Test.Specs.ErrorSpec qualified import Test.Specs.ExplainSpec qualified +import Test.Specs.FunctionsSpec qualified import Test.Specs.HealthSpec qualified import Test.Specs.MetricsSpec qualified import Test.Specs.MutationSpec qualified import Test.Specs.QuerySpec qualified import Test.Specs.SchemaSpec qualified -import Test.Specs.UDFSpec qualified import Test.TestHelpers (AgentTestSpec) import Prelude @@ -51,7 +51,7 @@ tests testData testConfig edgeCasesTestData capabilitiesResponse@API.Capabilitie for_ (API._cMutations _crCapabilities) \_ -> Test.Specs.MutationSpec.spec testData edgeCasesTestData _crCapabilities for_ (API._cUserDefinedFunctions _crCapabilities) \_ -> do usesDataset functionsTemplate do - Test.Specs.UDFSpec.spec testConfig _crCapabilities + Test.Specs.FunctionsSpec.spec testConfig _crCapabilities getCloneSchema :: Maybe API.Config -> API.DatasetTemplateName -> AgentIOClient -> IO API.SchemaResponse getCloneSchema mergeConfig datasetTemplate (AgentIOClient agentClient) = @@ -59,13 +59,13 @@ getCloneSchema mergeConfig datasetTemplate (AgentIOClient agentClient) = (createClone agentClient datasetTemplate) (deleteClone agentClient) ( \DatasetCloneInfo {..} -> - (agentClient // API._schema) testSourceName (mergeAgentConfig _dciAgentConfig mergeConfig) >>= guardSchemaResponse + (agentClient // API._schema) testSourceName (mergeAgentConfig _dciAgentConfig mergeConfig) (API.SchemaRequest mempty API.Everything) >>= guardSchemaResponse ) getChinookSchema :: API.Capabilities -> AgentConfig -> AgentIOClient -> IO API.SchemaResponse getChinookSchema API.Capabilities {..} agentConfig agentIOClient@(AgentIOClient agentClient) = do case agentConfig of - ManualConfig config -> (agentClient // API._schema) testSourceName config >>= guardSchemaResponse + ManualConfig config -> (agentClient // API._schema) testSourceName config (API.SchemaRequest mempty API.Everything) >>= guardSchemaResponse DatasetConfig mergeConfig -> if isJust _cDatasets then getCloneSchema mergeConfig chinookTemplate agentIOClient diff --git a/server/lib/dc-api/test/Test/AgentAPI.hs b/server/lib/dc-api/test/Test/AgentAPI.hs index 11e029afe15..e4d4851fe27 100644 --- a/server/lib/dc-api/test/Test/AgentAPI.hs +++ b/server/lib/dc-api/test/Test/AgentAPI.hs @@ -15,6 +15,7 @@ module Test.AgentAPI getHealth, getSourceHealth, getSchemaGuarded, + getSchemaGuarded', guardSchemaResponse, queryGuarded, queryExpectError, @@ -70,7 +71,12 @@ getSourceHealth = do getSchemaGuarded :: (HasBaseContext context, HasAgentTestContext context, HasDatasetContext context, MonadReader context m, MonadThrow m, MonadIO m) => AgentClientT m API.SchemaResponse getSchemaGuarded = do (sourceName, config) <- getSourceNameAndConfig - guardSchemaResponse =<< (client // API._schema) sourceName config + guardSchemaResponse =<< (client // API._schema) sourceName config (API.SchemaRequest mempty API.Everything) + +getSchemaGuarded' :: (HasBaseContext context, HasAgentTestContext context, HasDatasetContext context, MonadReader context m, MonadThrow m, MonadIO m) => API.SchemaRequest -> AgentClientT m API.SchemaResponse +getSchemaGuarded' schemaRequest = do + (sourceName, config) <- getSourceNameAndConfig + guardSchemaResponse =<< (client // API._schema) sourceName config schemaRequest guardSchemaResponse :: (MonadThrow m) => Union API.SchemaResponses -> m API.SchemaResponse guardSchemaResponse = API.schemaCase defaultAction successAction errorAction diff --git a/server/lib/dc-api/test/Test/Data.hs b/server/lib/dc-api/test/Test/Data.hs index a1f3455427b..556d20d21d8 100644 --- a/server/lib/dc-api/test/Test/Data.hs +++ b/server/lib/dc-api/test/Test/Data.hs @@ -658,7 +658,7 @@ functionField :: API.SchemaResponse -> TestConfig -> API.TableName -> API.Functi functionField schemaResponse@API.SchemaResponse {..} testConfig defaultTableName functionName columnName = columnField schemaResponse testConfig tableName columnName where - tableName = fromMaybe defaultTableName (functionReturnType ^? API._FunctionReturnsTable) + tableName = fromMaybe defaultTableName (functionReturnType ^? _Just . API._FunctionReturnsTable) functionReturnType = maybe (error $ "Can't find the function " <> show functionName <> " in " <> show (API._fiName <$> _srFunctions)) API._fiReturns functionInfo functionInfo = find (\API.FunctionInfo {..} -> _fiName == functionName) _srFunctions diff --git a/server/lib/dc-api/test/Test/Specs/UDFSpec.hs b/server/lib/dc-api/test/Test/Specs/FunctionsSpec.hs similarity index 83% rename from server/lib/dc-api/test/Test/Specs/UDFSpec.hs rename to server/lib/dc-api/test/Test/Specs/FunctionsSpec.hs index 33aacbb9bb7..6a270e110f4 100644 --- a/server/lib/dc-api/test/Test/Specs/UDFSpec.hs +++ b/server/lib/dc-api/test/Test/Specs/FunctionsSpec.hs @@ -1,26 +1,26 @@ {-# LANGUAGE ConstraintKinds #-} {-# LANGUAGE OverloadedStrings #-} --- | Module containing tests for user-defined-functions aka. "UDFs". -module Test.Specs.UDFSpec (spec) where +-- | Module containing tests for user-defined-functions +module Test.Specs.FunctionsSpec (spec) where -------------------------------------------------------------------------------- import Command (TestConfig) -import Control.Lens ((?~)) +import Control.Lens ((<&>), (?~)) import Control.Lens.Lens ((&)) import Control.Monad.Catch (MonadThrow) import Control.Monad.IO.Class (MonadIO) import Control.Monad.Reader (MonadReader) import Data.Aeson (Value (..)) import Data.HashMap.Strict qualified as HashMap -import Data.List (sort) +import Data.List (sort, sortOn) import Data.Set qualified as Set import Data.Text qualified as Text import GHC.Stack (HasCallStack) import Hasura.Backends.DataConnector.API import Hasura.Backends.DataConnector.API qualified as API -import Test.AgentAPI (getSchemaGuarded, queryGuarded) +import Test.AgentAPI (getSchemaGuarded, getSchemaGuarded', queryGuarded) import Test.AgentClient (HasAgentClient, runAgentClientT) import Test.AgentDatasets (HasDatasetContext) import Test.AgentTestContext (HasAgentTestContext) @@ -50,6 +50,36 @@ spec testConfig API.Capabilities {} = describe "supports functions" $ preloadAge functionNames <- (extractFunctionNames . API._srFunctions) <$> getPreloadedAgentSchema functionNames `jsonShouldBe` expectedFunctionNames + it "returns the specified functions from the Functions dataset when filtered" do + preloadedSchema <- getPreloadedAgentSchema + let FunctionsTestData {..} = mkFunctionsTestData preloadedSchema testConfig + extractFunctionNames = sort . fmap API._fiName + desiredFunctions = [_ftdFibonacciFunctionName] + filters = mempty {API._sfOnlyFunctions = Just desiredFunctions} + + functionNames <- extractFunctionNames . API._srFunctions <$> getSchemaGuarded' (API.SchemaRequest filters API.BasicInfo) + functionNames `jsonShouldBe` desiredFunctions + + it "returns the no functions when filtered with an empty list" do + let filters = mempty {API._sfOnlyFunctions = Just []} + + functionInfos <- API._srFunctions <$> getSchemaGuarded' (API.SchemaRequest filters API.BasicInfo) + functionInfos `jsonShouldBe` [] + + it "returns only Function names and types when using basic_info detail level" $ do + preloadedSchema <- getPreloadedAgentSchema + + let FunctionsTestData {..} = mkFunctionsTestData preloadedSchema testConfig + expectedFunctionNames = [_ftdFibonacciFunctionName, _ftdSearchArticlesFunctionName] + + functionInfos <- sortOn API._fiName . API._srFunctions <$> getSchemaGuarded' (API.SchemaRequest mempty API.BasicInfo) + + let expectedFunctionInfos = + expectedFunctionNames + <&> (\functionName -> API.FunctionInfo functionName API.FRead Nothing Nothing [] Nothing) + + functionInfos `jsonShouldBe` expectedFunctionInfos + it "can query for a list Fibonacci numbers using the fibonacci function" $ do preloadedSchema <- getPreloadedAgentSchema diff --git a/server/lib/dc-api/test/Test/Specs/SchemaSpec.hs b/server/lib/dc-api/test/Test/Specs/SchemaSpec.hs index b95da87acab..809f12e7c4f 100644 --- a/server/lib/dc-api/test/Test/Specs/SchemaSpec.hs +++ b/server/lib/dc-api/test/Test/Specs/SchemaSpec.hs @@ -4,7 +4,7 @@ module Test.Specs.SchemaSpec (spec) where -------------------------------------------------------------------------------- -import Control.Lens ((%~), (.~), (?~)) +import Control.Lens ((%~), (.~), (<&>), (?~)) import Control.Lens.At (at) import Control.Lens.Lens ((&)) import Control.Monad (forM_) @@ -18,11 +18,11 @@ import Data.Foldable (find) import Data.HashMap.Strict qualified as HashMap import Data.List (sort, sortOn) import Data.List.NonEmpty qualified as NonEmpty -import Data.Maybe (isJust, isNothing) +import Data.Maybe (isJust, isNothing, mapMaybe) import Data.Text qualified as Text import GHC.Stack (HasCallStack) import Hasura.Backends.DataConnector.API qualified as API -import Test.AgentAPI (getSchemaGuarded) +import Test.AgentAPI (getSchemaGuarded, getSchemaGuarded') import Test.AgentClient (HasAgentClient, runAgentClientT) import Test.AgentDatasets (HasDatasetContext) import Test.AgentTestContext (HasAgentTestContext) @@ -49,6 +49,46 @@ spec TestData {..} API.Capabilities {..} = describe "schema API" $ preloadAgentS let expectedTableNames = extractTableNames _tdSchemaTables tableNames `jsonShouldBe` expectedTableNames + it "returns the specified Chinook tables when filtered" $ do + let desiredTables = [_tdCustomersTableName, _tdInvoicesTableName, _tdInvoiceLinesTableName, _tdTracksTableName] + let filters = mempty {API._sfOnlyTables = Just desiredTables} + tableNames <- sort . fmap API._tiName . API._srTables <$> getSchemaGuarded' (API.SchemaRequest filters API.Everything) + + tableNames `jsonShouldBe` desiredTables + + it "returns no tables when filtered with an empty list" $ do + let filters = mempty {API._sfOnlyTables = Just []} + tableInfos <- API._srTables <$> getSchemaGuarded' (API.SchemaRequest filters API.Everything) + + tableInfos `jsonShouldBe` [] + + it "returns only Chinook table names and types when using basic_info detail level" $ do + tableInfos <- sortOn API._tiName . API._srTables <$> getSchemaGuarded' (API.SchemaRequest mempty API.BasicInfo) + + let expectedTableInfos = + _tdSchemaTables + <&> (\API.TableInfo {..} -> API.TableInfo _tiName _tiType [] Nothing (API.ForeignKeys mempty) Nothing False False False) + & sortOn API._tiName + + tableInfos `jsonShouldBe` expectedTableInfos + + it "can filter tables while using basic_info detail level" $ do + let desiredTables = [_tdAlbumsTableName, _tdArtistsTableName] + let filters = mempty {API._sfOnlyTables = Just desiredTables} + tableInfos <- sortOn API._tiName . API._srTables <$> getSchemaGuarded' (API.SchemaRequest filters API.BasicInfo) + + let expectedTableInfos = + _tdSchemaTables + & mapMaybe + ( \API.TableInfo {..} -> + if _tiName `elem` desiredTables + then Just $ API.TableInfo _tiName _tiType [] Nothing (API.ForeignKeys mempty) Nothing False False False + else Nothing + ) + & sortOn API._tiName + + tableInfos `jsonShouldBe` expectedTableInfos + testPerTable "returns the correct columns in the Chinook tables" $ \expectedTable actualTable -> do -- We remove some properties here so that we don't compare them since they vary between agent implementations let extractJsonForComparison table = diff --git a/server/lib/test-harness/src/Harness/Backend/DataConnector/Mock/Server.hs b/server/lib/test-harness/src/Harness/Backend/DataConnector/Mock/Server.hs index 633cf9203ae..a7ca64b116d 100644 --- a/server/lib/test-harness/src/Harness/Backend/DataConnector/Mock/Server.hs +++ b/server/lib/test-harness/src/Harness/Backend/DataConnector/Mock/Server.hs @@ -866,8 +866,8 @@ mockCapabilitiesHandler mcfg = liftIO $ do cfg <- I.readIORef mcfg pure $ inject $ SOP.I $ _capabilitiesResponse cfg -mockSchemaHandler :: I.IORef MockConfig -> I.IORef (Maybe AgentRequest) -> I.IORef (Maybe API.Config) -> API.SourceName -> API.Config -> Handler (Union API.SchemaResponses) -mockSchemaHandler mcfg mRecordedRequest mRecordedRequestConfig _sourceName requestConfig = liftIO $ do +mockSchemaHandler :: I.IORef MockConfig -> I.IORef (Maybe AgentRequest) -> I.IORef (Maybe API.Config) -> API.SourceName -> API.Config -> API.SchemaRequest -> Handler (Union API.SchemaResponses) +mockSchemaHandler mcfg mRecordedRequest mRecordedRequestConfig _sourceName requestConfig _schemaRequest = liftIO $ do cfg <- I.readIORef mcfg I.writeIORef mRecordedRequest (Just Schema) I.writeIORef mRecordedRequestConfig (Just requestConfig) diff --git a/server/src-lib/Hasura/Backends/DataConnector/Adapter/Metadata.hs b/server/src-lib/Hasura/Backends/DataConnector/Adapter/Metadata.hs index 210057ee3d3..73267d633ce 100644 --- a/server/src-lib/Hasura/Backends/DataConnector/Adapter/Metadata.hs +++ b/server/src-lib/Hasura/Backends/DataConnector/Adapter/Metadata.hs @@ -134,11 +134,11 @@ arityJsonAggSelect = \case functionReturnTypeFromAPI :: (MonadError QErr m) => DC.FunctionName -> - (Maybe (FunctionReturnType 'DataConnector), API.FunctionReturnType) -> + (Maybe (FunctionReturnType 'DataConnector), Maybe API.FunctionReturnType) -> m DC.TableName functionReturnTypeFromAPI funcGivenName = \case (Just (DC.FunctionReturnsTable t), _) -> pure t - (_, API.FunctionReturnsTable t) -> pure (Witch.into t) + (_, Just (API.FunctionReturnsTable t)) -> pure (Witch.into t) _ -> throw400 NotSupported $ "Function " @@ -177,7 +177,7 @@ buildFunctionInfo' objid <- case (_fcResponse, returnType) of (Just (DC.FunctionReturnsTable t), _) -> pure $ SOSourceObj sourceName $ mkAnyBackend $ SOITable @'DataConnector t - (_, API.FunctionReturnsTable t) -> pure $ SOSourceObj sourceName $ mkAnyBackend $ SOITable @'DataConnector (Witch.into t) + (_, Just (API.FunctionReturnsTable t)) -> pure $ SOSourceObj sourceName $ mkAnyBackend $ SOITable @'DataConnector (Witch.into t) _ -> throw400 NotSupported $ "Function " @@ -205,6 +205,10 @@ buildFunctionInfo' else IAUserProvided arg functionReturnType <- functionReturnTypeFromAPI funcName (_fcResponse, returnType) + jsonAggSelect <- + arityJsonAggSelect + <$> infoSet + `onNothing` throw400 NotSupported ("Function " <> tshow funcName <> " is missing a response cardinality") let funcInfo = FunctionInfo @@ -219,7 +223,7 @@ buildFunctionInfo' _fiReturnType = functionReturnType, _fiDescription = infoDesc, _fiPermissions = permissionMap, - _fiJsonAggSelect = arityJsonAggSelect infoSet, + _fiJsonAggSelect = jsonAggSelect, _fiComment = funcComment } pure $ (funcInfo, SchemaDependency objid DRTable) @@ -336,8 +340,9 @@ resolveDatabaseMetadata' :: SourceMetadata 'DataConnector -> DC.SourceConfig -> m (Either QErr (DBObjectsIntrospection 'DataConnector)) -resolveDatabaseMetadata' logger SourceMetadata {_smName} sourceConfig = runExceptT do - API.SchemaResponse {..} <- requestDatabaseSchema logger _smName sourceConfig +resolveDatabaseMetadata' logger sourceMetadata@SourceMetadata {_smName} sourceConfig = runExceptT do + let schemaRequest = makeTrackedItemsOnlySchemaRequest sourceMetadata + API.SchemaResponse {..} <- requestDatabaseSchema logger _smName sourceConfig schemaRequest let logicalModels = maybe mempty (InsOrdHashMap.fromList . map toLogicalModelMetadata . toList) _srObjectTypes tables = HashMap.fromList $ do @@ -390,17 +395,29 @@ resolveDatabaseMetadata' logger SourceMetadata {_smName} sourceConfig = runExcep _rsLogicalModels = logicalModels } +makeTrackedItemsOnlySchemaRequest :: SourceMetadata 'DataConnector -> API.SchemaRequest +makeTrackedItemsOnlySchemaRequest SourceMetadata {..} = + API.SchemaRequest + { _srFilters = + API.SchemaFilters + { _sfOnlyTables = Just $ Witch.into <$> InsOrdHashMap.keys _smTables, + _sfOnlyFunctions = Just $ Witch.into <$> InsOrdHashMap.keys _smFunctions + }, + _srDetailLevel = API.Everything + } + requestDatabaseSchema :: (MonadIO m, MonadBaseControl IO m, MonadError QErr m) => Logger Hasura -> SourceName -> DC.SourceConfig -> + API.SchemaRequest -> m API.SchemaResponse -requestDatabaseSchema logger sourceName sourceConfig = do +requestDatabaseSchema logger sourceName sourceConfig schemaRequest = do transformedSourceConfig <- transformSourceConfig sourceConfig Nothing ignoreTraceT . flip runAgentClientT (AgentClientContext logger (DC._scEndpoint transformedSourceConfig) (DC._scManager transformedSourceConfig) (DC._scTimeoutMicroseconds transformedSourceConfig) Nothing) - $ Client.schema sourceName (DC._scConfig transformedSourceConfig) + $ Client.schema sourceName (DC._scConfig transformedSourceConfig) schemaRequest getFieldType :: Bool -> API.ColumnType -> LogicalModelType 'DataConnector getFieldType isNullable = \case @@ -635,14 +652,14 @@ listAllTables' :: (CacheRM m, Has (Logger Hasura) r, MonadIO m, MonadBaseControl listAllTables' sourceName = do (logger :: Logger Hasura) <- asks getter sourceConfig <- askSourceConfig @'DataConnector sourceName - schemaResponse <- requestDatabaseSchema logger sourceName sourceConfig + schemaResponse <- requestDatabaseSchema logger sourceName sourceConfig (API.SchemaRequest mempty API.BasicInfo) pure $ fmap (Witch.from . API._tiName) $ API._srTables schemaResponse listAllTrackables' :: (CacheRM m, Has (Logger Hasura) r, MonadIO m, MonadBaseControl IO m, MonadReader r m, MonadError QErr m, MetadataM m) => SourceName -> m (TrackableInfo 'DataConnector) listAllTrackables' sourceName = do (logger :: Logger Hasura) <- asks getter sourceConfig <- askSourceConfig @'DataConnector sourceName - schemaResponse <- requestDatabaseSchema logger sourceName sourceConfig + schemaResponse <- requestDatabaseSchema logger sourceName sourceConfig (API.SchemaRequest mempty API.BasicInfo) let functions = fmap (\fi -> TrackableFunctionInfo (Witch.into (API._fiName fi)) (getVolatility (API._fiFunctionType fi))) $ API._srFunctions schemaResponse let tables = fmap (TrackableTableInfo . Witch.into . API._tiName) $ API._srTables schemaResponse pure diff --git a/server/src-lib/Hasura/Backends/DataConnector/Agent/Client.hs b/server/src-lib/Hasura/Backends/DataConnector/Agent/Client.hs index e848f3a870a..955df1384c8 100644 --- a/server/src-lib/Hasura/Backends/DataConnector/Agent/Client.hs +++ b/server/src-lib/Hasura/Backends/DataConnector/Agent/Client.hs @@ -105,9 +105,9 @@ capabilities = do defaultAction = throw400 DataConnectorError "Unexpected data connector capabilities response - Unexpected Type" capabilitiesGuard = API.capabilitiesCase defaultAction pure errorAction -schema :: (MonadIO m, MonadTrace m, MonadError QErr m) => RQL.SourceName -> API.Config -> AgentClientT m API.SchemaResponse -schema sourceName config = do - schemaGuard =<< (genericClient // API._schema) (toTxt sourceName) config +schema :: (MonadIO m, MonadTrace m, MonadError QErr m) => RQL.SourceName -> API.Config -> API.SchemaRequest -> AgentClientT m API.SchemaResponse +schema sourceName config schemaRequest = do + schemaGuard =<< (genericClient // API._schema) (toTxt sourceName) config schemaRequest where errorAction e = throw400WithDetail (mapErrorType $ API._crType e) (API._crMessage e) (API._crDetails e) defaultAction = throw400 DataConnectorError "Unexpected data connector schema response - Unexpected Type" diff --git a/server/src-test/Hasura/Backends/DataConnector/API/V0/SchemaSpec.hs b/server/src-test/Hasura/Backends/DataConnector/API/V0/SchemaSpec.hs index ea003a8c12f..9b621548157 100644 --- a/server/src-test/Hasura/Backends/DataConnector/API/V0/SchemaSpec.hs +++ b/server/src-test/Hasura/Backends/DataConnector/API/V0/SchemaSpec.hs @@ -4,8 +4,11 @@ module Hasura.Backends.DataConnector.API.V0.SchemaSpec (spec) where import Data.Aeson.QQ.Simple (aesonQQ) +import Hasura.Backends.DataConnector.API.V0.Function +import Hasura.Backends.DataConnector.API.V0.FunctionSpec (genFunctionName) import Hasura.Backends.DataConnector.API.V0.Schema -import Hasura.Backends.DataConnector.API.V0.TableSpec (genTableInfo) +import Hasura.Backends.DataConnector.API.V0.Table +import Hasura.Backends.DataConnector.API.V0.TableSpec (genTableInfo, genTableName) import Hasura.Generator.Common (defaultRange) import Hasura.Prelude import Hedgehog @@ -15,10 +18,44 @@ import Test.Hspec spec :: Spec spec = do + describe "SchemaRequest" $ do + describe "Minimal" $ do + testToFromJSONToSchema + (SchemaRequest mempty Everything) + [aesonQQ| {} |] + describe "Full" $ do + testToFromJSONToSchema + (SchemaRequest (SchemaFilters (Just [TableName ["my_table"]]) (Just [FunctionName ["my_function"]])) BasicInfo) + [aesonQQ| + { "filters": { + "only_tables": [["my_table"]], + "only_functions": [["my_function"]] + }, + "detail_level": "basic_info" + } + |] + jsonOpenApiProperties genSchemaRequest + describe "SchemaResponse" $ do testToFromJSONToSchema (SchemaResponse [] [] Nothing) [aesonQQ|{"tables": []}|] jsonOpenApiProperties genSchemaResponse +genSchemaRequest :: (MonadGen m) => m SchemaRequest +genSchemaRequest = + SchemaRequest + <$> genSchemaFilters + <*> genDetailLevel + +genSchemaFilters :: (MonadGen m) => m SchemaFilters +genSchemaFilters = + SchemaFilters + <$> Gen.maybe (Gen.list defaultRange genTableName) + <*> Gen.maybe (Gen.list defaultRange genFunctionName) + +genDetailLevel :: (MonadGen m) => m DetailLevel +genDetailLevel = + Gen.element [Everything, BasicInfo] + genSchemaResponse :: Gen SchemaResponse genSchemaResponse = do tables <- Gen.list defaultRange genTableInfo