From 8d6b9f70f1b19b7d45d177a6cf0690fabff32b9f Mon Sep 17 00:00:00 2001 From: Lyndon Maydwell Date: Tue, 17 Jan 2023 15:47:40 +1000 Subject: [PATCH] Datasets implementation for dataconnectors PR-URL: https://github.com/hasura/graphql-engine-mono/pull/7502 GitOrigin-RevId: 7ca9a16aa2b27f4efb1263c6415e5a718ca8ced8 --- dc-agents/README.md | 53 ++++++ dc-agents/dc-api-types/package.json | 2 +- dc-agents/dc-api-types/src/agent.openapi.json | 156 ++++++++++++++++++ dc-agents/dc-api-types/src/index.ts | 7 + .../dc-api-types/src/models/Capabilities.ts | 2 + dc-agents/dc-api-types/src/models/Config.ts | 5 + .../src/models/DatasetCapabilities.ts | 7 + .../src/models/DatasetDeleteResponse.ts | 11 ++ .../src/models/DatasetGetResponse.ts | 11 ++ .../src/models/DatasetPostRequest.ts | 10 ++ .../src/models/DatasetPostResponse.ts | 10 ++ .../src/models/DatasetTemplateName.ts | 5 + dc-agents/package-lock.json | 10 +- dc-agents/reference/README.md | 2 + dc-agents/reference/package-lock.json | 4 +- dc-agents/reference/package.json | 2 +- dc-agents/reference/src/capabilities.ts | 3 +- dc-agents/reference/src/config.ts | 7 + dc-agents/reference/src/data/index.ts | 4 +- dc-agents/reference/src/datasets.ts | 34 ++++ dc-agents/reference/src/index.ts | 41 ++++- dc-agents/sqlite/.gitignore | 1 + dc-agents/sqlite/README.md | 18 ++ dc-agents/sqlite/package-lock.json | 4 +- dc-agents/sqlite/package.json | 2 +- dc-agents/sqlite/src/capabilities.ts | 3 +- dc-agents/sqlite/src/datasets.ts | 79 +++++++++ dc-agents/sqlite/src/environment.ts | 7 +- dc-agents/sqlite/src/index.ts | 38 ++++- .../src/Test/DataConnector/MetadataApiSpec.hs | 4 +- server/lib/dc-api/dc-api.cabal | 1 + .../src/Hasura/Backends/DataConnector/API.hs | 29 +++- .../Hasura/Backends/DataConnector/API/V0.hs | 2 + .../DataConnector/API/V0/Capabilities.hs | 16 +- .../DataConnector/API/V0/ConfigSchema.hs | 15 +- .../Backends/DataConnector/API/V0/Dataset.hs | 127 ++++++++++++++ .../Backend/DataConnector/Mock/Server.hs | 12 +- .../DataConnector/API/V0/CapabilitiesSpec.hs | 4 + 38 files changed, 714 insertions(+), 34 deletions(-) create mode 100644 dc-agents/dc-api-types/src/models/Config.ts create mode 100644 dc-agents/dc-api-types/src/models/DatasetCapabilities.ts create mode 100644 dc-agents/dc-api-types/src/models/DatasetDeleteResponse.ts create mode 100644 dc-agents/dc-api-types/src/models/DatasetGetResponse.ts create mode 100644 dc-agents/dc-api-types/src/models/DatasetPostRequest.ts create mode 100644 dc-agents/dc-api-types/src/models/DatasetPostResponse.ts create mode 100644 dc-agents/dc-api-types/src/models/DatasetTemplateName.ts create mode 100644 dc-agents/reference/src/datasets.ts create mode 100644 dc-agents/sqlite/src/datasets.ts create mode 100644 server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Dataset.hs diff --git a/dc-agents/README.md b/dc-agents/README.md index f4fb65407dd..11cb1f60f28 100644 --- a/dc-agents/README.md +++ b/dc-agents/README.md @@ -258,6 +258,20 @@ The preference would be to support the highest level of atomicity possible (ie ` The agent can also specify whether or not it supports `returning` data from mutations. This refers to the ability to return the data that was mutated by mutation operations (for example, the updated rows in an update, or the deleted rows in a delete). +### Dataset Capabilities + +The agent can declare whether it supports datasets (ie. api for creating/cloning schemas). If it supports datasets, it needs to declare a `datasets` capability: + +```json +{ + "capabilities": { + "datasets": { } + } +} +``` + +See [Datasets](#datasets) for information on how this capability is used. + ### Schema The `GET /schema` endpoint is called whenever the metadata is (re)loaded by `graphql-engine`. It returns the following JSON object: @@ -2160,3 +2174,42 @@ Breaking down the properties in the `delete`-typed mutation operation: * `returning_fields`: This specifies a list of fields to return in the response. The property takes the same format as the `fields` property on Queries. It is expected that the specified fields will be returned for all rows affected by the deletion (ie. all deleted rows). Delete operations return responses that are the same as insert and update operations, except the affected rows in `returning` are the deleted rows instead. + +### Datasets + +The `/datasets` resource is available to use in order to create new databases/schemas from templates. + +Datasets are represented by abstract names referencing database-schema templates that can be cloned from and clones that can be used via config and deleted. This feature is required for testing the mutations feature, but may also have non-test uses - for example - spinning up interactive demo projects. + +The `/datasets/:name` resource has the following methods: + +* `GET /datasets/:template_name` -> `{"exists": true|false}` +* `POST /datasets/:clone_name {"from": template_name}` -> `{"config": {...}}` +* `DELETE /datasets/:clone_name` -> `{"message": "success"}` + +The `POST` method is the most significant way to interact with the API. It allows for cloning a dataset template to a new name. The new name can be used to delete the dataset, and the config returned from the POST API call can be used as the config header for non-dataset interactions such as queries. + +The following diagram shows the interactions between the various datatypes and resource methods: + +```mermaid +flowchart TD; + NAME["Dataset Name"] --> GET["GET /datasets/templates/:template_name"]; + style NAME stroke:#0f3,stroke-width:2px + NAME -- clone_name --> POST; + NAME -- from --> POST["POST /datasets/clones/:clone_name { from: TEMPLATE_NAME }"]; + GET --> EXISTS["{ exists: true }"]; + GET --> EXISTSF["{ exists: false }"]; + GET --> FAILUREG["400"]; + style FAILUREG stroke:#f33,stroke-width:2px + POST --> FAILUREP["400"]; + style FAILUREP stroke:#f33,stroke-width:2px + NAME --> DELETE["DELETE /datasets/clones/:clone_name"]; + POST --> CONFIG["Source Config"]; + style CONFIG stroke:#0f3,stroke-width:2px + DELETE --> SUCCESSD["{ message: 'success' }"]; + DELETE --> FAILURED["400"]; + style FAILURED stroke:#f33,stroke-width:2px + CONFIG --> SCHEMA["POST /schema"]; + CONFIG --> QUERY["POST /query"]; + CONFIG --> MUTATION["POST /mutation"]; +``` diff --git a/dc-agents/dc-api-types/package.json b/dc-agents/dc-api-types/package.json index 0ab609fcaad..4d350ea8b89 100644 --- a/dc-agents/dc-api-types/package.json +++ b/dc-agents/dc-api-types/package.json @@ -1,6 +1,6 @@ { "name": "@hasura/dc-api-types", - "version": "0.21.0", + "version": "0.22.0", "description": "Hasura GraphQL Engine Data Connector Agent API types", "author": "Hasura (https://github.com/hasura/graphql-engine)", "license": "Apache-2.0", diff --git a/dc-agents/dc-api-types/src/agent.openapi.json b/dc-agents/dc-api-types/src/agent.openapi.json index 93c8ae94790..cc57b27fd43 100644 --- a/dc-agents/dc-api-types/src/agent.openapi.json +++ b/dc-agents/dc-api-types/src/agent.openapi.json @@ -370,6 +370,103 @@ } } } + }, + "/datasets/templates/{template_name}": { + "get": { + "parameters": [ + { + "in": "path", + "name": "template_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json;charset=utf-8": { + "schema": { + "$ref": "#/components/schemas/DatasetGetResponse" + } + } + }, + "description": "" + }, + "404": { + "description": "`template_name` not found" + } + } + } + }, + "/datasets/clones/{clone_name}": { + "post": { + "parameters": [ + { + "in": "path", + "name": "clone_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json;charset=utf-8": { + "schema": { + "$ref": "#/components/schemas/DatasetPostRequest" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json;charset=utf-8": { + "schema": { + "$ref": "#/components/schemas/DatasetPostResponse" + } + } + }, + "description": "" + }, + "400": { + "description": "Invalid `body`" + }, + "404": { + "description": "`clone_name` not found" + } + } + }, + "delete": { + "parameters": [ + { + "in": "path", + "name": "clone_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json;charset=utf-8": { + "schema": { + "$ref": "#/components/schemas/DatasetDeleteResponse" + } + } + }, + "description": "" + }, + "404": { + "description": "`clone_name` not found" + } + } + } } }, "components": { @@ -404,6 +501,9 @@ "data_schema": { "$ref": "#/components/schemas/DataSchemaCapabilities" }, + "datasets": { + "$ref": "#/components/schemas/DatasetCapabilities" + }, "explain": { "$ref": "#/components/schemas/ExplainCapabilities" }, @@ -573,6 +673,7 @@ "MetricsCapabilities": {}, "ExplainCapabilities": {}, "RawCapabilities": {}, + "DatasetCapabilities": {}, "ConfigSchemaResponse": { "nullable": false, "properties": { @@ -2350,6 +2451,61 @@ "query" ], "type": "object" + }, + "DatasetGetResponse": { + "properties": { + "exists": { + "description": "Message detailing if the dataset exists", + "type": "boolean" + } + }, + "required": [ + "exists" + ], + "type": "object" + }, + "DatasetPostResponse": { + "properties": { + "config": { + "$ref": "#/components/schemas/Config" + } + }, + "required": [ + "config" + ], + "type": "object" + }, + "Config": { + "additionalProperties": { + "additionalProperties": true + }, + "type": "object" + }, + "DatasetPostRequest": { + "properties": { + "from": { + "$ref": "#/components/schemas/DatasetTemplateName" + } + }, + "required": [ + "from" + ], + "type": "object" + }, + "DatasetTemplateName": { + "type": "string" + }, + "DatasetDeleteResponse": { + "properties": { + "message": { + "description": "The named dataset to clone from", + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" } } } diff --git a/dc-agents/dc-api-types/src/index.ts b/dc-agents/dc-api-types/src/index.ts index 8be8dfa53a2..255310a686a 100644 --- a/dc-agents/dc-api-types/src/index.ts +++ b/dc-agents/dc-api-types/src/index.ts @@ -27,9 +27,16 @@ export type { ComparisonCapabilities } from './models/ComparisonCapabilities'; export type { ComparisonColumn } from './models/ComparisonColumn'; export type { ComparisonOperators } from './models/ComparisonOperators'; export type { ComparisonValue } from './models/ComparisonValue'; +export type { Config } from './models/Config'; export type { ConfigSchemaResponse } from './models/ConfigSchemaResponse'; export type { Constraint } from './models/Constraint'; export type { DataSchemaCapabilities } from './models/DataSchemaCapabilities'; +export type { DatasetCapabilities } from './models/DatasetCapabilities'; +export type { DatasetDeleteResponse } from './models/DatasetDeleteResponse'; +export type { DatasetGetResponse } from './models/DatasetGetResponse'; +export type { DatasetPostRequest } from './models/DatasetPostRequest'; +export type { DatasetPostResponse } from './models/DatasetPostResponse'; +export type { DatasetTemplateName } from './models/DatasetTemplateName'; export type { DeleteCapabilities } from './models/DeleteCapabilities'; export type { DeleteMutationOperation } from './models/DeleteMutationOperation'; export type { ErrorResponse } from './models/ErrorResponse'; diff --git a/dc-agents/dc-api-types/src/models/Capabilities.ts b/dc-agents/dc-api-types/src/models/Capabilities.ts index 91d5d0c9459..a99344399b7 100644 --- a/dc-agents/dc-api-types/src/models/Capabilities.ts +++ b/dc-agents/dc-api-types/src/models/Capabilities.ts @@ -4,6 +4,7 @@ import type { ComparisonCapabilities } from './ComparisonCapabilities'; import type { DataSchemaCapabilities } from './DataSchemaCapabilities'; +import type { DatasetCapabilities } from './DatasetCapabilities'; import type { ExplainCapabilities } from './ExplainCapabilities'; import type { MetricsCapabilities } from './MetricsCapabilities'; import type { MutationCapabilities } from './MutationCapabilities'; @@ -16,6 +17,7 @@ import type { SubscriptionCapabilities } from './SubscriptionCapabilities'; export type Capabilities = { comparisons?: ComparisonCapabilities; data_schema?: DataSchemaCapabilities; + datasets?: DatasetCapabilities; explain?: ExplainCapabilities; metrics?: MetricsCapabilities; mutations?: MutationCapabilities; diff --git a/dc-agents/dc-api-types/src/models/Config.ts b/dc-agents/dc-api-types/src/models/Config.ts new file mode 100644 index 00000000000..db940c4b5b7 --- /dev/null +++ b/dc-agents/dc-api-types/src/models/Config.ts @@ -0,0 +1,5 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type Config = Record; diff --git a/dc-agents/dc-api-types/src/models/DatasetCapabilities.ts b/dc-agents/dc-api-types/src/models/DatasetCapabilities.ts new file mode 100644 index 00000000000..68e52a80c32 --- /dev/null +++ b/dc-agents/dc-api-types/src/models/DatasetCapabilities.ts @@ -0,0 +1,7 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type DatasetCapabilities = { +}; + diff --git a/dc-agents/dc-api-types/src/models/DatasetDeleteResponse.ts b/dc-agents/dc-api-types/src/models/DatasetDeleteResponse.ts new file mode 100644 index 00000000000..f8ff4fee2b2 --- /dev/null +++ b/dc-agents/dc-api-types/src/models/DatasetDeleteResponse.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type DatasetDeleteResponse = { + /** + * The named dataset to clone from + */ + message: string; +}; + diff --git a/dc-agents/dc-api-types/src/models/DatasetGetResponse.ts b/dc-agents/dc-api-types/src/models/DatasetGetResponse.ts new file mode 100644 index 00000000000..7c0f061511a --- /dev/null +++ b/dc-agents/dc-api-types/src/models/DatasetGetResponse.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type DatasetGetResponse = { + /** + * Message detailing if the dataset exists + */ + exists: boolean; +}; + diff --git a/dc-agents/dc-api-types/src/models/DatasetPostRequest.ts b/dc-agents/dc-api-types/src/models/DatasetPostRequest.ts new file mode 100644 index 00000000000..02bed7f6b0e --- /dev/null +++ b/dc-agents/dc-api-types/src/models/DatasetPostRequest.ts @@ -0,0 +1,10 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { DatasetTemplateName } from './DatasetTemplateName'; + +export type DatasetPostRequest = { + from: DatasetTemplateName; +}; + diff --git a/dc-agents/dc-api-types/src/models/DatasetPostResponse.ts b/dc-agents/dc-api-types/src/models/DatasetPostResponse.ts new file mode 100644 index 00000000000..19952a8ec4b --- /dev/null +++ b/dc-agents/dc-api-types/src/models/DatasetPostResponse.ts @@ -0,0 +1,10 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { Config } from './Config'; + +export type DatasetPostResponse = { + config: Config; +}; + diff --git a/dc-agents/dc-api-types/src/models/DatasetTemplateName.ts b/dc-agents/dc-api-types/src/models/DatasetTemplateName.ts new file mode 100644 index 00000000000..b5a57a9fbb0 --- /dev/null +++ b/dc-agents/dc-api-types/src/models/DatasetTemplateName.ts @@ -0,0 +1,5 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type DatasetTemplateName = string; diff --git a/dc-agents/package-lock.json b/dc-agents/package-lock.json index 9f4c5d85391..ece5a589e4e 100644 --- a/dc-agents/package-lock.json +++ b/dc-agents/package-lock.json @@ -24,7 +24,7 @@ }, "dc-api-types": { "name": "@hasura/dc-api-types", - "version": "0.21.0", + "version": "0.22.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", @@ -1197,7 +1197,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^7.0.0", - "@hasura/dc-api-types": "0.21.0", + "@hasura/dc-api-types": "0.22.0", "fastify": "^3.29.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", @@ -1781,7 +1781,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.21.0", + "@hasura/dc-api-types": "0.22.0", "fastify": "^4.4.0", "fastify-metrics": "^9.2.1", "nanoid": "^3.3.4", @@ -3125,7 +3125,7 @@ "version": "file:reference", "requires": { "@fastify/cors": "^7.0.0", - "@hasura/dc-api-types": "0.21.0", + "@hasura/dc-api-types": "0.22.0", "@tsconfig/node16": "^1.0.3", "@types/node": "^16.11.49", "@types/xml2js": "^0.4.11", @@ -3514,7 +3514,7 @@ "version": "file:sqlite", "requires": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.21.0", + "@hasura/dc-api-types": "0.22.0", "@tsconfig/node16": "^1.0.3", "@types/node": "^16.11.49", "@types/sqlite3": "^3.1.8", diff --git a/dc-agents/reference/README.md b/dc-agents/reference/README.md index 783574b4108..6b3bee21306 100644 --- a/dc-agents/reference/README.md +++ b/dc-agents/reference/README.md @@ -24,6 +24,8 @@ More specifically, the `ChinookData.xml.gz` file is a GZipped version of https:/ The `schema-tables.json` is manually derived from the schema of the data as can be seen from the `CREATE TABLE` etc DML statements in the various per-database-vendor SQL scripts that can be found in `/ChinookDatabase/DataSources` in that repo. +The datasets can be operated on via the `/datasets` resources as described in `dc-agents/README.md`. + ## Configuration The reference agent supports some configuration properties that can be set via the `value` property of `configuration` on a source in Hasura metadata. The configuration is passed to the agent on each request via the `X-Hasura-DataConnector-Config` header. diff --git a/dc-agents/reference/package-lock.json b/dc-agents/reference/package-lock.json index 108fc707399..014a0e5a6ae 100644 --- a/dc-agents/reference/package-lock.json +++ b/dc-agents/reference/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^7.0.0", - "@hasura/dc-api-types": "0.21.0", + "@hasura/dc-api-types": "0.22.0", "fastify": "^3.29.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", @@ -44,7 +44,7 @@ } }, "node_modules/@hasura/dc-api-types": { - "version": "0.21.0", + "version": "0.22.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", diff --git a/dc-agents/reference/package.json b/dc-agents/reference/package.json index 20035ac358b..96597ba952a 100644 --- a/dc-agents/reference/package.json +++ b/dc-agents/reference/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@fastify/cors": "^7.0.0", - "@hasura/dc-api-types": "0.21.0", + "@hasura/dc-api-types": "0.22.0", "fastify": "^3.29.0", "mathjs": "^11.0.0", "pino-pretty": "^8.0.0", diff --git a/dc-agents/reference/src/capabilities.ts b/dc-agents/reference/src/capabilities.ts index 08695f55ef0..6c08bef346a 100644 --- a/dc-agents/reference/src/capabilities.ts +++ b/dc-agents/reference/src/capabilities.ts @@ -57,7 +57,8 @@ const capabilities: Capabilities = { supports_relations: true } }, - scalar_types: scalarTypes + scalar_types: scalarTypes, + datasets: {} } export const capabilitiesResponse: CapabilitiesResponse = { diff --git a/dc-agents/reference/src/config.ts b/dc-agents/reference/src/config.ts index 9baf944c68b..0c2eda74204 100644 --- a/dc-agents/reference/src/config.ts +++ b/dc-agents/reference/src/config.ts @@ -6,6 +6,7 @@ export type Casing = "pascal_case" | "lowercase"; export type Config = { tables: string[] | null schema: string | null + db: string | null table_name_casing: Casing column_name_casing: Casing } @@ -17,6 +18,7 @@ export const getConfig = (request: FastifyRequest): Config => { return { tables: config.tables ?? null, schema: config.schema ?? null, + db: config.db ?? null, table_name_casing: config.table_name_casing ?? "pascal_case", column_name_casing: config.column_name_casing ?? "pascal_case", } @@ -38,6 +40,11 @@ export const configSchema: ConfigSchemaResponse = { type: "string", nullable: true }, + db: { + description: "Name of the db. Omit to use the default db.", + type: "string", + nullable: true + }, table_name_casing: { $ref: "#/other_schemas/Casing" }, diff --git a/dc-agents/reference/src/data/index.ts b/dc-agents/reference/src/data/index.ts index 8ee1e2d3f91..525c97fd043 100644 --- a/dc-agents/reference/src/data/index.ts +++ b/dc-agents/reference/src/data/index.ts @@ -31,8 +31,8 @@ const parseNumbersInNumericColumns = (schema: SchemaResponse) => { }; } -export const loadStaticData = async (): Promise => { - const gzipReadStream = fs.createReadStream(__dirname + "/ChinookData.xml.gz"); +export const loadStaticData = async (name: string): Promise => { + const gzipReadStream = fs.createReadStream(__dirname + "/" + name); const unzipStream = stream.pipeline(gzipReadStream, zlib.createGunzip(), () => { }); const xmlStr = (await streamToBuffer(unzipStream)).toString("utf16le"); const xml = await xml2js.parseStringPromise(xmlStr, { explicitArray: false, emptyTag: () => null, valueProcessors: [parseNumbersInNumericColumns(schema)] }); diff --git a/dc-agents/reference/src/datasets.ts b/dc-agents/reference/src/datasets.ts new file mode 100644 index 00000000000..52693404428 --- /dev/null +++ b/dc-agents/reference/src/datasets.ts @@ -0,0 +1,34 @@ +import { DatasetDeleteResponse, DatasetGetResponse, DatasetPostRequest, DatasetPostResponse, } from '@hasura/dc-api-types'; +import { loadStaticData, StaticData } from './data'; + +export async function getDataset(name: string): Promise { + const safePath = mkPath(name); + const data = await loadStaticData(safePath); // TODO: Could make this more efficient, but this works for now! + if(data) { + return { exists: true }; + } else { + return { exists: false }; + } +} + +export async function cloneDataset(store: Record, name: string, body: DatasetPostRequest): Promise { + const safePathName = mkPath(body.from); + const data = await loadStaticData(safePathName); + store[`$${name}`] = data; + return { config: { db: `$${name}` } }; +} + +export async function deleteDataset(store: Record, name: string): Promise { + const exists = store[`$${name}`]; + if(exists) { + delete store[`$${name}`]; + return {message: "success"}; + } else { + throw(Error("Dataset does not exist.")); + } +} + +function mkPath(name: string): string { + const base = name.replace(/\//g,''); // TODO: Can this be made safer? + return `${base}.xml.gz`; +} diff --git a/dc-agents/reference/src/index.ts b/dc-agents/reference/src/index.ts index 7e45f66f012..24192e549bc 100644 --- a/dc-agents/reference/src/index.ts +++ b/dc-agents/reference/src/index.ts @@ -1,14 +1,15 @@ import Fastify from 'fastify'; import FastifyCors from '@fastify/cors'; -import { filterAvailableTables, getSchema, getTable, loadStaticData } from './data'; +import { filterAvailableTables, getSchema, getTable, loadStaticData, StaticData } from './data'; import { queryData } from './query'; import { getConfig } from './config'; import { capabilitiesResponse } from './capabilities'; -import { CapabilitiesResponse, SchemaResponse, QueryRequest, QueryResponse } from '@hasura/dc-api-types'; +import { CapabilitiesResponse, SchemaResponse, QueryRequest, QueryResponse, DatasetDeleteResponse, DatasetPostRequest, DatasetGetResponse, DatasetPostResponse } from '@hasura/dc-api-types'; +import { cloneDataset, deleteDataset, getDataset } from './datasets'; const port = Number(process.env.PORT) || 8100; const server = Fastify({ logger: { prettyPrint: true } }); -let staticData = {}; +let staticData : Record = {}; server.register(FastifyCors, { // Accept all origins of requests. This must be modified in @@ -33,10 +34,40 @@ server.get<{ Reply: SchemaResponse }>("/schema", async (request, _response) => { server.post<{ Body: QueryRequest, Reply: QueryResponse }>("/query", async (request, _response) => { server.log.info({ headers: request.headers, query: request.body, }, "query.request"); const config = getConfig(request); - const data = filterAvailableTables(staticData, config); + // Prefix '$' to disambiguate from default datasets. + const dbName = config.db ? `$${config.db}` : '@default'; + const data = filterAvailableTables(staticData[dbName], config); return queryData(getTable(data, config), request.body); }); +// Methods on dataset resources. +// +// Examples: +// +// > curl -H 'content-type: application/json' -XGET localhost:8100/datasets/ChinookData +// {"exists": true} +// +server.get<{ Params: { name: string, }, Reply: DatasetGetResponse }>("/datasets/templates/:name", async (request, _response) => { + server.log.info({ headers: request.headers, query: request.body, }, "datasets.templates.get"); + return getDataset(request.params.name); +}); + +// > curl -H 'content-type: application/json' -XPOST localhost:8100/datasets/foo -d '{"from": "ChinookData"}' +// {"config":{"db":"$foo"}} +// +server.post<{ Params: { name: string, }, Body: DatasetPostRequest, Reply: DatasetPostResponse }>("/datasets/clones/:name", async (request, _response) => { + server.log.info({ headers: request.headers, query: request.body, }, "datasets.clones.post"); + return cloneDataset(staticData, request.params.name, request.body); +}); + +// > curl -H 'content-type: application/json' -XDELETE 'localhost:8100/datasets/foo' +// {"message":"success"} +// +server.delete<{ Params: { name: string, }, Reply: DatasetDeleteResponse }>("/datasets/clones/:name", async (request, _response) => { + server.log.info({ headers: request.headers, query: request.body, }, "datasets.clones.delete"); + return deleteDataset(staticData, request.params.name); +}); + server.get("/health", async (request, response) => { server.log.info({ headers: request.headers, query: request.body, }, "health.request"); response.statusCode = 204; @@ -49,7 +80,7 @@ process.on('SIGINT', () => { const start = async () => { try { - staticData = await loadStaticData(); + staticData = {'@default' : await loadStaticData("ChinookData.xml.gz")}; await server.listen(port, "0.0.0.0"); } catch (err) { diff --git a/dc-agents/sqlite/.gitignore b/dc-agents/sqlite/.gitignore index 593b38be420..93b3e876b00 100644 --- a/dc-agents/sqlite/.gitignore +++ b/dc-agents/sqlite/.gitignore @@ -1,3 +1,4 @@ node_modules dist ./*.sqlite +./dataset_clones/ \ No newline at end of file diff --git a/dc-agents/sqlite/README.md b/dc-agents/sqlite/README.md index 90f181145c0..8ab6c009622 100644 --- a/dc-agents/sqlite/README.md +++ b/dc-agents/sqlite/README.md @@ -68,6 +68,11 @@ Note: Boolean flags `{FLAG}` can be provided as `1`, `true`, `yes`, or omitted a | `LOG_LEVEL` | `fatal` \| `error` \| `info` \| `debug` \| `trace` \| `silent` | `info` | The minimum log level to output | | `METRICS` | `{FLAG}` | `false` | Enables a `/metrics` prometheus metrics endpoint. | `QUERY_LENGTH_LIMIT` | `INT` | `Infinity` | Puts a limit on the length of generated SQL before execution. | +| `DATASETS` | `{FLAG}` | `false` | Enable dataset operations | +| `DATASET_DELETE` | `{FLAG}` | `false` | Enable `DELETE /datasets/:name` | +| `DATASET_TEMPLATES` | `DIRECTORY` | `./dataset_templates` | Directory to clone datasets from. | +| `DATASET_CLONES` | `DIRECTORY` | `./dataset_clones` | Directory to clone datasets to. | + ## Agent usage @@ -95,6 +100,19 @@ The dataset used for testing the reference agent is sourced from: * https://raw.githubusercontent.com/lerocha/chinook-database/master/ChinookDatabase/DataSources/Chinook_Sqlite.sql +### Datasets + +Datasets support is enabled via the ENV variables: + +* `DATASETS` +* `DATASET_DELETE` +* `DATASET_TEMPLATES` +* `DATASET_CLONES` + +Templates will be looked up at `${DATASET_TEMPLATES}/${template_name}.db`. + +Clones will be copied to `${DATASET_CLONES}/${clone_name}.db`. + ## Testing Changes to the Agent Run: diff --git a/dc-agents/sqlite/package-lock.json b/dc-agents/sqlite/package-lock.json index 8dc56cda71b..ecae46f7cad 100644 --- a/dc-agents/sqlite/package-lock.json +++ b/dc-agents/sqlite/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.21.0", + "@hasura/dc-api-types": "0.22.0", "fastify": "^4.4.0", "fastify-metrics": "^9.2.1", "nanoid": "^3.3.4", @@ -54,7 +54,7 @@ "license": "MIT" }, "node_modules/@hasura/dc-api-types": { - "version": "0.21.0", + "version": "0.22.0", "license": "Apache-2.0", "devDependencies": { "@tsconfig/node16": "^1.0.3", diff --git a/dc-agents/sqlite/package.json b/dc-agents/sqlite/package.json index 4d12032eb50..1230e560455 100644 --- a/dc-agents/sqlite/package.json +++ b/dc-agents/sqlite/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@fastify/cors": "^8.1.0", - "@hasura/dc-api-types": "0.21.0", + "@hasura/dc-api-types": "0.22.0", "fastify-metrics": "^9.2.1", "fastify": "^4.4.0", "nanoid": "^3.3.4", diff --git a/dc-agents/sqlite/src/capabilities.ts b/dc-agents/sqlite/src/capabilities.ts index 0b63c331e14..1ac7d322e0b 100644 --- a/dc-agents/sqlite/src/capabilities.ts +++ b/dc-agents/sqlite/src/capabilities.ts @@ -1,5 +1,5 @@ import { configSchema } from "./config" -import { METRICS, MUTATIONS } from "./environment" +import { DATASETS, METRICS, MUTATIONS } from "./environment" import { CapabilitiesResponse, ScalarTypeCapabilities } from "@hasura/dc-api-types" @@ -129,6 +129,7 @@ export const capabilitiesResponse: CapabilitiesResponse = { ), explain: {}, raw: {}, + ... (DATASETS ? { datasets: {} } : {}), ... (METRICS ? { metrics: {} } : {}) }, } diff --git a/dc-agents/sqlite/src/datasets.ts b/dc-agents/sqlite/src/datasets.ts new file mode 100644 index 00000000000..a991d58ca88 --- /dev/null +++ b/dc-agents/sqlite/src/datasets.ts @@ -0,0 +1,79 @@ +import { connect, SqlLogger } from './db'; +import { DatasetDeleteResponse, DatasetGetResponse, DatasetPostRequest, DatasetPostResponse } from '@hasura/dc-api-types'; +import { promises, existsSync } from 'fs'; +import { DATASET_CLONES, DATASET_DELETE, DATASET_TEMPLATES } from "./environment"; +import path from 'path'; + +export async function getDataset(template_name: string): Promise { + const path = mkTemplatePath(template_name); + if(existsSync(path)) { + const stats = await promises.stat(path); + if(stats.isFile()) { + return { exists: true }; + } else { + return { exists: false }; + } + } else { + return { exists: false }; + } +} + +export async function cloneDataset(logger: SqlLogger, clone_name: string, body: DatasetPostRequest): Promise { + const fromPath = mkTemplatePath(body.from); + const toPath = mkClonePath(clone_name); + const fromStats = await promises.stat(fromPath); + const exists = existsSync(toPath); + if(fromStats.isFile() && ! exists) { + // Check if this is a real SQLite DB + const db = connect({ db: fromPath, explicit_main_schema: false, tables: [], meta: false }, logger); + if(db) { + db.close(); + } else { + throw(Error("Dataset is not an SQLite Database!")) + } + await promises.cp(fromPath, toPath); + return { config: { db: toPath } }; + } else if(exists) { + throw(Error("Dataset already exists!")) + } else { + throw(Error("Can't Clone!")) + } +} + +export async function deleteDataset(clone_name: string): Promise { + if(DATASET_DELETE) { + const path = mkClonePath(clone_name); + const exists = existsSync(path); + if(exists) { + const stats = await promises.stat(path); + if(stats.isFile()) { + await promises.rm(path); + return { message: "success" }; + } else { + throw(Error("Dataset is not a file.")); + } + } else { + throw(Error("Dataset does not exist.")); + } + } else { + throw(Error("Dataset deletion not available.")); + } +} + +function mkTemplatePath(name: string): string { + const parsed = path.parse(name); + const safeName = parsed.base; + if(name != safeName) { + throw(Error(`Template name ${name} is not valid.`)); + } + return path.join(DATASET_TEMPLATES, safeName); +} + +function mkClonePath(name: string): string { + const parsed = path.parse(name); + const safeName = parsed.base; + if(name != safeName) { + throw(Error(`Template name ${name} is not valid.`)); + } + return path.join(DATASET_CLONES, safeName); +} diff --git a/dc-agents/sqlite/src/environment.ts b/dc-agents/sqlite/src/environment.ts index 16fe170da2f..d06c85fa2d8 100644 --- a/dc-agents/sqlite/src/environment.ts +++ b/dc-agents/sqlite/src/environment.ts @@ -38,4 +38,9 @@ export const DB_PRIVATECACHE = envToBool('DB_PRIVATECACHE'); export const DEBUGGING_TAGS = envToBool('DEBUGGING_TAGS'); export const QUERY_LENGTH_LIMIT = envToNum('QUERY_LENGTH_LIMIT', Infinity); -export const MUTATIONS = envToBool('MUTATIONS'); \ No newline at end of file +export const MUTATIONS = envToBool('MUTATIONS'); + +export const DATASETS = envToBool('DATASETS'); +export const DATASET_DELETE = envToBool('DATASET_DELETE'); +export const DATASET_TEMPLATES = envToString('DATASET_TEMPLATES', "./dataset_templates"); +export const DATASET_CLONES = envToString('DATASET_CLONES', "./dataset_clones"); diff --git a/dc-agents/sqlite/src/index.ts b/dc-agents/sqlite/src/index.ts index 6ac9bfbb1d8..8018108d7ff 100644 --- a/dc-agents/sqlite/src/index.ts +++ b/dc-agents/sqlite/src/index.ts @@ -4,12 +4,13 @@ import { getSchema } from './schema'; import { explain, queryData } from './query'; import { getConfig, tryGetConfig } from './config'; import { capabilitiesResponse } from './capabilities'; -import { QueryResponse, SchemaResponse, QueryRequest, CapabilitiesResponse, ExplainResponse, RawRequest, RawResponse, ErrorResponse, MutationRequest, MutationResponse } from '@hasura/dc-api-types'; +import { QueryResponse, SchemaResponse, QueryRequest, CapabilitiesResponse, ExplainResponse, RawRequest, RawResponse, ErrorResponse, MutationRequest, MutationResponse, DatasetGetResponse, DatasetPostResponse, DatasetDeleteResponse, DatasetPostRequest, DatasetTemplateName } from '@hasura/dc-api-types'; import { connect } from './db'; import metrics from 'fastify-metrics'; import prometheus from 'prom-client'; import { runRawOperation } from './raw'; -import { LOG_LEVEL, METRICS, MUTATIONS, PERMISSIVE_CORS, PRETTY_PRINT_LOGS } from './environment'; +import { DATASETS, DATASET_DELETE, LOG_LEVEL, METRICS, MUTATIONS, PERMISSIVE_CORS, PRETTY_PRINT_LOGS } from './environment'; +import { cloneDataset, deleteDataset, getDataset } from './datasets'; const port = Number(process.env.PORT) || 8100; @@ -143,7 +144,7 @@ server.post<{ Body: QueryRequest, Reply: ExplainResponse}>("/explain", async (re return explain(config, sqlLogger, request.body); }); -if (MUTATIONS) { +if(MUTATIONS) { server.post<{ Body: MutationRequest, Reply: MutationResponse}>("/mutation", async (request, _response) => { server.log.info({ headers: request.headers, query: request.body, }, "mutation.request"); throw Error("Mutations not yet implemented"); @@ -170,6 +171,34 @@ server.get("/health", async (request, response) => { } }); +// Data-Set Features - Names must match files in the associated datasets directory. +// If they exist then they are tracked for the purposes of this feature in SQLite. +if(DATASETS) { + server.get<{ Params: { template_name: DatasetTemplateName, }, Reply: DatasetGetResponse }>("/datasets/templates/:template_name", async (request, _response) => { + server.log.info({ headers: request.headers, query: request.body, }, "datasets.templates.get"); + const result = await getDataset(request.params.template_name); + if(! result.exists) { + _response.statusCode = 404; + } + return result; + }); + + // TODO: The name param here should be a DatasetCloneName, but this isn't being code-generated. + server.post<{ Params: { clone_name: string, }, Body: DatasetPostRequest, Reply: DatasetPostResponse }>("/datasets/clones/:clone_name", async (request, _response) => { + server.log.info({ headers: request.headers, query: request.body, }, "datasets.clones.post"); + return cloneDataset(sqlLogger, request.params.clone_name, request.body); + }); + + // Only allow deletion if this is explicitly supported by ENV configuration + if(DATASET_DELETE) { + // TODO: The name param here should be a DatasetCloneName, but this isn't being code-generated. + server.delete<{ Params: { clone_name: string, }, Reply: DatasetDeleteResponse }>("/datasets/clones/:clone_name", async (request, _response) => { + server.log.info({ headers: request.headers, query: request.body, }, "datasets.clones.delete"); + return deleteDataset(request.params.clone_name); + }); + } +} + server.get("/", async (request, response) => { response.type('text/html'); return ` @@ -190,6 +219,9 @@ server.get("/", async (request, response) => {
  • POST /raw - Raw Query Handler
  • GET /health - Healthcheck
  • GET /metrics - Prometheus formatted metrics +
  • GET /datasets/{NAME} - Information on Dataset +
  • POST /datasets/{NAME} - Create a Dataset +
  • DELETE /datasets/{NAME} - Delete a Dataset diff --git a/server/lib/api-tests/src/Test/DataConnector/MetadataApiSpec.hs b/server/lib/api-tests/src/Test/DataConnector/MetadataApiSpec.hs index fa68ba1bb37..50fb55c3dcd 100644 --- a/server/lib/api-tests/src/Test/DataConnector/MetadataApiSpec.hs +++ b/server/lib/api-tests/src/Test/DataConnector/MetadataApiSpec.hs @@ -230,15 +230,15 @@ schemaInspectionTests opts = describe "Schema and Source Inspection" $ do ) -- Note: These fields are backend specific so we ignore their values and just verify their shapes: <&> Lens.set (key "config_schema_response" . key "other_schemas") J.Null <&> Lens.set (key "config_schema_response" . key "config_schema") J.Null + <&> Lens.set (key "capabilities" . _Object . Lens.at "datasets") Nothing <&> Lens.set (key "options" . key "uri") J.Null - <&> Lens.set (_Object . Lens.at "display_name") (Just J.Null) + <&> Lens.set (_Object . Lens.at "display_name") Nothing ) [yaml| capabilities: *backendCapabilities config_schema_response: config_schema: null other_schemas: null - display_name: null options: uri: null |] diff --git a/server/lib/dc-api/dc-api.cabal b/server/lib/dc-api/dc-api.cabal index d192afdf2c9..d25149b7bbd 100644 --- a/server/lib/dc-api/dc-api.cabal +++ b/server/lib/dc-api/dc-api.cabal @@ -88,6 +88,7 @@ library Hasura.Backends.DataConnector.API.V0.Scalar Hasura.Backends.DataConnector.API.V0.Schema Hasura.Backends.DataConnector.API.V0.Table + Hasura.Backends.DataConnector.API.V0.Dataset other-modules: Hasura.Backends.DataConnector.API.V0.Name diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API.hs index 061a1c2df5c..c18a19baef9 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API.hs @@ -149,6 +149,27 @@ type RawApi = :> ReqBody '[JSON] V0.RawRequest :> Post '[JSON] V0.RawResponse +type DatasetGetApi = + "datasets" + :> "templates" + :> Capture "template_name" DatasetTemplateName + :> Get '[JSON] V0.DatasetGetResponse + +type DatasetPostApi = + "datasets" + :> "clones" + :> Capture "clone_name" DatasetCloneName + :> ReqBody '[JSON] V0.DatasetPostRequest + :> Post '[JSON] V0.DatasetPostResponse + +type DatasetDeleteApi = + "datasets" + :> "clones" + :> Capture "clone_name" DatasetCloneName + :> Delete '[JSON] V0.DatasetDeleteResponse + +type DatasetApi = DatasetGetApi :<|> DatasetPostApi :<|> DatasetDeleteApi + data Prometheus -- NOTE: This seems like quite a brittle definition and we may want to be @@ -189,13 +210,17 @@ data Routes mode = Routes -- | 'GET /metrics' _metrics :: mode :- MetricsApi, -- | 'GET /raw' - _raw :: mode :- RawApi + _raw :: mode :- RawApi, + -- | 'GET /datasets/:template_name' + -- 'POST /datasets/:clone_name' + -- 'DELETE /datasets/:clone_name' + _datasets :: mode :- DatasetApi } deriving stock (Generic) -- | servant-openapi3 does not (yet) support NamedRoutes so we need to compose the -- API the old-fashioned way using :<|> for use by @toOpenApi@ -type Api = CapabilitiesApi :<|> SchemaApi :<|> QueryApi :<|> ExplainApi :<|> MutationApi :<|> HealthApi :<|> MetricsApi :<|> RawApi +type Api = CapabilitiesApi :<|> SchemaApi :<|> QueryApi :<|> ExplainApi :<|> MutationApi :<|> HealthApi :<|> MetricsApi :<|> RawApi :<|> DatasetApi -- | Provide an OpenApi 3.0 schema for the API openApiSchema :: OpenApi diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0.hs index 6b9c855053c..fbdea09d90e 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0.hs @@ -14,6 +14,7 @@ module Hasura.Backends.DataConnector.API.V0 module Scalar, module Schema, module Table, + module Dataset, ) where @@ -21,6 +22,7 @@ import Hasura.Backends.DataConnector.API.V0.Aggregate as Aggregate import Hasura.Backends.DataConnector.API.V0.Capabilities as Capabilities import Hasura.Backends.DataConnector.API.V0.Column as Column import Hasura.Backends.DataConnector.API.V0.ConfigSchema as ConfigSchema +import Hasura.Backends.DataConnector.API.V0.Dataset as Dataset import Hasura.Backends.DataConnector.API.V0.ErrorResponse as ErrorResponse import Hasura.Backends.DataConnector.API.V0.Explain as Explain import Hasura.Backends.DataConnector.API.V0.Expression as Expression diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs index 91766d68550..71c34656697 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Capabilities.hs @@ -29,6 +29,7 @@ module Hasura.Backends.DataConnector.API.V0.Capabilities MetricsCapabilities (..), ExplainCapabilities (..), RawCapabilities (..), + DatasetCapabilities (..), CapabilitiesResponse (..), ) where @@ -81,14 +82,15 @@ data Capabilities = Capabilities _cComparisons :: Maybe ComparisonCapabilities, _cMetrics :: Maybe MetricsCapabilities, _cExplain :: Maybe ExplainCapabilities, - _cRaw :: Maybe RawCapabilities + _cRaw :: Maybe RawCapabilities, + _cDatasets :: Maybe DatasetCapabilities } deriving stock (Eq, Show, Generic) deriving anyclass (NFData, Hashable) deriving (FromJSON, ToJSON, ToSchema) via Autodocodec Capabilities defaultCapabilities :: Capabilities -defaultCapabilities = Capabilities defaultDataSchemaCapabilities Nothing Nothing Nothing mempty Nothing Nothing Nothing Nothing Nothing +defaultCapabilities = Capabilities defaultDataSchemaCapabilities Nothing Nothing Nothing mempty Nothing Nothing Nothing Nothing Nothing Nothing instance HasCodec Capabilities where codec = @@ -104,6 +106,7 @@ instance HasCodec Capabilities where <*> optionalField "metrics" "The agent's metrics capabilities" .= _cMetrics <*> optionalField "explain" "The agent's explain capabilities" .= _cExplain <*> optionalField "raw" "The agent's raw query capabilities" .= _cRaw + <*> optionalField "datasets" "The agent's dataset capabilities" .= _cDatasets data DataSchemaCapabilities = DataSchemaCapabilities { _dscSupportsPrimaryKeys :: Bool, @@ -419,6 +422,15 @@ instance HasCodec RawCapabilities where codec = object "RawCapabilities" $ pure RawCapabilities +data DatasetCapabilities = DatasetCapabilities {} + deriving stock (Eq, Ord, Show, Generic, Data) + deriving anyclass (NFData, Hashable) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec DatasetCapabilities + +instance HasCodec DatasetCapabilities where + codec = + object "DatasetCapabilities" $ pure DatasetCapabilities + data CapabilitiesResponse = CapabilitiesResponse { _crCapabilities :: Capabilities, _crConfigSchemaResponse :: ConfigSchemaResponse, diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/ConfigSchema.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/ConfigSchema.hs index 4f4396d89cc..101685dc0c3 100644 --- a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/ConfigSchema.hs +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/ConfigSchema.hs @@ -2,6 +2,7 @@ module Hasura.Backends.DataConnector.API.V0.ConfigSchema ( Config (..), + emptyConfig, ConfigSchemaResponse (..), validateConfigAgainstConfigSchema, ) @@ -11,10 +12,12 @@ import Autodocodec qualified import Control.DeepSeq (NFData) import Control.Lens ((%~), (&), (.~), (^?)) import Data.Aeson (FromJSON (..), Object, ToJSON (..), Value (..), eitherDecode, encode, object, withObject, (.:), (.=), ()) +import Data.Aeson.KeyMap (empty) import Data.Aeson.Lens (AsValue (..), key, members, values) -import Data.Aeson.Types (JSONPathElement (..)) +import Data.Aeson.Types (JSONPathElement (..), emptyObject) import Data.Bifunctor (first) import Data.ByteString.Lazy qualified as BSL +import Data.Data (Data) import Data.HashMap.Strict.InsOrd qualified as InsOrdHashMap import Data.Hashable (Hashable) import Data.Maybe (fromMaybe) @@ -28,9 +31,17 @@ import Servant.API (FromHttpApiData (..), ToHttpApiData (..)) import Prelude newtype Config = Config {unConfig :: Object} - deriving stock (Eq, Show, Ord) + deriving stock (Eq, Show, Ord, Data) deriving newtype (Hashable, NFData, ToJSON, FromJSON) +emptyConfig :: Config +emptyConfig = Config empty + +instance Autodocodec.HasCodec Config where + codec = + Autodocodec.named "Config" $ + Autodocodec.dimapCodec Config unConfig Autodocodec.codec + instance FromHttpApiData Config where parseUrlPiece = first Text.pack . eitherDecode . BSL.fromStrict . Text.encodeUtf8 parseHeader = first Text.pack . eitherDecode . BSL.fromStrict diff --git a/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Dataset.hs b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Dataset.hs new file mode 100644 index 00000000000..95a6b00d8e2 --- /dev/null +++ b/server/lib/dc-api/src/Hasura/Backends/DataConnector/API/V0/Dataset.hs @@ -0,0 +1,127 @@ +{-# LANGUAGE OverloadedLists #-} +{-# LANGUAGE TemplateHaskell #-} + +module Hasura.Backends.DataConnector.API.V0.Dataset + ( DatasetTemplateName (..), + DatasetCloneName (..), + DatasetGetResponse (..), + DatasetPostRequest (..), + DatasetPostResponse (..), + DatasetDeleteResponse (..), + datasetGetSuccess, + datasetDeleteSuccess, + -- | Lenses + unDatasetTemplateName, + unDatasetCloneName, + dsExists, + dspFrom, + dspConfig, + dsdMessage, + ) +where + +import Autodocodec.Extended +import Autodocodec.OpenAPI () +import Control.Lens ((&), (?~)) +import Control.Lens.TH (makeLenses) +import Data.Aeson (FromJSON, ToJSON, Value) +import Data.Data (Data) +import Data.HashMap.Strict qualified as H +import Data.OpenApi (HasType (type_), OpenApiType (OpenApiString), ToParamSchema, ToSchema) +import Data.OpenApi.Internal.ParamSchema (ToParamSchema (toParamSchema)) +import Data.Text (Text) +import GHC.Generics (Generic) +import Hasura.Backends.DataConnector.API.V0.ConfigSchema qualified as Config +import Servant.API (FromHttpApiData (parseUrlPiece), ToHttpApiData (toUrlPiece)) +import Prelude + +newtype DatasetTemplateName = DatasetTemplateName + { _unDatasetTemplateName :: Text + } + deriving stock (Eq, Ord, Show, Data) + deriving newtype (ToParamSchema, ToHttpApiData, FromHttpApiData) + deriving (ToJSON, FromJSON, ToSchema) via Autodocodec DatasetTemplateName + +instance HasCodec DatasetTemplateName where + codec = + named "DatasetTemplateName" $ + dimapCodec DatasetTemplateName _unDatasetTemplateName codec + +$(makeLenses ''DatasetTemplateName) + +newtype DatasetCloneName = DatasetCloneName + { _unDatasetCloneName :: Text + } + deriving stock (Eq, Ord, Show, Data) + deriving newtype (ToParamSchema, ToHttpApiData, FromHttpApiData) + deriving (ToJSON, FromJSON, ToSchema) via Autodocodec DatasetCloneName + +instance HasCodec DatasetCloneName where + codec = + named "DatasetCloneName" $ + dimapCodec DatasetCloneName _unDatasetCloneName codec + +$(makeLenses ''DatasetCloneName) + +-- | Request Dataset Info +data DatasetGetResponse = DatasetGetResponse + { _dsExists :: Bool + } + deriving stock (Eq, Ord, Show, Data) + deriving (ToJSON, FromJSON, ToSchema) via Autodocodec DatasetGetResponse + +datasetGetSuccess :: DatasetGetResponse +datasetGetSuccess = DatasetGetResponse True + +instance HasCodec DatasetGetResponse where + codec = + object "DatasetGetResponse" $ + DatasetGetResponse + <$> requiredField "exists" "Message detailing if the dataset exists" .= _dsExists + +$(makeLenses ''DatasetGetResponse) + +-- | Create a new Dataset +data DatasetPostRequest = DatasetPostRequest + {_dspFrom :: DatasetTemplateName} + deriving stock (Eq, Ord, Show, Generic, Data) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec DatasetPostRequest + +instance HasCodec DatasetPostRequest where + codec = + object "DatasetPostRequest" $ + DatasetPostRequest + <$> requiredField "from" "The named dataset to clone from" .= _dspFrom + +$(makeLenses ''DatasetPostRequest) + +data DatasetPostResponse = DatasetPostResponse + {_dspConfig :: Config.Config} + deriving stock (Eq, Ord, Show, Generic, Data) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec DatasetPostResponse + +instance HasCodec DatasetPostResponse where + codec = + object "DatasetPostResponse" $ + DatasetPostResponse + <$> requiredField "config" "A config to connect to the cloned dataset" .= _dspConfig + +$(makeLenses ''DatasetPostResponse) + +-- | Delete a Dataset +data DatasetDeleteResponse = DatasetDeleteResponse + { _dsdMessage :: Text + } + deriving stock (Eq, Ord, Show, Generic, Data) + deriving (FromJSON, ToJSON, ToSchema) via Autodocodec DatasetDeleteResponse + +datasetDeleteSuccess :: DatasetDeleteResponse +datasetDeleteSuccess = DatasetDeleteResponse "success" + +instance HasCodec DatasetDeleteResponse where + codec = + object "DatasetDeleteResponse" $ + DatasetDeleteResponse + <$> requiredField "message" "The named dataset to clone from" .= _dsdMessage + +$(makeLenses ''DatasetDeleteResponse) diff --git a/server/lib/test-harness/src/Harness/Backend/DataConnector/Mock/Server.hs b/server/lib/test-harness/src/Harness/Backend/DataConnector/Mock/Server.hs index 5bdde4aac75..6b4f3ce00fd 100644 --- a/server/lib/test-harness/src/Harness/Backend/DataConnector/Mock/Server.hs +++ b/server/lib/test-harness/src/Harness/Backend/DataConnector/Mock/Server.hs @@ -1,4 +1,5 @@ {-# LANGUAGE QuasiQuotes #-} +{-# LANGUAGE TypeOperators #-} -- | Mock Agent Warp server backend module Harness.Backend.DataConnector.Mock.Server @@ -67,7 +68,8 @@ capabilities = }, API._cMetrics = Just API.MetricsCapabilities {}, API._cExplain = Just API.ExplainCapabilities {}, - API._cRaw = Just API.RawCapabilities {} + API._cRaw = Just API.RawCapabilities {}, + API._cDatasets = Just API.DatasetCapabilities {} }, _crConfigSchemaResponse = API.ConfigSchemaResponse @@ -806,6 +808,13 @@ metricsHandler = pure "# NOTE: Metrics would go here." rawHandler :: API.SourceName -> API.Config -> API.RawRequest -> Handler API.RawResponse rawHandler _ _ _ = pure $ API.RawResponse [] -- NOTE: Raw query response would go here. +datasetHandler :: (API.DatasetTemplateName -> Handler API.DatasetGetResponse) :<|> ((API.DatasetCloneName -> API.DatasetPostRequest -> Handler API.DatasetPostResponse) :<|> (API.DatasetCloneName -> Handler API.DatasetDeleteResponse)) +datasetHandler = datasetGetHandler :<|> datasetPostHandler :<|> datasetDeleteHandler + where + datasetGetHandler _ = pure $ API.datasetGetSuccess + datasetPostHandler _ _ = pure $ API.DatasetPostResponse API.emptyConfig + datasetDeleteHandler _ = pure $ API.datasetDeleteSuccess + dcMockableServer :: I.IORef MockConfig -> I.IORef (Maybe AgentRequest) -> I.IORef (Maybe API.Config) -> Server API.Api dcMockableServer mcfg mRecordedRequest mRecordedRequestConfig = mockCapabilitiesHandler mcfg @@ -816,6 +825,7 @@ dcMockableServer mcfg mRecordedRequest mRecordedRequestConfig = :<|> healthcheckHandler :<|> metricsHandler :<|> rawHandler + :<|> datasetHandler mockAgentPort :: Warp.Port mockAgentPort = 65006 diff --git a/server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs b/server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs index 4275bdbd289..09a879b92eb 100644 --- a/server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs +++ b/server/src-test/Hasura/Backends/DataConnector/API/V0/CapabilitiesSpec.hs @@ -130,6 +130,9 @@ genExplainCapabilities = pure ExplainCapabilities {} genRawCapabilities :: MonadGen m => m RawCapabilities genRawCapabilities = pure RawCapabilities {} +genDatasetCapabilities :: MonadGen m => m DatasetCapabilities +genDatasetCapabilities = pure DatasetCapabilities {} + genCapabilities :: Gen Capabilities genCapabilities = Capabilities @@ -143,6 +146,7 @@ genCapabilities = <*> Gen.maybe genMetricsCapabilities <*> Gen.maybe genExplainCapabilities <*> Gen.maybe genRawCapabilities + <*> Gen.maybe genDatasetCapabilities emptyConfigSchemaResponse :: ConfigSchemaResponse emptyConfigSchemaResponse = ConfigSchemaResponse mempty mempty