From d40304d6cb8cd1937aaa79c5d323ae54a39f25f9 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Thu, 26 Oct 2023 20:57:54 -0700 Subject: [PATCH] add _async_query api doc and swagger Signed-off-by: YANGDB --- docs/dev/API/README.md | 91 ++++++++++++++++++++++++++++++++++++++ docs/dev/API/swagger.yaml | 92 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 docs/dev/API/README.md create mode 100644 docs/dev/API/swagger.yaml diff --git a/docs/dev/API/README.md b/docs/dev/API/README.md new file mode 100644 index 0000000000..9187a84b75 --- /dev/null +++ b/docs/dev/API/README.md @@ -0,0 +1,91 @@ +# Async Query Interface API Documentation + +The Async Query Interface API enables the execution of queries on top of a Spark engine. This is specifically designed to work with the `S3Glue` data source connector and allows queries to be run asynchronously. + +The Async Query Interface is especially useful for running complex, time-consuming queries without blocking the main thread. The API provides endpoints for query creation, result retrieval, and query cancellation. + +--- + +## Table of Contents +1. [Configurations](#configurations) +2. [Async Query Creation](#async-query-creation) +3. [Query Result Retrieval](#query-result-retrieval) +4. [Query Cancellation](#query-cancellation) +5. [API Mock Tests](#api-mock-tests) + +--- + +## Configurations + +Before using the Async Query APIs, the Spark Execution Engine Config for Async Queries must be set under `plugins.query.executionengine.spark.config` in the cluster settings. For more details on this configuration, refer to the introductory section of this document. + +--- + +## Async Query Creation + +To create an async query, you can use the `POST /_plugins/_async_query` endpoint. + +**Examples** + +- [Query Creation Sample Request](../system/query_creation_request.json) +- [Query Creation Sample Response](../system/query_creation_response.json) + +--- + +## Query Result Retrieval + +To fetch the result of a previously created async query, use the `GET /_plugins/_async_query/{queryId}` endpoint. + +**Examples** + +- [Query Result Sample Request](../system/query_result_request.json) +- [Query Result Sample Response](../system/query_result_response.json) + +--- + +## Query Cancellation + +To cancel a running or queued query, use the `DELETE /_plugins/_async_query/{queryId}` endpoint. + +**Examples** + +- [Query Cancellation Sample Request](../system/query_cancellation_request.json) +- [Query Cancellation Sample Response](../system/query_cancellation_response.json) + +--- + +_This API is designed to work seamlessly with AWS EMRServerless as the Spark execution engine._ + +--- + +## API Mock Tests + +The [Swagger YAML file](swagger.yaml) describes the API. You can visualize the REST API using any of the numerous [online editors](https://editor.swagger.io/). + +### Setup Mock Server + +To test the API, you can use a Swagger-based mock library like Stoplight Prism. + +#### Running the Swagger Mock Server + +```bash +npm install -g @stoplight/prism-cli +``` + +Once installed, run the server using: + +```bash +prism mock swagger.yaml +``` + +#### Running the Swagger Mock Server as Docker + +Run the following one-liner Docker command: + +```bash +docker run -p 9200:4010 -v "$(pwd)/docs/API:/api" --name integ-prism -t stoplight/prism mock -h 0.0.0.0 /api/swagger.yaml +``` + +After the server has started, you can initiate a CURL request to test any of the API endpoints. + +![Swagger API Screenshot](../../img/swagger-api.png) diff --git a/docs/dev/API/swagger.yaml b/docs/dev/API/swagger.yaml new file mode 100644 index 0000000000..b07418a5d8 --- /dev/null +++ b/docs/dev/API/swagger.yaml @@ -0,0 +1,92 @@ +openapi: "3.0.0" +info: + title: "Async Query Interface Endpoints" + version: "1.0.0" + +paths: + /_plugins/_async_query: + post: + summary: "Create an async query" + description: "If the security plugin is enabled, this API can only be invoked by users with specific permissions." + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + datasource: + type: string + lang: + type: string + query: + type: string + sessionId: + type: string + responses: + "200": + description: "Query created successfully" + content: + application/json: + schema: + type: object + properties: + queryId: + type: string + sessionId: + type: string + + /_plugins/_async_query/{queryId}: + get: + summary: "Fetch async query result" + description: "If the security plugin is enabled, this API can only be invoked by users with specific permissions." + parameters: + - in: path + name: queryId + required: true + schema: + type: string + responses: + "200": + description: "Query result" + content: + application/json: + schema: + type: object + properties: + status: + type: string + schema: + type: array + items: + type: object + properties: + name: + type: string + type: + type: string + datarows: + type: array + items: + type: array + items: + type: string + total: + type: integer + size: + type: integer + + delete: + summary: "Cancel async query" + description: "If the security plugin is enabled, this API can only be invoked by users with specific permissions." + parameters: + - in: path + name: queryId + required: true + schema: + type: string + responses: + "200": + description: "Query cancelled successfully" +servers: + - url: "http://ec2-18-237-133-156.us-west-2.compute.amazonaws.com:9200" \ No newline at end of file