Merge pull request #465 from SpectraLogic/4_1_x_API

Updating main branch from 4.0 to 4.1 API generation

ds3-autogen-c/src/main/resources/templates/header-templates/ds3_h.ftl

@@ -10,8 +10,10 @@
 #include <stdint.h>
 #include <string.h>
 #include <curl/curl.h>
+#include "ds3_bool.h"
 #include "ds3_string.h"
 #include "ds3_string_multimap.h"
+#include "ds3_uint64_string_map.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,10 +55,6 @@ typedef struct {
 
 typedef struct _ds3_metadata ds3_metadata;
 
-typedef enum {
-    False, True
-}ds3_bool;
-
 typedef enum {
     HTTP_GET,
     HTTP_PUT,
@@ -115,6 +113,12 @@ typedef struct {
 </#list>
 <#-- **************************************** -->
 
+typedef struct {
+    ds3_metadata* metadata;
+    ds3_checksum_type *blob_checksum_type;
+    ds3_uint64_string_map* blob_checksums;
+}ds3_head_object_response;
+
 <#-- **************************************** -->
 <#-- Generate all Structs                     -->
 <#list getStructs() as structEntry>
@@ -153,6 +157,7 @@ typedef struct _ds3_client {
 
 LIBRARY_API void ds3_request_free(ds3_request* request);
 LIBRARY_API void ds3_error_free(ds3_error* error);
+LIBRARY_API void ds3_head_object_response_free(ds3_head_object_response* response);
 LIBRARY_API void ds3_multipart_upload_part_response_free(ds3_multipart_upload_part_response* response);
 LIBRARY_API void ds3_complete_multipart_upload_response_free(ds3_complete_multipart_upload_response* response);
 LIBRARY_API void ds3_delete_objects_response_free(ds3_delete_objects_response* response);
@@ -214,7 +219,7 @@ LIBRARY_API ds3_request* ds3_init_head_bucket_request(const char *const bucket_n
 LIBRARY_API ds3_error* ds3_head_bucket_request(const ds3_client* client, const ds3_request* request);
     <#elseif requestEntry.getName() == "ds3_head_object_request">
 LIBRARY_API ds3_request* ds3_init_head_object_request(const char* bucket_name, const char *const object_name);
-LIBRARY_API ds3_error* ds3_head_object_request(const ds3_client* client, const ds3_request* request, ds3_metadata** _metadata);
+LIBRARY_API ds3_error* ds3_head_object_request(const ds3_client* client, const ds3_request* request, ds3_head_object_response** response);
     <#else>
         <#include "RequestCommentDocumentation.ftl">
         <#include "InitRequestPrototype.ftl">

ds3-autogen-c/src/main/resources/templates/request-templates/HeadObjectRequest.ftl

@@ -1,4 +1,4 @@
-ds3_error* ds3_head_object_request(const ds3_client* client, const ds3_request* request, ds3_metadata** _metadata) {
+ds3_error* ds3_head_object_request(const ds3_client* client, const ds3_request* request, ds3_head_object_response** response) {
     ds3_error* error;
     ds3_string_multimap* return_headers;
     ds3_metadata* metadata;
@@ -13,8 +13,11 @@ ds3_error* ds3_head_object_request(const ds3_client* client, const ds3_request*
     error = _internal_request_dispatcher(client, request, NULL, NULL, NULL, NULL, &return_headers);
 
     if (error == NULL) {
-        metadata = _init_metadata(return_headers);
-        *_metadata = metadata;
+        ds3_head_object_response* response_ptr = g_new0(ds3_head_object_response, 1);
+        response_ptr->metadata = _init_metadata(return_headers);
+        response_ptr->blob_checksum_type = get_blob_checksum_type(client->log, return_headers);
+        response_ptr->blob_checksums = get_blob_checksums(client->log, return_headers);
+        *response = response_ptr;
         ds3_string_multimap_free(return_headers);
     }
 

ds3-autogen-c/src/main/resources/templates/source-templates/ds3_c.ftl

@@ -12,6 +12,7 @@
 #include "ds3_connection.h"
 #include "ds3_request.h"
 #include "ds3_string_multimap_impl.h"
+#include "ds3_uint64_string_map.h"
 #include "ds3_utils.h"
 
 #ifdef _WIN32

ds3-autogen-c/src/main/resources/templates/source-templates/ds3_requests.ftl

@@ -8,7 +8,9 @@
 #include "ds3.h"
 #include "ds3_net.h"
 #include "ds3_request.h"
+#include "ds3_response_header_utils.h"
 #include "ds3_string_multimap_impl.h"
+#include "ds3_uint64_string_map.h"
 #include "ds3_utils.h"
 
 #ifdef _WIN32

ds3-autogen-c/src/main/resources/templates/source-templates/free_custom_types.ftl

@@ -34,6 +34,23 @@ void ds3_delete_objects_response_free(ds3_delete_objects_response* response) {
     g_free(response);
 }
 
+void ds3_head_object_response_free(ds3_head_object_response* response) {
+    if (response == NULL) {
+        return;
+    }
+    if (response->blob_checksum_type != NULL) {
+        g_free(response->blob_checksum_type);
+    }
+    if (response->metadata != NULL) {
+        ds3_metadata_free(response->metadata);
+    }
+    if (response->blob_checksums != NULL) {
+        ds3_uint64_string_map_free(response->blob_checksums);
+    }
+
+    g_free(response);
+}
+
 void ds3_paging_free(ds3_paging* paging) {
     g_free(paging);
 }

ds3-autogen-c/src/main/resources/templates/source-templates/internal_request_processors.ftl

@@ -69,7 +69,7 @@ static ds3_error* _get_request_xml_nodes(
     return NULL;
 }
 
-static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_response* obj_list, object_list_type list_type, ds3_job_chunk_client_processing_order_guarantee order) {
+static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_response* obj_list, object_list_type list_type) {
     char size_buff[STRING_BUFFER_SIZE];
     xmlDocPtr doc;
     ds3_bulk_object_response* obj;
@@ -80,17 +80,6 @@ static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_resp
     doc = xmlNewDoc((xmlChar*)"1.0");
     objects_node = xmlNewNode(NULL, (xmlChar*) "Objects");
 
-    if (list_type == BULK_GET) {
-        if (order == DS3_JOB_CHUNK_CLIENT_PROCESSING_ORDER_GUARANTEE_NONE) {
-            xmlSetProp(objects_node, (xmlChar*) "ChunkClientProcessingOrderGuarantee", (const xmlChar *) "NONE");
-        } else if (order == DS3_JOB_CHUNK_CLIENT_PROCESSING_ORDER_GUARANTEE_IN_ORDER) {
-            xmlSetProp(objects_node, (xmlChar*) "ChunkClientProcessingOrderGuarantee", (const xmlChar *) "IN_ORDER");
-        } else {
-            return NULL;
-        }
-    }
-
-
     for (obj_index = 0; obj_index < obj_list->num_objects; obj_index++) {
         obj = obj_list->objects[obj_index];
         memset(size_buff, 0, sizeof(size_buff));
@@ -177,7 +166,7 @@ static ds3_error* _init_request_payload(const ds3_request* _request,
             if (request->object_list == NULL || request->object_list->num_objects == 0) {
                 return ds3_create_error(DS3_ERROR_MISSING_ARGS, "The bulk command requires a list of objects to process");
             }
-            doc = _generate_xml_bulk_objects_list(request->object_list, operation_type, request->chunk_ordering);
+            doc = _generate_xml_bulk_objects_list(request->object_list, operation_type);
             break;
 
         case COMPLETE_MPU:

ds3-autogen-c/src/test/java/com/spectralogic/ds3autogen/c/CCodeGeneratorSpectraS3Requests_Test.java

@@ -194,4 +194,47 @@ public void testGenerateSpectraS3GetBucketRequestPrototype() throws IOException,
         final String expectedOutput = "LIBRARY_API ds3_error* ds3_get_bucket_request(const ds3_client* client, const ds3_request* request, ds3_list_bucket_result_response** response);";
         assertEquals(expectedOutput, output);
     }
+
+    @Test
+    public void testHeadObjectRequest() throws IOException, ParseException, TemplateModelException {
+        final String inputSpecFile = "/input/HeadObjectRequest.xml";
+        final TestFileUtilsImpl fileUtils = new TestFileUtilsImpl();
+        final Ds3SpecParser parser = new Ds3SpecParserImpl();
+        final Ds3ApiSpec spec = parser.getSpec(CCodeGenerator_Test.class.getResourceAsStream(inputSpecFile));
+
+        final Source source = SourceConverter.toSource(CCodeGenerator.getAllEnums(spec),
+                CCodeGenerator.getAllStructs(spec, ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of()),
+                CCodeGenerator.getAllRequests(spec, new Ds3DocSpecEmptyImpl()));
+        final CCodeGenerator codeGenerator = new CCodeGenerator();
+        codeGenerator.processTemplate(source, "source-templates/ds3_requests.ftl", fileUtils.getOutputStream());
+
+        final ByteArrayOutputStream bstream = (ByteArrayOutputStream) fileUtils.getOutputStream();
+        final String output = new String(bstream.toByteArray());
+
+        assertTrue(output.contains("ds3_error* ds3_head_object_request(const ds3_client* client, const ds3_request* request, ds3_head_object_response** response) {\n" +
+                "    ds3_error* error;\n" +
+                "    ds3_string_multimap* return_headers;\n" +
+                "    ds3_metadata* metadata;\n" +
+                "\n" +
+                "    int num_slashes = num_chars_in_ds3_str(request->path, '/');\n" +
+                "    if (num_slashes < 2 || ((num_slashes == 2) && ('/' == request->path->value[request->path->size-1]))) {\n" +
+                "        return ds3_create_error(DS3_ERROR_MISSING_ARGS, \"The object name parameter is required.\");\n" +
+                "    } else if (g_ascii_strncasecmp(request->path->value, \"//\", 2) == 0) {\n" +
+                "        return ds3_create_error(DS3_ERROR_MISSING_ARGS, \"The bucket name parameter is required.\");\n" +
+                "    }\n" +
+                "\n" +
+                "    error = _internal_request_dispatcher(client, request, NULL, NULL, NULL, NULL, &return_headers);\n" +
+                "\n" +
+                "    if (error == NULL) {\n" +
+                "        ds3_head_object_response* response_ptr = g_new0(ds3_head_object_response, 1);\n" +
+                "        response_ptr->metadata = _init_metadata(return_headers);\n" +
+                "        response_ptr->blob_checksum_type = get_blob_checksum_type(client->log, return_headers);\n" +
+                "        response_ptr->blob_checksums = get_blob_checksums(client->log, return_headers);\n" +
+                "        *response = response_ptr;\n" +
+                "        ds3_string_multimap_free(return_headers);\n" +
+                "    }\n" +
+                "\n" +
+                "    return error;\n" +
+                "}"));
+    }
 }

ds3-autogen-c/src/test/resources/input/HeadObjectRequest.xml

@@ -0,0 +1,27 @@
+<Data>
+    <Contract>
+        <RequestHandlers>
+            <RequestHandler Classification="amazons3" Name="com.spectralogic.s3.server.handler.reqhandler.amazons3.HeadObjectRequestHandler">
+                <Request BucketRequirement="REQUIRED" HttpVerb="HEAD" IncludeIdInPath="false" ObjectRequirement="REQUIRED">
+                    <OptionalQueryParams/>
+                    <RequiredQueryParams/>
+                </Request>
+                <ResponseCodes>
+                    <ResponseCode>
+                        <Code>200</Code>
+                        <ResponseTypes>
+                            <ResponseType Type="null"/>
+                        </ResponseTypes>
+                    </ResponseCode>
+                    <ResponseCode>
+                        <Code>404</Code>
+                        <ResponseTypes>
+                            <ResponseType Type="com.spectralogic.s3.server.domain.HttpErrorResultApiBean"/>
+                        </ResponseTypes>
+                    </ResponseCode>
+                </ResponseCodes>
+                <Version>1.CF182CD57551902A475553F26582BC78</Version>
+            </RequestHandler>
+        </RequestHandlers>
+    </Contract>
+</Data>

ds3-autogen-go/src/main/java/com/spectralogic/ds3autogen/go/GoCodeGenerator.java

@@ -32,10 +32,7 @@
 import com.spectralogic.ds3autogen.go.generators.parser.JobListParserGenerator;
 import com.spectralogic.ds3autogen.go.generators.parser.TypeParserModelGenerator;
 import com.spectralogic.ds3autogen.go.generators.request.*;
-import com.spectralogic.ds3autogen.go.generators.response.BaseResponseGenerator;
-import com.spectralogic.ds3autogen.go.generators.response.GetObjectResponseGenerator;
-import com.spectralogic.ds3autogen.go.generators.response.NoResponseGenerator;
-import com.spectralogic.ds3autogen.go.generators.response.ResponseModelGenerator;
+import com.spectralogic.ds3autogen.go.generators.response.*;
 import com.spectralogic.ds3autogen.go.generators.type.BaseTypeGenerator;
 import com.spectralogic.ds3autogen.go.generators.type.TypeModelGenerator;
 import com.spectralogic.ds3autogen.go.models.client.Client;
@@ -215,6 +212,9 @@ static ResponseModelGenerator<?> getResponseGenerator(final Ds3Request ds3Reques
         if (isGetObjectAmazonS3Request(ds3Request)) {
             return new GetObjectResponseGenerator();
         }
+        if (isHeadObjectRequest(ds3Request)) {
+            return new HeadObjectResponseGenerator();
+        }
         if (!hasResponsePayload(ds3Request.getDs3ResponseCodes())) {
             return new NoResponseGenerator();
         }

ds3-autogen-go/src/main/kotlin/com/spectralogic/ds3autogen/go/generators/response/HeadObjectResponseGenerator.kt

@@ -0,0 +1,49 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2018 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3autogen.go.generators.response
+
+import com.google.common.collect.ImmutableList
+import com.spectralogic.ds3autogen.api.models.apispec.Ds3ResponseCode
+import com.spectralogic.ds3autogen.go.models.response.ResponseCode
+import com.spectralogic.ds3autogen.go.utils.goIndent
+
+class HeadObjectResponseGenerator : BaseResponseGenerator() {
+
+    override fun toResponsePayloadStruct(expectedResponseCodes: ImmutableList<Ds3ResponseCode>?): String {
+        return "BlobChecksumType ChecksumType\n" +
+                goIndent(1) + "BlobChecksums map[int64]string"
+    }
+
+    /**
+     * Converts a Ds3ResponseCode into a ResponseCode model which contains the Go
+     * code for parsing the specified response.
+     */
+    override fun toResponseCode(ds3ResponseCode: Ds3ResponseCode, responseName: String): ResponseCode {
+        if (ds3ResponseCode.code == 200) {
+            val parsingCode = "checksumType, err := getBlobChecksumType(webResponse.Header())\n" +
+                    goIndent(2) + "if err != nil {\n" +
+                    goIndent(3) + "return nil, err\n" +
+                    goIndent(2) + "}\n" +
+                    goIndent(2) + "checksumMap, err := getBlobChecksumMap(webResponse.Header())\n" +
+                    goIndent(2) + "if err != nil {\n" +
+                    goIndent(3) + "return nil, err\n" +
+                    goIndent(2) + "}\n" +
+                    goIndent(2) + "return &$responseName{BlobChecksumType: checksumType, BlobChecksums: checksumMap, Headers: webResponse.Header()}, nil"
+            return ResponseCode(ds3ResponseCode.code, parsingCode)
+        }
+        return toStandardResponseCode(ds3ResponseCode, responseName)
+    }
+}

ds3-autogen-go/src/test/java/com/spectralogic/ds3autogen/go/GoCodeGenerator_Test.java

@@ -22,6 +22,7 @@
 import com.spectralogic.ds3autogen.go.generators.request.*;
 import com.spectralogic.ds3autogen.go.generators.response.BaseResponseGenerator;
 import com.spectralogic.ds3autogen.go.generators.response.GetObjectResponseGenerator;
+import com.spectralogic.ds3autogen.go.generators.response.HeadObjectResponseGenerator;
 import com.spectralogic.ds3autogen.go.generators.response.NoResponseGenerator;
 import org.hamcrest.MatcherAssert;
 import org.junit.Test;
@@ -104,9 +105,11 @@ public void getResponseGeneratorTest() {
         // Amazon Get Object
         assertThat(getResponseGenerator(getRequestAmazonS3GetObject()), instanceOf(GetObjectResponseGenerator.class));
 
+        // Head Object
+        assertThat(getResponseGenerator(getHeadObjectRequest()), instanceOf(HeadObjectResponseGenerator.class));
+
         // Commands with no response payload
         assertThat(getResponseGenerator(getRequestDeleteNotification()), instanceOf(NoResponseGenerator.class));
-        assertThat(getResponseGenerator(getHeadObjectRequest()), instanceOf(NoResponseGenerator.class));
 
         // Commands with a standard response payload
         assertThat(getResponseGenerator(getRequestAmazonS3GetObject()), instanceOf(BaseResponseGenerator.class));

ds3-autogen-go/src/test/java/com/spectralogic/ds3autogen/go/GoFunctionalTests.java

@@ -36,7 +36,7 @@
 public class GoFunctionalTests {
 
     private final static Logger LOG = LoggerFactory.getLogger(GoFunctionalTests.class);
-    private final static GeneratedCodeLogger CODE_LOGGER = new GeneratedCodeLogger(FileTypeToLog.REQUEST, LOG);
+    private final static GeneratedCodeLogger CODE_LOGGER = new GeneratedCodeLogger(FileTypeToLog.RESPONSE, LOG);
 
     @Test
     public void simpleRequestNoPayload() throws IOException, TemplateModelException {
@@ -1306,4 +1306,42 @@ public void putBucketSpectraS3Test() throws IOException, TemplateModelException
         assertTrue(requestCode.contains("func (putBucketSpectraS3Request *PutBucketSpectraS3Request) WithUserId(userId string) *PutBucketSpectraS3Request {"));
         assertTrue(requestCode.contains("putBucketSpectraS3Request.UserId = &userId"));
     }
+
+    @Test
+    public void headObjectTest() throws IOException, TemplateModelException {
+        final String requestName = "HeadObjectRequest";
+        final FileUtils fileUtils = mock(FileUtils.class);
+        final GoTestCodeUtil codeGenerator = new GoTestCodeUtil(fileUtils, requestName);
+
+        codeGenerator.generateCode(fileUtils, "/input/headObjectRequest.xml");
+
+        // Verify Request file was generated
+        final String requestCode = codeGenerator.getRequestCode();
+        CODE_LOGGER.logFile(requestCode, FileTypeToLog.REQUEST);
+        assertTrue(hasContent(requestCode));
+
+        assertTrue(requestCode.contains("type HeadObjectRequest struct {"));
+        assertTrue(requestCode.contains("BucketName string"));
+        assertTrue(requestCode.contains("ObjectName string"));
+
+        assertTrue(requestCode.contains("func NewHeadObjectRequest(bucketName string, objectName string) *HeadObjectRequest {"));
+        assertTrue(requestCode.contains("BucketName: bucketName,"));
+        assertTrue(requestCode.contains("ObjectName: objectName,"));
+
+        // Verify Response file was generated
+        final String responseCode = codeGenerator.getResponseCode();
+        CODE_LOGGER.logFile(responseCode, FileTypeToLog.RESPONSE);
+        assertTrue(hasContent(responseCode));
+
+        assertTrue(responseCode.contains("type HeadObjectResponse struct {"));
+        assertTrue(responseCode.contains("BlobChecksumType ChecksumType"));
+        assertTrue(responseCode.contains("BlobChecksums map[int64]string"));
+        assertTrue(responseCode.contains("Headers *http.Header"));
+
+        assertTrue(responseCode.contains("func NewHeadObjectResponse(webResponse WebResponse) (*HeadObjectResponse, error) {"));
+        assertTrue(responseCode.contains("expectedStatusCodes := []int { 200 }"));
+        assertTrue(responseCode.contains("checksumType, err := getBlobChecksumType(webResponse.Header())"));
+        assertTrue(responseCode.contains("checksumMap, err := getBlobChecksumMap(webResponse.Header())"));
+        assertTrue(responseCode.contains("return &HeadObjectResponse{BlobChecksumType: checksumType, BlobChecksums: checksumMap, Headers: webResponse.Header()}, nil"));
+    }
 }