Merge pull request #484 from RachelTucker/python-get-object-buffer

Updating python generation to support settable buffer size in get object

ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/PythonCodeGenerator.java

@@ -25,6 +25,7 @@
 import com.spectralogic.ds3autogen.api.models.docspec.Ds3DocSpec;
 import com.spectralogic.ds3autogen.python.generators.client.BaseClientGenerator;
 import com.spectralogic.ds3autogen.python.generators.client.ClientModelGenerator;
+import com.spectralogic.ds3autogen.python.generators.client.GetObjectCommandGenerator;
 import com.spectralogic.ds3autogen.python.generators.request.*;
 import com.spectralogic.ds3autogen.python.generators.response.*;
 import com.spectralogic.ds3autogen.python.generators.type.BaseTypeGenerator;
@@ -38,6 +39,7 @@
 import com.spectralogic.ds3autogen.utils.Helper;
 import com.spectralogic.ds3autogen.utils.collections.GuavaCollectors;
 import freemarker.template.*;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -146,10 +148,20 @@ protected static ImmutableList<BaseClient> toClientCommands(
      * Generates the python model for a client command
      */
     protected static BaseClient toClientCommand(final Ds3Request ds3Request, final Ds3DocSpec docSpec) {
-        final ClientModelGenerator<?> clientGenerator = new BaseClientGenerator();
+        final ClientModelGenerator<?> clientGenerator = getClientGenerator(ds3Request);
         return clientGenerator.generate(ds3Request, docSpec);
     }
 
+    /**
+     * Retrieves the appropriate generator for creating the specified client command
+     */
+    static ClientModelGenerator<?> getClientGenerator(final Ds3Request ds3Request) {
+        if (isGetObjectAmazonS3Request(ds3Request)) {
+            return new GetObjectCommandGenerator();
+        }
+        return new BaseClientGenerator();
+    }
+
     /**
      * Generates the python models for type descriptors, which are used to describe expected responses
      */
@@ -257,33 +269,33 @@ public BaseRequestGenerator getPutObjectRequestGenerator() {
     /**
      * Converts all Ds3Requests into the python response handler models
      */
-    protected static ImmutableList<BaseResponse> toResponseModelList(
+    protected ImmutableList<BaseResponse> toResponseModelList(
             final ImmutableList<Ds3Request> ds3Requests) {
         if (isEmpty(ds3Requests)) {
             return ImmutableList.of();
         }
         return ds3Requests.stream()
-                .map(PythonCodeGenerator::toResponseModel)
+                .map(this::toResponseModel)
                 .collect(GuavaCollectors.immutableList());
     }
 
     /**
      * Converts a Ds3Request into a python response handler model
      */
-    protected static BaseResponse toResponseModel(final Ds3Request ds3Request) {
+    protected BaseResponse toResponseModel(final Ds3Request ds3Request) {
         final ResponseModelGenerator<?> responseGenerator = getResponseGenerator(ds3Request);
         return responseGenerator.generate(ds3Request);
     }
 
     /**
      * Retrieves the Response Generator associated with the Ds3Request
      */
-    protected static ResponseModelGenerator<?> getResponseGenerator(final Ds3Request ds3Request) {
+    protected ResponseModelGenerator<?> getResponseGenerator(final Ds3Request ds3Request) {
         if (supportsPaginationRequest(ds3Request)) {
             return new PaginationResponseGenerator();
         }
         if (isGetObjectAmazonS3Request(ds3Request)) {
-            return new GetObjectResponseGenerator();
+            return getGetObjectResponseGenerator();
         }
         if (isHeadObjectRequest(ds3Request)) {
             return new HeadObjectResponseGenerator();
@@ -293,4 +305,12 @@ protected static ResponseModelGenerator<?> getResponseGenerator(final Ds3Request
         }
         return new BaseResponseGenerator();
     }
+
+    /**
+     * Retrieves the Python 2.7 compatible generator for the GetObjectResponse
+     */
+    @Override
+    public BaseResponseGenerator getGetObjectResponseGenerator() {
+        return new GetObjectResponseGenerator();
+    }
 }

ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/generators/client/BaseClientGenerator.java

@@ -25,31 +25,43 @@
 import static com.spectralogic.ds3autogen.utils.NormalizingContractNamesUtil.removePath;
 import static com.spectralogic.ds3autogen.utils.NormalizingContractNamesUtil.toResponseName;
 
-public class BaseClientGenerator implements ClientModelGenerator<BaseClient> {
+public class BaseClientGenerator implements ClientModelGenerator<BaseClient>, ClientModelGeneratorUtils {
 
     @Override
     public BaseClient generate(final Ds3Request ds3Request, final Ds3DocSpec docSpec) {
         final String commandName = toPythonCommandName(ds3Request.getName());
         final String requestType = removePath(ds3Request.getName());
         final String responseName = toResponseName(ds3Request.getName());
         final String documentation = toDocumentation(ds3Request.getName(), docSpec);
+        final String funcParams = getFunctionParameters();
+        final String responseParams = getResponseParameters();
 
-        return new BaseClient(commandName, requestType, responseName, documentation);
+        return new BaseClient(commandName, requestType, responseName, documentation, funcParams, responseParams);
     }
 
     /**
      * Creates the client documentation for the request
      * @param requestName The request name with path
      */
-    protected static String toDocumentation(final String requestName, final Ds3DocSpec docSpec) {
+    static String toDocumentation(final String requestName, final Ds3DocSpec docSpec) {
         return toCommandDocs(removePath(requestName), docSpec, 1);
     }
 
     /**
      * Converts the Ds3Request name into the command name used in the client
      * ex: GetBucketRequest -> get_bucket
      */
-    protected static String toPythonCommandName(final String requestName) {
+    static String toPythonCommandName(final String requestName) {
         return camelToUnderscore(toCommandName(requestName));
     }
+
+    @Override
+    public String getFunctionParameters() {
+        return "self, request";
+    }
+
+    @Override
+    public String getResponseParameters() {
+        return "self.net_client.get_response(request), request";
+    }
 }

ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/generators/client/ClientModelGeneratorUtils.java

@@ -0,0 +1,10 @@
+package com.spectralogic.ds3autogen.python.generators.client;
+
+public interface ClientModelGeneratorUtils {
+
+    /** Gets a string containing a comma separated list of the command's function parameters */
+    String getFunctionParameters();
+
+    /** Gets a string containing a comma separated list of parameters passed to the response constructor */
+    String getResponseParameters();
+}

ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/generators/client/GetObjectCommandGenerator.java

@@ -0,0 +1,29 @@
+/*
+ * ******************************************************************************
+ *   Copyright 2014-2018 Spectra Logic Corporation. All Rights Reserved.
+ *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+ *   this file except in compliance with the License. A copy of the License is located at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file.
+ *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ *   specific language governing permissions and limitations under the License.
+ * ****************************************************************************
+ */
+
+package com.spectralogic.ds3autogen.python.generators.client;
+
+public class GetObjectCommandGenerator extends BaseClientGenerator {
+
+    @Override
+    public String getFunctionParameters() {
+        return "self, request, buffer_size=1048576";
+    }
+
+    @Override
+    public String getResponseParameters() {
+        return "self.net_client.get_response(request), request, buffer_size";
+    }
+}

ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/generators/response/GetObjectResponseGenerator.java

@@ -31,12 +31,19 @@ public class GetObjectResponseGenerator extends BaseResponseGenerator {
     public String toParseResponsePayload(final Ds3Request ds3Request) {
         return "stream = self.request.stream\n" +
                 pythonIndent(2) + "try:\n" +
-                pythonIndent(3) + "bytes_read = response.read()\n" +
+                pythonIndent(3) + "bytes_read = response.read(self.buffer_size)\n" +
                 pythonIndent(3) + "while bytes_read:\n" +
                 pythonIndent(4) + "stream.write(bytes_read)\n" +
-                pythonIndent(4) + "bytes_read = response.read()\n" +
+                pythonIndent(4) + "bytes_read = response.read(self.buffer_size)\n" +
                 pythonIndent(2) + "finally:\n" +
                 pythonIndent(3) + "stream.close()\n" +
                 pythonIndent(3) + "response.close()\n";
     }
+
+    @Override
+    public String toInitResponse() {
+        return "def __init__(self, response, request, buffer_size=None):\n" +
+                pythonIndent(2) + "self.buffer_size = buffer_size\n" +
+                pythonIndent(2) + "super(self.__class__, self).__init__(response, request)\n";
+    }
 }

ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/model/client/BaseClient.java

@@ -21,12 +21,22 @@ public class BaseClient {
     private final String requestType;
     private final String responseName;
     private final String documentation;
+    private final String functionParams; /** comma separated list of function parameters */
+    private final String responseParams; /** comma separated list of parameters passed to response */
 
-    public BaseClient(final String commandName, final String requestType, final String responseName, final String documentation) {
+    public BaseClient(
+            final String commandName,
+            final String requestType,
+            final String responseName,
+            final String documentation,
+            final String functionParams,
+            final String responseParams) {
         this.commandName = commandName;
         this.requestType = requestType;
         this.responseName = responseName;
         this.documentation = documentation;
+        this.functionParams = functionParams;
+        this.responseParams = responseParams;
     }
 
     public String getCommandName() {
@@ -44,4 +54,12 @@ public String getResponseName() {
     public String getDocumentation() {
         return documentation;
     }
+
+    public String getFunctionParams() {
+        return functionParams;
+    }
+
+    public String getResponseParams() {
+        return responseParams;
+    }
 }

ds3-autogen-python/src/main/kotlin/com/spectralogic/ds3autogen/python/PythonCodeGeneratorInterface.kt

@@ -21,6 +21,7 @@ import com.spectralogic.ds3autogen.api.models.apispec.Ds3Request
 import com.spectralogic.ds3autogen.api.models.apispec.Ds3Type
 import com.spectralogic.ds3autogen.api.models.docspec.Ds3DocSpec
 import com.spectralogic.ds3autogen.python.generators.request.BaseRequestGenerator
+import com.spectralogic.ds3autogen.python.generators.response.BaseResponseGenerator
 import com.spectralogic.ds3autogen.python.model.request.BaseRequest
 import freemarker.template.Configuration
 import freemarker.template.Template
@@ -48,4 +49,6 @@ interface PythonCodeGeneratorInterface {
 
     fun toRequestModelList(ds3Requests: ImmutableList<Ds3Request>,
                            docSpec: Ds3DocSpec) : ImmutableList<BaseRequest>
+
+    fun getGetObjectResponseGenerator() : BaseResponseGenerator
 }

ds3-autogen-python/src/main/resources/tmpls/python/commands/client/client_class.ftl

@@ -9,8 +9,8 @@ class Client(object):
 
 <#list clientCommands as cmd>
     ${cmd.documentation}
-    def ${cmd.commandName}(self, request):
+    def ${cmd.commandName}(${cmd.functionParams}):
         if not isinstance(request, ${cmd.requestType}):
             raise TypeError('request for ${cmd.commandName} should be of type ${cmd.requestType} but was ' + request.__class__.__name__)
-        return ${cmd.responseName}(self.net_client.get_response(request), request)
+        return ${cmd.responseName}(${cmd.responseParams})
 </#list>

ds3-autogen-python/src/test/java/com.spectralogic.ds3autogen.python/PythonCodeGenerator_Test.java

@@ -23,6 +23,8 @@
 import com.spectralogic.ds3autogen.api.models.apispec.Ds3Type;
 import com.spectralogic.ds3autogen.api.models.enums.HttpVerb;
 import com.spectralogic.ds3autogen.docspec.Ds3DocSpecEmptyImpl;
+import com.spectralogic.ds3autogen.python.generators.client.BaseClientGenerator;
+import com.spectralogic.ds3autogen.python.generators.client.GetObjectCommandGenerator;
 import com.spectralogic.ds3autogen.python.generators.request.*;
 import com.spectralogic.ds3autogen.python.generators.response.BaseResponseGenerator;
 import com.spectralogic.ds3autogen.python.generators.response.GetObjectResponseGenerator;
@@ -172,22 +174,22 @@ public void toTypeDescriptorList_FullMap_Test() {
 
     @Test
     public void getResponseGenerator_Test() {
-        assertThat(getResponseGenerator(createDs3RequestTestData("com.test.EmptyRequest", Classification.spectrads3)),
+        assertThat(generator.getResponseGenerator(createDs3RequestTestData("com.test.EmptyRequest", Classification.spectrads3)),
                 instanceOf(BaseResponseGenerator.class));
 
-        assertThat(getResponseGenerator(getRequestAmazonS3GetObject()), instanceOf(GetObjectResponseGenerator.class));
+        assertThat(generator.getResponseGenerator(getRequestAmazonS3GetObject()), instanceOf(GetObjectResponseGenerator.class));
 
-        assertThat(getResponseGenerator(getHeadObjectRequest()), instanceOf(HeadResponseGenerator.class));
+        assertThat(generator.getResponseGenerator(getHeadObjectRequest()), instanceOf(HeadResponseGenerator.class));
 
-        assertThat(getResponseGenerator(getObjectsDetailsRequest()), instanceOf(PaginationResponseGenerator.class));
+        assertThat(generator.getResponseGenerator(getObjectsDetailsRequest()), instanceOf(PaginationResponseGenerator.class));
 
-        assertThat(getResponseGenerator(getUsersSpectraS3Request()), instanceOf(PaginationResponseGenerator.class));
+        assertThat(generator.getResponseGenerator(getUsersSpectraS3Request()), instanceOf(PaginationResponseGenerator.class));
     }
 
     @Test
     public void toResponseModel_Test() {
         final Ds3Request request = getBucketRequest();
-        final BaseResponse result = toResponseModel(request);
+        final BaseResponse result = generator.toResponseModel(request);
         assertThat(result.getName(), is("GetBucketResponseHandler"));
         assertThat(result.getCodes().size(), is(1));
 
@@ -198,20 +200,20 @@ public void toResponseModel_Test() {
 
     @Test
     public void toResponseModelList_NullList_Test() {
-        final ImmutableList<BaseResponse> result = toResponseModelList(null);
+        final ImmutableList<BaseResponse> result = generator.toResponseModelList(null);
         assertThat(result.size(), is(0));
     }
 
     @Test
     public void toResponseModelList_EmptyList_Test() {
-        final ImmutableList<BaseResponse> result = toResponseModelList(ImmutableList.of());
+        final ImmutableList<BaseResponse> result = generator.toResponseModelList(ImmutableList.of());
         assertThat(result.size(), is(0));
     }
 
     @Test
     public void toResponseModelList_Test() {
         final ImmutableList<Ds3Request> requests = ImmutableList.of(getBucketRequest());
-        final ImmutableList<BaseResponse> result = toResponseModelList(requests);
+        final ImmutableList<BaseResponse> result = generator.toResponseModelList(requests);
         assertThat(result.size(), is(1));
         assertThat(result.get(0).getName(), is("GetBucketResponseHandler"));
     }
@@ -250,4 +252,10 @@ public void toClientCommands_Test() {
         assertThat(result.get(1).getResponseName(), is("PutBucketSpectraS3Response"));
         assertThat(result.get(1).getCommandName(), is("put_bucket_spectra_s3"));
     }
+
+    @Test
+    public void getClientGenerator_Test() {
+        assertThat(getClientGenerator(getRequestAmazonS3GetObject()), instanceOf(GetObjectCommandGenerator.class));
+        assertThat(getClientGenerator(getRequestSpectraS3GetObject()), instanceOf(BaseClientGenerator.class));
+    }
 }

ds3-autogen-python/src/test/java/com.spectralogic.ds3autogen.python/generators/client/BaseClientGenerator_Test.java

@@ -27,6 +27,8 @@
 
 public class BaseClientGenerator_Test {
 
+    private static final BaseClientGenerator generator = new BaseClientGenerator();
+
     private static Ds3DocSpec getTestDocSpec() {
         return new Ds3DocSpecImpl(
                 ImmutableMap.of(
@@ -57,4 +59,16 @@ public void toDocumentation_Test() {
         assertThat(toDocumentation("com.test.TestOneRequest", docSpec), is(expected));
         assertThat(toDocumentation("TestOneRequest", docSpec), is(expected));
     }
+
+    @Test
+    public void getFunctionParametersTest() {
+        final String expected = "self, request";
+        assertThat(generator.getFunctionParameters(), is(expected));
+    }
+
+    @Test
+    public void getResponseParametersTest() {
+        final String expected = "self.net_client.get_response(request), request";
+        assertThat(generator.getResponseParameters(), is(expected));
+    }
 }