From b03712390c3165203e152739eed0894431b4f487 Mon Sep 17 00:00:00 2001 From: GregoryTravis Date: Wed, 27 Sep 2023 10:02:32 -0400 Subject: [PATCH] Improve HTTP tests (#7847) * simple-httpbin encodes response using the Content-encoding header value * Return sent body verbatim --- .../lib/Standard/Base/0.0.0-dev/src/Data.enso | 39 ++++- .../Base/0.0.0-dev/src/Network/HTTP.enso | 141 ++++++++++++------ .../0.0.0-dev/src/Network/HTTP/Header.enso | 10 +- .../0.0.0-dev/src/Network/HTTP/Request.enso | 34 ----- .../src/Network/HTTP/Request_Body.enso | 15 +- test/Tests/src/Network/Http_Spec.enso | 135 +++++++---------- .../reviewed-licenses/Apache_License_V2.0 | 1 - .../reviewed-licenses/Apache_License_V2.0 | 1 - .../java/org/enso/shttp/SimpleHTTPBin.java | 2 +- .../{DummyHandler.java => TestHandler.java} | 51 ++++++- 10 files changed, 245 insertions(+), 184 deletions(-) delete mode 100644 tools/legal-review/engine/reviewed-licenses/Apache_License_V2.0 delete mode 100644 tools/legal-review/project-manager/reviewed-licenses/Apache_License_V2.0 rename tools/simple-httpbin/src/main/java/org/enso/shttp/{DummyHandler.java => TestHandler.java} (65%) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index fb58dad07e73..c0fd6a912a68 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -192,13 +192,6 @@ fetch (uri:(URI | Text)) (method:HTTP_Method=HTTP_Method.Get) (headers:(Vector ( - try_auto_parse: If successful should the body be attempted to be parsed to an Enso native object. - ! Specifying Content Types - - If the `body` parameter specifies an explicit content type, then it is an - error to also specify additional `Content-Type` headers in the `headers` - parameter. (It is not an error to specify multiple `Content-Type` values in - `headers`, however.) - ! Supported Body Types - Request_Body.Text: Sends a text string, with optional encoding and content @@ -215,6 +208,38 @@ fetch (uri:(URI | Text)) (method:HTTP_Method=HTTP_Method.Get) (headers:(Vector ( - File: shorthand for `Request_Body.Binary that_file`. - Any other Enso object: shorthand for `Request_Body.Json that_object`. + ! Specifying Content Types + + If the `body` parameter specifies an explicit content type, then it is an + error to also specify additional `Content-Type` headers in the `headers` + parameter. (It is not an error to specify multiple `Content-Type` values in + `headers`, however.) + + ! Default Content Types + + The following specifies the default content type for each `Request_Body` + type. + + - Request_Body.Text: `text/plain` + - Request_Body.Json: `application/json` + - Request_Body.Binary: `application/octet-stream` + - Request_Body.Form_Data: + If `url_encoded` is True: `application/x-www-form-urlencoded` + If `url_encoded` is False: `multipart/form-data` + - Request_Body.Empty: No content type is sent + - Text: `text/plain` + - File: `application/octet-stream` + - Any other Enso object: `application/json` + + ! Specifying Text Encodings + + Text encoding can be specified in the `encoding` parameter to the + `Request_Body.Text` constructor. This value will be added to the + `Content-Type` header. + + If a value for `encoding` is specified, but no value for `content_type` is + specified, then `"text/plain"` is used as the content type. + > Example Write a text string to an HTTP endpoint. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso index 2a135b6e9072..eb3d05bf9870 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso @@ -2,6 +2,7 @@ import project.Any.Any import project.Data.Map.Map import project.Data.Pair.Pair import project.Data.Set.Set +import project.Data.Text.Encoding.Encoding import project.Data.Text.Text import project.Data.Time.Duration.Duration import project.Data.Vector.Vector @@ -95,56 +96,60 @@ type HTTP handle_request_error = Panic.catch JException handler=(cause-> Error.throw (Request_Error.Error 'IllegalArgumentException' cause.payload.getMessage)) - resolved_req = req.resolve_content_type - resolved_req.if_not_error <| Panic.recover Any <| handle_request_error <| check_output_context <| + Panic.recover Any <| handle_request_error <| check_output_context <| body_publishers = HttpRequest.BodyPublishers builder = HttpRequest.newBuilder # set uri - uri = case resolved_req.uri of - _ : Text -> resolved_req.uri.to_uri - _ : URI -> resolved_req.uri + uri = case req.uri of + _ : Text -> req.uri.to_uri + _ : URI -> req.uri builder.uri uri.internal_uri - # Generate body publisher, possibly generating additional headers - body_publisher_and_headers = case resolved_req.body of - Request_Body.Text text encoding _ -> - ## We ignore the content_type field of the request body, - since any value it had was moved to the headers list in resolve_content_type - headers = [Header.new "Content-Encoding" encoding.character_set] - Pair.new (body_publishers.ofString text encoding.to_java_charset) headers - Request_Body.Json x -> - json = x.to_json - json.if_not_error <| - Pair.new (body_publishers.ofString json) [Header.application_json] - Request_Body.Binary file -> - path = Path.of file.path - Pair.new (body_publishers.ofFile path) [Header.application_octet_stream] - Request_Body.Form_Data form_data url_encoded -> - build_form_body_publisher form_data url_encoded - Request_Body.Empty -> - Pair.new (body_publishers.noBody) [] - _ -> - Error.throw (Illegal_Argument.Error ("Unsupported POST body: " + resolved_req.body.to_display_text + "; this is a bug in the Data library")) - - # Send request - body_publisher_and_headers.if_not_error <| - body_publisher = body_publisher_and_headers.first - additional_headers = body_publisher_and_headers.second - - # set method and body - builder.method resolved_req.method.to_http_method_name body_publisher - - # set headers - all_headers = resolved_req.headers + additional_headers - all_headers.map h-> builder.header h.name h.value - - http_request = builder.build - body_handler = HttpResponse.BodyHandlers . ofInputStream - - response = Response.Value (self.internal_http_client.send http_request body_handler) - if error_on_failure_code.not || response.code.is_success then response else - Error.throw (Request_Error.Error "Status Code" ("Request failed with status code: " + response.code.to_text + ". " + response.body.decode_as_text)) + headers = resolve_headers req + + headers.if_not_error <| + # Generate body publisher and optional form content boundary + body_publisher_and_boundary = case req.body of + Request_Body.Text text encoding _ -> + body_publisher = case encoding of + Nothing -> body_publishers.ofString text + _ : Encoding -> body_publishers.ofString text encoding.to_java_charset + Pair.new body_publisher Nothing + Request_Body.Json x -> + json = x.to_json + json.if_not_error <| + Pair.new (body_publishers.ofString json) Nothing + Request_Body.Binary file -> + path = Path.of file.path + Pair.new (body_publishers.ofFile path) Nothing + Request_Body.Form_Data form_data url_encoded -> + build_form_body_publisher form_data url_encoded + Request_Body.Empty -> + Pair.new (body_publishers.noBody) Nothing + _ -> + Error.throw (Illegal_Argument.Error ("Unsupported POST body: " + req.body.to_display_text + "; this is a bug in the Data library")) + + # Send request + body_publisher_and_boundary.if_not_error <| + body_publisher = body_publisher_and_boundary.first + boundary = body_publisher_and_boundary.second + + boundary_header_list = if boundary.is_nothing then [] else [Header.multipart_form_data boundary] + + # set method and body + builder.method req.method.to_http_method_name body_publisher + + # set headers + all_headers = headers + boundary_header_list + all_headers.map h-> builder.header h.name h.value + + http_request = builder.build + body_handler = HttpResponse.BodyHandlers . ofInputStream + + response = Response.Value (self.internal_http_client.send http_request body_handler) + if error_on_failure_code.not || response.code.is_success then response else + Error.throw (Request_Error.Error "Status Code" ("Request failed with status code: " + response.code.to_text + ". " + response.body.decode_as_text)) ## PRIVATE Static helper for get-like methods @@ -203,10 +208,52 @@ parse_headers headers = _ : Header -> h _ -> Error.throw (Illegal_Argument.Error "Invalid header type - all values must be Vector, Pair or Header (got "+(Meta.get_simple_type_name h)+").") +## PRIVATE + If either encoding or content type is specified in the Request_Body, that is used as the content type header. + If encoding is specified without content type, "text/plain" is used as the content type. + It is an error to specify the content type in both the request body and the header list. + If the body is not Request_Body.Empty, and no content type is specified, a default is used. +resolve_headers : Request -> Vector Header +resolve_headers req = + is_content_type_header h = h.name . equals_ignore_case Header.content_type_header_name + + # Check for content type and encoding in the Request_Body. + request_body_content_type_header = case req.body of + Request_Body.Text _ request_body_encoding request_body_content_type -> + if request_body_content_type.is_nothing && request_body_encoding.is_nothing then Nothing else + content_type = request_body_content_type.if_nothing "text/plain" + encoding = request_body_encoding.if_nothing Encoding.utf_8 + Header.content_type content_type encoding=encoding + _ -> Nothing + + ## Raise error if content type is specified in both the request body and + header list; otherwise, add any Request_Body settings to the header list. + has_content_type_header_in_list = req.headers.any is_content_type_header + + content_type_additions = case request_body_content_type_header.is_nothing.not && has_content_type_header_in_list of + True -> Error.throw (Illegal_Argument.Error "Cannot specify Content-Type/encoding in both the request body and request headers") + False -> + if request_body_content_type_header.is_nothing then [] else [request_body_content_type_header] + + content_type_additions.if_not_error <| + all_headers = req.headers + content_type_additions + + # Add default content type, if one is not specified and the body is not Request_Body.Empty. + contains_content_type = all_headers.any is_content_type_header + is_body_empty = case req.body of + Request_Body.Empty -> True + _ -> False + default_content_type = if is_body_empty || contains_content_type then [] else + default = req.body.default_content_type_header + if default.is_nothing then [] else [default] + + all_headers + default_content_type + ## PRIVATE Build a BodyPublisher from the given form data. -build_form_body_publisher : Map Text (Text | File) -> Boolean -> Pair BodyPublisher [Header] + The pair's second value is a content boundary in the case of a `multipart/form-data` form; otherwise, Nothing +build_form_body_publisher : Map Text (Text | File) -> Boolean -> Pair BodyPublisher Text build_form_body_publisher (form_data:(Map Text (Text | File))) (url_encoded:Boolean=False) = case url_encoded of True -> body_builder = Http_Utils.urlencoded_body_builder @@ -214,7 +261,7 @@ build_form_body_publisher (form_data:(Map Text (Text | File))) (url_encoded:Bool case value of _ : Text -> body_builder.add_part_text key value _ : File -> body_builder.add_part_file key value.path - Pair.new body_builder.build [] + Pair.new body_builder.build Nothing False -> body_builder = Http_Utils.multipart_body_builder form_data.map_with_key key-> value-> @@ -222,7 +269,7 @@ build_form_body_publisher (form_data:(Map Text (Text | File))) (url_encoded:Bool _ : Text -> body_builder.add_part_text key value _ : File -> body_builder.add_part_file key value.path boundary = body_builder.get_boundary - Pair.new body_builder.build [Header.multipart_form_data boundary] + Pair.new body_builder.build boundary ## PRIVATE fetch_methods : Set HTTP_Method diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso index e5171ac880a9..9ba183018f4b 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso @@ -1,4 +1,5 @@ import project.Data.Numbers.Integer +import project.Data.Text.Encoding.Encoding import project.Data.Text.Text import project.Nothing.Nothing from project.Data.Boolean import Boolean, False, True @@ -102,6 +103,9 @@ type Header Arguments: - value: The value for the content type header. + - encoding: The `Encoding` to use as the `charset` in the content-type + value. If encoding is `Nothing`, then the `charset` is not added to the + header valye. > Example Create a content type header containing "my_type". @@ -109,8 +113,10 @@ type Header import Standard.Base.Network.HTTP.Header.Header example_content_type = Header.content_type "my_type" - content_type : Text -> Header - content_type value = Header.Value Header.content_type_header_name value + content_type : Text -> Encoding -> Header + content_type value encoding=Nothing = + charset = if encoding.is_nothing then "" else "; charset="+encoding.character_set + Header.Value Header.content_type_header_name value+charset ## Header "Content-Type: application/json". diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request.enso index 24c5cd3765db..48f779821eb4 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request.enso @@ -242,37 +242,3 @@ type Request _ : Map -> parts new_body = Request_Body.Form_Data form_data Request.Value self.method self.uri self.headers new_body . with_headers [Header.application_x_www_form_urlencoded] - - ## PRIVATE - - Canonicalize any `Content-Type` values specified in the request body and header list. - It is an error to specify content type in both the body and the header list. - If neither is specified, then content type defaults to 'text/plain'. - (Multiple `Content-Type` headers in the header list is not an error.) - - The effect of this method is to check for errors, then move the body - content type, if any, to the headers list. - resolve_content_type : Request - resolve_content_type self = - is_content_type_header header = header.name.equals_ignore_case Header.content_type_header_name - - body_content_type = case self.body of - Request_Body.Text _ _ content_type -> content_type - _ -> Nothing - header_content_types = self.headers.filter is_content_type_header - - case body_content_type.is_nothing of - True -> case header_content_types.is_empty of - True -> - self.with_header Header.content_type_header_name "text/plain" - False -> - self - False -> case header_content_types.is_empty of - True -> - header = Header.content_type self.body.content_type - new_body = case self.body of - Request_Body.Text text encoding _ -> Request_Body.Text text encoding Nothing - _ -> Panic.throw (Illegal_State.Error "Unexpected request body content type. This is a bug in the HTTP library.") - Request.new self.method self.uri self.headers new_body . with_header header.name header.value - False -> - Error.throw (Illegal_Argument.Error "Cannot specify Content-Type in both the request body and request headers") diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request_Body.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request_Body.enso index 58fead11dc65..bec6264b5b6e 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request_Body.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request_Body.enso @@ -2,6 +2,7 @@ import project.Any.Any import project.Data.Map.Map import project.Data.Text.Encoding.Encoding import project.Data.Text.Text +import project.Network.HTTP.Header.Header import project.Nothing.Nothing import project.System.File.File from project.Data.Boolean import Boolean, False, True @@ -15,7 +16,7 @@ type Request_Body - text: The plain text in the request body. - encoding: the text encoding to send as a Content-Encoding header - content_type: the content_type to send as a Content-Type header - Text (text:Text) (encoding:Encoding=Encoding.utf_8) (content_type:(Text|Nothing)=Nothing) + Text (text:Text) (encoding:(Encoding|Nothing)=Nothing) (content_type:(Text|Nothing)=Nothing) ## Request body with an object to be sent as JSON. @@ -40,6 +41,18 @@ type Request_Body ## Empty request body; used for GET Empty + ## PRIVATE + For `Request_Body.Form_Data url_encoded=False`, this returns `Nothing`, + because the content type must include a content boundary. + default_content_type_header : Header | Nothing + default_content_type_header self = + case self of + Request_Body.Text _ _ _ -> Header.content_type "text/plain" encoding=Encoding.utf_8 + Request_Body.Json _ -> Header.content_type "application/json" + Request_Body.Binary _ -> Header.content_type "application/octet-stream" + Request_Body.Form_Data _ url_encoded -> if url_encoded then Header.application_x_www_form_urlencoded else Nothing + Request_Body.Empty -> Nothing + Request_Body.from (that:Text) = Request_Body.Text that Request_Body.from (that:File) = Request_Body.Binary that Request_Body.from (that:Any) = Request_Body.Json that diff --git a/test/Tests/src/Network/Http_Spec.enso b/test/Tests/src/Network/Http_Spec.enso index 85e02060439d..9090b7a03857 100644 --- a/test/Tests/src/Network/Http_Spec.enso +++ b/test/Tests/src/Network/Http_Spec.enso @@ -8,6 +8,7 @@ import Standard.Base.Network.HTTP.Request_Body.Request_Body import Standard.Base.Network.HTTP.Request_Error import Standard.Base.Network.Proxy.Proxy import Standard.Base.Runtime.Context +from Standard.Base.Network.HTTP import resolve_headers import Standard.Test.Extensions from Standard.Test import Test, Test_Suite @@ -62,7 +63,6 @@ spec = { "headers": { "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", "Content-Length": "0" }, "origin": "127.0.0.1", @@ -102,7 +102,6 @@ spec = { "headers": { "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", "Content-Length": "0" }, "origin": "127.0.0.1", @@ -145,9 +144,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "text/plain; charset=UTF-8", "Content-Length": "11" }, "origin": "127.0.0.1", @@ -156,7 +154,6 @@ spec = "form": null, "files": null, "data": "hello world", - "json": null, "args": {} } response . should_equal expected_response @@ -168,7 +165,7 @@ spec = { "headers": { "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "application/json", "Content-Length": "20" }, "origin": "127.0.0.1", @@ -177,7 +174,6 @@ spec = "form": null, "files": null, "data": "{\\"a\\":\\"asdf\\",\\"b\\":123}", - "json": null, "args": {} } response . should_equal expected_response @@ -189,7 +185,7 @@ spec = { "headers": { "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "application/json", "Content-Length": "20" }, "origin": "127.0.0.1", @@ -198,7 +194,6 @@ spec = "form": null, "files": null, "data": "{\\"a\\":\\"asdf\\",\\"b\\":123}", - "json": null, "args": {} } response . should_equal expected_response @@ -209,7 +204,7 @@ spec = { "headers": { "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "application/json", "Content-Length": "50" }, "origin": "127.0.0.1", @@ -218,7 +213,6 @@ spec = "form": null, "files": null, "data": "{\\"type\\":\\"Test_Type\\",\\"constructor\\":\\"Aaa\\",\\"s\\":\\"abc\\"}", - "json": null, "args": {} } response . should_equal expected_response @@ -229,7 +223,7 @@ spec = { "headers": { "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "application/json", "Content-Length": "47" }, "origin": "127.0.0.1", @@ -238,7 +232,6 @@ spec = "form": null, "files": null, "data": "{\\"type\\":\\"Test_Type\\",\\"constructor\\":\\"Bbb\\",\\"i\\":12}", - "json": null, "args": {} } response . should_equal expected_response @@ -252,9 +245,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-16LE", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "text/plain; charset=UTF-16LE", "Content-Length": "24" }, "origin": "127.0.0.1", @@ -262,8 +254,7 @@ spec = "method": "POST", "form": null, "files": null, - "data": "H\\u0000e\\u0000l\\u0000l\\u0000o\\u0000 \\u0000W\\u0000o\\u0000r\\u0000l\\u0000d\\u0000!\\u0000", - "json": null, + "data": "Hello World!", "args": {} } response . should_equal expected_response @@ -273,9 +264,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/csv", + "Content-Type": "text/csv; charset=UTF-8", "Content-Length": "6" }, "origin": "127.0.0.1", @@ -283,8 +273,7 @@ spec = "method": "POST", "form": null, "files": null, - "data": "a,b,c", - "json": null, + "data": "a,b,c\\n", "args": {} } response . should_equal expected_response @@ -292,35 +281,21 @@ spec = Test.specify "Can perform a File POST" <| test_file = enso_project.data / "sample.txt" response = Data.post url_post (Request_Body.Binary test_file) - response.at "headers" . at "Content-Type" . should_equal "text/plain" - response.at "data" . should_equal 'Cupcake ipsum dolor sit amet. Caramels tootsie roll cake ice cream. Carrot cake apple pie gingerbread chocolate cake pudding tart souffl\u00E9 jelly beans gummies.' + response.at "headers" . at "Content-Type" . should_equal "application/octet-stream" + expected_text = test_file.read_text + response . at "data" . should_equal expected_text Test.specify "Can perform a binary File POST" <| test_file = enso_project.data / "sample.png" response = Data.post url_post (Request_Body.Binary test_file) - expected_response = Json.parse <| ''' - { - "headers": { - "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", - "Content-Length": "26905" - }, - "origin": "127.0.0.1", - "url": "", - "method": "POST", - "form": null, - "files": null, - "data": "\uFFFDPNG", - "json": null, - "args": {} - } - response . should_equal expected_response + response.at "headers" . at "Content-Type" . should_equal "application/octet-stream" + response.at "data" . should_start_with '\uFFFDPNG' Test.specify "Can perform a url-encoded form POST" <| test_file = enso_project.data / "sample.txt" form_data = Map.from_vector [["key", "val"], ["a_file", test_file]] response = Data.post url_post (Request_Body.Form_Data form_data url_encoded=True) - response.at "headers" . at "Content-Type" . should_equal "text/plain" + response.at "headers" . at "Content-Type" . should_equal "application/x-www-form-urlencoded" response.at "data" . replace "%0D%" "%" . should_equal 'key=val&a_file=Cupcake+ipsum+dolor+sit+amet.+Caramels+tootsie+roll+cake+ice+cream.+Carrot+cake+apple+pie+gingerbread+chocolate+cake+pudding+tart+souffl%C3%A9+jelly+beans+gummies.%0A%0ATootsie+roll+chupa+chups+muffin+croissant+fruitcake+jujubes+danish+cotton+candy+danish.+Oat+cake+chocolate+fruitcake+halvah+icing+oat+cake+toffee+powder.+Pastry+drag%C3%A9e+croissant.+Ice+cream+candy+canes+dessert+muffin+sugar+plum+tart+jujubes.%0A' Test.specify "Can perform a multipart form POST" <| @@ -328,23 +303,23 @@ spec = form_data = Map.from_vector [["key", "val"], ["a_file", test_file]] response = Data.post url_post (Request_Body.Form_Data form_data) response_json = response - response_json.at "headers" . at "Content-Type" . should_equal "text/plain" + response_json.at "headers" . at "Content-Type" . should_start_with "multipart/form-data; boundary=" response_json.at "data" . is_empty . should_be_false Test.specify "Can perform a File POST with auto-conversion" <| test_file = enso_project.data / "sample.txt" response = Data.post url_post test_file - response.at "headers" . at "Content-Type" . should_equal "text/plain" - response.at "data" . should_equal 'Cupcake ipsum dolor sit amet. Caramels tootsie roll cake ice cream. Carrot cake apple pie gingerbread chocolate cake pudding tart souffl\u00E9 jelly beans gummies.' + response.at "headers" . at "Content-Type" . should_equal "application/octet-stream" + expected_text = test_file.read_text + response . at "data" . should_equal expected_text Test.specify "Can perform a Text POST with auto-conversion" <| response = Data.post url_post "hello world" expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "text/plain; charset=UTF-8", "Content-Length": "11" }, "origin": "127.0.0.1", @@ -353,7 +328,6 @@ spec = "form": null, "files": null, "data": "hello world", - "json": null, "args": {} } response . should_equal expected_response @@ -363,9 +337,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "text/plain; charset=UTF-8", "Content-Length": "11" }, "origin": "127.0.0.1", @@ -374,7 +347,6 @@ spec = "form": null, "files": null, "data": "hello world", - "json": null, "args": {} } response . should_equal expected_response @@ -384,9 +356,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "application/diff", + "Content-Type": "application/diff; charset=UTF-8", "Content-Length": "11" }, "origin": "127.0.0.1", @@ -395,7 +366,6 @@ spec = "form": null, "files": null, "data": "hello world", - "json": null, "args": {} } response . should_equal expected_response @@ -406,7 +376,6 @@ spec = { "headers": { "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", "Content-Length": "0" }, "origin": "127.0.0.1", @@ -415,7 +384,6 @@ spec = "form": null, "files": null, "data": "", - "json": null, "args": {} } response . should_equal expected_response @@ -425,9 +393,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "text/plain; charset=UTF-8", "Content-Length": "11" }, "origin": "127.0.0.1", @@ -436,7 +403,6 @@ spec = "form": null, "files": null, "data": "hello world", - "json": null, "args": {} } response.decode_as_json . should_equal expected_response @@ -446,9 +412,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "text/plain; charset=UTF-8", "Content-Length": "11", "Custom": "asdf" }, @@ -458,7 +423,6 @@ spec = "form": null, "files": null, "data": "hello world", - "json": null, "args": {} } response . should_equal expected_response @@ -494,9 +458,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "application/json", + "Content-Type": "application/json; charset=UTF-8", "Content-Length": "23" }, "origin": "127.0.0.1", @@ -505,7 +468,6 @@ spec = "form": null, "files": null, "data": "{\\"a\\": \\"asdf\\", \\"b\\": 123}", - "json": {"a": "asdf", "b": 123}, "args": {} } response . should_equal expected_response @@ -515,7 +477,6 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", "Content-Type": "application/json", "Content-Length": "23" @@ -526,7 +487,6 @@ spec = "form": null, "files": null, "data": "{\\"a\\": \\"asdf\\", \\"b\\": 123}", - "json": {"a": "asdf", "b": 123}, "args": {} } response . should_equal expected_response @@ -536,7 +496,6 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", "Content-Type": "application/json", "Content-Length": "23" @@ -547,7 +506,6 @@ spec = "form": null, "files": null, "data": "{\\"a\\": \\"asdf\\", \\"b\\": 123}", - "json": {"a": "asdf", "b": 123}, "args": {} } response . should_equal expected_response @@ -557,9 +515,8 @@ spec = expected_response = Json.parse <| ''' { "headers": { - "Content-Encoding": "UTF-8", "User-Agent": "Java-http-client/17.0.7", - "Content-Type": "text/plain", + "Content-Type": "text/plain; charset=UTF-8", "Content-Length": "23" }, "origin": "127.0.0.1", @@ -568,7 +525,6 @@ spec = "form": null, "files": null, "data": "{\\"a\\": \\"asdf\\", \\"b\\": 123}", - "json": null, "args": {} } response . should_equal expected_response @@ -576,22 +532,35 @@ spec = Test.specify "Cannot specify content type in both body and headers" <| Data.post url_post (Request_Body.Text "hello world" content_type="text/plain") headers=[Header.content_type "application/json"] . should_fail_with Illegal_Argument - Test.specify "Header resolution" <| - expected0 = Request.new HTTP_Method.Get "" [Header.content_type "text/plain"] (Request_Body.Text "") - Request.new HTTP_Method.Get "" [] (Request_Body.Text "") . resolve_content_type . should_equal expected0 + Test.specify "Cannot specify content type (implicitly via explicit text encoding) in both body and headers" <| + Data.post url_post (Request_Body.Text "hello world" encoding=Encoding.utf_8) headers=[Header.content_type "application/json"] . should_fail_with Illegal_Argument + + Test.group "Header resolution" <| + Test.specify "Default content type and encoding" <| + expected = [Header.content_type "text/plain; charset=UTF-8"] + resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "")) . should_contain_the_same_elements_as expected + + Test.specify "Content type specified in body" <| + expected = [Header.content_type "application/json; charset=UTF-8"] + resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" content_type="application/json")) . should_contain_the_same_elements_as expected - expected1 = Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "") - Request.new HTTP_Method.Get "" [] (Request_Body.Text "" content_type="application/json") . resolve_content_type . should_equal expected1 + Test.specify "Content type specified in header list" <| + expected = [Header.content_type "application/json"] + resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "")) . should_contain_the_same_elements_as expected - expected2 = Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "") - Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "") . resolve_content_type . should_equal expected2 + Test.specify "Text encoding specified in body" <| + expected = [Header.content_type "text/plain; charset=UTF-16LE"] + resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" encoding=Encoding.utf_16_le)) . should_contain_the_same_elements_as expected - Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "" content_type="text/plain") . resolve_content_type . should_fail_with Illegal_Argument + Test.specify "Can't specify content type in both places" <| + resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "" content_type="text/plain")) . should_fail_with Illegal_Argument - expected3 = Request.new HTTP_Method.Get "" [Header.new "some" "header", Header.content_type "application/json"] (Request_Body.Text "") - Request.new HTTP_Method.Get "" [Header.new "some" "header"] (Request_Body.Text "" content_type="application/json") . resolve_content_type . should_equal expected3 + Test.specify "Custom header" <| + expected = [Header.new "some" "header", Header.content_type "application/json; charset=UTF-8"] + resolve_headers (Request.new HTTP_Method.Get "" [Header.new "some" "header"] (Request_Body.Text "" content_type="application/json")) . should_contain_the_same_elements_as expected - expected4 = Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "") - Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "") . resolve_content_type . should_equal expected4 + Test.specify "Multiple content types in header list are ok" <| + expected = [Header.content_type "application/json", Header.content_type "text/plain"] + resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "")) . should_contain_the_same_elements_as expected main = Test_Suite.run_main spec diff --git a/tools/legal-review/engine/reviewed-licenses/Apache_License_V2.0 b/tools/legal-review/engine/reviewed-licenses/Apache_License_V2.0 deleted file mode 100644 index ff46ef6ff419..000000000000 --- a/tools/legal-review/engine/reviewed-licenses/Apache_License_V2.0 +++ /dev/null @@ -1 +0,0 @@ -tools/legal-review/license-texts/APACHE2.0 diff --git a/tools/legal-review/project-manager/reviewed-licenses/Apache_License_V2.0 b/tools/legal-review/project-manager/reviewed-licenses/Apache_License_V2.0 deleted file mode 100644 index ff46ef6ff419..000000000000 --- a/tools/legal-review/project-manager/reviewed-licenses/Apache_License_V2.0 +++ /dev/null @@ -1 +0,0 @@ -tools/legal-review/license-texts/APACHE2.0 diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java index d021036b2c65..863268e1fdb4 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/SimpleHTTPBin.java @@ -56,7 +56,7 @@ public static void main(String[] args) { server = new SimpleHTTPBin(host, port); for (HttpMethod method : HttpMethod.values()) { String path = "/" + method.toString().toLowerCase(); - server.addHandler(path, new DummyHandler()); + server.addHandler(path, new TestHandler()); } final SimpleHTTPBin server1 = server; diff --git a/tools/simple-httpbin/src/main/java/org/enso/shttp/DummyHandler.java b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java similarity index 65% rename from tools/simple-httpbin/src/main/java/org/enso/shttp/DummyHandler.java rename to tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java index 4945272ced67..ba122f6612ca 100644 --- a/tools/simple-httpbin/src/main/java/org/enso/shttp/DummyHandler.java +++ b/tools/simple-httpbin/src/main/java/org/enso/shttp/TestHandler.java @@ -2,22 +2,29 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; -import java.io.BufferedReader; +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStreamReader; +import java.io.InputStream; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.text.StringEscapeUtils; -public class DummyHandler implements HttpHandler { +public class TestHandler implements HttpHandler { private static final Set ignoredHeaders = Set.of("Host"); + private static final Pattern textEncodingRegex = Pattern.compile(".*; charset=([^;]+).*"); + @Override public void handle(HttpExchange exchange) throws IOException { boolean first = true; String contentType = null; + String textEncoding = "UTF-8"; HttpMethod meth = HttpMethod.valueOf(exchange.getRequestMethod()); String response; @@ -44,6 +51,10 @@ public void handle(HttpExchange exchange) throws IOException { } if (entry.getKey().equals("Content-type")) { contentType = entry.getValue().get(0); + String parsedTextEncoding = parseTextEncoding(contentType); + if (parsedTextEncoding != null) { + textEncoding = parsedTextEncoding; + } } } response += "\n"; @@ -56,14 +67,11 @@ public void handle(HttpExchange exchange) throws IOException { || meth == HttpMethod.PUT || meth == HttpMethod.PATCH) { boolean isJson = contentType != null && contentType.equals("application/json"); - InputStreamReader isr = new InputStreamReader(exchange.getRequestBody(), "utf-8"); - BufferedReader br = new BufferedReader(isr); - String value = br.readLine(); response += " \"form\": null,\n"; response += " \"files\": null,\n"; + String value = readBody(exchange.getRequestBody(), textEncoding); response += " \"data\": \"" + (value == null ? "" : StringEscapeUtils.escapeJson(value)) + "\",\n"; - response += " \"json\": " + (isJson ? value : "null") + ",\n"; } response += " \"args\": {}\n"; response += "}"; @@ -74,6 +82,35 @@ public void handle(HttpExchange exchange) throws IOException { os.close(); } + private String readBody(InputStream inputStream, String encoding) { + BufferedInputStream bis = new BufferedInputStream(inputStream); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try { + while (true) { + int c = bis.read(); + if (c == -1) { + break; + } + baos.write((byte) c); + } + } catch (IOException ie) { + } + try { + return baos.toString(encoding); + } catch (UnsupportedEncodingException uee) { + return "unsupported encoding"; + } + } + + private String parseTextEncoding(String contentType) { + Matcher matcher = textEncodingRegex.matcher(contentType); + if (matcher.matches()) { + return matcher.group(1); + } else { + return null; + } + } + private String formatHeaderKey(String key) { int idx = key.indexOf('-'); if (idx != -1 && key.length() >= idx) {