Improve HTTP tests (#7847)

* simple-httpbin encodes response using the Content-encoding header value
* Return sent body verbatim

distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso

@@ -192,13 +192,6 @@ fetch (uri:(URI | Text)) (method:HTTP_Method=HTTP_Method.Get) (headers:(Vector (
    - try_auto_parse:  If successful should the body be attempted to be parsed to
      an Enso native object.
 
-   ! Specifying Content Types
-
-     If the `body` parameter specifies an explicit content type, then it is an
-     error to also specify additional `Content-Type` headers in the `headers`
-     parameter. (It is not an error to specify multiple `Content-Type` values in
-     `headers`, however.)
-
    ! Supported Body Types
 
      - Request_Body.Text: Sends a text string, with optional encoding and content
@@ -215,6 +208,38 @@ fetch (uri:(URI | Text)) (method:HTTP_Method=HTTP_Method.Get) (headers:(Vector (
      - File: shorthand for `Request_Body.Binary that_file`.
      - Any other Enso object: shorthand for `Request_Body.Json that_object`.
 
+   ! Specifying Content Types
+
+     If the `body` parameter specifies an explicit content type, then it is an
+     error to also specify additional `Content-Type` headers in the `headers`
+     parameter. (It is not an error to specify multiple `Content-Type` values in
+     `headers`, however.)
+
+   ! Default Content Types
+
+     The following specifies the default content type for each `Request_Body`
+     type.
+
+     - Request_Body.Text: `text/plain`
+     - Request_Body.Json: `application/json`
+     - Request_Body.Binary: `application/octet-stream`
+     - Request_Body.Form_Data:
+           If `url_encoded` is True: `application/x-www-form-urlencoded`
+           If `url_encoded` is False: `multipart/form-data`
+     - Request_Body.Empty: No content type is sent
+     - Text: `text/plain`
+     - File: `application/octet-stream`
+     - Any other Enso object: `application/json`
+
+   ! Specifying Text Encodings
+
+     Text encoding can be specified in the `encoding` parameter to the
+     `Request_Body.Text` constructor. This value will be added to the
+     `Content-Type` header.
+
+     If a value for `encoding` is specified, but no value for `content_type` is
+     specified, then `"text/plain"` is used as the content type.
+
    > Example
      Write a text string to an HTTP endpoint.
 

distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso

@@ -2,6 +2,7 @@ import project.Any.Any
 import project.Data.Map.Map
 import project.Data.Pair.Pair
 import project.Data.Set.Set
+import project.Data.Text.Encoding.Encoding
 import project.Data.Text.Text
 import project.Data.Time.Duration.Duration
 import project.Data.Vector.Vector
@@ -95,56 +96,60 @@ type HTTP
         handle_request_error =
             Panic.catch JException handler=(cause-> Error.throw (Request_Error.Error 'IllegalArgumentException' cause.payload.getMessage))
 
-        resolved_req = req.resolve_content_type
-        resolved_req.if_not_error <| Panic.recover Any <| handle_request_error <| check_output_context <|
+        Panic.recover Any <| handle_request_error <| check_output_context <|
             body_publishers = HttpRequest.BodyPublishers
             builder = HttpRequest.newBuilder
 
             # set uri
-            uri = case resolved_req.uri of
-                _ : Text -> resolved_req.uri.to_uri
-                _ : URI -> resolved_req.uri
+            uri = case req.uri of
+                _ : Text -> req.uri.to_uri
+                _ : URI -> req.uri
             builder.uri uri.internal_uri
 
-            # Generate body publisher, possibly generating additional headers
-            body_publisher_and_headers = case resolved_req.body of
-                Request_Body.Text text encoding _ ->
-                    ## We ignore the content_type field of the request body,
-                       since any value it had was moved to the headers list in resolve_content_type
-                    headers = [Header.new "Content-Encoding" encoding.character_set]
-                    Pair.new (body_publishers.ofString text encoding.to_java_charset) headers
-                Request_Body.Json x ->
-                    json = x.to_json
-                    json.if_not_error <|
-                        Pair.new (body_publishers.ofString json) [Header.application_json]
-                Request_Body.Binary file ->
-                    path = Path.of file.path
-                    Pair.new (body_publishers.ofFile path) [Header.application_octet_stream]
-                Request_Body.Form_Data form_data url_encoded ->
-                    build_form_body_publisher form_data url_encoded
-                Request_Body.Empty ->
-                    Pair.new (body_publishers.noBody) []
-                _ ->
-                    Error.throw (Illegal_Argument.Error ("Unsupported POST body: " + resolved_req.body.to_display_text + "; this is a bug in the Data library"))
-
-            # Send request
-            body_publisher_and_headers.if_not_error <|
-                body_publisher = body_publisher_and_headers.first
-                additional_headers = body_publisher_and_headers.second
-
-                # set method and body
-                builder.method resolved_req.method.to_http_method_name body_publisher
-
-                # set headers
-                all_headers = resolved_req.headers + additional_headers
-                all_headers.map h-> builder.header h.name h.value
-
-                http_request = builder.build
-                body_handler = HttpResponse.BodyHandlers . ofInputStream
-
-                response = Response.Value (self.internal_http_client.send http_request body_handler)
-                if error_on_failure_code.not || response.code.is_success then response else
-                    Error.throw (Request_Error.Error "Status Code" ("Request failed with status code: " + response.code.to_text + ". " + response.body.decode_as_text))
+            headers = resolve_headers req
+
+            headers.if_not_error <|
+                # Generate body publisher and optional form content boundary
+                body_publisher_and_boundary = case req.body of
+                    Request_Body.Text text encoding _ ->
+                        body_publisher = case encoding of
+                            Nothing -> body_publishers.ofString text
+                            _ : Encoding ->  body_publishers.ofString text encoding.to_java_charset
+                        Pair.new body_publisher Nothing
+                    Request_Body.Json x ->
+                        json = x.to_json
+                        json.if_not_error <|
+                            Pair.new (body_publishers.ofString json) Nothing
+                    Request_Body.Binary file ->
+                        path = Path.of file.path
+                        Pair.new (body_publishers.ofFile path) Nothing
+                    Request_Body.Form_Data form_data url_encoded ->
+                        build_form_body_publisher form_data url_encoded
+                    Request_Body.Empty ->
+                        Pair.new (body_publishers.noBody) Nothing
+                    _ ->
+                        Error.throw (Illegal_Argument.Error ("Unsupported POST body: " + req.body.to_display_text + "; this is a bug in the Data library"))
+
+                # Send request
+                body_publisher_and_boundary.if_not_error <|
+                    body_publisher = body_publisher_and_boundary.first
+                    boundary = body_publisher_and_boundary.second
+
+                    boundary_header_list = if boundary.is_nothing then [] else [Header.multipart_form_data boundary]
+
+                    # set method and body
+                    builder.method req.method.to_http_method_name body_publisher
+
+                    # set headers
+                    all_headers = headers + boundary_header_list
+                    all_headers.map h-> builder.header h.name h.value
+
+                    http_request = builder.build
+                    body_handler = HttpResponse.BodyHandlers . ofInputStream
+
+                    response = Response.Value (self.internal_http_client.send http_request body_handler)
+                    if error_on_failure_code.not || response.code.is_success then response else
+                        Error.throw (Request_Error.Error "Status Code" ("Request failed with status code: " + response.code.to_text + ". " + response.body.decode_as_text))
 
     ## PRIVATE
        Static helper for get-like methods
@@ -203,26 +208,68 @@ parse_headers headers =
         _ : Header -> h
         _ -> Error.throw (Illegal_Argument.Error "Invalid header type - all values must be Vector, Pair or Header (got "+(Meta.get_simple_type_name h)+").")
 
+## PRIVATE
+   If either encoding or content type is specified in the Request_Body, that is used as the content type header.
+   If encoding is specified without content type, "text/plain" is used as the content type.
+   It is an error to specify the content type in both the request body and the header list.
+   If the body is not Request_Body.Empty, and no content type is specified, a default is used.
+resolve_headers : Request -> Vector Header
+resolve_headers req =
+    is_content_type_header h = h.name . equals_ignore_case Header.content_type_header_name
+
+    # Check for content type and encoding in the Request_Body.
+    request_body_content_type_header = case req.body of
+        Request_Body.Text _ request_body_encoding request_body_content_type ->
+            if request_body_content_type.is_nothing && request_body_encoding.is_nothing then Nothing else
+                content_type = request_body_content_type.if_nothing "text/plain"
+                encoding = request_body_encoding.if_nothing Encoding.utf_8
+                Header.content_type content_type encoding=encoding
+        _ -> Nothing
+
+    ## Raise error if content type is specified in both the request body and
+       header list; otherwise, add any Request_Body settings to the header list.
+    has_content_type_header_in_list = req.headers.any is_content_type_header
+
+    content_type_additions = case request_body_content_type_header.is_nothing.not && has_content_type_header_in_list of
+        True -> Error.throw (Illegal_Argument.Error "Cannot specify Content-Type/encoding in both the request body and request headers")
+        False ->
+            if request_body_content_type_header.is_nothing then [] else [request_body_content_type_header]
+
+    content_type_additions.if_not_error <|
+        all_headers = req.headers + content_type_additions
+
+        # Add default content type, if one is not specified and the body is not Request_Body.Empty.
+        contains_content_type = all_headers.any is_content_type_header
+        is_body_empty = case req.body of
+            Request_Body.Empty -> True
+            _ -> False
+        default_content_type = if is_body_empty || contains_content_type then [] else
+            default = req.body.default_content_type_header
+            if default.is_nothing then [] else [default]
+
+        all_headers + default_content_type
+
 ## PRIVATE
 
    Build a BodyPublisher from the given form data.
-build_form_body_publisher : Map Text (Text | File) -> Boolean -> Pair BodyPublisher [Header]
+   The pair's second value is a content boundary in the case of a `multipart/form-data` form; otherwise, Nothing
+build_form_body_publisher : Map Text (Text | File) -> Boolean -> Pair BodyPublisher Text
 build_form_body_publisher (form_data:(Map Text (Text | File))) (url_encoded:Boolean=False) = case url_encoded of
     True ->
         body_builder = Http_Utils.urlencoded_body_builder
         form_data.map_with_key key-> value->
             case value of
                 _ : Text -> body_builder.add_part_text key value
                 _ : File -> body_builder.add_part_file key value.path
-        Pair.new body_builder.build []
+        Pair.new body_builder.build Nothing
     False ->
         body_builder = Http_Utils.multipart_body_builder
         form_data.map_with_key key-> value->
             case value of
                 _ : Text -> body_builder.add_part_text key value
                 _ : File -> body_builder.add_part_file key value.path
         boundary = body_builder.get_boundary
-        Pair.new body_builder.build [Header.multipart_form_data boundary]
+        Pair.new body_builder.build boundary
 
 ## PRIVATE
 fetch_methods : Set HTTP_Method

distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso

@@ -1,4 +1,5 @@
 import project.Data.Numbers.Integer
+import project.Data.Text.Encoding.Encoding
 import project.Data.Text.Text
 import project.Nothing.Nothing
 from project.Data.Boolean import Boolean, False, True
@@ -102,15 +103,20 @@ type Header
 
        Arguments:
        - value: The value for the content type header.
+       - encoding: The `Encoding` to use as the `charset` in the content-type
+         value. If encoding is `Nothing`, then the `charset` is not added to the
+         header valye.
 
        > Example
          Create a content type header containing "my_type".
 
              import Standard.Base.Network.HTTP.Header.Header
 
              example_content_type = Header.content_type "my_type"
-    content_type : Text -> Header
-    content_type value = Header.Value Header.content_type_header_name value
+    content_type : Text -> Encoding -> Header
+    content_type value encoding=Nothing =
+        charset = if encoding.is_nothing then "" else "; charset="+encoding.character_set
+        Header.Value Header.content_type_header_name value+charset
 
     ## Header "Content-Type: application/json".
 

distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request.enso

@@ -242,37 +242,3 @@ type Request
             _ : Map -> parts
         new_body = Request_Body.Form_Data form_data
         Request.Value self.method self.uri self.headers new_body . with_headers [Header.application_x_www_form_urlencoded]
-
-    ## PRIVATE
-
-       Canonicalize any `Content-Type` values specified in the request body and header list.
-       It is an error to specify content type in both the body and the header list.
-       If neither is specified, then content type defaults to 'text/plain'.
-       (Multiple `Content-Type` headers in the header list is not an error.)
-
-       The effect of this method is to check for errors, then move the body
-       content type, if any, to the headers list.
-    resolve_content_type : Request
-    resolve_content_type self =
-        is_content_type_header header = header.name.equals_ignore_case Header.content_type_header_name
-
-        body_content_type = case self.body of
-            Request_Body.Text _ _ content_type -> content_type
-            _ -> Nothing
-        header_content_types = self.headers.filter is_content_type_header
-
-        case body_content_type.is_nothing of
-            True -> case header_content_types.is_empty of
-                True ->
-                    self.with_header Header.content_type_header_name "text/plain"
-                False ->
-                    self
-            False -> case header_content_types.is_empty of
-                True ->
-                    header = Header.content_type self.body.content_type
-                    new_body = case self.body of
-                        Request_Body.Text text encoding _ -> Request_Body.Text text encoding Nothing
-                        _ -> Panic.throw (Illegal_State.Error "Unexpected request body content type. This is a bug in the HTTP library.")
-                    Request.new self.method self.uri self.headers new_body . with_header header.name header.value
-                False ->
-                    Error.throw (Illegal_Argument.Error "Cannot specify Content-Type in both the request body and request headers")

distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Request_Body.enso

@@ -2,6 +2,7 @@ import project.Any.Any
 import project.Data.Map.Map
 import project.Data.Text.Encoding.Encoding
 import project.Data.Text.Text
+import project.Network.HTTP.Header.Header
 import project.Nothing.Nothing
 import project.System.File.File
 from project.Data.Boolean import Boolean, False, True
@@ -15,7 +16,7 @@ type Request_Body
        - text: The plain text in the request body.
        - encoding: the text encoding to send as a Content-Encoding header
        - content_type: the content_type to send as a Content-Type header
-    Text (text:Text) (encoding:Encoding=Encoding.utf_8) (content_type:(Text|Nothing)=Nothing)
+    Text (text:Text) (encoding:(Encoding|Nothing)=Nothing) (content_type:(Text|Nothing)=Nothing)
 
     ## Request body with an object to be sent as JSON.
 
@@ -40,6 +41,18 @@ type Request_Body
     ## Empty request body; used for GET
     Empty
 
+    ## PRIVATE
+       For `Request_Body.Form_Data url_encoded=False`, this returns `Nothing`,
+       because the content type must include a content boundary.
+    default_content_type_header : Header | Nothing
+    default_content_type_header self =
+        case self of
+            Request_Body.Text _ _ _ -> Header.content_type "text/plain" encoding=Encoding.utf_8
+            Request_Body.Json _ -> Header.content_type "application/json"
+            Request_Body.Binary _ -> Header.content_type "application/octet-stream"
+            Request_Body.Form_Data _ url_encoded -> if url_encoded then Header.application_x_www_form_urlencoded else Nothing
+            Request_Body.Empty -> Nothing
+
 Request_Body.from (that:Text) = Request_Body.Text that
 Request_Body.from (that:File) = Request_Body.Binary that
 Request_Body.from (that:Any) = Request_Body.Json that