requests: Support unicode in json and data payloads.

`json.dumps()` returns a str object, and if the JSON data contains unicode
then computing the length of this str -- which is used for "Content-Length"
-- will return the number of characters which will be shorter than the JSON
payload that is sent as bytes.  The server will then read a truncated body.

Fix that by converting any str payload into bytes.  This fix will also
convert any str object passed in by the user as the `data` argument, and
that actually seems to match the upstream requests behaviour (even if it's
documented that `data` should be bytes).

Note that `bytes(str, "utf-8")` does not make a copy of the str data, it
just makes a small bytes object that wraps the same underlying data.  So
it's relatively efficient.

Fixes issue #251.

Signed-off-by: Damien George <damien@micropython.org>

Author: dpgeorge

python-ecosys/requests/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.11.0", pypi="requests")
+metadata(version="0.11.1", pypi="requests")
 
 package("requests")

python-ecosys/requests/requests/__init__.py

@@ -122,8 +122,11 @@ def request(
             if chunked_data:
                 if "Transfer-Encoding" not in headers and "Content-Length" not in headers:
                     headers["Transfer-Encoding"] = "chunked"
-            elif "Content-Length" not in headers:
-                headers["Content-Length"] = str(len(data))
+            else:
+                if isinstance(data, str):
+                    data = bytes(data, "utf-8")
+                if "Content-Length" not in headers:
+                    headers["Content-Length"] = str(len(data))
 
         if "Connection" not in headers:
             headers["Connection"] = "close"

python-ecosys/requests/test_requests.py

@@ -84,6 +84,55 @@ def test_post_json():
     ), format_message(response)
 
 
+def test_post_json_unicode():
+    response = requests.request("POST", "http://example.com", json="aαbβcγdδ")
+
+    assert response.raw._write_buffer.getvalue() == (
+        b"POST / HTTP/1.0\r\n"
+        + b"Connection: close\r\n"
+        + b"Content-Type: application/json\r\n"
+        + b"Host: example.com\r\n"
+        + b"Content-Length: 14\r\n\r\n"
+        + b'"aαbβcγdδ"'
+    ), format_message(response)
+
+
+def test_post_data_str():
+    response = requests.request("POST", "http://example.com", data="body")
+
+    assert response.raw._write_buffer.getvalue() == (
+        b"POST / HTTP/1.0\r\n"
+        + b"Content-Length: 4\r\n"
+        + b"Host: example.com\r\n"
+        + b"Connection: close\r\n\r\n"
+        + b"body"
+    ), format_message(response)
+
+
+def test_post_data_str_unicode():
+    response = requests.request("POST", "http://example.com", data="aαbβcγdδ")
+
+    assert response.raw._write_buffer.getvalue() == (
+        b"POST / HTTP/1.0\r\n"
+        + b"Content-Length: 12\r\n"
+        + b"Host: example.com\r\n"
+        + b"Connection: close\r\n\r\n"
+        + b"aαbβcγdδ"
+    ), format_message(response)
+
+
+def test_post_data_bytes():
+    response = requests.request("POST", "http://example.com", data=b"body\x01\x02\x03\xff")
+
+    assert response.raw._write_buffer.getvalue() == (
+        b"POST / HTTP/1.0\r\n"
+        + b"Content-Length: 8\r\n"
+        + b"Host: example.com\r\n"
+        + b"Connection: close\r\n\r\n"
+        + b"body\x01\x02\x03\xff"
+    ), response.raw._write_buffer.getvalue()
+
+
 def test_post_chunked_data():
     def chunks():
         yield "test"
@@ -157,6 +206,10 @@ def test_do_not_modify_headers_argument():
 test_get_auth()
 test_get_custom_header()
 test_post_json()
+test_post_json_unicode()
+test_post_data_str()
+test_post_data_str_unicode()
+test_post_data_bytes()
 test_post_chunked_data()
 test_overwrite_get_headers()
 test_overwrite_post_json_headers()