Fix Windows and Apple transport and add shared data URL decoding

CMakeLists.txt

@@ -8,6 +8,9 @@ include(FetchContent)
 FetchContent_Declare(arcana.cpp
     GIT_REPOSITORY https://github.com/microsoft/arcana.cpp.git
     GIT_TAG        7c6be8aaa29effbc8ee1a2217388fb6e399150d2)
+FetchContent_Declare(base-n
+    GIT_REPOSITORY https://github.com/azawadzki/base-n.git
+    GIT_TAG        7573e77c0b9b0e8a5fb63d96dbde212c921993b4)
 FetchContent_Declare(AndroidExtensions
     GIT_REPOSITORY https://github.com/BabylonJS/AndroidExtensions.git
     GIT_TAG        2e85a8d43b89246c460112c9e5546ad54b6e87b4)
@@ -31,6 +34,7 @@ endif()
 project(UrlLib)
 
 FetchContent_MakeAvailable_With_Message(arcana.cpp)
+FetchContent_MakeAvailable_With_Message(base-n)
 
 if(ANDROID)
     FetchContent_MakeAvailable_With_Message(AndroidExtensions)
@@ -81,6 +85,7 @@ endif()
 set(SOURCES
     "Include/UrlLib/UrlLib.h"
     "Source/UrlRequest_Base.h"
+    "Source/DataUrl.h"
     "Source/UrlRequest_Shared.h"
     "Source/WebSocket_Base.h"
     "Source/WebSocket_Shared.h"
@@ -90,6 +95,7 @@ add_library(UrlLib ${SOURCES})
 
 target_include_directories(UrlLib PUBLIC "Include")
 target_include_directories(UrlLib PRIVATE "Source")
+target_include_directories(UrlLib PRIVATE "${base-n_SOURCE_DIR}/include")
 
 target_link_libraries(UrlLib
     PUBLIC arcana

Include/UrlLib/UrlLib.h

@@ -61,6 +61,7 @@ namespace UrlLib
 
         void Abort();
 
+        // data: GET URLs are decoded in the shared layer (see README.md for the supported contract).
         void Open(UrlMethod method, const std::string& url);
 
         // Registers a resolver for a non-transport URL scheme such as "blob". Registration is
@@ -69,6 +70,8 @@ namespace UrlLib
         // SendAsync() time, so every consumer (fetch, XMLHttpRequest, image / video src, texture
         // loaders, ...) resolves such URLs uniformly through UrlRequest instead of each carrying
         // its own branch.
+        // A registered "data" resolver overrides the built-in decoder until unregistered.
+        // Shared resolvers observe Abort() before invocation, returning a cancelled task.
         //
         // A resolver that reports the URL as not handled -- or that throws -- surfaces as a
         // transport-style failure (status stays None, with an error symbol recorded); an exception

NOTICE.md

@@ -71,6 +71,30 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE
 ```
 
+## base-n
+
+```
+Copyright (C) 2012 Andrzej Zawadzki (azawadzki@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
+
 ## boost
 
 ```

README.md

@@ -46,6 +46,71 @@ Platform support: the Apple (`NSURLSession`) and Linux (`libcurl`) backends popu
 these accessors today; the Windows and Android backends currently always report
 empty/zero (contributions welcome — the plumbing in `UrlRequest_Base.h` is shared).
 
+## In-memory data URLs
+
+`UrlRequest` provides a built-in, platform-independent `data:` GET decoder through the
+same shared scheme-resolution path as registered resolvers. No network transport or
+consumer-specific image/fetch wrapper is involved. The supported contract is:
+
+* `data:[type/subtype[;name=value...]][;base64],payload`, with a required comma.
+  An omitted type defaults to `text/plain;charset=US-ASCII`; the `;charset=...`
+  shorthand uses `text/plain`. Explicit MIME types use ASCII HTTP token syntax;
+  parameter values are nonempty tokens or ASCII quoted strings with backslash escapes.
+  Type/subtype and parameter names are lowercased; parameter values are retained.
+  Invalid or unsupported MIME syntax is rejected, not replaced by a guessed MIME type.
+* Scheme and the final `;base64` marker are ASCII case-insensitive. Surrounding ASCII
+  whitespace in the metadata, and spaces between that semicolon and `base64`, are allowed.
+  The payload is percent-decoded to **bytes** before optional base64 decoding. Literal
+  `+` is preserved; incomplete or non-hex percent escapes remain literal, as in URL percent
+  decoding. Raw non-ASCII payload bytes are preserved; callers should supply UTF-8 or
+  percent-encoded bytes, not expect locale-dependent URI unescaping.
+* Base64 uses the standard alphabet and forgiving-base64 rules: ASCII whitespace is ignored,
+  padding may be omitted, and unused trailing bits are ignored. Bad alphabet characters,
+  misplaced/excess padding, and lengths congruent to one modulo four are errors. Base64url
+  (`-`/`_`) is not supported.
+* A literal `?` belongs to the payload (or to the metadata if before the comma), not a
+  separate query to discard. A literal `#` starts a fragment, which is excluded from decoding
+  and `ResponseUrl()`. Encode these bytes as `%3F`/`%23` to include them literally.
+* Success is status 200 / `"OK"`, with a `content-type` header and the decoded body, including
+  empty bodies. `Buffer` returns exact bytes; `String` also preserves those bytes (including
+  NUL), without charset transcoding, matching the shared resolver contract.
+* Decoding is synchronous at `SendAsync()`, once per `Open()`. Malformed/unsupported input
+  completes normally with status 0, empty body/headers, and a diagnostic `DataUrlInvalid`
+  error. Non-GET requests similarly report `DataUrlUnsupportedMethod`; other decoder
+  exceptions report `DataUrlFailed`. No invalid payload is passed to the platform transport.
+  `Abort()` before sending prevents shared resolution and returns an operation-cancelled task,
+  as for cancellation-guarded Windows continuations. There is no asynchronous decoding to
+  interrupt; aborting an already-completed request does not erase its response. As with the
+  existing transport cancellation source, reopening an aborted request does not reset cancellation.
+
+This is a bounded byte/MIME transport, not an implementation of all Fetch behavior: it
+does not implement browser URL canonicalization, permissive MIME-parser recovery,
+charset conversion, streaming, CORS, or a payload-size policy. Supply a serialized URL
+without surrounding control characters. A resolver explicitly registered for `"data"`
+overrides the built-in decoder; unregistering it restores the built-in behavior.
+Base64 decoding reuses the pinned `base-n` helper, with validation before invoking its
+otherwise permissive decoder.
+
+On Windows, a successful HTTP response without a Content-Type header leaves that header
+absent rather than dereferencing the optional WinRT value or inventing a MIME type.
+Existing Windows HTTP response behavior is otherwise unchanged: successful 2xx responses normalize
+to 200 after body reading; non-success responses retain their wire status and do not read
+their body or publish response headers.
+
+Windows POST matches Content-Type case-insensitively and sets it on the content headers,
+not the request headers. Bodies are sent byte-for-byte without UTF-8 transcoding or an
+automatically added charset. Omitted Content-Type stays absent; supplied MIME parameters
+are preserved, and an invalid supplied MIME type reports a transport error rather than success.
+
+Apple POST also preserves body bytes, using length-aware NSData rather than a
+NUL-terminated UTF-8 string. The shared POST wire tests run on Windows and Apple;
+Linux explicitly skips them because its backend does not implement POST yet.
+
+Android POST writes raw byte-array chunks to the connection's output stream instead of
+converting the body to a Java string. This preserves embedded NULs and UTF-8 bytes while
+keeping Content-Length consistent with the supplied body. Android coverage comes from
+JsRuntimeHost's Fetch/XHR HTTP transport tests.
+
 ## Contributing
 
 Please read [CONTRIBUTING.md](./CONTRIBUTING.md) for details on our code of conduct, and 

Source/DataUrl.h

@@ -0,0 +1,225 @@
+#pragma once
+
+#include <UrlLib/UrlLib.h>
+#include <basen.hpp>
+
+#include <algorithm>
+#include <stdexcept>
+#include <string_view>
+
+namespace UrlLib::Detail
+{
+    inline char AsciiLower(char value)
+    {
+        return value >= 'A' && value <= 'Z' ? static_cast<char>(value + ('a' - 'A')) : value;
+    }
+
+    inline bool IsAsciiWhitespace(char value)
+    {
+        return value == ' ' || value == '\t' || value == '\n' || value == '\r' || value == '\f';
+    }
+
+    inline std::string_view TrimAsciiWhitespace(std::string_view value)
+    {
+        while (!value.empty() && IsAsciiWhitespace(value.front()))
+        {
+            value.remove_prefix(1);
+        }
+        while (!value.empty() && IsAsciiWhitespace(value.back()))
+        {
+            value.remove_suffix(1);
+        }
+        return value;
+    }
+
+    inline int HexDigit(char value)
+    {
+        if (value >= '0' && value <= '9')
+        {
+            return value - '0';
+        }
+        value = AsciiLower(value);
+        return value >= 'a' && value <= 'f' ? value - 'a' + 10 : -1;
+    }
+
+    template<typename Container>
+    inline void PercentDecodeBytes(std::string_view value, Container& bytes)
+    {
+        bytes.reserve(value.size());
+        for (size_t index = 0; index < value.size(); ++index)
+        {
+            if (value[index] == '%' && index + 2 < value.size())
+            {
+                const int high = HexDigit(value[index + 1]);
+                const int low = HexDigit(value[index + 2]);
+                if (high >= 0 && low >= 0)
+                {
+                    bytes.push_back(static_cast<typename Container::value_type>((high << 4) | low));
+                    index += 2;
+                    continue;
+                }
+            }
+            // URL percent decoding preserves incomplete/non-hex escapes and does not map '+' to space.
+            bytes.push_back(static_cast<typename Container::value_type>(static_cast<unsigned char>(value[index])));
+        }
+    }
+
+    inline bool IsMimeToken(char value)
+    {
+        return (value >= 'a' && value <= 'z') || (value >= 'A' && value <= 'Z') ||
+            (value >= '0' && value <= '9') ||
+            std::string_view{"!#$%&'*+-.^_`|~"}.find(value) != std::string_view::npos;
+    }
+
+    inline std::string DataContentType(std::string_view metadata)
+    {
+        if (metadata.empty())
+        {
+            return "text/plain;charset=US-ASCII";
+        }
+        std::string type{metadata.front() == ';' ? "text/plain" : ""};
+        type.append(metadata);
+        const auto separator = type.find(';');
+        const std::string_view essence{type.data(), separator == std::string::npos ? type.size() : separator};
+        const auto slash = essence.find('/');
+        if (slash == std::string_view::npos || slash == 0 || slash + 1 == essence.size() ||
+            !std::all_of(essence.begin(), essence.begin() + slash, IsMimeToken) ||
+            !std::all_of(essence.begin() + slash + 1, essence.end(), IsMimeToken))
+        {
+            throw std::invalid_argument{"data URL requires a valid MIME type"};
+        }
+        std::transform(type.begin(), type.begin() + essence.size(), type.begin(), AsciiLower);
+
+        size_t index = essence.size();
+        while (index < type.size())
+        {
+            ++index; // semicolon
+            while (index < type.size() && type[index] == ' ')
+            {
+                ++index;
+            }
+            const size_t name = index;
+            while (index < type.size() && IsMimeToken(type[index]))
+            {
+                type[index] = AsciiLower(type[index]);
+                ++index;
+            }
+            if (index == name || index == type.size() || type[index++] != '=')
+            {
+                throw std::invalid_argument{"data URL requires name=value MIME parameters"};
+            }
+            if (index < type.size() && type[index] == '"')
+            {
+                ++index;
+                bool closed = false;
+                while (index < type.size())
+                {
+                    auto value = static_cast<unsigned char>(type[index++]);
+                    if (value == '"')
+                    {
+                        closed = true;
+                        break;
+                    }
+                    if (value == '\\' && index < type.size())
+                    {
+                        value = static_cast<unsigned char>(type[index++]);
+                    }
+                    if (value < 0x20 || value >= 0x7f)
+                    {
+                        throw std::invalid_argument{"data URL MIME parameter contains a non-ASCII/control byte"};
+                    }
+                }
+                if (!closed)
+                {
+                    throw std::invalid_argument{"data URL MIME parameter has an unterminated quote"};
+                }
+            }
+            else
+            {
+                const size_t start = index;
+                while (index < type.size() && IsMimeToken(type[index]))
+                {
+                    ++index;
+                }
+                if (index == start)
+                {
+                    throw std::invalid_argument{"data URL MIME parameter has an empty/invalid value"};
+                }
+            }
+            if (index < type.size() && type[index] != ';')
+            {
+                throw std::invalid_argument{"data URL MIME parameter has an invalid delimiter"};
+            }
+        }
+        return type;
+    }
+
+    inline UrlSchemeResolverResult ResolveDataUrl(const std::string& url)
+    {
+        const std::string_view source = std::string_view{url}.substr(0, url.find('#'));
+        const auto comma = source.find(',', 5);
+        if (comma == std::string_view::npos)
+        {
+            throw std::invalid_argument{"data URL is missing its comma separator"};
+        }
+        auto metadata = TrimAsciiWhitespace(source.substr(5, comma - 5));
+        bool base64 = false;
+        if (metadata.size() >= 6)
+        {
+            auto marker = metadata.substr(metadata.size() - 6);
+            if (std::equal(marker.begin(), marker.end(), "base64",
+                [](char left, char right) { return AsciiLower(left) == right; }))
+            {
+                const auto prefix = TrimAsciiWhitespace(metadata.substr(0, metadata.size() - 6));
+                if (!prefix.empty() && prefix.back() == ';')
+                {
+                    base64 = true;
+                    metadata = prefix.substr(0, prefix.size() - 1);
+                }
+            }
+        }
+
+        UrlSchemeResolverResult result;
+        result.contentType = DataContentType(metadata);
+        const auto payload = source.substr(comma + 1);
+        std::vector<std::byte> body;
+        if (base64)
+        {
+            std::string normalized;
+            std::string_view encoded = payload;
+            if (payload.find('%') != std::string_view::npos ||
+                std::any_of(payload.begin(), payload.end(), IsAsciiWhitespace))
+            {
+                PercentDecodeBytes(payload, normalized);
+                normalized.erase(std::remove_if(normalized.begin(), normalized.end(), IsAsciiWhitespace), normalized.end());
+                encoded = normalized;
+            }
+            if (encoded.size() % 4 == 0)
+            {
+                for (int padding = 0; padding < 2 && !encoded.empty() && encoded.back() == '='; ++padding)
+                {
+                    encoded.remove_suffix(1);
+                }
+            }
+            if (encoded.size() % 4 == 1 ||
+                !std::all_of(encoded.begin(), encoded.end(), [](char value) {
+                    return (value >= 'A' && value <= 'Z') || (value >= 'a' && value <= 'z') ||
+                        (value >= '0' && value <= '9') || value == '+' || value == '/';
+                }))
+            {
+                throw std::invalid_argument{"data URL contains invalid base64"};
+            }
+            // base-n skips malformed input, so validate the forgiving-base64 grammar before decoding.
+            body.resize(encoded.size() / 4 * 3 + encoded.size() % 4 * 3 / 4);
+            bn::decode_b64(encoded.begin(), encoded.end(), reinterpret_cast<char*>(body.data()));
+        }
+        else
+        {
+            PercentDecodeBytes(payload, body);
+        }
+        result.body = std::make_shared<const std::vector<std::byte>>(std::move(body));
+        result.handled = true;
+        result.statusCode = UrlStatusCode::Ok;
+        return result;
+    }
+}