http support for posix

This commit is contained in:
Ken Van Hoeylandt 2026-10-15 21:20:27 +02:00
parent 625c8383f3
commit 6f3e76ea36
8 changed files with 611 additions and 16 deletions

View File

@ -15,8 +15,9 @@ if (DEFINED ENV{ESP_IDF_VERSION})
lwip
mbedtls
)
list(FILTER SOURCE_FILES EXCLUDE REGEX ".*download_mock\\.cpp$")
list(FILTER SOURCE_FILES EXCLUDE REGEX ".*download_posix\\.cpp$")
else ()
list(APPEND REQUIRES_LIST mbedtls)
list(FILTER SOURCE_FILES EXCLUDE REGEX ".*download_esp\\.cpp$")
endif ()

View File

@ -0,0 +1,24 @@
// SPDX-License-Identifier: Apache-2.0
#pragma once
#include <cstddef>
#include <string>
struct HttpUrl {
std::string host;
std::string port;
/** The path and query, "/" when the URL has none */
std::string path;
};
/** Parses an https:// URL. The port is 443 when the URL has none. */
bool http_parse_https_url(const std::string& url, HttpUrl& out);
/** Parses a status line such as "HTTP/1.1 200 OK" */
bool http_parse_status_line(const std::string& line, int& statusCode);
/** Parses a header line such as "Content-Length: 42". The name is returned in lower case, the value without surrounding whitespace. */
bool http_parse_header_line(const std::string& line, std::string& name, std::string& value);
/** Parses the size line of a chunk in a chunked body, e.g. "1a2b" or "1a2b;name=value" */
bool http_parse_chunk_size(const std::string& line, size_t& size);

View File

@ -1,9 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
#include <http/private/download.h>
HttpDownloadEvent http_download_run(const std::string&, const std::string&, const std::string&, HttpDownloadLink* link) {
if (http_download_is_cancelled(link)) {
return http_download_make_cancelled_event();
}
return http_download_make_error_event("HTTP downloads are not supported on this platform");
}

View File

@ -0,0 +1,392 @@
// SPDX-License-Identifier: Apache-2.0
#include <http/private/download.h>
#include <http/private/http_parsing.h>
#include <mbedtls/ssl.h>
#include <mbedtls/x509_crt.h>
#include <psa/crypto.h>
#include <tactility/log.h>
#include <cerrno>
#include <cstdio>
#include <cstring>
#include <netdb.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <unistd.h>
namespace {
constexpr auto* TAG = "http-download";
constexpr int TIMEOUT_SECONDS = 5;
constexpr size_t BUFFER_SIZE = 512;
// A write to a closed connection must fail instead of raising SIGPIPE. macOS has no MSG_NOSIGNAL, but SO_NOSIGPIPE on the socket.
#ifdef MSG_NOSIGNAL
constexpr int SEND_FLAGS = MSG_NOSIGNAL;
#else
constexpr int SEND_FLAGS = 0;
#endif
int bio_send(void* context, const unsigned char* data, size_t size) {
const int fd = *static_cast<int*>(context);
const ssize_t sent = send(fd, data, size, SEND_FLAGS);
if (sent >= 0) {
return static_cast<int>(sent);
}
if (errno == EINTR) {
return MBEDTLS_ERR_SSL_WANT_WRITE;
}
return (errno == EAGAIN || errno == EWOULDBLOCK) ? MBEDTLS_ERR_SSL_TIMEOUT : MBEDTLS_ERR_SSL_INTERNAL_ERROR;
}
int bio_receive(void* context, unsigned char* data, size_t size) {
const int fd = *static_cast<int*>(context);
const ssize_t received = recv(fd, data, size, 0);
if (received >= 0) {
return static_cast<int>(received);
}
if (errno == EINTR) {
return MBEDTLS_ERR_SSL_WANT_READ;
}
// The socket's receive timeout elapsed
return (errno == EAGAIN || errno == EWOULDBLOCK) ? MBEDTLS_ERR_SSL_TIMEOUT : MBEDTLS_ERR_SSL_INTERNAL_ERROR;
}
// RAII: the TLS session, the socket and the certificate are released on every exit path
class TlsConnection {
int fd = -1;
bool handshakeDone = false;
mbedtls_x509_crt ca {};
mbedtls_ssl_config config {};
mbedtls_ssl_context ssl {};
// Bytes that readLine() read beyond the line it returned
std::string pending;
public:
TlsConnection() {
mbedtls_x509_crt_init(&ca);
mbedtls_ssl_config_init(&config);
mbedtls_ssl_init(&ssl);
}
~TlsConnection() {
if (handshakeDone) {
mbedtls_ssl_close_notify(&ssl);
}
if (fd >= 0) {
close(fd);
}
mbedtls_ssl_free(&ssl);
mbedtls_ssl_config_free(&config);
mbedtls_x509_crt_free(&ca);
}
bool loadCertificate(const std::string& certPath) {
auto* file = fopen(certPath.c_str(), "rb");
if (file == nullptr) {
return false;
}
fseek(file, 0, SEEK_END);
const long size = ftell(file);
fseek(file, 0, SEEK_SET);
if (size <= 0) {
fclose(file);
return false;
}
// A PEM certificate is parsed as a NUL-terminated string
std::string certificate(static_cast<size_t>(size) + 1, '\0');
const size_t readBytes = fread(certificate.data(), 1, static_cast<size_t>(size), file);
fclose(file);
if (readBytes != static_cast<size_t>(size)) {
return false;
}
return mbedtls_x509_crt_parse(&ca, reinterpret_cast<const unsigned char*>(certificate.data()), certificate.size()) == 0;
}
bool connectSocket(const HttpUrl& url) {
addrinfo hints {};
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
addrinfo* addresses = nullptr;
if (getaddrinfo(url.host.c_str(), url.port.c_str(), &hints, &addresses) != 0) {
LOG_E(TAG, "Failed to resolve %s", url.host.c_str());
return false;
}
const timeval timeout { .tv_sec = TIMEOUT_SECONDS, .tv_usec = 0 };
for (auto* address = addresses; address != nullptr; address = address->ai_next) {
fd = socket(address->ai_family, address->ai_socktype, address->ai_protocol);
if (fd < 0) {
continue;
}
setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout));
#ifdef SO_NOSIGPIPE
const int enabled = 1;
setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &enabled, sizeof(enabled));
#endif
int result;
do {
result = ::connect(fd, address->ai_addr, address->ai_addrlen);
} while (result != 0 && errno == EINTR);
if (result == 0) {
break;
}
close(fd);
fd = -1;
}
freeaddrinfo(addresses);
return fd >= 0;
}
bool handshake(const HttpUrl& url, HttpDownloadLink* link) {
if (mbedtls_ssl_config_defaults(&config, MBEDTLS_SSL_IS_CLIENT, MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT) != 0) {
return false;
}
mbedtls_ssl_conf_authmode(&config, MBEDTLS_SSL_VERIFY_REQUIRED);
mbedtls_ssl_conf_ca_chain(&config, &ca, nullptr);
// Same as the ESP32 implementation
mbedtls_ssl_conf_max_tls_version(&config, MBEDTLS_SSL_VERSION_TLS1_2);
if (mbedtls_ssl_setup(&ssl, &config) != 0 || mbedtls_ssl_set_hostname(&ssl, url.host.c_str()) != 0) {
return false;
}
mbedtls_ssl_set_bio(&ssl, &fd, bio_send, bio_receive, nullptr);
int result;
while ((result = mbedtls_ssl_handshake(&ssl)) != 0) {
if ((result != MBEDTLS_ERR_SSL_WANT_READ && result != MBEDTLS_ERR_SSL_WANT_WRITE) || http_download_is_cancelled(link)) {
LOG_E(TAG, "TLS handshake with %s failed: -0x%04x", url.host.c_str(), static_cast<unsigned>(-result));
return false;
}
}
handshakeDone = true;
return true;
}
bool writeAll(const std::string& data) {
size_t offset = 0;
while (offset < data.size()) {
const int written = mbedtls_ssl_write(&ssl, reinterpret_cast<const unsigned char*>(data.data()) + offset, data.size() - offset);
if (written > 0) {
offset += static_cast<size_t>(written);
} else if (written != MBEDTLS_ERR_SSL_WANT_READ && written != MBEDTLS_ERR_SSL_WANT_WRITE) {
return false;
}
}
return true;
}
/** @return the number of bytes read, 0 when the server closed the connection, or -1 on an error */
int read(char* buffer, size_t size) {
if (!pending.empty()) {
const size_t count = std::min(size, pending.size());
memcpy(buffer, pending.data(), count);
pending.erase(0, count);
return static_cast<int>(count);
}
while (true) {
const int result = mbedtls_ssl_read(&ssl, reinterpret_cast<unsigned char*>(buffer), size);
if (result >= 0) {
return result;
}
if (result == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) {
return 0;
}
if (result != MBEDTLS_ERR_SSL_WANT_READ && result != MBEDTLS_ERR_SSL_WANT_WRITE) {
return -1;
}
}
}
/** Reads a line that ends with CRLF, without the CRLF */
bool readLine(std::string& line) {
constexpr size_t MAX_LINE_LENGTH = 8192;
while (true) {
const auto end = pending.find("\r\n");
if (end != std::string::npos) {
line = pending.substr(0, end);
pending.erase(0, end + 2);
return true;
}
if (pending.size() > MAX_LINE_LENGTH) {
return false;
}
char buffer[BUFFER_SIZE];
const int result = mbedtls_ssl_read(&ssl, reinterpret_cast<unsigned char*>(buffer), sizeof(buffer));
if (result > 0) {
pending.append(buffer, static_cast<size_t>(result));
} else if (result != MBEDTLS_ERR_SSL_WANT_READ && result != MBEDTLS_ERR_SSL_WANT_WRITE) {
return false;
}
}
}
};
// RAII: a file that's removed unless commit() renamed it onto the target path
class TempFile {
std::string tempPath;
FILE* file = nullptr;
public:
explicit TempFile(const std::string& targetPath) : tempPath(targetPath + ".tmp") {
file = fopen(tempPath.c_str(), "wb");
}
~TempFile() {
if (file != nullptr) {
fclose(file);
remove(tempPath.c_str());
}
}
bool isOpen() const { return file != nullptr; }
bool write(const char* data, size_t size) { return fwrite(data, 1, size, file) == size; }
/** Closes the file and replaces the target path with it */
bool commit(const std::string& targetPath) {
const bool closed = fclose(file) == 0;
file = nullptr;
if (!closed || rename(tempPath.c_str(), targetPath.c_str()) != 0) {
remove(tempPath.c_str());
return false;
}
return true;
}
};
}
HttpDownloadEvent http_download_run(const std::string& url, const std::string& certPath, const std::string& targetPath, HttpDownloadLink* link) {
if (http_download_is_cancelled(link)) {
return http_download_make_cancelled_event();
}
HttpUrl parsedUrl;
if (!http_parse_https_url(url, parsedUrl)) {
return http_download_make_error_event("Unsupported URL");
}
if (psa_crypto_init() != PSA_SUCCESS) {
return http_download_make_error_event("Failed to initialize crypto");
}
TlsConnection connection;
if (!connection.loadCertificate(certPath)) {
return http_download_make_error_event("Failed to read certificate file");
}
if (!connection.connectSocket(parsedUrl) || !connection.handshake(parsedUrl, link)) {
return http_download_make_error_event("Failed to open connection");
}
const std::string request =
"GET " + parsedUrl.path + " HTTP/1.1\r\n"
"Host: " + parsedUrl.host + "\r\n"
"User-Agent: Tactility\r\n"
"Accept-Encoding: identity\r\n"
"Connection: close\r\n"
"\r\n";
if (!connection.writeAll(request)) {
return http_download_make_error_event("Failed to send request");
}
std::string line;
int statusCode = 0;
if (!connection.readLine(line) || !http_parse_status_line(line, statusCode)) {
return http_download_make_error_event("Failed to fetch response headers");
}
// -1 means the length is unknown: the body is chunked or ends when the server closes the connection
long contentLength = -1;
bool chunked = false;
while (true) {
if (!connection.readLine(line)) {
return http_download_make_error_event("Failed to fetch response headers", statusCode);
}
if (line.empty()) {
break;
}
std::string name;
std::string value;
if (!http_parse_header_line(line, name, value)) {
continue;
}
if (name == "content-length") {
contentLength = strtol(value.c_str(), nullptr, 10);
} else if (name == "transfer-encoding" && value.find("chunked") != std::string::npos) {
chunked = true;
}
}
if (statusCode < 200 || statusCode >= 300) {
return http_download_make_error_event("Server response is not OK", statusCode);
}
if (http_download_is_cancelled(link)) {
return http_download_make_cancelled_event(statusCode);
}
// Downloaded to a sibling ".tmp" file first and only renamed onto targetPath on success, so a
// failed/cancelled download never leaves a partial file at targetPath.
TempFile file(targetPath);
if (!file.isOpen()) {
return http_download_make_error_event("Failed to open target file", statusCode);
}
char buffer[BUFFER_SIZE];
bool cancelled = false;
// Copies up to `count` body bytes to the file. A count of -1 copies until the server closes the connection.
// @return an error message, or nullptr when the bytes were copied or the download was cancelled
auto copyBody = [&](long count) -> const char* {
while (count != 0) {
if (http_download_is_cancelled(link)) {
cancelled = true;
return nullptr;
}
const size_t wanted = (count < 0) ? sizeof(buffer) : std::min(sizeof(buffer), static_cast<size_t>(count));
const int received = connection.read(buffer, wanted);
if (received < 0) {
return "Failed to read response data";
}
if (received == 0) {
return (count < 0) ? nullptr : "Response body was incomplete";
}
if (!file.write(buffer, static_cast<size_t>(received))) {
return "Failed to write downloaded data";
}
if (count > 0) {
count -= received;
}
}
return nullptr;
};
const char* error = nullptr;
if (chunked) {
while (error == nullptr && !cancelled) {
size_t chunkSize = 0;
if (!connection.readLine(line) || !http_parse_chunk_size(line, chunkSize)) {
error = "Failed to read response data";
} else if (chunkSize == 0) {
// The trailer ends with an empty line
while (connection.readLine(line) && !line.empty()) {}
break;
} else if ((error = copyBody(static_cast<long>(chunkSize))) == nullptr && (!connection.readLine(line) || !line.empty())) {
error = "Failed to read response data";
}
}
} else {
error = copyBody(contentLength);
}
if (cancelled) {
return http_download_make_cancelled_event(statusCode);
}
if (error != nullptr) {
return http_download_make_error_event(error, statusCode);
}
if (!file.commit(targetPath)) {
return http_download_make_error_event("Failed to finalize downloaded file", statusCode);
}
return http_download_make_success_event(statusCode);
}

View File

@ -0,0 +1,128 @@
// SPDX-License-Identifier: Apache-2.0
#include <http/private/http_parsing.h>
#include <cctype>
#include <cstdlib>
#include <string_view>
namespace {
std::string trim(const std::string& text) {
const auto start = text.find_first_not_of(" \t");
if (start == std::string::npos) {
return "";
}
const auto end = text.find_last_not_of(" \t");
return text.substr(start, end - start + 1);
}
bool is_digits(const std::string& text) {
if (text.empty()) {
return false;
}
for (const char c : text) {
if (!std::isdigit(static_cast<unsigned char>(c))) {
return false;
}
}
return true;
}
}
bool http_parse_https_url(const std::string& url, HttpUrl& out) {
constexpr std::string_view scheme = "https://";
if (url.compare(0, scheme.size(), scheme) != 0) {
return false;
}
const auto authorityStart = scheme.size();
const auto pathStart = url.find_first_of("/?#", authorityStart);
const std::string authority = url.substr(authorityStart, pathStart == std::string::npos ? std::string::npos : pathStart - authorityStart);
if (authority.empty() || authority.find('@') != std::string::npos) {
return false;
}
// An IPv6 address is enclosed in brackets, as it contains colons itself
std::string host;
std::string rest;
if (authority[0] == '[') {
const auto close = authority.find(']');
if (close == std::string::npos) {
return false;
}
host = authority.substr(1, close - 1);
rest = authority.substr(close + 1);
} else {
const auto colon = authority.find(':');
host = authority.substr(0, colon);
rest = colon == std::string::npos ? "" : authority.substr(colon);
}
if (host.empty()) {
return false;
}
std::string port = "443";
if (!rest.empty()) {
if (rest[0] != ':' || !is_digits(rest.substr(1))) {
return false;
}
port = rest.substr(1);
}
std::string path = pathStart == std::string::npos ? "/" : url.substr(pathStart);
// The fragment isn't sent to the server
path = path.substr(0, path.find('#'));
if (path.empty() || path[0] != '/') {
path.insert(0, "/");
}
out = { .host = host, .port = port, .path = path };
return true;
}
bool http_parse_status_line(const std::string& line, int& statusCode) {
if (line.compare(0, 5, "HTTP/") != 0) {
return false;
}
const auto space = line.find(' ');
if (space == std::string::npos) {
return false;
}
const std::string code = line.substr(space + 1, 3);
if (code.size() != 3 || !is_digits(code) || (line.size() > space + 4 && line[space + 4] != ' ')) {
return false;
}
statusCode = std::atoi(code.c_str());
return true;
}
bool http_parse_header_line(const std::string& line, std::string& name, std::string& value) {
const auto colon = line.find(':');
if (colon == std::string::npos || colon == 0) {
return false;
}
name = trim(line.substr(0, colon));
for (auto& c : name) {
c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
}
value = trim(line.substr(colon + 1));
return !name.empty();
}
bool http_parse_chunk_size(const std::string& line, size_t& size) {
const std::string digits = trim(line.substr(0, line.find(';')));
if (digits.empty() || digits.size() > sizeof(size_t) * 2) {
return false;
}
size_t result = 0;
for (const char c : digits) {
if (!std::isxdigit(static_cast<unsigned char>(c))) {
return false;
}
const char lower = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
result = result * 16 + static_cast<size_t>(lower <= '9' ? lower - '0' : lower - 'a' + 10);
}
size = result;
return true;
}

View File

@ -5,7 +5,7 @@ enable_language(C CXX ASM)
file(GLOB_RECURSE TEST_SOURCES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/source/*.cpp)
add_executable(HttpModuleTests EXCLUDE_FROM_ALL ${TEST_SOURCES})
target_include_directories(HttpModuleTests PRIVATE ${DOCTESTINC})
target_include_directories(HttpModuleTests PRIVATE ${DOCTESTINC} ${PROJECT_SOURCE_DIR}/../private)
add_test(NAME HttpModuleTests COMMAND HttpModuleTests)

View File

@ -4,7 +4,7 @@
TEST_CASE("http_download_start requires a subscribed subscription") {
HttpDownloadSubscription sub {};
CHECK_EQ(http_download_start("http://example.com/file", "/cert.pem", "/target.bin", &sub), ERROR_INVALID_ARGUMENT);
CHECK_EQ(http_download_start("https://example.com/file", "/nonexistent/cert.pem", "/target.bin", &sub), ERROR_INVALID_ARGUMENT);
}
TEST_CASE("http_download_poll times out before the download finishes") {
@ -21,13 +21,13 @@ TEST_CASE("http_download_poll times out before the download finishes") {
task_event_group_destruct(&event_group);
}
TEST_CASE("http_download_start on this (non-ESP-IDF) platform always fails the download") {
TEST_CASE("http_download_start fails the download when the certificate can't be read") {
TaskEventGroup event_group {};
task_event_group_construct(&event_group);
HttpDownloadSubscription sub {};
CHECK_EQ(http_download_subscribe(&sub, &event_group), ERROR_NONE);
CHECK_EQ(http_download_start("http://example.com/file", "/cert.pem", "/target.bin", &sub), ERROR_NONE);
CHECK_EQ(http_download_start("https://example.com/file", "/nonexistent/cert.pem", "/target.bin", &sub), ERROR_NONE);
CHECK_EQ(task_event_group_wait(&event_group, sub.bit, false, nullptr, pdMS_TO_TICKS(2000)), ERROR_NONE);
@ -55,7 +55,7 @@ TEST_CASE("http_download_unsubscribe is safe to call right after start, without
HttpDownloadSubscription sub {};
CHECK_EQ(http_download_subscribe(&sub, &event_group), ERROR_NONE);
CHECK_EQ(http_download_start("http://example.com/file", "/cert.pem", "/target.bin", &sub), ERROR_NONE);
CHECK_EQ(http_download_start("https://example.com/file", "/nonexistent/cert.pem", "/target.bin", &sub), ERROR_NONE);
// No cancel, no poll for the terminal event - just unsubscribe and tear down immediately,
// racing whatever the download task is doing.
@ -77,7 +77,7 @@ TEST_CASE("http_download_cancel before start makes the download finish as cancel
HttpDownloadSubscription sub {};
CHECK_EQ(http_download_subscribe(&sub, &event_group), ERROR_NONE);
CHECK_EQ(http_download_cancel(&sub), ERROR_NONE);
CHECK_EQ(http_download_start("http://example.com/file", "/cert.pem", "/target.bin", &sub), ERROR_NONE);
CHECK_EQ(http_download_start("https://example.com/file", "/nonexistent/cert.pem", "/target.bin", &sub), ERROR_NONE);
CHECK_EQ(task_event_group_wait(&event_group, sub.bit, false, nullptr, pdMS_TO_TICKS(2000)), ERROR_NONE);

View File

@ -0,0 +1,59 @@
#include "doctest.h"
#include <http/private/http_parsing.h>
TEST_CASE("http_parse_https_url splits the host, port and path") {
HttpUrl url;
REQUIRE(http_parse_https_url("https://cdn.example.org/apps/1.0/apps.json?x=1#top", url));
CHECK_EQ(url.host, "cdn.example.org");
CHECK_EQ(url.port, "443");
CHECK_EQ(url.path, "/apps/1.0/apps.json?x=1");
REQUIRE(http_parse_https_url("https://example.org:8443", url));
CHECK_EQ(url.port, "8443");
CHECK_EQ(url.path, "/");
REQUIRE(http_parse_https_url("https://[::1]:9000/file", url));
CHECK_EQ(url.host, "::1");
CHECK_EQ(url.port, "9000");
}
TEST_CASE("http_parse_https_url rejects other schemes and malformed URLs") {
HttpUrl url;
CHECK_FALSE(http_parse_https_url("http://example.org/", url));
CHECK_FALSE(http_parse_https_url("https://", url));
CHECK_FALSE(http_parse_https_url("https://example.org:abc/", url));
CHECK_FALSE(http_parse_https_url("https://user@example.org/", url));
}
TEST_CASE("http_parse_status_line reads the status code") {
int status = 0;
REQUIRE(http_parse_status_line("HTTP/1.1 200 OK", status));
CHECK_EQ(status, 200);
REQUIRE(http_parse_status_line("HTTP/1.1 404", status));
CHECK_EQ(status, 404);
CHECK_FALSE(http_parse_status_line("HTTP/1.1 20 OK", status));
CHECK_FALSE(http_parse_status_line("SSH-2.0 200", status));
}
TEST_CASE("http_parse_header_line lowercases the name and trims the value") {
std::string name;
std::string value;
REQUIRE(http_parse_header_line("Content-Length: 42 ", name, value));
CHECK_EQ(name, "content-length");
CHECK_EQ(value, "42");
CHECK_FALSE(http_parse_header_line("no colon here", name, value));
CHECK_FALSE(http_parse_header_line(": empty name", name, value));
}
TEST_CASE("http_parse_chunk_size reads hexadecimal sizes and ignores extensions") {
size_t size = 0;
REQUIRE(http_parse_chunk_size("1a2B", size));
CHECK_EQ(size, 0x1a2b);
REQUIRE(http_parse_chunk_size("0", size));
CHECK_EQ(size, 0);
REQUIRE(http_parse_chunk_size("ff;name=value", size));
CHECK_EQ(size, 0xff);
CHECK_FALSE(http_parse_chunk_size("", size));
CHECK_FALSE(http_parse_chunk_size("xyz", size));
}