const std = @import("std");const debug_trace = @import("../../core/shared/debug_trace.zig");const io_mod = @import("../../core/shared/io.zig");const url_policy = @import("url_policy.zig"); const Allocator = std.mem.Allocator;const IpAddress = std.Io.net.IpAddress;const posix = std.posix; pub const max_body_bytes: usize = 10 * 1024 * 1024;const max_redirect_hops: usize = 10;const default_hop_timeout_ms: i64 = 60 * 1000;const max_header_bytes: usize = 64 * 1024;const plain_transport_buffer_len: usize = 4096;const tls_transport_buffer_len: usize = std.crypto.tls.Client.min_buffer_len;const tls_allow_truncation_attacks = false; const FailureStage = enum {    connect,    tls_handshake,    request_write,    response_head,    response_body,}; const BodyFraming = union(enum) {    no_body,    content_length: usize,    chunked,    close_delimited,}; var default_resolver_ctx: u8 = 0;var default_connector_ctx: u8 = 0; pub const Deadline = struct {    deadline_ms: i64,}; pub const FetchOptions = struct {    max_body_bytes: usize = max_body_bytes,    deadline: ?Deadline = null,    cancel_flag: ?*std.atomic.Value(bool) = null,}; pub const Resolver = struct {    ctx: *anyopaque,    resolve: *const fn (*anyopaque, Allocator, []const u8, u16, FetchOptions) anyerror![]IpAddress,}; pub const Connector = struct {    ctx: *anyopaque,    get: *const fn (*anyopaque, Allocator, PinnedTarget, FetchOptions) anyerror!ConnectorResponse,}; pub const Transport = struct {    resolver: Resolver,    connector: Connector,}; pub fn defaultTransport() Transport {    return .{        .resolver = .{ .ctx = @ptrCast(&default_resolver_ctx), .resolve = resolveDefault },        .connector = .{ .ctx = @ptrCast(&default_connector_ctx), .get = connectDefault },    };} pub const PinnedTarget = struct {    url: url_policy.ValidatedUrl,    admitted_addresses: []const IpAddress,    tls_server_name: []const u8,    host_header: []const u8,}; pub const ConnectorResponse = struct {    status: std.http.Status,    body: []u8,    location: ?[]u8 = null,    content_encoding: ?[]u8 = null,    content_encoding_count: usize = 0,    content_type: ?[]u8 = null,     pub fn deinit(self: *ConnectorResponse, alloc: Allocator) void {        alloc.free(self.body);        if (self.location) |location| alloc.free(location);        if (self.content_encoding) |encoding| alloc.free(encoding);        if (self.content_type) |mime| alloc.free(mime);        self.* = .{ .status = .ok, .body = &.{} };    }}; pub const Success = struct {    final_url: []u8,    status: std.http.Status,    body: []u8,    content_type: ?[]u8 = null,     fn deinit(self: *Success, alloc: Allocator) void {        alloc.free(self.final_url);        alloc.free(self.body);        if (self.content_type) |mime| alloc.free(mime);        self.* = .{ .final_url = &.{}, .status = .ok, .body = &.{} };    }}; pub const FailureKind = enum {    non_success_status,    unexpected_content_encoding,}; pub const Failure = struct {    kind: FailureKind,    status: ?std.http.Status = null,    body: []u8 = &.{},    content_encoding: ?[]u8 = null,     fn deinit(self: *Failure, alloc: Allocator) void {        alloc.free(self.body);        if (self.content_encoding) |encoding| alloc.free(encoding);        self.* = .{ .kind = .non_success_status };    }}; pub const Result = union(enum) {    success: Success,    failure: Failure,    cross_host_redirect: []u8,     pub fn deinit(self: *Result, alloc: Allocator) void {        switch (self.*) {            .success => |*success| success.deinit(alloc),            .failure => |*failure| failure.deinit(alloc),            .cross_host_redirect => |url| alloc.free(url),        }        self.* = .{ .failure = .{ .kind = .non_success_status } };    }}; pub fn fetch(alloc: Allocator, initial: url_policy.ValidatedUrl, options: FetchOptions, transport: Transport) anyerror!Result {    var current = try cloneUrl(alloc, initial);    defer current.deinit(alloc);     var hop: usize = 0;    while (hop <= max_redirect_hops) : (hop += 1) {        const hop_options = hopOptions(options);        const addresses = try resolveAddresses(alloc, current, hop_options, transport.resolver);        defer alloc.free(addresses);         if (addresses.len == 0) return error.NoAddressReturned;        for (addresses) |address| {            if (!url_policy.isPublicAddress(address)) return error.NonPublicDnsAnswer;        }         const host_header = try hostHeader(alloc, current);        defer alloc.free(host_header);         const pinned: PinnedTarget = .{            .url = current,            .admitted_addresses = addresses,            .tls_server_name = current.canonical_host,            .host_header = host_header,        };         var response = try transport.connector.get(transport.connector.ctx, alloc, pinned, hop_options);        defer response.deinit(alloc);         if (isRedirectStatus(response.status)) {            const location = response.location orelse return error.MissingRedirectLocation;            var redirect = try url_policy.redirectTarget(alloc, current, location);            defer redirect.deinit(alloc);             switch (redirect.kind) {                .follow => {                    current.deinit(alloc);                    current = try cloneUrl(alloc, redirect.target.?);                    continue;                },                .cross_host => {                    return .{ .cross_host_redirect = try alloc.dupe(u8, redirect.cross_host_url.?) };                },            }        }         if (response.content_encoding_count > 1)            return unexpectedContentEncoding(alloc, response.status, response.content_encoding orelse "multiple Content-Encoding fields");         const content_encoding = if (response.content_encoding) |encoding|            parseContentEncoding(encoding) orelse                return unexpectedContentEncoding(alloc, response.status, encoding)        else            std.http.ContentEncoding.identity;         if (content_encoding != .identity) {            if (response.status.class() != .success)                return unexpectedContentEncoding(alloc, response.status, response.content_encoding.?);             const decoded_body = decodeContentEncoding(alloc, content_encoding, response.body, options.max_body_bytes) catch |err| switch (err) {                error.OutOfMemory, error.BodyTooLarge => return err,                else => return unexpectedContentEncoding(alloc, response.status, response.content_encoding.?),            };            errdefer alloc.free(decoded_body);            const final_url = try alloc.dupe(u8, current.retrieval_url);            errdefer alloc.free(final_url);            const content_type = if (response.content_type) |mime| try alloc.dupe(u8, mime) else null;            errdefer if (content_type) |mime| alloc.free(mime);            return .{ .success = .{                .final_url = final_url,                .status = response.status,                .body = decoded_body,                .content_type = content_type,            } };        }         if (response.status.class() != .success) {            return .{ .failure = .{                .kind = .non_success_status,                .status = response.status,                .body = try alloc.dupe(u8, response.body),            } };        }         return .{ .success = .{            .final_url = try alloc.dupe(u8, current.retrieval_url),            .status = response.status,            .body = try alloc.dupe(u8, response.body),            .content_type = if (response.content_type) |mime| try alloc.dupe(u8, mime) else null,        } };    }     return error.RedirectLimitExceeded;} fn unexpectedContentEncoding(alloc: Allocator, status: std.http.Status, encoding: []const u8) !Result {    return .{ .failure = .{        .kind = .unexpected_content_encoding,        .status = status,        .content_encoding = try alloc.dupe(u8, encoding),    } };} fn parseContentEncoding(encoding: []const u8) ?std.http.ContentEncoding {    if (std.ascii.eqlIgnoreCase(encoding, "identity")) return .identity;    if (std.ascii.eqlIgnoreCase(encoding, "gzip")) return .gzip;    if (std.ascii.eqlIgnoreCase(encoding, "deflate")) return .deflate;    if (std.ascii.eqlIgnoreCase(encoding, "zstd")) return .zstd;    return null;} fn decodeContentEncoding(    alloc: Allocator,    content_encoding: std.http.ContentEncoding,    encoded: []const u8,    cap: usize,) ![]u8 {    const scratch_len: usize = switch (content_encoding) {        .gzip, .deflate => std.compress.flate.max_window_len,        .zstd => std.compress.zstd.default_window_len + std.compress.zstd.block_size_max,        .identity, .compress => unreachable,    };    const scratch = try alloc.alloc(u8, scratch_len);    defer alloc.free(scratch);     var encoded_reader: std.Io.Reader = .fixed(encoded);    var decompressor: std.http.Decompress = undefined;    const decoded_reader = std.http.Decompress.init(&decompressor, &encoded_reader, scratch, content_encoding);    var decoded: std.ArrayList(u8) = .empty;    errdefer decoded.deinit(alloc);    const overflow_limit = std.math.add(usize, cap, 1) catch return error.BodyTooLarge;    decoded_reader.appendRemaining(alloc, &decoded, .limited(overflow_limit)) catch |err| switch (err) {        error.StreamTooLong => return error.BodyTooLarge,        else => return err,    };    if (decoded.items.len > cap) return error.BodyTooLarge;    return decoded.toOwnedSlice(alloc);} fn hopOptions(options: FetchOptions) FetchOptions {    var effective = normalizedOptions(options);    if (effective.deadline == null) {        effective.deadline = .{ .deadline_ms = monotonicMillis() + default_hop_timeout_ms };    }    return effective;} fn normalizedOptions(options: FetchOptions) FetchOptions {    return .{        .max_body_bytes = if (options.max_body_bytes == 0) max_body_bytes else options.max_body_bytes,        .deadline = options.deadline,        .cancel_flag = options.cancel_flag,    };} fn resolveAddresses(alloc: Allocator, target: url_policy.ValidatedUrl, options: FetchOptions, resolver: Resolver) ![]IpAddress {    if (target.literal_address) |literal| {        var address = literal;        address.setPort(target.port);        const out = try alloc.alloc(IpAddress, 1);        out[0] = address;        return out;    }    return resolver.resolve(resolver.ctx, alloc, target.canonical_host, target.port, normalizedOptions(options));} fn cloneUrl(alloc: Allocator, source: url_policy.ValidatedUrl) !url_policy.ValidatedUrl {    return .{        .scheme = source.scheme,        .canonical_host = try alloc.dupe(u8, source.canonical_host),        .port = source.port,        .explicit_port = source.explicit_port,        .path_query = try alloc.dupe(u8, source.path_query),        .retrieval_url = try alloc.dupe(u8, source.retrieval_url),        .literal_address = source.literal_address,    };} fn hostHeader(alloc: Allocator, target: url_policy.ValidatedUrl) ![]u8 {    const host = try hostForHeader(alloc, target.canonical_host);    defer alloc.free(host);    if (target.explicit_port) |port| return std.fmt.allocPrint(alloc, "{s}:{d}", .{ host, port });    return alloc.dupe(u8, host);} fn hostForHeader(alloc: Allocator, canonical_host: []const u8) ![]u8 {    if (std.mem.findScalar(u8, canonical_host, ':') == null) return alloc.dupe(u8, canonical_host);    return std.fmt.allocPrint(alloc, "[{s}]", .{canonical_host});} fn isRedirectStatus(status: std.http.Status) bool {    return status == .moved_permanently or        status == .found or        status == .temporary_redirect or        status == .permanent_redirect;} fn resolveDefault(_: *anyopaque, alloc: Allocator, host: []const u8, port: u16, options: FetchOptions) anyerror![]IpAddress {    try checkControl(options);    const host_name = try std.Io.net.HostName.init(host);    if (options.deadline) |deadline| return resolveWithDeadline(alloc, host_name, port, deadline, options);    return collectLookup(alloc, host_name, port, options);} const ResolveSelect = union(enum) {    lookup: anyerror![]IpAddress,    timeout: anyerror!void,}; fn resolveWithDeadline(alloc: Allocator, host_name: std.Io.net.HostName, port: u16, deadline: Deadline, options: FetchOptions) anyerror![]IpAddress {    const zio = io_mod.getIo();    var select_buffer: [2]ResolveSelect = undefined;    var select: std.Io.Select(ResolveSelect) = .init(zio, &select_buffer);     try select.concurrent(.lookup, collectLookup, .{ alloc, host_name, port, options });    try select.concurrent(.timeout, waitForDeadline, .{deadline});     const result = try select.await();    switch (result) {        .lookup => |lookup_result| {            select.cancelDiscard();            return lookup_result;        },        .timeout => |timeout_result| {            cancelResolveSelect(alloc, &select);            try timeout_result;            return error.Timeout;        },    }} fn collectLookup(alloc: Allocator, host_name: std.Io.net.HostName, port: u16, options: FetchOptions) anyerror![]IpAddress {    try checkControl(options);    const zio = io_mod.getIo();    var lookup_buffer: [64]std.Io.net.HostName.LookupResult = undefined;    var lookup_queue: std.Io.Queue(std.Io.net.HostName.LookupResult) = .init(&lookup_buffer);    try std.Io.net.HostName.lookup(host_name, zio, &lookup_queue, .{ .port = port });     var addresses: std.ArrayList(IpAddress) = .empty;    errdefer addresses.deinit(alloc);    while (lookup_queue.getOne(zio)) |item| {        switch (item) {            .address => |address| try addresses.append(alloc, address),            .canonical_name => {},        }    } else |err| switch (err) {        error.Closed => {},        error.Canceled => return error.Canceled,    }    try checkControl(options);    if (addresses.items.len == 0) return error.NoAddressReturned;    return addresses.toOwnedSlice(alloc);} fn waitForDeadline(deadline: Deadline) anyerror!void {    const timestamp: std.Io.Clock.Timestamp = .{        .clock = .awake,        .raw = std.Io.Timestamp.fromNanoseconds(@as(i96, deadline.deadline_ms) * std.time.ns_per_ms),    };    try timestamp.wait(io_mod.getIo());    return error.Timeout;} fn cancelResolveSelect(alloc: Allocator, select: *std.Io.Select(ResolveSelect)) void {    while (select.cancel()) |item| switch (item) {        .lookup => |lookup_result| {            const addresses = lookup_result catch continue;            alloc.free(addresses);        },        .timeout => {},    };} const Dialer = struct {    ctx: *anyopaque,    connect_fn: *const fn (*anyopaque, IpAddress, FetchOptions) anyerror!posix.fd_t,}; fn connectDefaultDialer(_: *anyopaque, address: IpAddress, options: FetchOptions) anyerror!posix.fd_t {    return connectPinned(address, options);} fn connectAdmitted(addresses: []const IpAddress, options: FetchOptions, dialer: Dialer) anyerror!posix.fd_t {    if (addresses.len == 0) return error.NoAddressReturned;     for (addresses, 0..) |address, index| {        try checkControl(options);        const fd = dialer.connect_fn(dialer.ctx, address, options) catch |err| {            traceDialFailure(index, addresses.len, err);            if (!isRetryableConnectError(err)) return err;            if (index + 1 == addresses.len) return err;            try checkControl(options);            continue;        };        return fd;    }    unreachable;} fn isRetryableConnectError(err: anyerror) bool {    return switch (err) {        error.ConnectionRefused,        error.ConnectionResetByPeer,        error.NetworkUnreachable,        error.HostUnreachable,        error.ConnectionFailed,        error.UnexpectedClose,        error.Timeout,        => true,        else => false,    };} fn connectDefault(_: *anyopaque, alloc: Allocator, target: PinnedTarget, options: FetchOptions) anyerror!ConnectorResponse {    const effective = normalizedOptions(options);    const dialer: Dialer = .{        .ctx = @ptrCast(&default_connector_ctx),        .connect_fn = connectDefaultDialer,    };    const fd = connectAdmitted(target.admitted_addresses, effective, dialer) catch |err| {        traceFailure(.connect, err);        return err;    };    defer closeFd(fd);     return switch (target.url.scheme) {        .http => try fetchPlain(alloc, fd, target, effective),        .https => try fetchTls(alloc, fd, target, effective),    };} fn fetchPlain(alloc: Allocator, fd: posix.fd_t, target: PinnedTarget, options: FetchOptions) anyerror!ConnectorResponse {    var writer: PlainDeadlineWriter = undefined;    writer.init(fd, options);    writeRequest(&writer.interface, target, options) catch |err| {        const root = unwrapWriteFailure(err, writer.err);        traceFailure(.request_write, root);        return root;    };    writer.interface.flush() catch |err| {        const root = unwrapWriteFailure(err, writer.err);        traceFailure(.request_write, root);        return root;    };     var reader: PlainDeadlineReader = undefined;    reader.init(fd, options);    var failure_stage: FailureStage = .response_head;    return readResponse(alloc, &reader.interface, options.max_body_bytes, options, &failure_stage) catch |err| {        const root = unwrapReadFailure(err, null, reader.err);        traceFailure(failure_stage, root);        return root;    };} fn fetchTls(alloc: Allocator, fd: posix.fd_t, target: PinnedTarget, options: FetchOptions) anyerror!ConnectorResponse {    const zio = io_mod.getIo();     var ca_bundle: std.crypto.Certificate.Bundle = .empty;    defer ca_bundle.deinit(alloc);    var ca_lock: std.Io.RwLock = .init;    const now = std.Io.Clock.real.now(zio);    ca_bundle.rescan(alloc, zio, now) catch |err| {

Lines 1–500 of 3854. Load more