components/net/tests/http_loader.rs - main - githubmirror/servo

components/net/tests/http_loader.rs
2 166 строк71 KB
Narfinger
net/script: Allow Content-Length header to inform buffers downstream (#45758)
10 июл 2026, 13:10
10 июл 2026, 13:10a876251
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

use std::collections::HashMap;
use std::io::Write;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::{Duration, Instant};

use content_security_policy as csp;
use cookie::Cookie as CookiePair;
use crossbeam_channel::{Receiver, unbounded};
use devtools_traits::{
    ChromeToDevtoolsControlMsg, DevtoolsControlMsg, HttpRequest as DevtoolsHttpRequest,
    HttpResponse as DevtoolsHttpResponse, NetworkEvent,
};
use embedder_traits::AuthenticationResponse;
use flate2::Compression;
use flate2::write::{GzEncoder, ZlibEncoder};
use headers::authorization::Basic;
use headers::{
    Authorization, ContentLength, Date, HeaderMapExt, Host, StrictTransportSecurity, UserAgent,
};
use http::header::{self, HeaderMap, HeaderValue, WWW_AUTHENTICATE};
use http::uri::Authority;
use http::{HeaderName, Method, StatusCode};
use http_body_util::combinators::BoxBody;
use hyper::body::{Body, Bytes, Incoming};
use hyper::{Request as HyperRequest, Response as HyperResponse};
use net::cookie::ServoCookie;
use net::cookie_storage::CookieStorage;
use net::fetch::methods::{self};
use net::http_loader::{determine_requests_referrer, serialize_origin};
use net::resource_thread::AuthCacheEntry;
use net::test::DECODER_BUFFER_SIZE;
use net_traits::blob_url_store::UrlWithBlobClaim;
use net_traits::http_status::HttpStatus;
use net_traits::request::{
    CredentialsMode, Destination, Referrer, Request, RequestBuilder, RequestMode,
    TraversableForUserPrompts, create_request_body_with_content,
};
use net_traits::response::{Response, ResponseBody};
use net_traits::{CookieSource, FetchTaskTarget, NetworkError, ReferrerPolicy, get_current_locale};
use parking_lot::{Mutex, RwLock};
use servo_base::id::{TEST_PIPELINE_ID, TEST_WEBVIEW_ID};
use servo_url::{ImmutableOrigin, ServoUrl};
use url::Url;

use crate::{
    create_generic_embedder_proxy_and_receiver, fetch, fetch_with_context, make_body, make_server,
    make_ssl_server, mock_origin, new_fetch_context, receive_credential_prompt_msgs,
    replace_host_table, spawn_blocking_task,
};

fn assert_cookie_for_domain(
    cookie_jar: &RwLock<CookieStorage>,
    domain: &str,
    cookie: Option<&str>,
) {
    let mut cookie_jar = cookie_jar.write();
    let url = ServoUrl::parse(&*domain).unwrap();
    let cookies = cookie_jar.cookies_for_url(&url, CookieSource::HTTP);
    assert_eq!(cookies.as_ref().map(|c| &**c), cookie);
}

fn recv_http_request(devtools_port: &Receiver<DevtoolsControlMsg>) -> DevtoolsHttpRequest {
    match devtools_port.recv().unwrap() {
        DevtoolsControlMsg::FromChrome(ChromeToDevtoolsControlMsg::NetworkEvent(_, net_event)) => {
            match net_event {
                NetworkEvent::HttpRequest(req) => req,
                NetworkEvent::HttpRequestUpdate(req) => req,
                other => panic!("Expected HttpRequest but got: {:?}", other),
            }
        },
        other => panic!("Expected NetworkEvent but got: {:?}", other),
    }
}

fn recv_all_network_events(devtools_port: Receiver<DevtoolsControlMsg>) -> Vec<NetworkEvent> {
    let mut events = vec![];
    while let Ok(msg) = devtools_port.recv() {
        match msg {
            DevtoolsControlMsg::FromChrome(ChromeToDevtoolsControlMsg::NetworkEvent(
                _,
                net_event,
            )) => events.push(net_event),
            other => panic!("Expected NetworkEvent but got: {:?}", other),
        }
    }
    events
}

pub fn expect_devtools_http_request(
    devtools_port: &Receiver<DevtoolsControlMsg>,
) -> (DevtoolsHttpRequest, DevtoolsHttpRequest) {
    (
        recv_http_request(devtools_port),
        recv_http_request(devtools_port),
    )
}

fn pluck<F: Fn(&NetworkEvent) -> bool>(events: &mut Vec<NetworkEvent>, f: F) -> NetworkEvent {
    let Some(idx) = events.iter().position(f) else {
        panic!("No matching network event")
    };
    events.remove(idx)
}

#[track_caller]
fn expect_request(events: &mut Vec<NetworkEvent>) -> DevtoolsHttpRequest {
    let event = pluck(events, |event| {
        matches!(event, NetworkEvent::HttpRequest(_))
    });
    match event {
        NetworkEvent::HttpRequest(req) => req,
        _ => unreachable!(),
    }
}

#[track_caller]
fn expect_request_update(events: &mut Vec<NetworkEvent>) -> DevtoolsHttpRequest {
    let event = pluck(events, |event| {
        matches!(event, NetworkEvent::HttpRequestUpdate(_))
    });
    match event {
        NetworkEvent::HttpRequestUpdate(req) => req,
        _ => unreachable!(),
    }
}

#[track_caller]
fn expect_response(events: &mut Vec<NetworkEvent>) -> DevtoolsHttpResponse {
    let event = pluck(events, |event| {
        matches!(event, NetworkEvent::HttpResponse(_))
    });
    match event {
        NetworkEvent::HttpResponse(resp) => resp,
        _ => unreachable!(),
    }
}

pub fn expect_devtools_http_response(
    devtools_port: &Receiver<DevtoolsControlMsg>,
) -> DevtoolsHttpResponse {
    loop {
        match devtools_port.recv().unwrap() {
            DevtoolsControlMsg::FromChrome(ChromeToDevtoolsControlMsg::NetworkEvent(
                _,
                net_event_response,
            )) => match net_event_response {
                NetworkEvent::HttpResponse(httpresponse) => return httpresponse,
                NetworkEvent::SecurityInfo(_) => continue,
                other => panic!("Expected HttpResponse but got: {:?}", other),
            },
            other => panic!("Expected NetworkEvent but got: {:?}", other),
        }
    }
}

pub fn devtools_response_with_body(
    devtools_port: &Receiver<DevtoolsControlMsg>,
) -> DevtoolsHttpResponse {
    let devhttpresponses = vec![
        expect_devtools_http_response(devtools_port),
        expect_devtools_http_response(devtools_port),
    ];
    return devhttpresponses
        .into_iter()
        .find(|resp| resp.body.is_some())
        .expect("One of the responses should have a body");
}

#[test]
fn test_check_default_headers_loaded_in_every_request() {
    let expected_headers = Arc::new(Mutex::new(None));
    let expected_headers_clone = expected_headers.clone();
    let handler = move |request: HyperRequest<Incoming>,
                        _: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
        assert_eq!(
            request.headers().clone(),
            expected_headers_clone.lock().take().unwrap()
        );
    };
    let (server, url) = make_server(handler);

    let mut headers = HeaderMap::new();

    headers.insert(
        header::ACCEPT_ENCODING,
        HeaderValue::from_static("gzip, deflate, br, zstd"),
    );

    headers.typed_insert(Host::from(
        format!("{}:{}", url.host_str().unwrap(), url.port().unwrap())
            .parse::<Authority>()
            .unwrap(),
    ));

    headers.insert(
        header::ACCEPT,
        HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
    );

    headers.insert(header::ACCEPT_LANGUAGE, get_current_locale().1.clone());

    headers.typed_insert::<UserAgent>(crate::DEFAULT_USER_AGENT.parse().unwrap());

    // Append fetch metadata headers
    headers.insert(
        HeaderName::from_static("sec-fetch-dest"),
        HeaderValue::from_static("document"),
    );
    headers.insert(
        HeaderName::from_static("sec-fetch-mode"),
        HeaderValue::from_static("no-cors"),
    );
    headers.insert(
        HeaderName::from_static("sec-fetch-site"),
        HeaderValue::from_static("same-origin"),
    );
    headers.insert(
        HeaderName::from_static("sec-fetch-user"),
        HeaderValue::from_static("?1"),
    );

    *expected_headers.lock() = Some(headers.clone());

    // Testing for method.GET
    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .destination(Destination::Document)
        .origin(url.clone().origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = dbg!(fetch(request, None));
    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );

    // Testing for method.POST
    let mut post_headers = headers.clone();
    post_headers.typed_insert(ContentLength(0 as u64));
    let url_str = url.as_str();
    // request gets header "Origin: http://example.com" but expected_headers has
    // "Origin: http://example.com/" which do not match for equality so strip trailing '/'
    post_headers.insert(
        header::ORIGIN,
        HeaderValue::from_str(&url_str[..url_str.len() - 1]).unwrap(),
    );
    *expected_headers.lock() = Some(post_headers);
    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::POST)
        .destination(Destination::Document)
        .origin(url.clone().origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);
    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );

    let _ = server.close();
}

#[test]
fn test_load_when_request_is_not_get_or_head_and_there_is_no_body_content_length_should_be_set_to_0()
 {
    let handler = move |request: HyperRequest<Incoming>,
                        _: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
        assert_eq!(
            request.headers().typed_get::<ContentLength>(),
            Some(ContentLength(0))
        );
    };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::POST)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);
    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );

    let _ = server.close();
}

#[test]
fn test_request_and_response_data_with_network_messages() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response
                .headers_mut()
                .typed_insert(Host::from("foo.bar".parse::<Authority>().unwrap()));
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut request_headers = HeaderMap::new();
    request_headers.typed_insert(Host::from("bar.foo".parse::<Authority>().unwrap()));
    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .headers(request_headers)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let (devtools_chan, devtools_port) = unbounded();
    let response = fetch(request, Some(devtools_chan));
    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );

    let _ = server.close();

    // notification received from devtools
    let devhttprequests = expect_devtools_http_request(&devtools_port);
    let devhttpresponse = devtools_response_with_body(&devtools_port);

    // Creating default headers for request
    let mut headers = HeaderMap::new();

    headers.insert(
        header::ACCEPT,
        HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
    );

    headers.insert(header::ACCEPT_LANGUAGE, get_current_locale().1.clone());

    headers.typed_insert::<UserAgent>(crate::DEFAULT_USER_AGENT.parse().unwrap());

    headers.insert(
        header::ACCEPT_ENCODING,
        HeaderValue::from_static("gzip, deflate, br, zstd"),
    );

    // Append fetch metadata headers
    headers.insert(
        HeaderName::from_static("sec-fetch-dest"),
        HeaderValue::from_static("document"),
    );
    headers.insert(
        HeaderName::from_static("sec-fetch-mode"),
        HeaderValue::from_static("no-cors"),
    );
    headers.insert(
        HeaderName::from_static("sec-fetch-site"),
        HeaderValue::from_static("cross-site"),
    );
    headers.insert(
        HeaderName::from_static("sec-fetch-user"),
        HeaderValue::from_static("?1"),
    );

    let httprequest = DevtoolsHttpRequest {
        url: url.url(),
        method: Method::GET,
        headers: headers,
        body: Some(vec![].into()),
        pipeline_id: TEST_PIPELINE_ID,
        started_date_time: devhttprequests.1.started_date_time,
        time_stamp: devhttprequests.1.time_stamp,
        connect_time: devhttprequests.1.connect_time,
        send_time: devhttprequests.1.send_time,
        destination: Destination::Document,
        is_xhr: false,
        browsing_context_id: TEST_WEBVIEW_ID.into(),
    };

    let content = "Yay!";
    let mut response_headers = HeaderMap::new();
    response_headers.typed_insert(ContentLength(content.len() as u64));
    response_headers.typed_insert(Host::from("foo.bar".parse::<Authority>().unwrap()));
    response_headers.typed_insert(
        devhttpresponse
            .headers
            .as_ref()
            .unwrap()
            .typed_get::<Date>()
            .unwrap()
            .clone(),
    );

    let httpresponse = DevtoolsHttpResponse {
        headers: Some(response_headers),
        status: HttpStatus::default(),
        body: Some(content.as_bytes().to_vec().into()),
        from_cache: false,
        pipeline_id: TEST_PIPELINE_ID,
        browsing_context_id: TEST_WEBVIEW_ID.into(),
    };

    assert_eq!(devhttprequests.1, httprequest);
    assert_eq!(devhttpresponse, httpresponse);
}

#[test]
fn test_request_and_response_message_from_devtool_without_pipeline_id() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response
                .headers_mut()
                .typed_insert(Host::from("foo.bar".parse::<Authority>().unwrap()));
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(None)
        .policy_container(Default::default())
        .build();

    let (devtools_chan, devtools_port) = unbounded();
    let response = fetch(request, Some(devtools_chan));
    assert!(response.actual_response().status.code().is_success());

    let _ = server.close();

    // notification received from devtools
    assert!(devtools_port.try_recv().is_err());
}

#[test]
fn test_redirected_request_to_devtools() {
    let post_handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(request.method(), Method::GET);
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (post_server, post_url) = make_server(post_handler);

    let post_redirect_url = post_url.clone();
    let pre_handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(request.method(), Method::POST);
            response.headers_mut().insert(
                header::LOCATION,
                HeaderValue::from_str(&post_redirect_url.to_string()).unwrap(),
            );
            *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
        };
    let (pre_server, pre_url) = make_server(pre_handler);

    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), pre_url.clone(), Referrer::NoReferrer)
        .method(Method::POST)
        .destination(Destination::Document)
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .origin(mock_origin())
        .policy_container(Default::default())
        .build();

    let (devtools_chan, devtools_port) = unbounded();
    fetch(request, Some(devtools_chan));

    let _ = pre_server.close();
    let _ = post_server.close();

    let mut events = recv_all_network_events(devtools_port);
    let first_request = expect_request(&mut events);
    let first_request_update = expect_request_update(&mut events);
    let first_response = expect_response(&mut events);

    assert_eq!(first_request.method, Method::POST);
    assert_eq!(first_request.url, pre_url.url());
    assert_eq!(
        first_response.status,
        HttpStatus::from(StatusCode::MOVED_PERMANENTLY)
    );
    assert_eq!(first_request.method, first_request_update.method);
    assert_eq!(first_request.url, first_request_update.url);

    let second_request = expect_request(&mut events);
    let second_request_update = expect_request_update(&mut events);
    let second_response = expect_response(&mut events);

    assert_eq!(second_request.method, Method::GET);
    assert_eq!(second_request.url, post_url.url());
    assert_eq!(second_response.status, HttpStatus::default());
    assert_eq!(second_request.method, second_request_update.method);
    assert_eq!(second_request.url, second_request_update.url);
}

#[test]
fn test_load_when_redirecting_from_a_post_should_rewrite_next_request_as_get() {
    let post_handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(request.method(), Method::GET);
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (post_server, post_url) = make_server(post_handler);

    let post_redirect_url = post_url.clone();
    let pre_handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(request.method(), Method::POST);
            response.headers_mut().insert(
                header::LOCATION,
                HeaderValue::from_str(&post_redirect_url.to_string()).unwrap(),
            );
            *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
        };
    let (pre_server, pre_url) = make_server(pre_handler);

    let request = RequestBuilder::new(None, pre_url.clone(), Referrer::NoReferrer)
        .method(Method::POST)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = pre_server.close();
    let _ = post_server.close();

    assert!(response.to_actual().status.code().is_success());
}

#[test]
fn test_load_should_decode_the_response_as_deflate_when_response_headers_have_content_encoding_deflate()
 {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response.headers_mut().insert(
                header::CONTENT_ENCODING,
                HeaderValue::from_static("deflate"),
            );
            let mut e = ZlibEncoder::new(Vec::new(), Compression::default());
            e.write(b"Yay!").unwrap();
            let encoded_content = e.finish().unwrap();
            *response.body_mut() = make_body(encoded_content);
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server.close();

    let internal_response = response.internal_response.unwrap();
    assert!(internal_response.status.clone().code().is_success());
    assert_eq!(
        *internal_response.body.lock(),
        ResponseBody::Done(b"Yay!".to_vec())
    );
}

#[test]
fn test_load_should_decode_the_response_as_gzip_when_response_headers_have_content_encoding_gzip() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response
                .headers_mut()
                .insert(header::CONTENT_ENCODING, HeaderValue::from_static("gzip"));
            let mut e = GzEncoder::new(Vec::new(), Compression::default());
            e.write(b"Yay!").unwrap();
            let encoded_content = e.finish().unwrap();
            *response.body_mut() = make_body(encoded_content);
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server.close();

    let internal_response = response.internal_response.unwrap();
    assert!(internal_response.status.clone().code().is_success());
    assert_eq!(
        *internal_response.body.lock(),
        ResponseBody::Done(b"Yay!".to_vec())
    );
}

#[test]
fn test_load_doesnt_send_request_body_on_any_redirect() {
    let post_handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(request.method(), Method::GET);
            assert_eq!(request.size_hint().exact(), Some(0));
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (post_server, post_url) = make_server(post_handler);

    let post_redirect_url = post_url.clone();
    let pre_handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(request.size_hint().exact(), Some(13));
            response.headers_mut().insert(
                header::LOCATION,
                HeaderValue::from_str(&post_redirect_url.to_string()).unwrap(),
            );
            *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
        };
    let (pre_server, pre_url) = make_server(pre_handler);

    let content = "Body on POST!";
    let request_body = create_request_body_with_content(content.to_string());

    let request = RequestBuilder::new(None, pre_url.clone(), Referrer::NoReferrer)
        .body(Some(request_body))
        .method(Method::POST)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = pre_server.close();
    let _ = post_server.close();

    assert!(response.to_actual().status.code().is_success());
}

#[test]
fn test_load_doesnt_add_host_to_hsts_list_when_url_is_http_even_if_hsts_headers_are_present() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response
                .headers_mut()
                .typed_insert(StrictTransportSecurity::excluding_subdomains(
                    Duration::from_secs(31536000),
                ));
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let mut context = new_fetch_context(None, None);
    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
    assert_eq!(
        context
            .state
            .hsts_list
            .read()
            .is_host_secure(url.host_str().unwrap()),
        false
    );
}

#[test]
fn test_load_sets_cookies_in_the_resource_manager_when_it_get_set_cookie_header_in_response() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response.headers_mut().insert(
                header::SET_COOKIE,
                HeaderValue::from_static("mozillaIs=theBest"),
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut context = new_fetch_context(None, None);

    assert_cookie_for_domain(&context.state.cookie_jar, url.as_str(), None);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );

    assert_cookie_for_domain(
        &context.state.cookie_jar,
        url.as_str(),
        Some("mozillaIs=theBest"),
    );
}

#[test]
fn test_load_sets_requests_cookies_header_for_url_by_getting_cookies_from_the_resource_manager() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(
                request.headers().get(header::COOKIE).unwrap().as_bytes(),
                b"mozillaIs=theBest"
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut context = new_fetch_context(None, None);

    {
        let mut cookie_jar = context.state.cookie_jar.write();
        let cookie = ServoCookie::new_wrapped(
            CookiePair::new("mozillaIs".to_owned(), "theBest".to_owned()),
            &url,
            CookieSource::HTTP,
        )
        .unwrap();
        cookie_jar.push(cookie, &url, CookieSource::HTTP);
    }

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_load_sends_cookie_if_nonhttp() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(
                request.headers().get(header::COOKIE).unwrap().as_bytes(),
                b"mozillaIs=theBest"
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut context = new_fetch_context(None, None);

    {
        let mut cookie_jar = context.state.cookie_jar.write();
        let cookie = ServoCookie::new_wrapped(
            CookiePair::new("mozillaIs".to_owned(), "theBest".to_owned()),
            &url,
            CookieSource::NonHTTP,
        )
        .unwrap();
        cookie_jar.push(cookie, &url, CookieSource::HTTP);
    }

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_cookie_set_with_httponly_should_not_be_available_using_getcookiesforurl() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response.headers_mut().insert(
                header::SET_COOKIE,
                HeaderValue::from_static("mozillaIs=theBest; HttpOnly"),
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut context = new_fetch_context(None, None);

    assert_cookie_for_domain(&context.state.cookie_jar, url.as_str(), None);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );

    assert_cookie_for_domain(
        &context.state.cookie_jar,
        url.as_str(),
        Some("mozillaIs=theBest"),
    );
    let mut cookie_jar = context.state.cookie_jar.write();
    assert!(
        cookie_jar
            .cookies_for_url(&url, CookieSource::NonHTTP)
            .is_none()
    );
}

#[test]
fn test_when_cookie_received_marked_secure_is_ignored_for_http() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response.headers_mut().insert(
                header::SET_COOKIE,
                HeaderValue::from_static("mozillaIs=theBest; Secure"),
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut context = new_fetch_context(None, None);

    assert_cookie_for_domain(&context.state.cookie_jar, url.as_str(), None);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    assert!(response.actual_response().status.code().is_success());

    assert_cookie_for_domain(&context.state.cookie_jar, url.as_str(), None);
}

#[test]
fn test_load_sets_content_length_to_length_of_request_body() {
    let content = "This is a request body";
    let content_bytes = content.as_bytes();
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let content_length = ContentLength(content_bytes.len() as u64);
            assert_eq!(
                request.headers().typed_get::<ContentLength>(),
                Some(content_length)
            );
            *response.body_mut() = make_body(content_bytes.to_vec());
        };
    let (server, url) = make_server(handler);

    let request_body = create_request_body_with_content(content.to_string());

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::POST)
        .body(Some(request_body))
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_load_uses_explicit_accept_from_headers_in_load_data() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(
                request
                    .headers()
                    .get(header::ACCEPT)
                    .unwrap()
                    .to_str()
                    .unwrap(),
                "text/html"
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut accept_headers = HeaderMap::new();
    accept_headers.insert(header::ACCEPT, HeaderValue::from_static("text/html"));
    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .headers(accept_headers)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_load_sets_default_accept_to_html_xhtml_xml_and_then_anything_else() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(
                request
                    .headers()
                    .get(header::ACCEPT)
                    .unwrap()
                    .to_str()
                    .unwrap(),
                "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_load_uses_explicit_accept_encoding_from_load_data_headers() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(
                request
                    .headers()
                    .get(header::ACCEPT_ENCODING)
                    .unwrap()
                    .to_str()
                    .unwrap(),
                "chunked"
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut accept_encoding_headers = HeaderMap::new();
    accept_encoding_headers.insert(header::ACCEPT_ENCODING, HeaderValue::from_static("chunked"));
    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .headers(accept_encoding_headers)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_load_sets_default_accept_encoding_to_gzip_and_deflate() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(
                request
                    .headers()
                    .get(header::ACCEPT_ENCODING)
                    .unwrap()
                    .to_str()
                    .unwrap(),
                "gzip, deflate, br, zstd"
            );
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_load_errors_when_there_a_redirect_loop() {
    let url_b_for_a = Arc::new(Mutex::new(None::<ServoUrl>));
    let url_b_for_a_clone = url_b_for_a.clone();
    let handler_a =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response.headers_mut().insert(
                header::LOCATION,
                HeaderValue::from_str(&url_b_for_a_clone.lock().as_ref().unwrap().to_string())
                    .unwrap(),
            );
            *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
        };
    let (server_a, url_a) = make_server(handler_a);

    let url_a_for_b = url_a.clone();
    let handler_b =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response.headers_mut().insert(
                header::LOCATION,
                HeaderValue::from_str(&url_a_for_b.to_string()).unwrap(),
            );
            *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
        };
    let (server_b, url_b) = make_server(handler_b);

    *url_b_for_a.lock() = Some(url_b.url());

    let request = RequestBuilder::new(None, url_a.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server_a.close();
    let _ = server_b.close();

    assert_eq!(
        response.get_network_error(),
        Some(&NetworkError::TooManyRedirects)
    );
}

#[test]
fn test_load_succeeds_with_a_redirect_loop() {
    let url_b_for_a = Arc::new(Mutex::new(None::<ServoUrl>));
    let url_b_for_a_clone = url_b_for_a.clone();
    let handled_a = AtomicBool::new(false);
    let handler_a =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            if !handled_a.swap(true, Ordering::SeqCst) {
                response.headers_mut().insert(
                    header::LOCATION,
                    HeaderValue::from_str(&url_b_for_a_clone.lock().as_ref().unwrap().to_string())
                        .unwrap(),
                );
                *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
            } else {
                *response.body_mut() = make_body(b"Success".to_vec());
            }
        };
    let (server_a, url_a) = make_server(handler_a);

    let url_a_for_b = url_a.clone();
    let handler_b =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response.headers_mut().insert(
                header::LOCATION,
                HeaderValue::from_str(&url_a_for_b.to_string()).unwrap(),
            );
            *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
        };
    let (server_b, url_b) = make_server(handler_b);

    *url_b_for_a.lock() = Some(url_b.url());

    let request = RequestBuilder::new(None, url_a.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server_a.close();
    let _ = server_b.close();

    let response = response.to_actual();
    assert_eq!(response.url_list, [url_a.url(), url_b.url(), url_a.url()]);
    assert_eq!(
        *response.body.lock(),
        ResponseBody::Done(b"Success".to_vec())
    );
}

#[test]
fn test_load_follows_a_redirect() {
    let post_handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(request.method(), Method::GET);
            *response.body_mut() = make_body(b"Yay!".to_vec());
        };
    let (post_server, post_url) = make_server(post_handler);

    let post_redirect_url = post_url.clone();
    let pre_handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert_eq!(request.method(), Method::GET);
            response.headers_mut().insert(
                header::LOCATION,
                HeaderValue::from_str(&post_redirect_url.to_string()).unwrap(),
            );
            *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
        };
    let (pre_server, pre_url) = make_server(pre_handler);

    let request = RequestBuilder::new(None, pre_url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = pre_server.close();
    let _ = post_server.close();

    let internal_response = response.internal_response.unwrap();
    assert!(internal_response.status.clone().code().is_success());
    assert_eq!(
        *internal_response.body.lock(),
        ResponseBody::Done(b"Yay!".to_vec())
    );
}

#[test]
fn test_redirect_from_x_to_y_provides_y_cookies_from_y() {
    let shared_url_y = Arc::new(Mutex::new(None::<ServoUrl>));
    let shared_url_y_clone = shared_url_y.clone();
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let path = request.uri().path();
            if path == "/com/" {
                assert_eq!(
                    request.headers().get(header::COOKIE).unwrap().as_bytes(),
                    b"mozillaIsNot=dotOrg"
                );
                let location = shared_url_y.lock().as_ref().unwrap().to_string();
                response.headers_mut().insert(
                    header::LOCATION,
                    HeaderValue::from_str(&location.to_string()).unwrap(),
                );
                *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
            } else if path == "/org/" {
                assert_eq!(
                    request.headers().get(header::COOKIE).unwrap().as_bytes(),
                    b"mozillaIs=theBest"
                );
                *response.body_mut() = make_body(b"Yay!".to_vec());
            } else {
                panic!("unexpected path {:?}", path)
            }
        };
    let (server, url) = make_server(handler);
    let port = url.port().unwrap();

    assert_eq!(url.host_str(), Some("localhost"));
    let ip = "127.0.0.1".parse().unwrap();
    let mut host_table = HashMap::new();
    host_table.insert("mozilla.com".to_owned(), ip);
    host_table.insert("mozilla.org".to_owned(), ip);

    replace_host_table(host_table);

    let url_x = ServoUrl::parse(&format!("http://mozilla.com:{}/com/", port)).unwrap();
    let url_y = ServoUrl::parse(&format!("http://mozilla.org:{}/org/", port)).unwrap();
    *shared_url_y_clone.lock() = Some(url_y.clone());

    let mut context = new_fetch_context(None, None);
    {
        let mut cookie_jar = context.state.cookie_jar.write();
        let cookie_x = ServoCookie::new_wrapped(
            CookiePair::new("mozillaIsNot".to_owned(), "dotOrg".to_owned()),
            &url_x,
            CookieSource::HTTP,
        )
        .unwrap();

        cookie_jar.push(cookie_x, &url_x, CookieSource::HTTP);

        let cookie_y = ServoCookie::new_wrapped(
            CookiePair::new("mozillaIs".to_owned(), "theBest".to_owned()),
            &url_y,
            CookieSource::HTTP,
        )
        .unwrap();
        cookie_jar.push(cookie_y, &url_y, CookieSource::HTTP);
    }

    let request = RequestBuilder::new(
        None,
        UrlWithBlobClaim::new(url_x.clone(), None),
        Referrer::NoReferrer,
    )
    .method(Method::GET)
    .destination(Destination::Document)
    .origin(mock_origin())
    .pipeline_id(Some(TEST_PIPELINE_ID))
    .credentials_mode(CredentialsMode::Include)
    .policy_container(Default::default())
    .build();

    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    let internal_response = response.internal_response.unwrap();
    assert!(internal_response.status.clone().code().is_success());
    assert_eq!(
        *internal_response.body.lock(),
        ResponseBody::Done(b"Yay!".to_vec())
    );
}

#[test]
fn test_redirect_from_x_to_x_provides_x_with_cookie_from_first_response() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let path = request.uri().path();
            if path == "/initial/" {
                response.headers_mut().insert(
                    header::SET_COOKIE,
                    HeaderValue::from_static("mozillaIs=theBest; path=/;"),
                );
                let location = "/subsequent/".to_string();
                response.headers_mut().insert(
                    header::LOCATION,
                    HeaderValue::from_str(&location.to_string()).unwrap(),
                );
                *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
            } else if path == "/subsequent/" {
                assert_eq!(
                    request.headers().get(header::COOKIE).unwrap().as_bytes(),
                    b"mozillaIs=theBest"
                );
                *response.body_mut() = make_body(b"Yay!".to_vec());
            } else {
                panic!("unexpected path {:?}", path)
            }
        };
    let (server, url) = make_server(handler);

    let url = url.join("/initial/").unwrap();

    let request = RequestBuilder::new(
        None,
        UrlWithBlobClaim::new(url.clone(), None),
        Referrer::NoReferrer,
    )
    .method(Method::GET)
    .destination(Destination::Document)
    .origin(mock_origin())
    .pipeline_id(Some(TEST_PIPELINE_ID))
    .credentials_mode(CredentialsMode::Include)
    .policy_container(Default::default())
    .build();

    let response = fetch(request, None);

    let _ = server.close();

    let internal_response = response.internal_response.unwrap();
    assert!(internal_response.status.clone().code().is_success());
    assert_eq!(
        *internal_response.body.lock(),
        ResponseBody::Done(b"Yay!".to_vec())
    );
}

#[test]
fn test_if_auth_creds_not_in_url_but_in_cache_it_sets_it() {
    let handler =
        move |request: HyperRequest<Incoming>,
              _response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let expected = Authorization::basic("username", "test");
            assert_eq!(
                request.headers().typed_get::<Authorization<Basic>>(),
                Some(expected)
            );
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .mode(RequestMode::Navigate)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let mut context = new_fetch_context(None, None);

    let auth_entry = AuthCacheEntry {
        user_name: "username".to_owned(),
        password: "test".to_owned(),
    };

    context
        .state
        .auth_cache
        .write()
        .entries
        .insert(url.origin().clone().ascii_serialization(), auth_entry);

    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_auth_ui_needs_www_auth() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            *response.status_mut() = StatusCode::UNAUTHORIZED;
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .mode(RequestMode::Navigate)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let response = fetch(request, None);

    let _ = server.close();

    assert_eq!(
        response.internal_response.unwrap().status,
        StatusCode::UNAUTHORIZED
    );
}

#[test]
fn test_determine_requests_referrer_shorter_than_4k() {
    let url_str = "http://username:password@example.com/such/short/referer?query#fragment";
    let referrer_source = ServoUrl::parse(url_str).unwrap();
    let current_url = ServoUrl::parse("http://example.com/current/url").unwrap();
    let referrer_policy = ReferrerPolicy::UnsafeUrl;

    let referer = determine_requests_referrer(referrer_policy, referrer_source, current_url);

    assert_eq!(
        referer.unwrap().as_str(),
        "http://example.com/such/short/referer?query"
    );
}

#[test]
fn test_determine_requests_referrer_longer_than_4k() {
    let long_url_str = format!(
        "http://username:password@example.com/such/{}/referer?query#fragment",
        "long".repeat(1024)
    );
    let referrer_source = ServoUrl::parse(&long_url_str).unwrap();
    let current_url = ServoUrl::parse("http://example.com/current/url").unwrap();
    let referrer_policy = ReferrerPolicy::UnsafeUrl;

    let referer = determine_requests_referrer(referrer_policy, referrer_source, current_url);

    assert_eq!(referer.unwrap().as_str(), "http://example.com/");
}

#[test]
fn test_fetch_compressed_response_update_count() {
    // contents of ../../tests/wpt/tests/fetch/content-encoding/br/resources/foo.text.br
    const DATA_BROTLI_COMPRESSED: [u8; 15] = [
        0xe1, 0x18, 0x48, 0xc1, 0x2f, 0x65, 0xf6, 0x16, 0x9f, 0x05, 0x01, 0xbb, 0x20, 0x00, 0x06,
    ];
    const DATA_DECOMPRESSED_LEN: usize = 10500;

    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            response
                .headers_mut()
                .insert(header::CONTENT_ENCODING, HeaderValue::from_static("br"));
            *response.body_mut() = make_body(DATA_BROTLI_COMPRESSED.to_vec());
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    struct FetchResponseCollector {
        sender: Option<tokio::sync::oneshot::Sender<usize>>,
        update_count: usize,
    }
    impl FetchTaskTarget for FetchResponseCollector {
        fn process_request_body(&mut self, _: &Request) {}
        fn process_response(&mut self, _: &Request, _: &Response) {}
        fn process_response_chunk(&mut self, _: &Request, _: Vec<u8>) {
            self.update_count += 1;
        }
        /// Fired when the response is fully fetched
        fn process_response_eof(&mut self, _: &Request, _: &Response) {
            let _ = self.sender.take().unwrap().send(self.update_count);
        }
        fn process_csp_violations(&mut self, _: &Request, _: Vec<csp::Violation>) {}

        fn process_response_length_hint(&mut self, _: &Request, _: usize) {}
    }

    let (sender, receiver) = tokio::sync::oneshot::channel();
    let mut target = FetchResponseCollector {
        sender: Some(sender),
        update_count: 0,
    };
    let response_update_count = spawn_blocking_task::<_, Response>(async move {
        methods::fetch(request, &mut target, &mut new_fetch_context(None, None)).await;
        receiver.await.unwrap()
    });

    server.close();

    const EXPECTED_UPDATE_COUNT: usize =
        (DATA_DECOMPRESSED_LEN + DECODER_BUFFER_SIZE - 1) / DECODER_BUFFER_SIZE;
    assert_eq!(response_update_count, EXPECTED_UPDATE_COUNT);
}

#[test]
fn test_origin_serialization_compatibility() {
    let ensure_serialiations_match = |url_string| {
        let url = Url::parse(url_string).unwrap();
        let origin = ImmutableOrigin::new(&url);
        let serialized = format!("{}", serialize_origin(&origin));
        assert_eq!(serialized, origin.ascii_serialization());
    };

    ensure_serialiations_match("https://example.com");
    ensure_serialiations_match("https://example.com:443");

    ensure_serialiations_match("http://example.com");
    ensure_serialiations_match("http://example.com:80");

    ensure_serialiations_match("https://example.com:1234");
    ensure_serialiations_match("http://example.com:1234");

    ensure_serialiations_match("data:,dataurltexta");
}

#[test]
fn test_user_credentials_prompt_when_proxy_authentication_is_required() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let expected = Authorization::basic("username", "test");
            if let Some(credentials) = request.headers().typed_get::<Authorization<Basic>>() {
                if credentials == expected {
                    *response.status_mut() = StatusCode::OK;
                } else {
                    *response.status_mut() = StatusCode::UNAUTHORIZED;
                }
            } else {
                *response.status_mut() = StatusCode::PROXY_AUTHENTICATION_REQUIRED;
            }
        };
    let (server, url) = make_server(handler);

    let mut request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .mode(RequestMode::Navigate)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    request.traversable_for_user_prompts =
        TraversableForUserPrompts::TraversableNavigable(Default::default());

    let (embedder_proxy, embedder_receiver) = create_generic_embedder_proxy_and_receiver();
    let _ = receive_credential_prompt_msgs(
        embedder_receiver,
        Some(AuthenticationResponse {
            username: "username".into(),
            password: "test".into(),
        }),
    );

    let mut context = new_fetch_context(None, Some(embedder_proxy));

    let response = fetch_with_context(request, &mut context);

    let _ = server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_success()
    );
}

#[test]
fn test_prompt_credentials_when_client_receives_unauthorized_response() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let expected = Authorization::basic("username", "test");
            if let Some(credentials) = request.headers().typed_get::<Authorization<Basic>>() {
                if credentials == expected {
                    *response.status_mut() = StatusCode::OK;
                } else {
                    *response.status_mut() = StatusCode::UNAUTHORIZED;
                }
            } else {
                *response.status_mut() = StatusCode::UNAUTHORIZED;
            }

            response
                .headers_mut()
                .insert(WWW_AUTHENTICATE, HeaderValue::from_static("Basic"));
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .mode(RequestMode::Navigate)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let (embedder_proxy, embedder_receiver) = create_generic_embedder_proxy_and_receiver();
    let _ = receive_credential_prompt_msgs(
        embedder_receiver,
        Some(AuthenticationResponse {
            username: "username".into(),
            password: "test".into(),
        }),
    );
    let mut context = new_fetch_context(None, Some(embedder_proxy));

    let response = fetch_with_context(request, &mut context);

    server.close();

    assert_eq!(
        response.internal_response.unwrap().status.code(),
        StatusCode::OK
    );
}

#[test]
fn test_dont_prompt_credentials_when_unauthorized_response_contains_no_www_authenticate_header() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            assert!(
                request
                    .headers()
                    .typed_get::<Authorization<Basic>>()
                    .is_none()
            );
            *response.status_mut() = StatusCode::UNAUTHORIZED;
        };

    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .mode(RequestMode::Navigate)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let (embedder_proxy, embedder_receiver) = create_generic_embedder_proxy_and_receiver();
    let handle = std::thread::spawn(move || {
        loop {
            let Ok(msg) = embedder_receiver.recv() else {
                return;
            };
            match msg {
                net::embedder::NetToEmbedderMsg::RequestAuthentication(..) => {
                    panic!(
                        "Should not have requested authentication as there's no www-authenticate header"
                    );
                },
                net::embedder::NetToEmbedderMsg::WebResourceRequested(..) => {},
                _ => unreachable!(),
            }
        }
    });
    let mut context = new_fetch_context(None, Some(embedder_proxy));

    let response = fetch_with_context(request, &mut context);

    server.close();

    assert_eq!(
        response.internal_response.unwrap().status.code(),
        StatusCode::UNAUTHORIZED
    );

    // Without this join we won't notice if the mock embedder thread panics!
    drop(context); // Dropping this context causes the embedder thread to exit.
    handle.join().unwrap();
}

#[test]
fn test_prompt_credentials_user_cancels_dialog_input() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let expected = Authorization::basic("username", "test");
            if let Some(credentials) = request.headers().typed_get::<Authorization<Basic>>() {
                if credentials == expected {
                    *response.status_mut() = StatusCode::OK;
                } else {
                    *response.status_mut() = StatusCode::UNAUTHORIZED;
                }
            } else {
                *response.status_mut() = StatusCode::UNAUTHORIZED;
            }
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .mode(RequestMode::Navigate)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let (embedder_proxy, embedder_receiver) = create_generic_embedder_proxy_and_receiver();
    let _ = receive_credential_prompt_msgs(embedder_receiver, None);
    let mut context = new_fetch_context(None, Some(embedder_proxy));

    let response = fetch_with_context(request, &mut context);

    server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_client_error()
    );
}

#[test]
fn test_prompt_credentials_user_input_incorrect_credentials() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let expected = Authorization::basic("username", "test");
            if let Some(credentials) = request.headers().typed_get::<Authorization<Basic>>() {
                if credentials == expected {
                    *response.status_mut() = StatusCode::OK;
                } else {
                    *response.status_mut() = StatusCode::UNAUTHORIZED;
                }
            } else {
                *response.status_mut() = StatusCode::UNAUTHORIZED;
            }
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .mode(RequestMode::Navigate)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let (embedder_proxy, embedder_receiver) = create_generic_embedder_proxy_and_receiver();
    let _ = receive_credential_prompt_msgs(
        embedder_receiver,
        Some(AuthenticationResponse {
            username: "test".into(),
            password: "test".into(),
        }),
    );
    let mut context = new_fetch_context(None, Some(embedder_proxy));

    let response = fetch_with_context(request, &mut context);

    server.close();

    assert!(
        response
            .internal_response
            .unwrap()
            .status
            .code()
            .is_client_error()
    );
}

#[test]
fn test_prompt_credentials_user_input_incorrect_mode() {
    let handler =
        move |request: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            let expected = Authorization::basic("test", "test");
            if let Some(credentials) = request.headers().typed_get::<Authorization<Basic>>() {
                if credentials == expected {
                    *response.status_mut() = StatusCode::OK;
                } else {
                    *response.status_mut() = StatusCode::UNAUTHORIZED;
                }
            } else {
                *response.status_mut() = StatusCode::UNAUTHORIZED;
            }
        };
    let (server, url) = make_server(handler);

    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .mode(RequestMode::SameOrigin)
        .origin(mock_origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .credentials_mode(CredentialsMode::Include)
        .policy_container(Default::default())
        .build();

    let (embedder_proxy, embedder_receiver) = create_generic_embedder_proxy_and_receiver();
    let _ = receive_credential_prompt_msgs(
        embedder_receiver,
        Some(AuthenticationResponse {
            username: "test".into(),
            password: "test".into(),
        }),
    );
    let mut context = new_fetch_context(None, Some(embedder_proxy));

    let response = fetch_with_context(request, &mut context);

    server.close();

    assert!(response.internal_response.is_none());
}

/// Collects all network events from the devtools channel.
/// Drains all available events after the fetch has completed.
fn collect_all_network_events(devtools_port: &Receiver<DevtoolsControlMsg>) -> Vec<NetworkEvent> {
    let mut events = Vec::new();

    // Drain all available events from the channel.
    // Since fetch has already completed, all events should be queued.
    while let Ok(msg) = devtools_port.try_recv() {
        if let DevtoolsControlMsg::FromChrome(ChromeToDevtoolsControlMsg::NetworkEvent(
            _,
            net_event,
        )) = msg
        {
            events.push(net_event);
        }
    }
    events
}

#[test]
fn test_security_info_for_https_connection() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            *response.body_mut() = make_body(b"Hello HTTPS!".to_vec());
        };

    let (server, mut url) = make_ssl_server(handler);
    url.as_mut_url().set_scheme("https").unwrap();

    let (devtools_sender, devtools_receiver) = unbounded();

    let mut context = new_fetch_context(Some(devtools_sender), None);

    // The server certificate is self-signed, so we need to add an override
    // so that the connection works properly.
    for certificate in server.certificates.as_ref().unwrap().iter() {
        context.state.override_manager.add_override(certificate);
    }

    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(url.clone().origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let response = fetch_with_context(request, &mut context);
    server.close();

    assert!(response.status.code().is_success());

    let events = collect_all_network_events(&devtools_receiver);
    let security_info_event = events.iter().find_map(|event| {
        if let NetworkEvent::SecurityInfo(info) = event {
            Some(info)
        } else {
            None
        }
    });

    assert!(
        security_info_event.is_some(),
        "Expected to receive a SecurityInfo event for HTTPS connection"
    );

    let security_info = security_info_event.unwrap();
    assert!(
        security_info.security_info.is_some(),
        "Expected security_info to contain TLS details for HTTPS connection"
    );

    let tls_info = security_info.security_info.as_ref().unwrap();
    assert!(
        tls_info.protocol_version.is_some(),
        "Expected protocol_version to be set for HTTPS connection"
    );

    let protocol = tls_info.protocol_version.as_ref().unwrap();
    assert!(
        protocol.starts_with("TLS"),
        "Expected TLS protocol version, got: {}",
        protocol
    );

    assert!(
        tls_info.cipher_suite.is_some(),
        "Expected cipher_suite to be set for HTTPS connection"
    );
}

#[test]
fn test_no_security_info_for_http_connection() {
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            *response.body_mut() = make_body(b"Hello HTTP!".to_vec());
        };

    let (server, url) = make_server(handler);

    let (devtools_sender, devtools_receiver) = unbounded();

    let request = RequestBuilder::new(Some(TEST_WEBVIEW_ID), url.clone(), Referrer::NoReferrer)
        .method(Method::GET)
        .body(None)
        .destination(Destination::Document)
        .origin(url.clone().origin())
        .pipeline_id(Some(TEST_PIPELINE_ID))
        .policy_container(Default::default())
        .build();

    let mut context = new_fetch_context(Some(devtools_sender), None);

    let response = fetch_with_context(request, &mut context);
    server.close();

    assert!(response.status.code().is_success());

    let events = collect_all_network_events(&devtools_receiver);
    let security_info_events: Vec<_> = events
        .iter()
        .filter_map(|event| {
            if let NetworkEvent::SecurityInfo(info) = event {
                Some(info)
            } else {
                None
            }
        })
        .collect();

    for security_info in &security_info_events {
        assert!(
            security_info.security_info.is_none(),
            "HTTP connection should not have TLS security info"
        );
    }
}

#[test]
fn test_stale_while_revalidate_serves_cached_and_revalidates_in_background() {
    let request_count = Arc::new(AtomicUsize::new(0));
    let request_count_clone = request_count.clone();
    let handler =
        move |_: HyperRequest<Incoming>,
              response: &mut HyperResponse<BoxBody<Bytes, hyper::Error>>| {
            request_count_clone.fetch_add(1, Ordering::SeqCst);
            response.headers_mut().insert(
                header::CACHE_CONTROL,
                HeaderValue::from_static("max-age=0, stale-while-revalidate=30"),
            );
            *response.body_mut() = make_body(b"content".to_vec());
        };
    let (server, url) = make_server(handler);

    let mut context = new_fetch_context(None, None);

    let build_request = || {
        RequestBuilder::new(None, url.clone(), Referrer::NoReferrer)
            .method(Method::GET)
            .destination(Destination::Document)
            .origin(url.clone().origin())
            .pipeline_id(Some(TEST_PIPELINE_ID))
            .policy_container(Default::default())
            .build()
    };

    let response = fetch_with_context(build_request(), &mut context);
    assert!(response.actual_response().status.code().is_success());
    assert_eq!(request_count.load(Ordering::SeqCst), 1);

    // the stored response is stale but within the stale-while-revalidate window, so
    // it is served from cache immediately and a background revalidation is spawned.
    let response = fetch_with_context(build_request(), &mut context);
    assert!(response.actual_response().status.code().is_success());

    let deadline = Instant::now() + Duration::from_secs(5);
    while request_count.load(Ordering::SeqCst) < 2 && Instant::now() < deadline {
        std::thread::sleep(Duration::from_millis(20));
    }
    assert_eq!(
        request_count.load(Ordering::SeqCst),
        2,
        "exactly one background revalidation should have hit the server"
    );

    let _ = server.close();
}