diff --git a/url/src/lib.rs b/url/src/lib.rs index fa2803681..634fb759f 100644 --- a/url/src/lib.rs +++ b/url/src/lib.rs @@ -257,6 +257,179 @@ pub struct ParseOptions<'a> { violation_fn: Option<&'a dyn Fn(SyntaxViolation)>, } +/// Path bytes that the general parser copies through verbatim: no +/// percent-encoding, no backslash translation, no segment normalization. +/// +/// Deliberately conservative. Every byte here must be one the general parser +/// leaves untouched in a special-scheme path; anything else falls back. +#[inline] +fn is_verbatim_path_byte(b: u8) -> bool { + matches!(b, + b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' + | b'-' | b'_' | b'.' | b'~' | b'/' | b'@' | b'+' | b',' | b'=' | b'$' + | b'&' | b';' | b':' | b'!' | b'*' | b'\'' | b'(' | b')') +} + +/// Query bytes the general parser copies through verbatim, i.e. the complement +/// of the special-query percent-encode set (controls, space, `"`, `#`, `<`, +/// `>`, `'`, and everything non-ASCII). +/// +/// `%` is deliberately included: the parser does not re-encode existing escapes +/// in a query, so `%2F` survives unchanged. +#[inline] +fn is_verbatim_query_byte(b: u8) -> bool { + b.is_ascii() && b > b' ' && b != 0x7F && !matches!(b, b'"' | b'#' | b'<' | b'>' | b'\'') +} + +/// Whether `host` is a domain the general parser would pass through unchanged: +/// already lowercase, plainly not an IP address, and needing no IDNA work. +fn is_verbatim_host(host: &[u8]) -> bool { + if !matches!(host.first(), Some(b'a'..=b'z')) { + return false; + } + for label in host.split(|&b| b == b'.') { + // Rejected rather than reasoning about how the general parser treats an + // empty label, which includes the trailing-dot form. + if label.is_empty() { + return false; + } + // `xn--` can decode to something invalid, so punycode always needs real + // IDNA validation. + if label.starts_with(b"xn--") { + return false; + } + if !label + .iter() + .all(|b| matches!(b, b'a'..=b'z' | b'0'..=b'9' | b'-')) + { + return false; + } + } + // A final label that starts with a digit makes the host "end in a number", + // which sends the general parser down the IPv4 path. + match host.rsplit(|&b| b == b'.').next() { + Some(last) => !matches!(last.first(), Some(b'0'..=b'9')), + None => false, + } +} + +/// Parse a simple absolute `http(s)` URL without running the general parser. +/// +/// Returns `None` whenever anything at all is unusual, so the caller falls back +/// to the full parser. Every input this accepts must produce a `Url` identical +/// to what the general parser would produce; that is enforced by the +/// differential tests below, not by this comment. +fn parse_simple_absolute(input: &str) -> Option { + let bytes = input.as_bytes(); + + let (scheme_end, host_start) = if bytes.starts_with(b"https://") { + (5u32, 8usize) + } else if bytes.starts_with(b"http://") { + (4u32, 7usize) + } else { + return None; + }; + + let rest = &bytes[host_start..]; + // Checked before the scan below so that the common declines -- non-ASCII + // and punycode hosts, IPv4's leading digit, IPv6, uppercase -- cost one + // compare instead of a walk to the path separator. + if !matches!(rest.first(), Some(b'a'..=b'z')) { + return None; + } + // Everything that could follow the host instead of '/' -- userinfo '@', a + // port ':', a query '?', a fragment '#' -- is rejected below by the host + // and path byte classes, so scanning for '/' alone is enough here. + let host_len = rest.iter().position(|&b| b == b'/').unwrap_or(rest.len()); + let (host, after_host) = rest.split_at(host_len); + if !is_verbatim_host(host) { + return None; + } + // Validating the path and locating the query is one pass, not two: a URL + // without a query is the common case and must not pay to be scanned twice. + // + // A query is only handled when a path precedes it. `https://host?q` gains a + // "/" in its serialization, so the input is no longer copied verbatim, and + // that shape is rare enough not to be worth a second output form. + let mut path_len = 0; + while path_len < after_host.len() { + let b = after_host[path_len]; + if b == b'?' { + break; + } + if !is_verbatim_path_byte(b) { + return None; + } + path_len += 1; + } + let (path, query) = if path_len < after_host.len() { + (&after_host[..path_len], Some(&after_host[path_len + 1..])) + } else { + (after_host, None) + }; + // '#' is outside both byte classes, so an input carrying a fragment is + // declined here rather than needing its own handling. + if let Some(query) = query { + if !query.iter().copied().all(is_verbatim_query_byte) { + return None; + } + } + // "." and ".." segments need the general parser's normalization. Rejecting + // every "/." covers "/./", "/../", and a trailing "/." or "/..". + if path.windows(2).any(|w| w == b"/.") { + return None; + } + + let host_end = host_start + host_len; + // A special URL with an empty path serializes with a "/" appended. That can + // only happen without a query, since a query requires a preceding path. + let serialization = if path.is_empty() { + let mut s = String::with_capacity(input.len() + 1); + s.push_str(input); + s.push('/'); + s + } else { + String::from(input) + }; + + // `query_start` indexes the '?' itself, not the first byte after it. + let query_start = match query { + Some(_) => Some(to_u32(host_end + path.len()).ok()?), + None => None, + }; + let host_start = to_u32(host_start).ok()?; + let host_end = to_u32(host_end).ok()?; + Some(Url { + serialization, + scheme_end, + username_end: host_start, + host_start, + host_end, + host: HostInternal::Domain, + port: None, + path_start: host_end, + query_start, + fragment_start: None, + }) +} + +/// Whether `input` begins with a URL scheme, i.e. `alpha *( alnum | "+" | "-" | "." ) ":"`. +/// +/// This only sizes the parse buffer, so it deliberately does not reproduce the +/// parser's full scheme handling: a wrong answer costs a little capacity, never +/// correctness. +fn starts_with_scheme(input: &str) -> bool { + let mut bytes = input.as_bytes().iter(); + if !matches!(bytes.next(), Some(c) if c.is_ascii_alphabetic()) { + return false; + } + bytes + .take_while( + |b| matches!(b, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'+' | b'-' | b'.' | b':'), + ) + .any(|&b| b == b':') +} + impl<'a> ParseOptions<'a> { /// Change the base URL /// @@ -303,8 +476,31 @@ impl<'a> ParseOptions<'a> { /// Parse an URL string with the configuration so far. pub fn parse(self, input: &str) -> Result { + // A base is not excluded here: the fast path only accepts inputs with + // exactly two slashes after the scheme, and the general parser consults + // the base only when fewer than two follow it (the "special relative" + // state in `parse_with_scheme`). So the base is provably ignored for + // everything the fast path accepts, which lets `Url::join` take it for + // absolute inputs. + // + // An encoding override does still change how a query is serialized, and + // a violation callback expects to be called, so both are excluded. + if self.encoding_override.is_none() && self.violation_fn.is_none() { + if let Some(url) = parse_simple_absolute(input) { + return Ok(url); + } + } + + // A relative reference is resolved against the base, so the result is + // roughly the base plus the reference; sizing for `input` alone makes + // every `Url::join` grow the buffer a realloc at a time. Absolute + // inputs ignore the base, so they keep their exact `input.len()`. + let capacity = match self.base_url { + Some(base) if !starts_with_scheme(input) => input.len() + base.serialization.len(), + _ => input.len(), + }; Parser { - serialization: String::with_capacity(input.len()), + serialization: String::with_capacity(capacity), base_url: self.base_url, query_encoding_override: self.encoding_override, violation_fn: self.violation_fn, @@ -3228,3 +3424,225 @@ impl Drop for UrlQuery<'_> { } } } + +#[cfg(test)] +mod fast_path_tests { + use super::*; + use std::eprintln; + + /// The general parser, with the fast path bypassed. + fn parse_general(input: &str) -> Result { + Parser { + serialization: String::with_capacity(input.len()), + base_url: None, + query_encoding_override: None, + violation_fn: None, + context: Context::UrlParser, + } + .parse_url(input) + } + + /// Whatever the fast path accepts, it must produce field-for-field exactly + /// what the general parser produces. + fn assert_agrees(input: &str) { + let fast = match parse_simple_absolute(input) { + Some(fast) => fast, + None => return, // declined; the general parser handles it + }; + let slow = parse_general(input).unwrap_or_else(|e| { + panic!( + "fast path accepted {:?} but general parser failed: {}", + input, e + ) + }); + assert_eq!( + fast.serialization, slow.serialization, + "serialization for {:?}", + input + ); + assert_eq!( + fast.scheme_end, slow.scheme_end, + "scheme_end for {:?}", + input + ); + assert_eq!( + fast.username_end, slow.username_end, + "username_end for {:?}", + input + ); + assert_eq!( + fast.host_start, slow.host_start, + "host_start for {:?}", + input + ); + assert_eq!(fast.host_end, slow.host_end, "host_end for {:?}", input); + assert_eq!(fast.host, slow.host, "host for {:?}", input); + assert_eq!(fast.port, slow.port, "port for {:?}", input); + assert_eq!( + fast.path_start, slow.path_start, + "path_start for {:?}", + input + ); + assert_eq!( + fast.query_start, slow.query_start, + "query_start for {:?}", + input + ); + assert_eq!( + fast.fragment_start, slow.fragment_start, + "fragment_start for {:?}", + input + ); + } + + #[test] + fn fast_path_matches_general_parser() { + const CASES: &[&str] = &[ + // Expected to take the fast path. + "https://example.com/bench", + "http://example.com/", + "https://example.com", + "https://deno.land/x/oak@v12.6.1/mod.ts", + "https://sub.domain.example.org/a/b/c.ts", + "https://ex-ample.com/a~b/c!d/e'f/(g)/h*i", + "https://a.b/c:d@e/f,g;h=i$j&k+l", + // Expected to be declined - each would need general-parser work. + "https://EXAMPLE.com/", // uppercase host + "https://example.com:8080/", // port + "https://user@example.com/", // userinfo + // Queries taking the fast path. + "https://example.com/a?q=1", + "https://example.com/?q=1", + "https://example.com/a?", + "https://example.com/a?q=1&r=2", + "https://example.com/a?q=%2F", + "https://example.com/a?q=a+b", + "https://example.com/a?a[]=1&b{}=2", + "https://example.com/a?q=/?:@!$&()*,;=", + // Queries the fast path declines. + "https://example.com/a?q=1#f", // fragment after query + "https://example.com/a?q= b", // space in query + "https://example.com/a?q=\"x\"", // quote in query + "https://example.com/a?q=", // angle brackets + "https://example.com/a?q='x'", // apostrophe (special-query only) + "https://example.com/a?q=café", // non-ASCII + "https://example.com?q=1", // query with no path + "https://example.com/a#frag", // fragment + "https://example.com/a%2Fb", // percent-encoding + "https://example.com/a b", // space needs encoding + "https://example.com/a\\b", // backslash + "https://example.com/./a", // dot segment + "https://example.com/../a", // dot-dot segment + "https://example.com/a/.", // trailing dot segment + "https://127.0.0.1/", // IPv4 + "https://[::1]/", // IPv6 + "https://example.123/", // host ends in a number + "https://xn--mgbh0fb.example/", // punycode + // Punycode that fails IDNA validation. The general parser rejects + // these, so a fast path that passed `xn--` through verbatim would + // wrongly accept them. + "https://xn--a.example/", + "https://xn--.example/", + "https://xn--0.example/", + "https://xn--zzz.example/", // valid punycode, round-trips unchanged + "https://مثال.example/", // non-ASCII + // Uppercase after a lowercase first byte, which the first-byte + // check alone does not catch. + "https://example.COM/", + "https://exAmple.com/", + "https://example.com/A/B", // uppercase is legal in a path + "https://example..com/", // empty label + "https://example.com./", // trailing dot + "https://.example.com/", // leading dot + "https://exa_mple.com/", // underscore + "https://example.com/a\tb", // tab + "https://example.com/a\nb", // newline + "ftp://example.com/", // other scheme + "HTTPS://example.com/", // uppercase scheme + "https:/example.com/", // single slash + "https://", // empty host + "https:///a", // empty host with path + "", + // Boundaries: the shortest accepted forms, and inputs shorter than + // the scheme literal, which must not index past the end. + "http://a", + "http://a/", + "https://a", + "http:/", + "http:", + "h", + "https:/", + "https://a/b", + // Hyphens and digits are legal inside a host, but not as its first + // byte nor as the first byte of its final label. + "https://a-b.c-d.example/", + "https://-example.com/", + "https://example-.com/", + "https://a1.b2.example/", + "https://example.c0m/", + "https://1example.com/", // digit-led host + "https://example.4/", // final label is a number + // Every byte the path class admits, in one path. + "https://example.com/aZ0-_.~/@+,=$&;:!*'()", + // Bytes just outside that class, each of which the general parser + // percent-encodes. + "https://example.com/a[b", + "https://example.com/a]b", + "https://example.com/a|b", + "https://example.com/a\"b", + "https://example.com/a^b", + "https://example.com/a`b", + "https://example.com/a{b", + "https://example.com/a}b", + "https://example.com/ab", + "https://example.com/a\rb", + // Dot segments in every position, and dots that do not form one. + "https://example.com/a/./b", + "https://example.com/a/../b", + "https://example.com/a/..", + "https://example.com/.", + "https://example.com/a.b/c", + "https://example.com/..a", + // Repeated and trailing separators. + "https://example.com//a", + "https://example.com/a//", + "https://example.com/a/", + // Non-ASCII in the path rather than the host. + "https://example.com/café", + // Long host and long path, to exercise both scans. + "https://aaaaaaaaaaaaaaaaaaaaaaaaaaaa.example.com/aaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbb", + ]; + for case in CASES { + assert_agrees(case); + } + } + + /// Exhaustive differential run over an external corpus, when one is given: + /// `URL_DIFF_CORPUS=/path/to/urls.txt cargo test` + #[test] + fn fast_path_matches_general_parser_on_corpus() { + let path = match std::env::var("URL_DIFF_CORPUS") { + Ok(path) => path, + Err(_) => return, + }; + let data = std::fs::read_to_string(&path).expect("corpus readable"); + let mut accepted = 0usize; + let mut total = 0usize; + for line in data.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + total += 1; + if parse_simple_absolute(line).is_some() { + accepted += 1; + } + assert_agrees(line); + } + eprintln!( + "corpus differential: {}/{} took the fast path", + accepted, total + ); + } +} diff --git a/url/tests/unit.rs b/url/tests/unit.rs index faeb7c5f4..423403f95 100644 --- a/url/tests/unit.rs +++ b/url/tests/unit.rs @@ -1412,3 +1412,216 @@ fn test_path_percent_encode() { let url = Url::parse("http://localhost/a}b").unwrap(); assert_eq!(url.path(), "/a%7Db"); } + +/// Setting a syntax-violation callback makes `ParseOptions::parse` skip the +/// fast path for simple absolute http(s) URLs, so parsing the same input with +/// and without one exercises both routes and must agree on every observable. +#[test] +fn test_fast_path_agrees_with_general_parser() { + const CASES: &[&str] = &[ + "https://example.com/bench", + "https://example.com", + "http://example.com/", + "https://deno.land/x/oak@v12.6.1/mod.ts", + "https://example.com/aZ0-_.~/@+,=$&;:!*'()", + "https://a-b.c-d.example/a/b/c", + "http://a", + "https://example.com//a//", + // Declined by the fast path; included so the test still covers them. + "https://EXAMPLE.com/", + "https://example.com:8080/a", + "https://user:pw@example.com/a", + "https://example.com/a?q=1&r=2", + "https://example.com/?q=1", + "https://example.com/a?", + "https://example.com/a?q=%2F&r=a+b", + "https://example.com/a?q=/?:@!$&()*,;=", + "https://example.com/a?q= b", + "https://example.com/a?q='x'", + "https://example.com/a?q=café", + "https://example.com?q=1", + "https://example.com/a?q=1#f", + "https://example.com/a#frag", + "https://example.com/a b", + "https://example.com/a%2Fb", + "https://example.com/a/../b", + "https://127.0.0.1/a", + "https://[::1]/a", + "https://xn--mgbh0fb.example/", + "https://xn--zzz.example/", + "https://مثال.example/", + "https://example.com/café", + // Uppercase after a lowercase first byte, which a first-byte check + // alone would not catch. + "https://example.COM/", + "https://exAmple.com/", + "https://example.com/A/B", + ]; + for input in CASES { + let fast = Url::parse(input).unwrap(); + let general = Url::options() + .syntax_violation_callback(Some(&|_| {})) + .parse(input) + .unwrap(); + assert_eq!(fast, general, "Url mismatch for {:?}", input); + assert_eq!(fast.as_str(), general.as_str(), "as_str for {:?}", input); + assert_eq!(fast.scheme(), general.scheme(), "scheme for {:?}", input); + assert_eq!(fast.host_str(), general.host_str(), "host for {:?}", input); + assert_eq!(fast.port(), general.port(), "port for {:?}", input); + assert_eq!(fast.path(), general.path(), "path for {:?}", input); + assert_eq!(fast.query(), general.query(), "query for {:?}", input); + assert_eq!( + fast.fragment(), + general.fragment(), + "fragment for {:?}", + input + ); + assert_eq!( + fast.username(), + general.username(), + "username for {:?}", + input + ); + assert_eq!( + fast.password(), + general.password(), + "password for {:?}", + input + ); + assert_eq!( + fast.path_segments().map(|s| s.collect::>()), + general.path_segments().map(|s| s.collect::>()), + "path_segments for {:?}", + input + ); + } +} + +/// A `Url` from the fast path must stay correct once mutated, which is what +/// catches a wrong component offset that the getters alone would not reveal. +#[test] +fn test_fast_path_url_is_mutable() { + let mut url = Url::parse("https://example.com/a/b").unwrap(); + url.set_query(Some("q=1")); + assert_eq!(url.as_str(), "https://example.com/a/b?q=1"); + url.set_fragment(Some("frag")); + assert_eq!(url.as_str(), "https://example.com/a/b?q=1#frag"); + url.set_path("/c"); + assert_eq!(url.as_str(), "https://example.com/c?q=1#frag"); + url.set_host(Some("other.example")).unwrap(); + assert_eq!(url.as_str(), "https://other.example/c?q=1#frag"); + url.set_port(Some(8080)).unwrap(); + assert_eq!(url.as_str(), "https://other.example:8080/c?q=1#frag"); + url.set_username("user").unwrap(); + assert_eq!(url.as_str(), "https://user@other.example:8080/c?q=1#frag"); + + let mut empty_path = Url::parse("https://example.com").unwrap(); + assert_eq!(empty_path.path(), "/"); + empty_path.set_path("/x"); + assert_eq!(empty_path.as_str(), "https://example.com/x"); +} + +/// `Url::join` sizes its buffer from the base plus the reference, which must +/// not change what it resolves to. +#[test] +fn test_join_resolution_is_unaffected_by_buffer_sizing() { + let base = Url::parse("https://example.com/a/b/c?x=1#y").unwrap(); + let cases: &[(&str, &str)] = &[ + ("./d.ts", "https://example.com/a/b/d.ts"), + ("../d.ts", "https://example.com/a/d.ts"), + ("../../d.ts", "https://example.com/d.ts"), + ("../../../d.ts", "https://example.com/d.ts"), + ("/d.ts", "https://example.com/d.ts"), + ("d.ts", "https://example.com/a/b/d.ts"), + ("", "https://example.com/a/b/c?x=1"), + ("#z", "https://example.com/a/b/c?x=1#z"), + ("?q=2", "https://example.com/a/b/c?q=2"), + ("//other.example/d", "https://other.example/d"), + ("http://other.example/d", "http://other.example/d"), + ("https://other.example/d", "https://other.example/d"), + ]; + for (input, expected) in cases { + let joined = base.join(input).unwrap(); + assert_eq!(joined.as_str(), *expected, "join({:?})", input); + // Resolving against a base must match parsing the result outright. + assert_eq!(joined, Url::parse(expected).unwrap(), "join({:?})", input); + } +} + +/// An absolute input ignores the base, so `join` must agree with `parse` for +/// every input the fast path accepts, whatever the base looks like. +#[test] +fn test_join_absolute_matches_parse_for_every_base() { + const BASES: &[&str] = &[ + "https://example.com/a/b/c?x=1#y", + "http://other.example/", + "https://user:pw@host.example:8080/p?q#f", + "file:///a/b/c", + "sc://%C3%B1/x", // non-special scheme + "data:text/plain,foo", // cannot-be-a-base + "https://xn--mgbh0fb.example/", + ]; + const INPUTS: &[&str] = &[ + // Shapes the fast path accepts. + "https://target.example/p", + "http://target.example/p/q?r=1", + "https://target.example", + "https://target.example/", + // Shapes it declines, which must still resolve identically. + "https://TARGET.example/p", + "https://target.example:9000/p", + "https://target.example/p#f", + "https://127.0.0.1/p", + "https://target.example/a/../b", + // Extra slashes: the base is still ignored, but not by the fast path. + "https:///target.example/p", + "https:////target.example/p", + ]; + for base in BASES { + let base = Url::parse(base).unwrap(); + for input in INPUTS { + let joined = base.join(input); + let parsed = Url::parse(input); + assert_eq!( + joined.as_ref().map(|u| u.as_str()), + parsed.as_ref().map(|u| u.as_str()), + "join({:?}) on base {:?}", + input, + base.as_str() + ); + assert_eq!( + joined, + parsed, + "join({:?}) on base {:?}", + input, + base.as_str() + ); + } + } +} + +/// Relative inputs must keep resolving against the base, i.e. the fast path +/// must not swallow them now that a base no longer disables it. +#[test] +fn test_join_relative_still_resolves_against_base() { + let base = Url::parse("https://example.com/a/b/c?x=1#y").unwrap(); + assert_eq!( + base.join("./d").unwrap().as_str(), + "https://example.com/a/b/d" + ); + assert_eq!(base.join("/d").unwrap().as_str(), "https://example.com/d"); + assert_eq!( + base.join("//other.example/d").unwrap().as_str(), + "https://other.example/d" + ); + // Same scheme with fewer than two slashes is the "special relative" state, + // which resolves against the base rather than being an absolute URL. + assert_eq!( + base.join("https:/d").unwrap().as_str(), + "https://example.com/d" + ); + assert_eq!( + base.join("https:d").unwrap().as_str(), + "https://example.com/a/b/d" + ); +}