Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 53 additions & 21 deletions arrow-cast/src/parse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,8 @@ fn to_timestamp_nanos(dt: NaiveDateTime) -> Result<i64, ArrowError> {
///
/// Internally, this function uses the `chrono` library for the time parsing
///
/// Leap seconds are rejected because Arrow time types cannot represent them.
///
/// ## Timezone / Offset Handling
///
/// This function does not support parsing strings with a timezone
Expand Down Expand Up @@ -404,20 +406,20 @@ fn string_to_time(s: &str) -> Option<NaiveTime> {
(h, None) => h,
};

// Handle leap second
let (second, nanoseconds) = match digits[4] * 10 + digits[5] {
60 => (59, nanoseconds + 1_000_000_000),
s => (s, nanoseconds),
};

NaiveTime::from_hms_nano_opt(
hour as _,
(digits[2] * 10 + digits[3]) as _,
second as _,
(digits[4] * 10 + digits[5]) as _,
nanoseconds,
)
}

fn string_to_time_formatted(string: &str, format: &str) -> Option<NaiveTime> {
let time = NaiveTime::parse_from_str(string, format).ok()?;
// Chrono represents leap seconds with a nanosecond value of at least 1_000_000_000.
(time.nanosecond() < 1_000_000_000).then_some(time)
}

/// Specialized parsing implementations to convert strings to Arrow types.
///
/// This is used by csv and json reader and can be used directly as well.
Expand Down Expand Up @@ -604,7 +606,7 @@ impl Parser for Time64NanosecondType {
}

fn parse_formatted(string: &str, format: &str) -> Option<Self::Native> {
let nt = NaiveTime::parse_from_str(string, format).ok()?;
let nt = string_to_time_formatted(string, format)?;
Some(nt.num_seconds_from_midnight() as i64 * 1_000_000_000 + nt.nanosecond() as i64)
}
}
Expand All @@ -629,7 +631,7 @@ impl Parser for Time64MicrosecondType {
}

fn parse_formatted(string: &str, format: &str) -> Option<Self::Native> {
let nt = NaiveTime::parse_from_str(string, format).ok()?;
let nt = string_to_time_formatted(string, format)?;
Some(nt.num_seconds_from_midnight() as i64 * 1_000_000 + nt.nanosecond() as i64 / 1_000)
}
}
Expand All @@ -654,7 +656,7 @@ impl Parser for Time32MillisecondType {
}

fn parse_formatted(string: &str, format: &str) -> Option<Self::Native> {
let nt = NaiveTime::parse_from_str(string, format).ok()?;
let nt = string_to_time_formatted(string, format)?;
Some(nt.num_seconds_from_midnight() as i32 * 1_000 + nt.nanosecond() as i32 / 1_000_000)
}
}
Expand All @@ -679,7 +681,7 @@ impl Parser for Time32SecondType {
}

fn parse_formatted(string: &str, format: &str) -> Option<Self::Native> {
let nt = NaiveTime::parse_from_str(string, format).ok()?;
let nt = string_to_time_formatted(string, format)?;
Some(nt.num_seconds_from_midnight() as i32 + nt.nanosecond() as i32 / 1_000_000_000)
}
}
Expand Down Expand Up @@ -2253,10 +2255,7 @@ mod tests {
assert_eq!(Time64NanosecondType::parse("1"), Some(1));

// leap second
assert_eq!(
Time64NanosecondType::parse("23:59:60"),
Some(86_400_000_000_000)
);
assert_eq!(Time64NanosecondType::parse("23:59:60"), None);

// custom format
assert_eq!(
Expand Down Expand Up @@ -2330,10 +2329,7 @@ mod tests {
assert_eq!(Time64MicrosecondType::parse("1"), Some(1));

// leap second
assert_eq!(
Time64MicrosecondType::parse("23:59:60"),
Some(86_400_000_000)
);
assert_eq!(Time64MicrosecondType::parse("23:59:60"), None);

// custom format
assert_eq!(
Expand Down Expand Up @@ -2380,7 +2376,7 @@ mod tests {
assert_eq!(Time32MillisecondType::parse("1"), Some(1));

// leap second
assert_eq!(Time32MillisecondType::parse("23:59:60"), Some(86_400_000));
assert_eq!(Time32MillisecondType::parse("23:59:60"), None);

// custom format
assert_eq!(
Expand Down Expand Up @@ -2410,7 +2406,7 @@ mod tests {
assert_eq!(Time32SecondType::parse("1"), Some(1));

// leap second
assert_eq!(Time32SecondType::parse("23:59:60"), Some(86400));
assert_eq!(Time32SecondType::parse("23:59:60"), None);

// custom format
assert_eq!(
Expand All @@ -2419,6 +2415,42 @@ mod tests {
);
}

#[test]
fn parse_time_boundaries() {
fn check<T: Parser>(units_per_second: i64)
where
T::Native: Into<i64>,
{
for (input, format, expected) in [
("00:00:00", "%H:%M:%S", Some(0)),
(
"23:59:59.999999999",
"%H:%M:%S%.f",
Some(86_400 * units_per_second - 1),
),
("23:59:60", "%H:%M:%S", None),
("23:59:60.5", "%H:%M:%S%.f", None),
("12:34:60", "%H:%M:%S", None),
("11:59:60 PM", "%I:%M:%S %p", None),
] {
assert_eq!(T::parse(input).map(Into::into), expected, "{input}");
assert_eq!(
T::parse_formatted(input, format).map(Into::into),
expected,
"{input} ({format})"
);
}
assert_eq!(T::parse(" 23:59:60 "), None);
}

check::<Time32SecondType>(1);
check::<Time32MillisecondType>(1_000);
check::<Time64MicrosecondType>(1_000_000);
check::<Time64NanosecondType>(1_000_000_000);

assert!(string_to_time_nanoseconds("23:59:60").is_err());
}

#[test]
fn test_string_to_time_invalid() {
let cases = [
Expand Down
20 changes: 14 additions & 6 deletions arrow-cast/tests/cast/date_time.rs
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ fn test_cast_string_to_time32second() {
let b = cast(array, &to_type).unwrap();
let c = b.as_primitive::<Time32SecondType>();
assert_eq!(29315, c.value(0));
assert_eq!(29340, c.value(1));
assert!(c.is_null(1));
assert!(c.is_null(2));
assert!(c.is_null(3));
assert!(c.is_null(4));
Expand All @@ -267,7 +267,7 @@ fn test_cast_string_to_time32second() {
let err = cast_with_options(array, &to_type, &options).unwrap_err();
assert_eq!(
err.to_string(),
"Cast error: Cannot cast string '08:08:61.091323414' to value of Time32(s) type"
"Cast error: Cannot cast string '08:08:60.091323414' to value of Time32(s) type"
);
}
}
Expand Down Expand Up @@ -300,7 +300,7 @@ fn test_cast_string_to_time32millisecond() {
let b = cast(array, &to_type).unwrap();
let c = b.as_primitive::<Time32MillisecondType>();
assert_eq!(29315091, c.value(0));
assert_eq!(29340091, c.value(1));
assert!(c.is_null(1));
assert!(c.is_null(2));
assert!(c.is_null(3));
assert!(c.is_null(4));
Expand All @@ -312,7 +312,7 @@ fn test_cast_string_to_time32millisecond() {
let err = cast_with_options(array, &to_type, &options).unwrap_err();
assert_eq!(
err.to_string(),
"Cast error: Cannot cast string '08:08:61.091323414' to value of Time32(ms) type"
"Cast error: Cannot cast string '08:08:60.091323414' to value of Time32(ms) type"
);
}
}
Expand All @@ -321,16 +321,19 @@ fn test_cast_string_to_time32millisecond() {
fn test_cast_string_to_time64microsecond() {
let a0 = Arc::new(StringViewArray::from(vec![
Some("08:08:35.091323414"),
Some("23:59:60"), // leap second
Some("Not a valid time"),
None,
])) as ArrayRef;
let a1 = Arc::new(StringArray::from(vec![
Some("08:08:35.091323414"),
Some("23:59:60"), // leap second
Some("Not a valid time"),
None,
])) as ArrayRef;
let a2 = Arc::new(LargeStringArray::from(vec![
Some("08:08:35.091323414"),
Some("23:59:60"), // leap second
Some("Not a valid time"),
None,
])) as ArrayRef;
Expand All @@ -341,6 +344,7 @@ fn test_cast_string_to_time64microsecond() {
assert_eq!(29315091323, c.value(0));
assert!(c.is_null(1));
assert!(c.is_null(2));
assert!(c.is_null(3));

let options = CastOptions {
safe: false,
Expand All @@ -349,7 +353,7 @@ fn test_cast_string_to_time64microsecond() {
let err = cast_with_options(array, &to_type, &options).unwrap_err();
assert_eq!(
err.to_string(),
"Cast error: Cannot cast string 'Not a valid time' to value of Time64(µs) type"
"Cast error: Cannot cast string '23:59:60' to value of Time64(µs) type"
);
}
}
Expand All @@ -358,16 +362,19 @@ fn test_cast_string_to_time64microsecond() {
fn test_cast_string_to_time64nanosecond() {
let a0 = Arc::new(StringViewArray::from(vec![
Some("08:08:35.091323414"),
Some("23:59:60"), // leap second
Some("Not a valid time"),
None,
])) as ArrayRef;
let a1 = Arc::new(StringArray::from(vec![
Some("08:08:35.091323414"),
Some("23:59:60"), // leap second
Some("Not a valid time"),
None,
])) as ArrayRef;
let a2 = Arc::new(LargeStringArray::from(vec![
Some("08:08:35.091323414"),
Some("23:59:60"), // leap second
Some("Not a valid time"),
None,
])) as ArrayRef;
Expand All @@ -378,6 +385,7 @@ fn test_cast_string_to_time64nanosecond() {
assert_eq!(29315091323414, c.value(0));
assert!(c.is_null(1));
assert!(c.is_null(2));
assert!(c.is_null(3));

let options = CastOptions {
safe: false,
Expand All @@ -386,7 +394,7 @@ fn test_cast_string_to_time64nanosecond() {
let err = cast_with_options(array, &to_type, &options).unwrap_err();
assert_eq!(
err.to_string(),
"Cast error: Cannot cast string 'Not a valid time' to value of Time64(ns) type"
"Cast error: Cannot cast string '23:59:60' to value of Time64(ns) type"
);
}
}
Expand Down