//! ### Research //! //! * **windows** //! - [`dirent.c`](https://github.com/git/git/blob/main/compat/win32/dirent.c#L31:L31) contains all implementation (seemingly) of opening directories and reading their entries, along with all path conversions (UTF-16 for windows). This is done on the fly so git can work with [in UTF-8](https://github.com/git/git/blob/main/compat/win32/dirent.c#L12:L12). //! - mingw [is used for the conversion](https://github.com/git/git/blob/main/compat/mingw.h#L579:L579) and it appears they handle surrogates during the conversion, maybe some sort of non-strict UTF-8 converter? Actually it uses [WideCharToMultiByte](https://docs.microsoft.com/en-us/windows/win32/api/stringapiset/nf-stringapiset-widechartomultibyte) //! under the hood which by now does fail if the UTF-8 would be invalid unicode, i.e. unicode pairs. //! - `OsString` on windows already stores strings as WTF-8, which supports [surrogate pairs](https://unicodebook.readthedocs.io/unicode_encodings.html), //! something that UTF-8 isn't allowed do it for security reasons, after all it's UTF-16 specific and exists only to extend //! the encodable code-points. //! - informative reading on [WTF-8](https://simonsapin.github.io/wtf-8/#motivation) which is the encoding used by Rust //! internally that deals with surrogates and non-wellformed surrogates (those that aren't in pairs). //! * **unix** //! - It uses [opendir](https://man7.org/linux/man-pages/man3/opendir.3.html) and [readdir](https://man7.org/linux/man-pages/man3/readdir.3.html) //! respectively. There is no encoding specified, except that these paths are null-terminated. //! //! ### Learnings //! //! Surrogate pairs are a way to extend the encodable value range in UTF-16 encodings, used primarily on windows and in Javascript. //! For a long time these codepoints used for surrogates, always to be used in pairs, were not assigned, until…they were for rare //! emojies and the likes. The unicode standard does not require surrogates to happen in pairs, even though by now unpaired surrogates //! in UTF-16 are considered ill-formed, which aren't supposed to be converted to UTF-8 for example. //! //! This is the reason we have to deal with `to_string_lossy()`, it's _just_ for that quirk. //! //! This also means the only platform ever eligible to see conversion errors is windows, and there it's only older pre-vista //! windows versions which incorrectly allow ill-formed UTF-16 strings. Newer versions don't perform such conversions anymore, for //! example when going from UTF-16 to UTF-8, they will trigger an error. //! //! ### Conclusions //! //! Since [WideCharToMultiByte](https://docs.microsoft.com/en-us/windows/win32/api/stringapiset/nf-stringapiset-widechartomultibyte) by now is //! fixed (Vista onward) to produce valid UTF-8, lone surrogate codepoints will cause failure, which `git` //! [doesn't care about](https://github.com/git/git/blob/main/compat/win32/dirent.c#L12:L12). //! //! We will, though, which means from now on we can just convert to UTF-8 on windows and bubble up errors where necessary, //! preventing potential mismatched surrogate pairs to ever be saved on disk by gitoxide. //! //! Even though the error only exists on older windows versions, we will represent it in the type system through fallible function calls. //! Callers may `.expect()` on the result to indicate they don't wish to handle this special and rare case. Note that servers should not //! ever get into a code-path which does panic though. use std::{ borrow::Cow, ffi::OsStr, path::{Path, PathBuf}, }; #[derive(Debug)] /// The error type returned by [`into_bytes()`] and others may suffer from failed conversions from or to bytes. pub struct Utf8Error; impl std::fmt::Display for Utf8Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("Could not convert to UTF8 or from UTF8 due to ill-formed input") } } impl std::error::Error for Utf8Error {} /// Like [`into_bytes()`], but takes `OsStr` as input for a lossless, but fallible, conversion. pub fn os_str_into_bytes(path: &OsStr) -> Result<&[u8], Utf8Error> { let path = into_bytes(Cow::Borrowed(path.as_ref()))?; match path { Cow::Borrowed(path) => Ok(path), Cow::Owned(_) => unreachable!("borrowed cows stay borrowed"), } } /// Convert the given path either into its raw bytes on unix or its UTF8 encoded counterpart on windows. /// /// On windows, if the source Path contains ill-formed, lone surrogates, the UTF-8 conversion will fail /// causing `Utf8Error` to be returned. pub fn into_bytes<'a>(path: impl Into>) -> Result, Utf8Error> { let path = path.into(); let utf8_bytes = match path { Cow::Owned(path) => Cow::Owned({ #[cfg(unix)] let p = { use std::os::unix::ffi::OsStringExt; path.into_os_string().into_vec() }; #[cfg(not(unix))] let p: Vec<_> = path.into_os_string().into_string().map_err(|_| Utf8Error)?.into(); p }), Cow::Borrowed(path) => Cow::Borrowed({ #[cfg(unix)] let p = { use std::os::unix::ffi::OsStrExt; path.as_os_str().as_bytes() }; #[cfg(not(unix))] let p = path.to_str().ok_or(Utf8Error)?.as_bytes(); p }), }; Ok(utf8_bytes) } /// Similar to [`into_bytes()`] but panics if malformed surrogates are encountered on windows. pub fn into_bytes_or_panic_on_windows<'a>(path: impl Into>) -> Cow<'a, [u8]> { into_bytes(path).expect("prefix path doesn't contain ill-formed UTF-8") } /// Given `input` bytes, produce a `Path` from them ignoring encoding entirely if on unix. /// /// On windows, the input is required to be valid UTF-8, which is guaranteed if we wrote it before. There are some potential /// git versions and windows installation which produce mal-formed UTF-16 if certain emojies are in the path. It's as rare as /// it sounds, but possible. pub fn from_byte_slice(input: &[u8]) -> Result<&Path, Utf8Error> { #[cfg(unix)] let p = { use std::os::unix::ffi::OsStrExt; OsStr::from_bytes(input).as_ref() }; #[cfg(not(unix))] let p = Path::new(std::str::from_utf8(input).map_err(|_| Utf8Error)?); Ok(p) } /// Similar to [`from_byte_slice()`], but takes either borrowed or owned `input`. pub fn from_bytes<'a>(input: impl Into>) -> Result, Utf8Error> { let input = input.into(); match input { Cow::Borrowed(input) => from_byte_slice(input).map(Cow::Borrowed), Cow::Owned(input) => from_byte_vec(input).map(Cow::Owned), } } /// Similar to [`from_byte_slice()`], but takes either borrowed or owned `input` as bstr. #[cfg(feature = "bstr")] pub fn from_bstr<'a>(input: impl Into>) -> Result, Utf8Error> { let input = input.into(); match input { Cow::Borrowed(input) => from_byte_slice(input).map(Cow::Borrowed), Cow::Owned(input) => from_byte_vec(input).map(Cow::Owned), } } /// Similar to [`from_byte_slice()`], but takes and produces owned data. pub fn from_byte_vec(input: impl Into>) -> Result { let input = input.into(); #[cfg(unix)] let p = { use std::os::unix::ffi::OsStringExt; std::ffi::OsString::from_vec(input).into() }; #[cfg(not(unix))] let p = PathBuf::from(String::from_utf8(input).map_err(|_| Utf8Error)?); Ok(p) } /// Similar to [`from_byte_vec()`], but will panic if there is ill-formed UTF-8 in the `input`. pub fn from_byte_vec_or_panic_on_windows(input: impl Into>) -> PathBuf { from_byte_vec(input).expect("well-formed UTF-8 on windows") } /// Similar to [`from_byte_slice()`], but will panic if there is ill-formed UTF-8 in the `input`. pub fn from_byte_slice_or_panic_on_windows(input: &[u8]) -> &Path { from_byte_slice(input).expect("well-formed UTF-8 on windows") } /// Methods to handle paths as bytes and do conversions between them. pub mod convert { use std::borrow::Cow; fn replace<'a>(path: impl Into>, find: u8, replace: u8) -> Cow<'a, [u8]> { let path = path.into(); match path { Cow::Owned(mut path) => { for b in path.iter_mut().filter(|b| **b == find) { *b = replace; } path.into() } Cow::Borrowed(path) => { if !path.contains(&find) { return path.into(); } let mut path = path.to_owned(); for b in path.iter_mut().filter(|b| **b == find) { *b = replace; } path.into() } } } /// Replaces windows path separators with slashes. pub fn to_native_separators<'a>(path: impl Into>) -> Cow<'a, [u8]> { #[cfg(not(windows))] let p = to_unix_separators(path); #[cfg(windows)] let p = to_windows_separators(path); p } /// Replaces windows path separators with slashes. pub fn to_unix_separators<'a>(path: impl Into>) -> Cow<'a, [u8]> { replace(path, b'\\', b'/') } /// Find backslashes and replace them with slashes, which typically resembles a unix path. /// /// No other transformation is performed, the caller must check other invariants. pub fn to_windows_separators<'a>(path: impl Into>) -> Cow<'a, [u8]> { replace(path, b'/', b'\\') } /// Obtain a `BStr` compatible `Cow` from one that is bytes. #[cfg(feature = "bstr")] pub fn into_bstr(path: Cow<'_, [u8]>) -> Cow<'_, bstr::BStr> { match path { Cow::Owned(p) => Cow::Owned(p.into()), Cow::Borrowed(p) => Cow::Borrowed(p.into()), } } }