From a42b385e9bee97b3354aafdb24579effe16d3dea Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Mon, 29 Jan 2024 09:37:06 -0500 Subject: [PATCH] puffin-client: add SimpleMetadataRaw (#1150) This adds what is effectively an owned wrapper around `Archived`. Normally, an `Archived` has to be used behind a pointer (since it has a lifetime attached to its underlying byte buffer), but we create a wrapper around it that owns the underlying buffer and provides free access to the archived type. This in effect creates an anchor point for the archived type and lets us pass it around easily. (There has to be an anchor point for it somewhere.) An alternative to this approach would be to store it as a file backed memory map. But in practice, we're dealing with small files, and just reading them on to the heap is likely to be faster. (Memory maps also have wildly different perf characteristics across platforms.) Note that this commit just defines the type. It isn't actually used anywhere yet. --- crates/puffin-client/src/error.rs | 6 + crates/puffin-client/src/lib.rs | 5 +- crates/puffin-client/src/registry_client.rs | 97 ++++++++++ crates/puffin-client/src/rkyvutil.rs | 195 ++++++++++++++++++++ 4 files changed, 301 insertions(+), 2 deletions(-) create mode 100644 crates/puffin-client/src/rkyvutil.rs diff --git a/crates/puffin-client/src/error.rs b/crates/puffin-client/src/error.rs index af0af048c..d519fd91c 100644 --- a/crates/puffin-client/src/error.rs +++ b/crates/puffin-client/src/error.rs @@ -125,4 +125,10 @@ pub enum ErrorKind { #[error("Unsupported `Content-Type` \"{1}\" for {0}. Expected JSON or HTML.")] UnsupportedMediaType(Url, String), + + #[error("Reading from cache archive failed: {0}")] + ArchiveRead(String), + + #[error("Writing to cache archive failed: {0}")] + ArchiveWrite(#[source] crate::rkyvutil::SerializerError), } diff --git a/crates/puffin-client/src/lib.rs b/crates/puffin-client/src/lib.rs index 93f5d9a2f..b63551a32 100644 --- a/crates/puffin-client/src/lib.rs +++ b/crates/puffin-client/src/lib.rs @@ -2,8 +2,8 @@ pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithC pub use error::{Error, ErrorKind}; pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexClient, FlatIndexError}; pub use registry_client::{ - read_metadata_async, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum, - VersionFiles, + read_metadata_async, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadataRaw, + SimpleMetadatum, VersionFiles, }; mod cache_headers; @@ -13,3 +13,4 @@ mod flat_index; mod html; mod registry_client; mod remote_metadata; +mod rkyvutil; diff --git a/crates/puffin-client/src/registry_client.rs b/crates/puffin-client/src/registry_client.rs index f440b4512..efc4279a3 100644 --- a/crates/puffin-client/src/registry_client.rs +++ b/crates/puffin-client/src/registry_client.rs @@ -616,6 +616,103 @@ impl IntoIterator for SimpleMetadata { } } +/// An owned archived type for `SimpleMetadata`. +/// +/// This type is effectively a `Archived`, but owns its buffer. +/// Constructing the type requires validating that the bytes are valid, but +/// subsequent accesses are free. +#[derive(Debug)] +pub struct SimpleMetadataRaw { + raw: rkyv::util::AlignedVec, +} + +impl SimpleMetadataRaw { + /// Create a new owned archived value from the raw aligned bytes of the + /// serialized representation of a `SimpleMetadata`. + /// + /// # Errors + /// + /// If the bytes fail validation (e.g., contains unaligned pointers or + /// strings aren't valid UTF-8), then this returns an error. + pub fn new(raw: rkyv::util::AlignedVec) -> Result { + // We convert the error to a simple string because... the error type + // does not implement Send. And I don't think we really need to keep + // the error type around anyway. + let _ = rkyv::validation::validators::check_archived_root::(&raw) + .map_err(|e| ErrorKind::ArchiveRead(e.to_string()))?; + Ok(SimpleMetadataRaw { raw }) + } + + /// Like `SimpleMetadataRaw::new`, but reads the value from the given + /// reader. + /// + /// Note that this consumes the entirety of the given reader. + /// + /// # Errors + /// + /// If the bytes fail validation (e.g., contains unaligned pointers or + /// strings aren't valid UTF-8), then this returns an error. + pub fn from_reader(mut rdr: R) -> Result { + let mut buf = rkyv::util::AlignedVec::with_capacity(1024); + buf.extend_from_reader(&mut rdr).map_err(ErrorKind::Io)?; + SimpleMetadataRaw::new(buf) + } + + /// Creates an owned archive value from the unarchived value. + /// + /// # Errors + /// + /// This can fail if creating an archive for the given type fails. + /// Currently, this, at minimum, includes cases where a `SimpleMetadata` + /// contains a `PathBuf` that is not valid UTF-8. + pub fn from_unarchived(unarchived: &SimpleMetadata) -> Result { + use rkyv::ser::Serializer; + + let mut serializer = crate::rkyvutil::Serializer::<4096>::new(); + serializer + .serialize_value(unarchived) + .map_err(ErrorKind::ArchiveWrite)?; + let raw = serializer.into_serializer().into_inner(); + Ok(SimpleMetadataRaw { raw }) + } + + /// Write the underlying bytes of this archived value to the given writer. + /// + /// # Errors + /// + /// Any failures from writing are returned to the caller. + pub fn write(&self, mut wtr: W) -> Result<(), Error> { + Ok(wtr.write_all(&self.raw).map_err(ErrorKind::Io)?) + } + + /// Returns this owned archive value as a borrowed archive value. + pub fn as_archived(&self) -> &rkyv::Archived { + // SAFETY: We've validated that our underlying buffer is a valid + // archive for SimpleMetadata in the constructor, so we can skip + // validation here. Since we don't mutate the buffer, this conversion + // is guaranteed to be correct. + unsafe { rkyv::archived_root::(&self.raw) } + } + + /// Returns the raw underlying bytes of this owned archive value. + /// + /// They are guaranteed to be a valid serialization of + /// `Archived`. + pub fn as_bytes(&self) -> &[u8] { + &self.raw + } + + /// Deserialize this owned archived value into the original + /// `SimpleMetadata`. + pub fn deserialize(&self) -> SimpleMetadata { + use rkyv::Deserialize; + + self.as_archived() + .deserialize(&mut rkyv::de::deserializers::SharedDeserializeMap::new()) + .expect("valid archive must deserialize correctly") + } +} + #[derive(Debug)] enum MediaType { Json, diff --git a/crates/puffin-client/src/rkyvutil.rs b/crates/puffin-client/src/rkyvutil.rs new file mode 100644 index 000000000..e1bc1e31b --- /dev/null +++ b/crates/puffin-client/src/rkyvutil.rs @@ -0,0 +1,195 @@ +/*! +Defines some helpers for use with `rkyv`. + +Principally, we define our own implementation of the `Serializer` trait. +This involves a fair bit of boiler plate, but it was largely copied from +`CompositeSerializer`. (Indeed, our serializer wraps a `CompositeSerializer`.) + +The motivation for doing this is to support the archiving of `PathBuf` types. +Namely, for reasons AG doesn't completely understand at the time of writing, +the serializers that rkyv bundled cannot handle the error returned by `PathBuf` +potentially failing to serialize. Namely, since `PathBuf` has a platform +dependent representation when its contents are not valid UTF-8, serialization +in `rkyv` requires that it be valid UTF-8. If it isn't, serialization will +fail. +*/ + +use std::convert::Infallible; + +use rkyv::{ + ser::serializers::{ + AlignedSerializer, AllocScratch, AllocScratchError, AllocSerializer, CompositeSerializer, + CompositeSerializerError, FallbackScratch, HeapScratch, SharedSerializeMap, + SharedSerializeMapError, + }, + util::AlignedVec, + Archive, ArchiveUnsized, Fallible, +}; + +pub struct Serializer { + composite: CompositeSerializer< + AlignedSerializer, + FallbackScratch, AllocScratch>, + SharedSerializeMap, + >, +} + +impl Serializer { + pub fn new() -> Serializer { + let composite = AllocSerializer::::default(); + Serializer { composite } + } + + pub fn into_serializer(self) -> AlignedSerializer { + self.composite.into_serializer() + } +} + +impl Fallible for Serializer { + type Error = SerializerError; +} + +impl rkyv::ser::Serializer for Serializer { + #[inline] + fn pos(&self) -> usize { + self.composite.pos() + } + + #[inline] + fn write(&mut self, bytes: &[u8]) -> Result<(), Self::Error> { + self.composite + .write(bytes) + .map_err(SerializerError::Composite) + } + + #[inline] + fn pad(&mut self, padding: usize) -> Result<(), Self::Error> { + self.composite + .pad(padding) + .map_err(SerializerError::Composite) + } + + #[inline] + fn align(&mut self, align: usize) -> Result { + self.composite + .align(align) + .map_err(SerializerError::Composite) + } + + #[inline] + fn align_for(&mut self) -> Result { + self.composite + .align_for::() + .map_err(SerializerError::Composite) + } + + #[inline] + unsafe fn resolve_aligned( + &mut self, + value: &T, + resolver: T::Resolver, + ) -> Result { + self.composite + .resolve_aligned::(value, resolver) + .map_err(SerializerError::Composite) + } + + #[inline] + unsafe fn resolve_unsized_aligned( + &mut self, + value: &T, + to: usize, + metadata_resolver: T::MetadataResolver, + ) -> Result { + self.composite + .resolve_unsized_aligned(value, to, metadata_resolver) + .map_err(SerializerError::Composite) + } +} + +impl rkyv::ser::ScratchSpace for Serializer { + #[inline] + unsafe fn push_scratch( + &mut self, + layout: std::alloc::Layout, + ) -> Result, Self::Error> { + self.composite + .push_scratch(layout) + .map_err(SerializerError::Composite) + } + + #[inline] + unsafe fn pop_scratch( + &mut self, + ptr: std::ptr::NonNull, + layout: std::alloc::Layout, + ) -> Result<(), Self::Error> { + self.composite + .pop_scratch(ptr, layout) + .map_err(SerializerError::Composite) + } +} + +impl rkyv::ser::SharedSerializeRegistry for Serializer { + #[inline] + fn get_shared_ptr(&self, value: *const u8) -> Option { + self.composite.get_shared_ptr(value) + } + + #[inline] + fn add_shared_ptr(&mut self, value: *const u8, pos: usize) -> Result<(), Self::Error> { + self.composite + .add_shared_ptr(value, pos) + .map_err(SerializerError::Composite) + } +} + +#[derive(Debug)] +pub enum SerializerError { + Composite(CompositeSerializerError), + AsString(rkyv::with::AsStringError), +} + +impl std::fmt::Display for SerializerError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match *self { + SerializerError::Composite(ref e) => e.fmt(f), + SerializerError::AsString(ref e) => e.fmt(f), + } + } +} + +impl std::error::Error for SerializerError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match *self { + SerializerError::Composite(ref e) => Some(e), + SerializerError::AsString(ref e) => Some(e), + } + } +} + +/// Provides a way to build a serializer error if converting an +/// `OsString`/`PathBuf` to a `String` fails. i.e., It's invalid UTF-8. +/// +/// This impl is the entire point of this module. For whatever reason, none of +/// the serializers in rkyv handle this particular error case. Apparently, the +/// only way to use `rkyv::with::AsString` with `PathBuf` is to create one's +/// own serializer and provide a `From` impl for the `AsStringError` type. +/// Specifically, from the [AsString] docs: +/// +/// > Regular serializers don’t support the custom error handling needed for +/// > this type by default. To use this wrapper, a custom serializer with an +/// > error type satisfying ::Error: From must be +/// > provided. +/// +/// If we didn't need to use `rkyv::with::AsString` (which we do for +/// serializing `PathBuf` at time of writing), then we could just +/// use an `AllocSerializer` directly (which is a type alias for +/// `CompositeSerializer<...>`. +/// +/// [AsString]: https://docs.rs/rkyv/0.7.43/rkyv/with/struct.AsString.html +impl From for SerializerError { + fn from(e: rkyv::with::AsStringError) -> SerializerError { + SerializerError::AsString(e) + } +}