puffin-client: add SimpleMetadataRaw (#1150)

This adds what is effectively an owned wrapper around
`Archived<SimpleMetadata>`. Normally, an `Archived<SimpleMetadata>`
has to be used behind a pointer (since it has a lifetime
attached to its underlying byte buffer), but we create a
wrapper around it that owns the underlying buffer and provides
free access to the archived type.

This in effect creates an anchor point for the archived type
and lets us pass it around easily. (There has to be an anchor
point for it somewhere.)

An alternative to this approach would be to store it as a file
backed memory map. But in practice, we're dealing with small
files, and just reading them on to the heap is likely to be
faster. (Memory maps also have wildly different perf characteristics
across platforms.)

Note that this commit just defines the type. It isn't actually
used anywhere yet.

crates/puffin-client/src/error.rs

@@ -125,4 +125,10 @@ pub enum ErrorKind {
 
     #[error("Unsupported `Content-Type` \"{1}\" for {0}. Expected JSON or HTML.")]
     UnsupportedMediaType(Url, String),
+
+    #[error("Reading from cache archive failed: {0}")]
+    ArchiveRead(String),
+
+    #[error("Writing to cache archive failed: {0}")]
+    ArchiveWrite(#[source] crate::rkyvutil::SerializerError),
 }

crates/puffin-client/src/lib.rs

@@ -2,8 +2,8 @@ pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithC
 pub use error::{Error, ErrorKind};
 pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexClient, FlatIndexError};
 pub use registry_client::{
-    read_metadata_async, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum,
-    VersionFiles,
+    read_metadata_async, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadataRaw,
+    SimpleMetadatum, VersionFiles,
 };
 
 mod cache_headers;
@@ -13,3 +13,4 @@ mod flat_index;
 mod html;
 mod registry_client;
 mod remote_metadata;
+mod rkyvutil;

crates/puffin-client/src/registry_client.rs

@@ -616,6 +616,103 @@ impl IntoIterator for SimpleMetadata {
     }
 }
 
+/// An owned archived type for `SimpleMetadata`.
+///
+/// This type is effectively a `Archived<SimpleMetadata>`, but owns its buffer.
+/// Constructing the type requires validating that the bytes are valid, but
+/// subsequent accesses are free.
+#[derive(Debug)]
+pub struct SimpleMetadataRaw {
+    raw: rkyv::util::AlignedVec,
+}
+
+impl SimpleMetadataRaw {
+    /// Create a new owned archived value from the raw aligned bytes of the
+    /// serialized representation of a `SimpleMetadata`.
+    ///
+    /// # Errors
+    ///
+    /// If the bytes fail validation (e.g., contains unaligned pointers or
+    /// strings aren't valid UTF-8), then this returns an error.
+    pub fn new(raw: rkyv::util::AlignedVec) -> Result<SimpleMetadataRaw, Error> {
+        // We convert the error to a simple string because... the error type
+        // does not implement Send. And I don't think we really need to keep
+        // the error type around anyway.
+        let _ = rkyv::validation::validators::check_archived_root::<SimpleMetadata>(&raw)
+            .map_err(|e| ErrorKind::ArchiveRead(e.to_string()))?;
+        Ok(SimpleMetadataRaw { raw })
+    }
+
+    /// Like `SimpleMetadataRaw::new`, but reads the value from the given
+    /// reader.
+    ///
+    /// Note that this consumes the entirety of the given reader.
+    ///
+    /// # Errors
+    ///
+    /// If the bytes fail validation (e.g., contains unaligned pointers or
+    /// strings aren't valid UTF-8), then this returns an error.
+    pub fn from_reader<R: std::io::Read>(mut rdr: R) -> Result<SimpleMetadataRaw, Error> {
+        let mut buf = rkyv::util::AlignedVec::with_capacity(1024);
+        buf.extend_from_reader(&mut rdr).map_err(ErrorKind::Io)?;
+        SimpleMetadataRaw::new(buf)
+    }
+
+    /// Creates an owned archive value from the unarchived value.
+    ///
+    /// # Errors
+    ///
+    /// This can fail if creating an archive for the given type fails.
+    /// Currently, this, at minimum, includes cases where a `SimpleMetadata`
+    /// contains a `PathBuf` that is not valid UTF-8.
+    pub fn from_unarchived(unarchived: &SimpleMetadata) -> Result<SimpleMetadataRaw, Error> {
+        use rkyv::ser::Serializer;
+
+        let mut serializer = crate::rkyvutil::Serializer::<4096>::new();
+        serializer
+            .serialize_value(unarchived)
+            .map_err(ErrorKind::ArchiveWrite)?;
+        let raw = serializer.into_serializer().into_inner();
+        Ok(SimpleMetadataRaw { raw })
+    }
+
+    /// Write the underlying bytes of this archived value to the given writer.
+    ///
+    /// # Errors
+    ///
+    /// Any failures from writing are returned to the caller.
+    pub fn write<W: std::io::Write>(&self, mut wtr: W) -> Result<(), Error> {
+        Ok(wtr.write_all(&self.raw).map_err(ErrorKind::Io)?)
+    }
+
+    /// Returns this owned archive value as a borrowed archive value.
+    pub fn as_archived(&self) -> &rkyv::Archived<SimpleMetadata> {
+        // SAFETY: We've validated that our underlying buffer is a valid
+        // archive for SimpleMetadata in the constructor, so we can skip
+        // validation here. Since we don't mutate the buffer, this conversion
+        // is guaranteed to be correct.
+        unsafe { rkyv::archived_root::<SimpleMetadata>(&self.raw) }
+    }
+
+    /// Returns the raw underlying bytes of this owned archive value.
+    ///
+    /// They are guaranteed to be a valid serialization of
+    /// `Archived<SimpleMetadata>`.
+    pub fn as_bytes(&self) -> &[u8] {
+        &self.raw
+    }
+
+    /// Deserialize this owned archived value into the original
+    /// `SimpleMetadata`.
+    pub fn deserialize(&self) -> SimpleMetadata {
+        use rkyv::Deserialize;
+
+        self.as_archived()
+            .deserialize(&mut rkyv::de::deserializers::SharedDeserializeMap::new())
+            .expect("valid archive must deserialize correctly")
+    }
+}
+
 #[derive(Debug)]
 enum MediaType {
     Json,

crates/puffin-client/src/rkyvutil.rs

@@ -0,0 +1,195 @@
+/*!
+Defines some helpers for use with `rkyv`.
+
+Principally, we define our own implementation of the `Serializer` trait.
+This involves a fair bit of boiler plate, but it was largely copied from
+`CompositeSerializer`. (Indeed, our serializer wraps a `CompositeSerializer`.)
+
+The motivation for doing this is to support the archiving of `PathBuf` types.
+Namely, for reasons AG doesn't completely understand at the time of writing,
+the serializers that rkyv bundled cannot handle the error returned by `PathBuf`
+potentially failing to serialize. Namely, since `PathBuf` has a platform
+dependent representation when its contents are not valid UTF-8, serialization
+in `rkyv` requires that it be valid UTF-8. If it isn't, serialization will
+fail.
+*/
+
+use std::convert::Infallible;
+
+use rkyv::{
+    ser::serializers::{
+        AlignedSerializer, AllocScratch, AllocScratchError, AllocSerializer, CompositeSerializer,
+        CompositeSerializerError, FallbackScratch, HeapScratch, SharedSerializeMap,
+        SharedSerializeMapError,
+    },
+    util::AlignedVec,
+    Archive, ArchiveUnsized, Fallible,
+};
+
+pub struct Serializer<const N: usize> {
+    composite: CompositeSerializer<
+        AlignedSerializer<AlignedVec>,
+        FallbackScratch<HeapScratch<N>, AllocScratch>,
+        SharedSerializeMap,
+    >,
+}
+
+impl<const N: usize> Serializer<N> {
+    pub fn new() -> Serializer<N> {
+        let composite = AllocSerializer::<N>::default();
+        Serializer { composite }
+    }
+
+    pub fn into_serializer(self) -> AlignedSerializer<AlignedVec> {
+        self.composite.into_serializer()
+    }
+}
+
+impl<const N: usize> Fallible for Serializer<N> {
+    type Error = SerializerError;
+}
+
+impl<const N: usize> rkyv::ser::Serializer for Serializer<N> {
+    #[inline]
+    fn pos(&self) -> usize {
+        self.composite.pos()
+    }
+
+    #[inline]
+    fn write(&mut self, bytes: &[u8]) -> Result<(), Self::Error> {
+        self.composite
+            .write(bytes)
+            .map_err(SerializerError::Composite)
+    }
+
+    #[inline]
+    fn pad(&mut self, padding: usize) -> Result<(), Self::Error> {
+        self.composite
+            .pad(padding)
+            .map_err(SerializerError::Composite)
+    }
+
+    #[inline]
+    fn align(&mut self, align: usize) -> Result<usize, Self::Error> {
+        self.composite
+            .align(align)
+            .map_err(SerializerError::Composite)
+    }
+
+    #[inline]
+    fn align_for<T>(&mut self) -> Result<usize, Self::Error> {
+        self.composite
+            .align_for::<T>()
+            .map_err(SerializerError::Composite)
+    }
+
+    #[inline]
+    unsafe fn resolve_aligned<T: Archive + ?Sized>(
+        &mut self,
+        value: &T,
+        resolver: T::Resolver,
+    ) -> Result<usize, Self::Error> {
+        self.composite
+            .resolve_aligned::<T>(value, resolver)
+            .map_err(SerializerError::Composite)
+    }
+
+    #[inline]
+    unsafe fn resolve_unsized_aligned<T: ArchiveUnsized + ?Sized>(
+        &mut self,
+        value: &T,
+        to: usize,
+        metadata_resolver: T::MetadataResolver,
+    ) -> Result<usize, Self::Error> {
+        self.composite
+            .resolve_unsized_aligned(value, to, metadata_resolver)
+            .map_err(SerializerError::Composite)
+    }
+}
+
+impl<const N: usize> rkyv::ser::ScratchSpace for Serializer<N> {
+    #[inline]
+    unsafe fn push_scratch(
+        &mut self,
+        layout: std::alloc::Layout,
+    ) -> Result<std::ptr::NonNull<[u8]>, Self::Error> {
+        self.composite
+            .push_scratch(layout)
+            .map_err(SerializerError::Composite)
+    }
+
+    #[inline]
+    unsafe fn pop_scratch(
+        &mut self,
+        ptr: std::ptr::NonNull<u8>,
+        layout: std::alloc::Layout,
+    ) -> Result<(), Self::Error> {
+        self.composite
+            .pop_scratch(ptr, layout)
+            .map_err(SerializerError::Composite)
+    }
+}
+
+impl<const N: usize> rkyv::ser::SharedSerializeRegistry for Serializer<N> {
+    #[inline]
+    fn get_shared_ptr(&self, value: *const u8) -> Option<usize> {
+        self.composite.get_shared_ptr(value)
+    }
+
+    #[inline]
+    fn add_shared_ptr(&mut self, value: *const u8, pos: usize) -> Result<(), Self::Error> {
+        self.composite
+            .add_shared_ptr(value, pos)
+            .map_err(SerializerError::Composite)
+    }
+}
+
+#[derive(Debug)]
+pub enum SerializerError {
+    Composite(CompositeSerializerError<Infallible, AllocScratchError, SharedSerializeMapError>),
+    AsString(rkyv::with::AsStringError),
+}
+
+impl std::fmt::Display for SerializerError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match *self {
+            SerializerError::Composite(ref e) => e.fmt(f),
+            SerializerError::AsString(ref e) => e.fmt(f),
+        }
+    }
+}
+
+impl std::error::Error for SerializerError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match *self {
+            SerializerError::Composite(ref e) => Some(e),
+            SerializerError::AsString(ref e) => Some(e),
+        }
+    }
+}
+
+/// Provides a way to build a serializer error if converting an
+/// `OsString`/`PathBuf` to a `String` fails. i.e., It's invalid UTF-8.
+///
+/// This impl is the entire point of this module. For whatever reason, none of
+/// the serializers in rkyv handle this particular error case. Apparently, the
+/// only way to use `rkyv::with::AsString` with `PathBuf` is to create one's
+/// own serializer and provide a `From` impl for the `AsStringError` type.
+/// Specifically, from the [AsString] docs:
+///
+/// > Regular serializers don’t support the custom error handling needed for
+/// > this type by default. To use this wrapper, a custom serializer with an
+/// > error type satisfying <S as Fallible>::Error: From<AsStringError> must be
+/// > provided.
+///
+/// If we didn't need to use `rkyv::with::AsString` (which we do for
+/// serializing `PathBuf` at time of writing), then we could just
+/// use an `AllocSerializer` directly (which is a type alias for
+/// `CompositeSerializer<...>`.
+///
+/// [AsString]: https://docs.rs/rkyv/0.7.43/rkyv/with/struct.AsString.html
+impl From<rkyv::with::AsStringError> for SerializerError {
+    fn from(e: rkyv::with::AsStringError) -> SerializerError {
+        SerializerError::AsString(e)
+    }
+}