Source dist metadata refactor (#468)

## Summary and motivation

For a given source dist, we store the metadata of each wheel built
through it in `built-wheel-metadata-v0/pypi/<source dist
filename>/metadata.json`. During resolution, we check the cache status
of the source dist. If it is fresh, we check `metadata.json` for a
matching wheel. If there is one we use that metadata, if there isn't, we
build one. If the source is stale, we build a wheel and override
`metadata.json` with that single wheel. This PR thereby ties the local
built wheel metadata cache to the freshness of the remote source dist.
This functionality is available through `SourceDistCachedBuilder`.

`puffin_installer::Builder`, `puffin_installer::Downloader` and
`Fetcher` are removed, instead there are now `FetchAndBuild` which calls
into the also new `SourceDistCachedBuilder`. `FetchAndBuild` is the new
main high-level abstraction: It spawns parallel fetching/building, for
wheel metadata it calls into the registry client, for wheel files it
fetches them, for source dists it calls `SourceDistCachedBuilder`. It
handles locks around builds, and newly added also inter-process file
locking for git operations.

Fetching and building source distributions now happens in parallel in
`pip-sync`, i.e. we don't have to wait for the largest wheel to be
downloaded to start building source distributions.

In a follow-up PR, I'll also clear built wheels when they've become
stale.

Another effect is that in a fully cached resolution, we need neither zip
reading nor email parsing.

Closes #473

## Source dist cache structure 

Entries by supported sources:
 * `<build wheel metadata cache>/pypi/foo-1.0.0.zip/metadata.json`
* `<build wheel metadata
cache>/<sha256(index-url)>/foo-1.0.0.zip/metadata.json`
* `<build wheel metadata
cache>/url/<sha256(url)>/foo-1.0.0.zip/metadata.json`
But the url filename does not need to be a valid source dist filename

(<https://github.com/search?q=path%3A**%2Frequirements.txt+master.zip&type=code>),
so it could also be the following and we have to take any string as
filename:
* `<build wheel metadata
cache>/url/<sha256(url)>/master.zip/metadata.json`

Example:
```text
# git source dist
pydantic-extra-types @ git+https://github.com/pydantic/pydantic-extra-types.git
# pypi source dist
django_allauth==0.51.0
# url source dist
werkzeug @ ff1904eb5e/werkzeug-3.0.1.tar.gz
```
will be stored as
```text
built-wheel-metadata-v0
├── git
│   └── 5c56bc1c58c34c11
│       └── 843b753e9e8cb74e83cac55598719b39a4d5ef1f
│           └── metadata.json
├── pypi
│   └── django-allauth-0.51.0.tar.gz
│       └── metadata.json
└── url
    └── 6781bd6440ae72c2
        └── werkzeug-3.0.1.tar.gz
            └── metadata.json
```

The inside of a `metadata.json`:
```json
{
  "data": {
    "django_allauth-0.51.0-py3-none-any.whl": {
      "metadata-version": "2.1",
      "name": "django-allauth",
      "version": "0.51.0",
      ...
    }
  }
}
```

crates/puffin-dispatch/src/lib.rs

@@ -8,7 +8,7 @@ use std::pin::Pin;
 
 use anyhow::Result;
 use anyhow::{bail, Context};
-use itertools::{Either, Itertools};
+use itertools::Itertools;
 use tracing::{debug, instrument};
 
 use distribution_types::Metadata;
@@ -16,7 +16,8 @@ use pep508_rs::Requirement;
 use platform_tags::Tags;
 use puffin_build::{SourceBuild, SourceBuildContext};
 use puffin_client::RegistryClient;
-use puffin_installer::{Builder, Downloader, InstallPlan, Installer, Unzipper};
+use puffin_distribution::DistributionDatabase;
+use puffin_installer::{InstallPlan, Installer, Unzipper};
 use puffin_interpreter::{Interpreter, Virtualenv};
 use puffin_resolver::{DistFinder, Manifest, ResolutionOptions, Resolver};
 use puffin_traits::BuildContext;
@@ -139,57 +140,34 @@ impl BuildContext for BuildDispatch {
             };
 
             // Download any missing distributions.
-            let downloads = if remote.is_empty() {
+            let wheels = if remote.is_empty() {
                 vec![]
             } else {
+                // TODO(konstin): Check that there is no endless recursion
+                let fetcher = DistributionDatabase::new(self.cache(), &tags, &self.client, self);
                 debug!(
-                    "Downloading build requirement{}: {}",
+                    "Downloading and building requirement{} for build: {}",
                     if remote.len() == 1 { "" } else { "s" },
                     remote.iter().map(ToString::to_string).join(", ")
                 );
-                Downloader::new(&self.client, &self.cache)
-                    .with_no_build(self.no_build)
-                    .download(remote)
+
+                fetcher
+                    .get_wheels(remote)
                     .await
-                    .context("Failed to download build dependencies")?
+                    .context("Failed to download and build distributions")?
             };
 
-            let (wheels, sdists): (Vec<_>, Vec<_>) =
-                downloads
-                    .into_iter()
-                    .partition_map(|download| match download {
-                        puffin_distribution::Download::Wheel(wheel) => Either::Left(wheel),
-                        puffin_distribution::Download::SourceDist(sdist) => Either::Right(sdist),
-                    });
-
-            // Build any missing source distributions.
-            let sdists = if sdists.is_empty() {
-                vec![]
-            } else {
-                debug!(
-                    "Building source distributions{}: {}",
-                    if sdists.len() == 1 { "" } else { "s" },
-                    sdists.iter().map(ToString::to_string).join(", ")
-                );
-                Builder::new(self)
-                    .build(sdists)
-                    .await
-                    .context("Failed to build source distributions")?
-            };
-
-            let downloads = wheels.into_iter().chain(sdists).collect::<Vec<_>>();
-
             // Unzip any downloaded distributions.
-            let unzips = if downloads.is_empty() {
+            let unzips = if wheels.is_empty() {
                 vec![]
             } else {
                 debug!(
                     "Unzipping build requirement{}: {}",
-                    if downloads.len() == 1 { "" } else { "s" },
-                    downloads.iter().map(ToString::to_string).join(", ")
+                    if wheels.len() == 1 { "" } else { "s" },
+                    wheels.iter().map(ToString::to_string).join(", ")
                 );
                 Unzipper::default()
-                    .unzip(downloads, &self.cache)
+                    .unzip(wheels, &self.cache)
                     .await
                     .context("Failed to unpack build dependencies")?
             };