Reject pyproject.toml files in uv pip compile -o (#12673)

## Summary

Closes https://github.com/astral-sh/uv/issues/12646.

crates/uv/src/commands/pip/compile.rs

@@ -107,6 +107,27 @@ pub(crate) async fn pip_compile(
     printer: Printer,
     preview: PreviewMode,
 ) -> Result<ExitStatus> {
+    // If the user provides a `pyproject.toml` or other TOML file as the output file, raise an
+    // error.
+    if output_file
+        .and_then(Path::file_name)
+        .is_some_and(|name| name.eq_ignore_ascii_case("pyproject.toml"))
+    {
+        return Err(anyhow!(
+            "`pyproject.toml` is not a supported output format for `{}` (only `requirements.txt`-style output is supported)",
+            "uv pip compile".green()
+        ));
+    }
+    if output_file
+        .and_then(Path::extension)
+        .is_some_and(|name| name.eq_ignore_ascii_case("toml"))
+    {
+        return Err(anyhow!(
+            "TOML is not a supported output format for `{}` (only `requirements.txt`-style output is supported)",
+            "uv pip compile".green()
+        ));
+    }
+
     // Respect `UV_PYTHON`
     if python.is_none() && python_version.is_none() {
         if let Ok(request) = std::env::var("UV_PYTHON") {

crates/uv/tests/it/pip_compile.rs

@@ -16234,3 +16234,38 @@ fn compile_quotes() -> Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn compile_invalid_output_file() -> Result<()> {
+    let context = TestContext::new("3.12");
+    let requirements_in = context.temp_dir.child("requirements.in");
+    requirements_in.write_str("anyio==3.7.0")?;
+
+    uv_snapshot!(context
+        .pip_compile()
+        .arg("requirements.in")
+        .arg("-o")
+        .arg("pyproject.toml"), @r"
+    success: false
+    exit_code: 2
+    ----- stdout -----
+
+    ----- stderr -----
+    error: `pyproject.toml` is not a supported output format for `uv pip compile` (only `requirements.txt`-style output is supported)
+    ");
+
+    uv_snapshot!(context
+        .pip_compile()
+        .arg("requirements.in")
+        .arg("-o")
+        .arg("uv.toml"), @r"
+    success: false
+    exit_code: 2
+    ----- stdout -----
+
+    ----- stderr -----
+    error: TOML is not a supported output format for `uv pip compile` (only `requirements.txt`-style output is supported)
+    ");
+
+    Ok(())
+}