gracefully handle 404 errors from crates.io

Author: syphar

Cargo.lock

@@ -786,7 +786,7 @@ impl RustwideBuilder {
                         .runtime
                         .block_on(self.registry_api.get_release_data(name, version))
                         {
-                        Ok(data) => Some(data),
+                        Ok(data) => data,
                         Err(err) => {
                             error!(%name, %version, ?err, "could not fetch releases-data");
                             None

crates/bin/docs_rs_builder/src/docbuilder/rustwide_builder.rs

@@ -4,6 +4,7 @@ use crate::{
     rustdoc::{download_static_files, find_static_paths, find_successful_build_targets},
     rustdoc_status::fetch_rustdoc_status,
 };
+use anyhow::anyhow;
 use anyhow::{Result, bail};
 use docs_rs_cargo_metadata::CargoMetadata;
 use docs_rs_database::releases::{
@@ -117,7 +118,10 @@ async fn import_test_release_inner(
         (files_list, source_size)
     };
 
-    let registry_data = registry_api.get_release_data(name, version).await?;
+    let registry_data = registry_api
+        .get_release_data(name, version)
+        .await?
+        .ok_or_else(|| anyhow!("registry data not found"))?;
 
     let rustdoc_dir = {
         info!("download & extract rustdoc archive...");

crates/bin/docs_rs_import_release/src/import.rs

@@ -18,3 +18,10 @@ serde = { workspace = true }
 sqlx = { workspace = true }
 tracing = { workspace = true }
 url = { workspace = true }
+
+[dev-dependencies]
+docs_rs_types = { path = "../docs_rs_types", features = ["testing"] }
+mime = { workspace = true }
+mockito = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true }

crates/lib/docs_rs_registry_api/Cargo.toml

@@ -6,7 +6,10 @@ use anyhow::{Context, Result, anyhow, bail};
 use chrono::{DateTime, Utc};
 use docs_rs_types::{KrateName, Version};
 use docs_rs_utils::{APP_USER_AGENT, retry_async};
-use reqwest::header::{ACCEPT, HeaderValue, USER_AGENT};
+use reqwest::{
+    StatusCode,
+    header::{ACCEPT, HeaderValue, USER_AGENT},
+};
 use serde::Deserialize;
 use tracing::instrument;
 use url::Url;
@@ -60,25 +63,7 @@ impl RegistryApi {
         &self,
         name: &KrateName,
         version: &Version,
-    ) -> Result<ReleaseData> {
-        let (release_time, yanked, downloads) = self
-            .get_release_time_yanked_downloads(name, version)
-            .await
-            .context(format!("Failed to get crate data for {name}-{version}"))?;
-
-        Ok(ReleaseData {
-            release_time,
-            yanked,
-            downloads,
-        })
-    }
-
-    /// Get release_time, yanked and downloads from the registry's API
-    async fn get_release_time_yanked_downloads(
-        &self,
-        name: &KrateName,
-        version: &Version,
-    ) -> Result<(DateTime<Utc>, bool, i32)> {
+    ) -> Result<Option<ReleaseData>> {
         let url = {
             let mut url = self.api_base.clone();
             url.path_segments_mut()
@@ -103,28 +88,43 @@ impl RegistryApi {
             downloads: i32,
         }
 
-        let response: Response = retry_async(
+        let response: Response = match retry_async(
             || async {
-                Ok(self
-                    .client
-                    .get(url.clone())
-                    .send()
-                    .await?
-                    .error_for_status()?)
+                Ok(
+                    match self
+                        .client
+                        .get(url.clone())
+                        .send()
+                        .await?
+                        .error_for_status()
+                    {
+                        Ok(resp) => Some(resp),
+                        Err(err) if matches!(err.status(), Some(StatusCode::NOT_FOUND)) => None,
+                        Err(err) => return Err(err.into()),
+                    },
+                )
             },
             self.max_retries,
         )
         .await?
-        .json()
-        .await?;
+        {
+            Some(resp) => resp.json().await?,
+            None => {
+                return Ok(None);
+            }
+        };
 
         let version = response
             .versions
             .into_iter()
             .find(|data| data.num == *version)
             .with_context(|| anyhow!("Could not find version in response"))?;
 
-        Ok((version.created_at, version.yanked, version.downloads))
+        Ok(Some(ReleaseData {
+            release_time: version.created_at,
+            yanked: version.yanked,
+            downloads: version.downloads,
+        }))
     }
 
     /// Fetch owners from the registry's API
@@ -241,3 +241,74 @@ impl RegistryApi {
         Ok(Search { crates, meta })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use anyhow::Result;
+    use docs_rs_types::testing::{KRATE, V1, V2};
+    use reqwest::header::CONTENT_TYPE;
+
+    #[tokio::test]
+    async fn test_get_release_data() -> Result<()> {
+        let mut server = mockito::Server::new_async().await;
+
+        let created = Utc::now();
+
+        let _m = server
+            .mock("GET", "/api/v1/crates/krate/versions")
+            .with_status(200)
+            .with_header(CONTENT_TYPE, mime::APPLICATION_JSON.as_ref())
+            .with_body(
+                serde_json::json!({
+                    "versions": [
+                        {
+                            "num": V1.to_string(),
+                            "created_at": created.to_rfc3339(),
+                            "yanked": false,
+                            "downloads": 42
+                        },
+                        {
+                            "num": V2.to_string(),
+                            "created_at": "2025-01-01T00:00:00Z",
+                            "yanked": true,
+                            "downloads": 22
+                        }
+                    ]
+                })
+                .to_string(),
+            )
+            .create_async()
+            .await;
+
+        let api = RegistryApi::new(server.url().parse().unwrap(), 0)?;
+
+        assert_eq!(
+            api.get_release_data(&KRATE, &V1).await?,
+            Some(ReleaseData {
+                release_time: created,
+                yanked: false,
+                downloads: 42
+            })
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_404_in_release_data_returns_none() -> Result<()> {
+        let mut server = mockito::Server::new_async().await;
+
+        let _m = server
+            .mock("GET", "/api/v1/crates/krate/versions")
+            .with_status(404)
+            .create_async()
+            .await;
+
+        let api = RegistryApi::new(server.url().parse().unwrap(), 0)?;
+
+        assert_eq!(api.get_release_data(&KRATE, &V1).await?, None,);
+
+        Ok(())
+    }
+}

crates/lib/docs_rs_registry_api/src/api.rs

@@ -7,7 +7,7 @@ pub struct CrateData {
     pub owners: Vec<CrateOwner>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub struct ReleaseData {
     pub release_time: DateTime<Utc>,
     pub yanked: bool,