From 2ebb55600f18fcc9f0f92be9dda4d8880043eb68 Mon Sep 17 00:00:00 2001 From: Jatin Suri Date: Thu, 26 Mar 2026 17:55:34 -0400 Subject: [PATCH 1/3] Fix manifest list detection for single-arch release images Detect manifest lists by checking the manifest type directly (Manifest::ML variant) instead of counting the number of architectures. This fixes scraping of release images that are manifest lists with a single architecture (e.g. OKD SCOS releases), which previously failed with 404 errors when the graph builder mistakenly tried to fetch manifest-reference digests as blobs. --- .../release_scrape_dockerv2/registry/mod.rs | 40 +++++++++++-------- docs/design/openshift.md | 10 +---- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs b/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs index 4e151ec75..3f4cf6c80 100644 --- a/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs +++ b/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs @@ -259,22 +259,27 @@ async fn get_manifest_layers( let (tag, manifest, manifestref) = get_manifest_and_ref(tag, repo.to_owned(), ®istry_client).await?; - // Try to read the architecture from the manifest - let arch = match manifest.architectures() { - Ok(archs) => { - if archs.len() == 1 { - archs.first().map(std::string::ToString::to_string) - } else { - Some(String::from("multi")) + // Determine whether this is a manifest list by checking the manifest type + // directly, rather than relying on the number of architectures. A manifest + // list with a single architecture (e.g. OKD SCOS releases) must still be + // treated as "multi" so that the caller resolves through to the actual + // image layers instead of trying to fetch manifest-reference digests as + // blobs. + let is_manifest_list = matches!(manifest, dkregistry::v2::manifest::Manifest::ML(_)); + + let arch = if is_manifest_list { + Some(String::from("multi")) + } else { + match manifest.architectures() { + Ok(archs) => archs.first().map(std::string::ToString::to_string), + Err(e) => { + error!( + "could not get architecture from manifest for tag {}: {}", + tag, e + ); + None } } - Err(e) => { - error!( - "could not get architecture from manifest for tag {}: {}", - tag, e - ); - None - } }; let layers_digests = manifest @@ -354,9 +359,10 @@ pub async fn fetch_releases( } }; - // if the image is multi arch, we will have to get one image from the manifest list and - // use its metadata, because manifest lists are just collections of manifests and don't - // have their own layers with metadata files. + // If the manifest is a manifest list (regardless of the number of + // architectures it contains), resolve one image from the list and use + // its layers, because manifest lists don't have their own layers with + // metadata files. if arch.as_ref().unwrap() == "multi" { let digest = layers_digests .first() diff --git a/docs/design/openshift.md b/docs/design/openshift.md index 5dcc622da..198d3a603 100644 --- a/docs/design/openshift.md +++ b/docs/design/openshift.md @@ -198,7 +198,7 @@ $ cat release-manifests/image-references | head -n 10 The architecture of a release image is determined by the following algorithm: -- check if it is a manifest list of length > 1 by [media-types](https://github.com/openshift/docker-distribution/blob/main/docs/spec/manifest-v2-2.md#media-types). +- check if it is a manifest list by [media-types](https://github.com/openshift/docker-distribution/blob/main/docs/spec/manifest-v2-2.md#media-types). ```console ### manifest list quay.io/openshift-release-dev/ocp-release:4.15.5-multi @@ -219,13 +219,7 @@ The architecture of a release image is determined by the following algorithm: s390x ``` -* If it is a manifest list of `length > 1`, then the architecture is `multi`. - -* If `length == 1` which is an very uncommon case in practice, it is counted as single-arch whose architecture can be calculated by the following command: - - ```console - $ curl -sH 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' https://quay.io/v2/openshift-release-dev/ocp-release/manifests/4.15.5-multi | jq '.manifests[0].platform.architecture' - ``` +* If it is a manifest list (any number of architectures), then the architecture is `multi`. The graph builder detects manifest lists by checking the manifest type directly (the `Manifest::ML` variant from dkregistry), rather than counting architectures. This ensures that manifest lists containing a single architecture (e.g. OKD SCOS releases with only amd64) are handled correctly — the graph builder resolves through the manifest list to the actual image and its layers, instead of mistakenly treating the manifest-reference digests as blob digests. * If it is an image, its architecture is showed in its blob like `s390x` in the last twos commands above unless `release-manifests/release-metadata` [extracted from the release image](#update-image) indicates otherwise: From 802e5db31cccf5f34ebfc27ea613c6cb2972293a Mon Sep 17 00:00:00 2001 From: Jatin Suri Date: Wed, 8 Apr 2026 15:40:36 -0400 Subject: [PATCH 2/3] Carry bool over instead of modfying arch check --- .../release_scrape_dockerv2/registry/mod.rs | 53 +++++++++---------- docs/design/openshift.md | 10 +++- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs b/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs index 3f4cf6c80..a283b33ef 100644 --- a/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs +++ b/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs @@ -254,32 +254,30 @@ async fn get_manifest_layers( tag: String, repo: &str, registry_client: &Client, -) -> Result<(Option, String, Vec), Error> { +) -> Result<(Option, String, Vec, bool), Error> { trace!("[{}] Fetching release", tag); let (tag, manifest, manifestref) = get_manifest_and_ref(tag, repo.to_owned(), ®istry_client).await?; - // Determine whether this is a manifest list by checking the manifest type - // directly, rather than relying on the number of architectures. A manifest - // list with a single architecture (e.g. OKD SCOS releases) must still be - // treated as "multi" so that the caller resolves through to the actual - // image layers instead of trying to fetch manifest-reference digests as - // blobs. - let is_manifest_list = matches!(manifest, dkregistry::v2::manifest::Manifest::ML(_)); - - let arch = if is_manifest_list { - Some(String::from("multi")) - } else { - match manifest.architectures() { - Ok(archs) => archs.first().map(std::string::ToString::to_string), - Err(e) => { - error!( - "could not get architecture from manifest for tag {}: {}", - tag, e - ); - None + // Check if manifest is a manifest list with a single arch + let is_manifest_list = matches!(&manifest, dkregistry::v2::manifest::Manifest::ML(_)); + + // Try to read the architecture from the manifest + let arch = match manifest.architectures() { + Ok(archs) => { + if archs.len() == 1 { + archs.first().map(std::string::ToString::to_string) + } else { + Some(String::from("multi")) } } + Err(e) => { + error!( + "could not get architecture from manifest for tag {}: {}", + tag, e + ); + None + } }; let layers_digests = manifest @@ -294,7 +292,7 @@ async fn get_manifest_layers( .rev() .collect(); - Ok((arch, manifestref, layers_digests)) + Ok((arch, manifestref, layers_digests, is_manifest_list)) } /// Fetches a vector of all release metadata from the given repository, hosted on the given @@ -338,7 +336,7 @@ pub async fn fetch_releases( let misses = cache_misses.clone(); async move { - let (arch, manifestref, mut layers_digests) = + let (arch, manifestref, mut layers_digests, is_manifest_list) = match get_manifest_layers(tag.to_owned(), &repo, ®istry_client).await { Ok(result) => result, Err(e) => { @@ -359,11 +357,10 @@ pub async fn fetch_releases( } }; - // If the manifest is a manifest list (regardless of the number of - // architectures it contains), resolve one image from the list and use - // its layers, because manifest lists don't have their own layers with - // metadata files. - if arch.as_ref().unwrap() == "multi" { + // if the image is multi arch, we will have to get one image from the manifest list and + // use its metadata, because manifest lists are just collections of manifests and don't + // have their own layers with metadata files. + if is_manifest_list { let digest = layers_digests .first() .map(std::string::ToString::to_string) @@ -373,7 +370,7 @@ pub async fn fetch_releases( ); // TODO: destructured assignments are unstable in current rust, after updating rust // change this to (_,_,layers_digests) and remove separate assignment from below. - let (_ml_arch, _ml_manifestref, ml_layers_digests) = + let (_ml_arch, _ml_manifestref, ml_layers_digests, _ml_is_manifest_list) = get_manifest_layers(digest, &repo, ®istry_client).await?; layers_digests = ml_layers_digests; } diff --git a/docs/design/openshift.md b/docs/design/openshift.md index 198d3a603..5dcc622da 100644 --- a/docs/design/openshift.md +++ b/docs/design/openshift.md @@ -198,7 +198,7 @@ $ cat release-manifests/image-references | head -n 10 The architecture of a release image is determined by the following algorithm: -- check if it is a manifest list by [media-types](https://github.com/openshift/docker-distribution/blob/main/docs/spec/manifest-v2-2.md#media-types). +- check if it is a manifest list of length > 1 by [media-types](https://github.com/openshift/docker-distribution/blob/main/docs/spec/manifest-v2-2.md#media-types). ```console ### manifest list quay.io/openshift-release-dev/ocp-release:4.15.5-multi @@ -219,7 +219,13 @@ The architecture of a release image is determined by the following algorithm: s390x ``` -* If it is a manifest list (any number of architectures), then the architecture is `multi`. The graph builder detects manifest lists by checking the manifest type directly (the `Manifest::ML` variant from dkregistry), rather than counting architectures. This ensures that manifest lists containing a single architecture (e.g. OKD SCOS releases with only amd64) are handled correctly — the graph builder resolves through the manifest list to the actual image and its layers, instead of mistakenly treating the manifest-reference digests as blob digests. +* If it is a manifest list of `length > 1`, then the architecture is `multi`. + +* If `length == 1` which is an very uncommon case in practice, it is counted as single-arch whose architecture can be calculated by the following command: + + ```console + $ curl -sH 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' https://quay.io/v2/openshift-release-dev/ocp-release/manifests/4.15.5-multi | jq '.manifests[0].platform.architecture' + ``` * If it is an image, its architecture is showed in its blob like `s390x` in the last twos commands above unless `release-manifests/release-metadata` [extracted from the release image](#update-image) indicates otherwise: From 4602596c0855bf3ab724554030149a0bc8873411 Mon Sep 17 00:00:00 2001 From: Jatin Suri Date: Wed, 8 Apr 2026 16:06:19 -0400 Subject: [PATCH 3/3] Added error handeling for a manifest list with no children --- .../release_scrape_dockerv2/registry/mod.rs | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs b/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs index a283b33ef..ab4c0df24 100644 --- a/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs +++ b/cincinnati/src/plugins/internal/graph_builder/release_scrape_dockerv2/registry/mod.rs @@ -361,18 +361,40 @@ pub async fn fetch_releases( // use its metadata, because manifest lists are just collections of manifests and don't // have their own layers with metadata files. if is_manifest_list { - let digest = layers_digests - .first() - .map(std::string::ToString::to_string) - .expect( - format!("no images referenced in ManifestList ref:{}", manifestref) - .as_str(), - ); + let digest = match layers_digests.first() { + Some(d) => d.to_string(), + None => { + error!( + "no images referenced in ManifestList for {}:{} ref:{}", + &repo, &tag, &manifestref + ); + skip_releases.fetch_add(1, Ordering::SeqCst); + return Ok(()); + } + }; // TODO: destructured assignments are unstable in current rust, after updating rust // change this to (_,_,layers_digests) and remove separate assignment from below. - let (_ml_arch, _ml_manifestref, ml_layers_digests, _ml_is_manifest_list) = - get_manifest_layers(digest, &repo, ®istry_client).await?; - layers_digests = ml_layers_digests; + match get_manifest_layers(digest, &repo, ®istry_client).await { + Ok((_ml_arch, _ml_manifestref, ml_layers_digests, _ml_is_manifest_list)) => { + layers_digests = ml_layers_digests; + } + Err(e) => { + if tag.contains(".sig") { + debug!( + "encountered a signature for child manifest {}:{}: {}, ignoring this image", + &repo, &tag, e + ); + sig_releases.fetch_add(1, Ordering::SeqCst); + } else { + error!( + "fetching child manifest from ManifestList for {}:{}: {}", + &repo, &tag, e + ); + skip_releases.fetch_add(1, Ordering::SeqCst); + } + return Ok(()); + } + }; } let release = match lookup_or_fetch(