diff --git a/minecode_pipelines/pipelines/mine_meson.py b/minecode_pipelines/pipelines/mine_meson.py new file mode 100644 index 00000000..4b55f4ac --- /dev/null +++ b/minecode_pipelines/pipelines/mine_meson.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import requests + +from minecode_pipelines.pipes import meson +from minecode_pipelines.pipelines import MineCodeBasePipeline + + +class MineMeson(MineCodeBasePipeline): + """Pipeline to mine Meson WrapDB packages and publish them to FederatedCode repo.""" + + MESON_WRAPDB_RELEASES_URL = ( + "https://raw.githubusercontent.com/mesonbuild/wrapdb/master/releases.json" + ) + + @classmethod + def steps(cls): + return ( + cls.check_federatedcode_eligibility, + cls.create_federatedcode_working_dir, + cls.fetch_wrapdb_releases, + cls.fetch_federation_config, + cls.mine_and_publish_packageurls, + cls.delete_working_dir, + ) + + def fetch_wrapdb_releases(self): + """Fetch the Meson WrapDB releases.json index.""" + try: + response = requests.get(self.MESON_WRAPDB_RELEASES_URL, timeout=30) + response.raise_for_status() + self.releases = response.json() + except Exception as e: + self.log(f"Failed to fetch releases.json: {e}") + self.releases = {} + + def packages_count(self): + """Return the number of packages in the WrapDB releases index. + + Used by MineCodeBasePipeline to report mining progress. + """ + return len(self.releases) if hasattr(self, "releases") and self.releases else 0 + + def mine_packageurls(self): + """Yield PackageURLs from Meson WrapDB releases.json.""" + return meson.mine_meson_packageurls(releases=self.releases, logger=self.log) diff --git a/minecode_pipelines/pipes/meson.py b/minecode_pipelines/pipes/meson.py new file mode 100644 index 00000000..7766bd4e --- /dev/null +++ b/minecode_pipelines/pipes/meson.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from packageurl import PackageURL + + +def get_meson_packages(package_name, package_data): + """ + Return a tuple of (base_purl, [versioned_purl_strings]) for a single + Meson WrapDB package entry from ``releases.json``. + + The ``package_data`` dict has the structure:: + + { + "dependency_names": ["dep1", "dep2"], + "versions": ["1.0.0-1", "1.0.0-2", ...] + } + + WrapDB versions use a ``-N`` suffix to denote build recipe revisions that + are specific to the WrapDB and do not exist upstream. + """ + base_purl = PackageURL(type="meson", name=package_name) + versions = package_data.get("versions") or [] + versioned_purls = [ + PackageURL( + type="meson", + name=package_name, + version=str(version), + ).to_string() + for version in versions + ] + return base_purl, versioned_purls + + +def mine_meson_packageurls(releases, logger): + """ + Yield ``(base_purl, [versioned_purl_strings])`` tuples from a + Meson WrapDB ``releases.json`` mapping. + """ + for package_name, package_data in releases.items(): + if not package_data: + continue + yield get_meson_packages( + package_name=package_name, + package_data=package_data, + ) diff --git a/minecode_pipelines/tests/pipes/test_meson.py b/minecode_pipelines/tests/pipes/test_meson.py new file mode 100644 index 00000000..da88ac69 --- /dev/null +++ b/minecode_pipelines/tests/pipes/test_meson.py @@ -0,0 +1,64 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +from pathlib import Path + +from django.test import TestCase + +from minecode_pipelines.pipes.meson import get_meson_packages + +DATA_DIR = Path(__file__).parent.parent / "test_data" / "meson" + + +class MesonPipeTests(TestCase): + def test_get_meson_packages_empty_versions(self): + """Test that get_meson_packages handles empty version lists.""" + package_data = { + "dependency_names": ["empty-pkg"], + "versions": [], + } + base_purl, versioned_purls = get_meson_packages("empty-pkg", package_data) + + self.assertEqual(str(base_purl), "pkg:meson/empty-pkg") + self.assertEqual(versioned_purls, []) + + def test_get_meson_packages_no_versions_key(self): + """Test that get_meson_packages handles missing versions key.""" + package_data = { + "dependency_names": ["no-ver"], + } + base_purl, versioned_purls = get_meson_packages("no-ver", package_data) + + self.assertEqual(str(base_purl), "pkg:meson/no-ver") + self.assertEqual(versioned_purls, []) + + def test_get_meson_packages_from_releases_json(self): + """Test parsing packages from the real WrapDB releases.json fixture + using data-driven expected output.""" + releases_path = DATA_DIR / "releases.json" + expected_path = DATA_DIR / "expected_purls.json" + + with open(releases_path, encoding="utf-8") as f: + releases = json.load(f) + + with open(expected_path, encoding="utf-8") as f: + expected = json.load(f) + + actual = {} + for package_name, package_data in releases.items(): + if not package_data: + continue + base_purl, versioned_purls = get_meson_packages( + package_name=package_name, + package_data=package_data, + ) + actual[str(base_purl)] = sorted(versioned_purls) + + self.assertEqual(actual, expected) diff --git a/minecode_pipelines/tests/test_data/meson/expected_purls.json b/minecode_pipelines/tests/test_data/meson/expected_purls.json new file mode 100644 index 00000000..57c69b49 --- /dev/null +++ b/minecode_pipelines/tests/test_data/meson/expected_purls.json @@ -0,0 +1,46 @@ +{ + "pkg:meson/adamyaxley-obfuscate": [ + "pkg:meson/adamyaxley-obfuscate@1.0.0-1" + ], + "pkg:meson/aklomp-base64": [ + "pkg:meson/aklomp-base64@0.5.2-1" + ], + "pkg:meson/apache-orc": [ + "pkg:meson/apache-orc@2.2.0-1", + "pkg:meson/apache-orc@2.2.1-1", + "pkg:meson/apache-orc@2.2.2-1" + ], + "pkg:meson/bzip2": [ + "pkg:meson/bzip2@1.0.8-1" + ], + "pkg:meson/catch2": [ + "pkg:meson/catch2@2.11.1-1", + "pkg:meson/catch2@2.11.3-1", + "pkg:meson/catch2@2.13.3-1", + "pkg:meson/catch2@2.13.3-2", + "pkg:meson/catch2@2.13.7-1", + "pkg:meson/catch2@2.13.8-1", + "pkg:meson/catch2@2.4.1-1", + "pkg:meson/catch2@2.5.0-1", + "pkg:meson/catch2@2.7.2-1", + "pkg:meson/catch2@2.8.0-1", + "pkg:meson/catch2@2.9.0-1", + "pkg:meson/catch2@3.1.0-1", + "pkg:meson/catch2@3.10.0-1", + "pkg:meson/catch2@3.11.0-1", + "pkg:meson/catch2@3.12.0-1", + "pkg:meson/catch2@3.13.0-1", + "pkg:meson/catch2@3.2.0-1", + "pkg:meson/catch2@3.3.2-1", + "pkg:meson/catch2@3.4.0-1", + "pkg:meson/catch2@3.5.2-1", + "pkg:meson/catch2@3.5.3-1", + "pkg:meson/catch2@3.5.4-1", + "pkg:meson/catch2@3.6.0-1", + "pkg:meson/catch2@3.7.0-1", + "pkg:meson/catch2@3.7.1-1", + "pkg:meson/catch2@3.8.0-1", + "pkg:meson/catch2@3.8.1-1", + "pkg:meson/catch2@3.9.1-1" + ] +} \ No newline at end of file diff --git a/minecode_pipelines/tests/test_data/meson/releases.json b/minecode_pipelines/tests/test_data/meson/releases.json new file mode 100644 index 00000000..f6f24724 --- /dev/null +++ b/minecode_pipelines/tests/test_data/meson/releases.json @@ -0,0 +1,72 @@ +{ + "adamyaxley-obfuscate": { + "dependency_names": [ + "Obfuscate" + ], + "versions": [ + "1.0.0-1" + ] + }, + "aklomp-base64": { + "dependency_names": [ + "base64" + ], + "versions": [ + "0.5.2-1" + ] + }, + "apache-orc": { + "dependency_names": [ + "orc" + ], + "versions": [ + "2.2.2-1", + "2.2.1-1", + "2.2.0-1" + ] + }, + "bzip2": { + "dependency_names": [ + "bzip2" + ], + "versions": [ + "1.0.8-1" + ] + }, + "catch2": { + "dependency_names": [ + "catch2", + "catch2-with-main" + ], + "versions": [ + "3.13.0-1", + "3.12.0-1", + "3.11.0-1", + "3.10.0-1", + "3.9.1-1", + "3.8.1-1", + "3.8.0-1", + "3.7.1-1", + "3.7.0-1", + "3.6.0-1", + "3.5.4-1", + "3.5.3-1", + "3.5.2-1", + "3.4.0-1", + "3.3.2-1", + "3.2.0-1", + "3.1.0-1", + "2.13.8-1", + "2.13.7-1", + "2.13.3-2", + "2.13.3-1", + "2.11.3-1", + "2.11.1-1", + "2.9.0-1", + "2.8.0-1", + "2.7.2-1", + "2.5.0-1", + "2.4.1-1" + ] + } +} \ No newline at end of file diff --git a/pyproject-minecode_pipelines.toml b/pyproject-minecode_pipelines.toml index 857597e6..5953ab8e 100644 --- a/pyproject-minecode_pipelines.toml +++ b/pyproject-minecode_pipelines.toml @@ -62,6 +62,7 @@ mine_cpan = "minecode_pipelines.pipelines.mine_cpan:MineCpan" mine_cran = "minecode_pipelines.pipelines.mine_cran:MineCran" mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift" mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer" +mine_meson = "minecode_pipelines.pipelines.mine_meson:MineMeson" [tool.bumpversion] current_version = "0.1.1"