Add Meson WrapDB mining pipeline #803

minecode_pipelines/pipelines/mine_meson.py

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+import requests
+
+from minecode_pipelines.pipes import meson
+from minecode_pipelines.pipelines import MineCodeBasePipeline
+
+
+class MineMeson(MineCodeBasePipeline):
+    """Pipeline to mine Meson WrapDB packages and publish them to FederatedCode repo."""
+
+    MESON_WRAPDB_RELEASES_URL = (
+        "https://raw.githubusercontent.com/mesonbuild/wrapdb/master/releases.json"
+    )
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.create_federatedcode_working_dir,
+            cls.fetch_wrapdb_releases,
+            cls.fetch_federation_config,
+            cls.mine_and_publish_packageurls,
+            cls.delete_working_dir,
+        )
+
+    def fetch_wrapdb_releases(self):
+        """Fetch the Meson WrapDB releases.json index."""
+        try:
+            response = requests.get(self.MESON_WRAPDB_RELEASES_URL, timeout=30)
+            response.raise_for_status()
+            self.releases = response.json()
+        except Exception as e:
+            self.log(f"Failed to fetch releases.json: {e}")
+            self.releases = {}
+
+    def packages_count(self):
+        """Return the number of packages in the WrapDB releases index.
+
+        Used by MineCodeBasePipeline to report mining progress.
+        """
+        return len(self.releases) if hasattr(self, "releases") and self.releases else 0
+
+    def mine_packageurls(self):
+        """Yield PackageURLs from Meson WrapDB releases.json."""
+        return meson.mine_meson_packageurls(releases=self.releases, logger=self.log)

minecode_pipelines/pipes/meson.py

@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from packageurl import PackageURL
+
+
+def get_meson_packages(package_name, package_data):
+    """
+    Return a tuple of (base_purl, [versioned_purl_strings]) for a single
+    Meson WrapDB package entry from ``releases.json``.
+
+    The ``package_data`` dict has the structure::
+
+        {
+            "dependency_names": ["dep1", "dep2"],
+            "versions": ["1.0.0-1", "1.0.0-2", ...]
+        }
+
+    WrapDB versions use a ``-N`` suffix to denote build recipe revisions that
+    are specific to the WrapDB and do not exist upstream.
+    """
+    base_purl = PackageURL(type="meson", name=package_name)
+    versions = package_data.get("versions") or []
+    versioned_purls = [
+        PackageURL(
+            type="meson",
+            name=package_name,
+            version=str(version),
+        ).to_string()
+        for version in versions
+    ]
+    return base_purl, versioned_purls
+
+
+def mine_meson_packageurls(releases, logger):
+    """
+    Yield ``(base_purl, [versioned_purl_strings])`` tuples from a
+    Meson WrapDB ``releases.json`` mapping.
+    """
+    for package_name, package_data in releases.items():
+        if not package_data:
+            continue
+        yield get_meson_packages(
+            package_name=package_name,
+            package_data=package_data,
+        )

minecode_pipelines/tests/pipes/test_meson.py

@@ -0,0 +1,64 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+from pathlib import Path
+
+from django.test import TestCase
+
+from minecode_pipelines.pipes.meson import get_meson_packages
+
+DATA_DIR = Path(__file__).parent.parent / "test_data" / "meson"
+
+
+class MesonPipeTests(TestCase):
+    def test_get_meson_packages_empty_versions(self):
+        """Test that get_meson_packages handles empty version lists."""
+        package_data = {
+            "dependency_names": ["empty-pkg"],
+            "versions": [],
+        }
+        base_purl, versioned_purls = get_meson_packages("empty-pkg", package_data)
+
+        self.assertEqual(str(base_purl), "pkg:meson/empty-pkg")
+        self.assertEqual(versioned_purls, [])
+
+    def test_get_meson_packages_no_versions_key(self):
+        """Test that get_meson_packages handles missing versions key."""
+        package_data = {
+            "dependency_names": ["no-ver"],
+        }
+        base_purl, versioned_purls = get_meson_packages("no-ver", package_data)
+
+        self.assertEqual(str(base_purl), "pkg:meson/no-ver")
+        self.assertEqual(versioned_purls, [])
+
+    def test_get_meson_packages_from_releases_json(self):
+        """Test parsing packages from the real WrapDB releases.json fixture
+        using data-driven expected output."""
+        releases_path = DATA_DIR / "releases.json"
+        expected_path = DATA_DIR / "expected_purls.json"
+
+        with open(releases_path, encoding="utf-8") as f:
+            releases = json.load(f)
+
+        with open(expected_path, encoding="utf-8") as f:
+            expected = json.load(f)
+
+        actual = {}
+        for package_name, package_data in releases.items():
+            if not package_data:
+                continue
+            base_purl, versioned_purls = get_meson_packages(
+                package_name=package_name,
+                package_data=package_data,
+            )
+            actual[str(base_purl)] = sorted(versioned_purls)
+
+        self.assertEqual(actual, expected)

minecode_pipelines/tests/test_data/meson/expected_purls.json

@@ -0,0 +1,46 @@
+{
+  "pkg:meson/adamyaxley-obfuscate": [
+    "pkg:meson/adamyaxley-obfuscate@1.0.0-1"
+  ],
+  "pkg:meson/aklomp-base64": [
+    "pkg:meson/aklomp-base64@0.5.2-1"
+  ],
+  "pkg:meson/apache-orc": [
+    "pkg:meson/apache-orc@2.2.0-1",
+    "pkg:meson/apache-orc@2.2.1-1",
+    "pkg:meson/apache-orc@2.2.2-1"
+  ],
+  "pkg:meson/bzip2": [
+    "pkg:meson/bzip2@1.0.8-1"
+  ],
+  "pkg:meson/catch2": [
+    "pkg:meson/catch2@2.11.1-1",
+    "pkg:meson/catch2@2.11.3-1",
+    "pkg:meson/catch2@2.13.3-1",
+    "pkg:meson/catch2@2.13.3-2",
+    "pkg:meson/catch2@2.13.7-1",
+    "pkg:meson/catch2@2.13.8-1",
+    "pkg:meson/catch2@2.4.1-1",
+    "pkg:meson/catch2@2.5.0-1",
+    "pkg:meson/catch2@2.7.2-1",
+    "pkg:meson/catch2@2.8.0-1",
+    "pkg:meson/catch2@2.9.0-1",
+    "pkg:meson/catch2@3.1.0-1",
+    "pkg:meson/catch2@3.10.0-1",
+    "pkg:meson/catch2@3.11.0-1",
+    "pkg:meson/catch2@3.12.0-1",
+    "pkg:meson/catch2@3.13.0-1",
+    "pkg:meson/catch2@3.2.0-1",
+    "pkg:meson/catch2@3.3.2-1",
+    "pkg:meson/catch2@3.4.0-1",
+    "pkg:meson/catch2@3.5.2-1",
+    "pkg:meson/catch2@3.5.3-1",
+    "pkg:meson/catch2@3.5.4-1",
+    "pkg:meson/catch2@3.6.0-1",
+    "pkg:meson/catch2@3.7.0-1",
+    "pkg:meson/catch2@3.7.1-1",
+    "pkg:meson/catch2@3.8.0-1",
+    "pkg:meson/catch2@3.8.1-1",
+    "pkg:meson/catch2@3.9.1-1"
+  ]
+}

minecode_pipelines/tests/test_data/meson/releases.json

@@ -0,0 +1,72 @@
+{
+    "adamyaxley-obfuscate": {
+        "dependency_names": [
+            "Obfuscate"
+        ],
+        "versions": [
+            "1.0.0-1"
+        ]
+    },
+    "aklomp-base64": {
+        "dependency_names": [
+            "base64"
+        ],
+        "versions": [
+            "0.5.2-1"
+        ]
+    },
+    "apache-orc": {
+        "dependency_names": [
+            "orc"
+        ],
+        "versions": [
+            "2.2.2-1",
+            "2.2.1-1",
+            "2.2.0-1"
+        ]
+    },
+    "bzip2": {
+        "dependency_names": [
+            "bzip2"
+        ],
+        "versions": [
+            "1.0.8-1"
+        ]
+    },
+    "catch2": {
+        "dependency_names": [
+            "catch2",
+            "catch2-with-main"
+        ],
+        "versions": [
+            "3.13.0-1",
+            "3.12.0-1",
+            "3.11.0-1",
+            "3.10.0-1",
+            "3.9.1-1",
+            "3.8.1-1",
+            "3.8.0-1",
+            "3.7.1-1",
+            "3.7.0-1",
+            "3.6.0-1",
+            "3.5.4-1",
+            "3.5.3-1",
+            "3.5.2-1",
+            "3.4.0-1",
+            "3.3.2-1",
+            "3.2.0-1",
+            "3.1.0-1",
+            "2.13.8-1",
+            "2.13.7-1",
+            "2.13.3-2",
+            "2.13.3-1",
+            "2.11.3-1",
+            "2.11.1-1",
+            "2.9.0-1",
+            "2.8.0-1",
+            "2.7.2-1",
+            "2.5.0-1",
+            "2.4.1-1"
+        ]
+    }
+}

pyproject-minecode_pipelines.toml

@@ -62,6 +62,7 @@ mine_cpan = "minecode_pipelines.pipelines.mine_cpan:MineCpan"
 mine_cran = "minecode_pipelines.pipelines.mine_cran:MineCran"
 mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift"
 mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer"
+mine_meson = "minecode_pipelines.pipelines.mine_meson:MineMeson"
 
 [tool.bumpversion]
 current_version = "0.1.1"