Skip to content

Commit 92ed3e8

Browse files
Adityakk9031Adityakk9031
committed
Add Meson WrapDB mining pipeline #803
Signed-off-by: Aditya kumar singh <143548997+Adityakk9031@users.noreply.github.com>
1 parent 594973b commit 92ed3e8

5 files changed

Lines changed: 271 additions & 0 deletions

File tree

‎minecode_pipelines/pipelines/mine_meson.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/aboutcode-org/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/aboutcode-org/scancode.io for support and download.
22+
23+
import json
24+
from pathlib import Path
25+
26+
from minecode_pipelines.pipes import meson
27+
from minecode_pipelines.pipelines import MineCodeBasePipeline
28+
from scanpipe.pipes import federatedcode
29+
30+
31+
class MineMeson(MineCodeBasePipeline):
32+
"""Pipeline to mine Meson WrapDB packages and publish them to FederatedCode repo."""
33+
34+
MESON_WRAPDB_REPO = "https://github.com/mesonbuild/wrapdb"
35+
36+
@classmethod
37+
def steps(cls):
38+
return (
39+
cls.check_federatedcode_eligibility,
40+
cls.create_federatedcode_working_dir,
41+
cls.clone_wrapdb_index,
42+
cls.fetch_federation_config,
43+
cls.mine_and_publish_packageurls,
44+
cls.delete_working_dir,
45+
)
46+
47+
def clone_wrapdb_index(self):
48+
"""Clone the Meson WrapDB repository."""
49+
self.wrapdb_repo = federatedcode.clone_repository(
50+
repo_url=self.MESON_WRAPDB_REPO,
51+
clone_path=self.working_path / "wrapdb",
52+
logger=self.log,
53+
)
54+
55+
def packages_count(self):
56+
releases_path = Path(self.wrapdb_repo.working_dir) / "releases.json"
57+
if not releases_path.exists():
58+
return 0
59+
with open(releases_path, encoding="utf-8") as f:
60+
releases = json.load(f)
61+
return len(releases)
62+
63+
def mine_packageurls(self):
64+
"""Yield PackageURLs from Meson WrapDB releases.json."""
65+
return meson.mine_meson_packageurls(
66+
wrapdb_repo=self.wrapdb_repo,
67+
logger=self.log,
68+
)

‎minecode_pipelines/pipes/meson.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
#
3+
# http://nexb.com and https://github.com/aboutcode-org/scancode.io
4+
# The ScanCode.io software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode.io is provided as-is without warranties.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
16+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
17+
# ScanCode.io should be considered or used as legal advice. Consult an Attorney
18+
# for any legal advice.
19+
#
20+
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
21+
# Visit https://github.com/aboutcode-org/scancode.io for support and download.
22+
23+
import json
24+
from pathlib import Path
25+
26+
from packageurl import PackageURL
27+
28+
29+
MESON_WRAPDB_RELEASES_URL = (
30+
"https://raw.githubusercontent.com/mesonbuild/wrapdb/master/releases.json"
31+
)
32+
33+
34+
def get_meson_packages(package_name, package_data):
35+
"""
36+
Return a tuple of (base_purl, [versioned_purl_strings]) for a single
37+
Meson WrapDB package entry from ``releases.json``.
38+
39+
The ``package_data`` dict has the structure::
40+
41+
{
42+
"dependency_names": ["dep1", "dep2"],
43+
"versions": ["1.0.0-1", "1.0.0-2", ...]
44+
}
45+
46+
WrapDB versions use a ``-N`` suffix to denote build recipe revisions that
47+
are specific to the WrapDB and do not exist upstream.
48+
"""
49+
base_purl = PackageURL(type="meson", name=package_name)
50+
versions = package_data.get("versions") or []
51+
versioned_purls = [
52+
PackageURL(
53+
type="meson",
54+
name=package_name,
55+
version=str(version),
56+
).to_string()
57+
for version in versions
58+
]
59+
return base_purl, versioned_purls
60+
61+
62+
def mine_meson_packageurls(wrapdb_repo, logger):
63+
"""
64+
Yield ``(base_purl, [versioned_purl_strings])`` tuples from a cloned
65+
Meson WrapDB repository by parsing its ``releases.json``.
66+
"""
67+
releases_path = Path(wrapdb_repo.working_dir) / "releases.json"
68+
if not releases_path.exists():
69+
logger(f"releases.json not found at {releases_path}")
70+
return
71+
72+
with open(releases_path, encoding="utf-8") as f:
73+
releases = json.load(f)
74+
75+
for package_name, package_data in releases.items():
76+
if not package_data:
77+
continue
78+
yield get_meson_packages(
79+
package_name=package_name,
80+
package_data=package_data,
81+
)

‎minecode_pipelines/tests/pipes/test_meson.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# purldb is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/aboutcode-org/purldb for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
import json
11+
from pathlib import Path
12+
13+
from django.test import TestCase
14+
15+
from minecode_pipelines.pipes.meson import get_meson_packages
16+
17+
DATA_DIR = Path(__file__).parent.parent / "test_data" / "meson"
18+
19+
20+
class MesonPipeTests(TestCase):
21+
def test_get_meson_packages_basic(self):
22+
"""Test that get_meson_packages correctly parses a single package entry."""
23+
package_data = {
24+
"dependency_names": ["ogg"],
25+
"versions": ["1.3.6-1", "1.3.5-3", "1.3.5-2", "1.3.5-1"],
26+
}
27+
base_purl, versioned_purls = get_meson_packages("ogg", package_data)
28+
29+
self.assertEqual(str(base_purl), "pkg:meson/ogg")
30+
self.assertEqual(len(versioned_purls), 4)
31+
self.assertIn("pkg:meson/ogg@1.3.6-1", versioned_purls)
32+
self.assertIn("pkg:meson/ogg@1.3.5-1", versioned_purls)
33+
34+
def test_get_meson_packages_empty_versions(self):
35+
"""Test that get_meson_packages handles empty version lists."""
36+
package_data = {
37+
"dependency_names": ["empty-pkg"],
38+
"versions": [],
39+
}
40+
base_purl, versioned_purls = get_meson_packages("empty-pkg", package_data)
41+
42+
self.assertEqual(str(base_purl), "pkg:meson/empty-pkg")
43+
self.assertEqual(versioned_purls, [])
44+
45+
def test_get_meson_packages_no_versions_key(self):
46+
"""Test that get_meson_packages handles missing versions key."""
47+
package_data = {
48+
"dependency_names": ["no-ver"],
49+
}
50+
base_purl, versioned_purls = get_meson_packages("no-ver", package_data)
51+
52+
self.assertEqual(str(base_purl), "pkg:meson/no-ver")
53+
self.assertEqual(versioned_purls, [])
54+
55+
def test_get_meson_packages_from_releases_json(self):
56+
"""Test parsing packages from the test releases.json fixture."""
57+
releases_path = DATA_DIR / "releases.json"
58+
with open(releases_path, encoding="utf-8") as f:
59+
releases = json.load(f)
60+
61+
all_results = []
62+
for package_name, package_data in releases.items():
63+
if not package_data:
64+
continue
65+
all_results.append(
66+
get_meson_packages(
67+
package_name=package_name,
68+
package_data=package_data,
69+
)
70+
)
71+
72+
self.assertEqual(len(all_results), 3) # ogg, zlib, catch2
73+
74+
# Check ogg
75+
ogg_base, ogg_purls = all_results[0]
76+
self.assertEqual(str(ogg_base), "pkg:meson/ogg")
77+
self.assertEqual(len(ogg_purls), 4)
78+
79+
# Check zlib
80+
zlib_base, zlib_purls = all_results[1]
81+
self.assertEqual(str(zlib_base), "pkg:meson/zlib")
82+
self.assertEqual(len(zlib_purls), 3)
83+
84+
# Check catch2
85+
catch2_base, catch2_purls = all_results[2]
86+
self.assertEqual(str(catch2_base), "pkg:meson/catch2")
87+
self.assertEqual(len(catch2_purls), 2)
88+
self.assertIn("pkg:meson/catch2@3.5.2-1", catch2_purls)

‎minecode_pipelines/tests/test_data/meson/releases.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"ogg": {
3+
"dependency_names": [
4+
"ogg"
5+
],
6+
"versions": [
7+
"1.3.6-1",
8+
"1.3.5-3",
9+
"1.3.5-2",
10+
"1.3.5-1"
11+
]
12+
},
13+
"zlib": {
14+
"dependency_names": [
15+
"zlib"
16+
],
17+
"versions": [
18+
"1.3.1-1",
19+
"1.2.13-2",
20+
"1.2.13-1"
21+
]
22+
},
23+
"catch2": {
24+
"dependency_names": [
25+
"catch2",
26+
"catch2-with-main"
27+
],
28+
"versions": [
29+
"3.5.2-1",
30+
"3.4.0-1"
31+
]
32+
}
33+
}

‎pyproject-minecode_pipelines.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ mine_cpan = "minecode_pipelines.pipelines.mine_cpan:MineCpan"
6262
mine_cran = "minecode_pipelines.pipelines.mine_cran:MineCran"
6363
mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift"
6464
mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer"
65+
mine_meson = "minecode_pipelines.pipelines.mine_meson:MineMeson"
6566

6667
[tool.bumpversion]
6768
current_version = "0.1.1"

0 commit comments

Comments
 (0)