diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e5b5350 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +NCORE_USERNAME="" +NCORE_PASSWORD="" +RSS_URL="" diff --git a/.env.example.sh b/.env.example.sh deleted file mode 100644 index b66b6bd..0000000 --- a/.env.example.sh +++ /dev/null @@ -1,3 +0,0 @@ -export NCORE_USERNAME="" -export NCORE_PASSWORD="" -export RSS_URL="" diff --git a/.gitignore b/.gitignore index a412646..2fa60eb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ ncoreparser.egg-info/ .venv .vscode .env.sh +.env *.torrent .tox* diff --git a/Makefile b/Makefile index 042055f..9050672 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ ACTIVATE = . .venv/bin/activate +define LOAD_ENV + @set -a; [ -f .env ] && source .env; set +a; +endef + .venv: rm -rf .venv python3 -m venv .venv @@ -21,10 +25,10 @@ test: $(ACTIVATE) && tox module-test: - $(ACTIVATE) && cd ./tests/test_module && NCORE_USERNAME="${NCORE_USERNAME}" NCORE_PASSWORD="${NCORE_PASSWORD}" RSS_URL="${RSS_URL}" tox + $(LOAD_ENV) $(ACTIVATE) && cd ./tests/test_module && NCORE_USERNAME="${NCORE_USERNAME}" NCORE_PASSWORD="${NCORE_PASSWORD}" RSS_URL="${RSS_URL}" tox manual-test: - $(ACTIVATE) && python -m tests.manual --user ${NCORE_USERNAME} \ + $(LOAD_ENV) $(ACTIVATE) && python -m tests.manual --user ${NCORE_USERNAME} \ --passw "${NCORE_PASSWORD}" --rss-feed "${RSS_URL}" git-tag: diff --git a/Readme.md b/Readme.md index 62d42d5..f4265be 100644 --- a/Readme.md +++ b/Readme.md @@ -31,8 +31,12 @@ if __name__ == "__main__": client.login("", "") for t_type in SearchParamType: - torrent = client.search(pattern="", type=t_type, number=1, - sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING)[0] + torrent = client.search( + pattern="", + type=t_type, + sort_by=ParamSort.SEEDERS, + sort_order=ParamSeq.DECREASING + )[0] print(torrent['title'], torrent['type'], torrent['size'], torrent['id']) client.logout() @@ -50,8 +54,12 @@ if __name__ == "__main__": client.login("", "") - torrent = client.search(pattern="Forrest gump", type=SearchParamType.HD_HUN, number=1, - sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING)[0] + torrent = client.search( + pattern="Forrest gump", + type=SearchParamType.HD_HUN, + sort_by=ParamSort.SEEDERS, + sort_order=ParamSeq.DECREASING + )[0] client.download(torrent, "/tmp") client.logout() @@ -88,8 +96,14 @@ if __name__ == "__main__": torrents = client.get_by_activity() for torrent in torrents: - print(torrent['title'], torrent['type'], torrent['size'], - torrent['id'], torrent['rate'], torrent['remaining']) + print( + torrent['title'], + torrent['type'], + torrent['size'], + torrent['id'], + torrent['rate'], + torrent['remaining'] + ) client.logout() ``` @@ -125,8 +139,12 @@ async def main(): await client.login("", "") for t_type in SearchParamType: - torrent = await client.search(pattern="", type=t_type, number=1, - sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING)[0] + torrent = await client.search( + pattern="", + type=t_type, + sort_by=ParamSort.SEEDERS, + sort_order=ParamSeq.DECREASING + )[0] print(torrent['title'], torrent['type'], torrent['size'], torrent['id']) await client.logout() diff --git a/ncoreparser/__init__.py b/ncoreparser/__init__.py index 9c4d1ec..ac2e8b4 100644 --- a/ncoreparser/__init__.py +++ b/ncoreparser/__init__.py @@ -5,3 +5,4 @@ from .error import NcoreDownloadError, NcoreParserError, NcoreCredentialError, NcoreConnectionError from .util import Size from .torrent import Torrent +from .types import SearchResult diff --git a/ncoreparser/client.py b/ncoreparser/client.py index 8422d0d..fb8cc7c 100644 --- a/ncoreparser/client.py +++ b/ncoreparser/client.py @@ -1,11 +1,12 @@ import os import httpx -from typing_extensions import Any, Generator, Union +from typing_extensions import Any, Generator, Union # pylint: disable=no-name-in-module from ncoreparser.data import URLs, SearchParamType, SearchParamWhere, ParamSort, ParamSeq from ncoreparser.error import NcoreConnectionError, NcoreCredentialError, NcoreDownloadError from ncoreparser.parser import TorrentsPageParser, TorrenDetailParser, RssParser, ActivityParser, RecommendedParser from ncoreparser.util import Size, check_login from ncoreparser.torrent import Torrent +from ncoreparser.types import SearchResult class Client: @@ -40,29 +41,23 @@ def search( where: SearchParamWhere = SearchParamWhere.NAME, sort_by: ParamSort = ParamSort.UPLOAD, sort_order: ParamSeq = ParamSeq.DECREASING, - number: Union[int, None] = None, - ) -> list[Torrent]: - page_count = 1 - torrents: list[Torrent] = [] - while number is None or len(torrents) < number: - url = URLs.DOWNLOAD_PATTERN.value.format( - page=page_count, - t_type=type.value, - sort=sort_by.value, - seq=sort_order.value, - pattern=pattern, - where=where.value, - ) - try: - request = self._client.get(url) - except Exception as e: - raise NcoreConnectionError(f"Error while searhing torrents. {e}") from e - new_torrents = [Torrent(**params) for params in self._page_parser.get_items(request.text)] - torrents.extend(new_torrents) - if number is None or len(new_torrents) == 0: - return torrents - page_count += 1 - return torrents[:number] + page: int = 1, + ) -> SearchResult: + url = URLs.DOWNLOAD_PATTERN.value.format( + page=page, + t_type=type.value, + sort=sort_by.value, + seq=sort_order.value, + pattern=pattern, + where=where.value, + ) + try: + request = self._client.get(url) + except Exception as e: + raise NcoreConnectionError(f"Error while searhing torrents. {e}") from e + torrents = [Torrent(**params) for params in self._page_parser.get_items(request.text)] + num_of_pages = self._page_parser.get_num_of_pages(request.text) + return SearchResult(torrents=torrents, num_of_pages=num_of_pages) @check_login def get_torrent(self, id: str, **ext_params: Any) -> Torrent: diff --git a/ncoreparser/client_async.py b/ncoreparser/client_async.py index 28f4f9b..cf20cb1 100644 --- a/ncoreparser/client_async.py +++ b/ncoreparser/client_async.py @@ -2,12 +2,13 @@ import os import httpx -from typing_extensions import Any, AsyncGenerator, Union +from typing_extensions import Any, AsyncGenerator, Union # pylint: disable=no-name-in-module from ncoreparser.data import URLs, SearchParamType, SearchParamWhere, ParamSort, ParamSeq from ncoreparser.error import NcoreConnectionError, NcoreCredentialError, NcoreDownloadError from ncoreparser.parser import TorrentsPageParser, TorrenDetailParser, RssParser, ActivityParser, RecommendedParser from ncoreparser.util import Size, check_login from ncoreparser.torrent import Torrent +from ncoreparser.types import SearchResult class AsyncClient: @@ -42,29 +43,23 @@ async def search( where: SearchParamWhere = SearchParamWhere.NAME, sort_by: ParamSort = ParamSort.UPLOAD, sort_order: ParamSeq = ParamSeq.DECREASING, - number: Union[int, None] = None, - ) -> list[Torrent]: - page_count = 1 - torrents: list[Torrent] = [] - while number is None or len(torrents) < number: - url = URLs.DOWNLOAD_PATTERN.value.format( - page=page_count, - t_type=type.value, - sort=sort_by.value, - seq=sort_order.value, - pattern=pattern, - where=where.value, - ) - try: - request = await self._client.get(url) - except Exception as e: - raise NcoreConnectionError(f"Error while searhing torrents. {e}") from e - new_torrents = [Torrent(**params) for params in self._page_parser.get_items(request.text)] - torrents.extend(new_torrents) - if number is None or len(new_torrents) == 0: - return torrents - page_count += 1 - return torrents[:number] + page: int = 1, + ) -> SearchResult: + url = URLs.DOWNLOAD_PATTERN.value.format( + page=page, + t_type=type.value, + sort=sort_by.value, + seq=sort_order.value, + pattern=pattern, + where=where.value, + ) + try: + request = await self._client.get(url) + except Exception as e: + raise NcoreConnectionError(f"Error while searhing torrents. {e}") from e + torrents = [Torrent(**params) for params in self._page_parser.get_items(request.text)] + num_of_pages = self._page_parser.get_num_of_pages(request.text) + return SearchResult(torrents=torrents, num_of_pages=num_of_pages) @check_login async def get_torrent(self, id: str, **ext_params: Any) -> Torrent: diff --git a/ncoreparser/parser.py b/ncoreparser/parser.py index 9d420ec..cbaf508 100644 --- a/ncoreparser/parser.py +++ b/ncoreparser/parser.py @@ -1,12 +1,14 @@ +import math import re import datetime -from typing_extensions import Generator, Any, Union +from typing_extensions import Generator, Any, Union # pylint: disable=no-name-in-module from ncoreparser.error import NcoreParserError from ncoreparser.util import parse_datetime, Size from ncoreparser.data import SearchParamType, get_detailed_param class TorrentsPageParser: + # pylint: disable=too-many-instance-attributes def __init__(self) -> None: self.type_pattern = re.compile( r'.*' @@ -19,6 +21,8 @@ def __init__(self) -> None: self.not_found_pattern = re.compile(r'
Nincs találat!
') self.seeders_pattern = re.compile(r'
([0-9]+)
') self.leechers_pattern = re.compile(r'
([0-9]+)
') + self.current_page_pattern = re.compile(r'(\d+).*?') + self.last_page_pattern = re.compile(r' int: + current_num_of_items_found = self.current_page_pattern.search(data) + last_page_found = self.last_page_pattern.search(data) + + num_of_pages = 0 + if current_num_of_items_found: + current_num_of_items = int(current_num_of_items_found.group(1)) + num_of_pages = math.ceil(current_num_of_items / 25) + if last_page_found: + last_page = int(last_page_found.group(1)) + num_of_pages = max(num_of_pages, last_page) + return num_of_pages + class TorrenDetailParser: def __init__(self) -> None: diff --git a/ncoreparser/types.py b/ncoreparser/types.py new file mode 100644 index 0000000..37782fd --- /dev/null +++ b/ncoreparser/types.py @@ -0,0 +1,8 @@ +from dataclasses import dataclass +from ncoreparser.torrent import Torrent + + +@dataclass +class SearchResult: + torrents: list[Torrent] + num_of_pages: int diff --git a/ncoreparser/util.py b/ncoreparser/util.py index e27630a..a5708b8 100644 --- a/ncoreparser/util.py +++ b/ncoreparser/util.py @@ -1,6 +1,6 @@ import datetime import functools -from typing_extensions import Self, Callable, Any, Union +from typing_extensions import Self, Callable, Any, Union # pylint: disable=no-name-in-module from ncoreparser.error import NcoreConnectionError diff --git a/pyproject.toml b/pyproject.toml index 8d9bdd9..58091d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "ncoreparser" -version = "3.2.0" +version = "4.0.0" description = "Package to download from ncore.pro" authors = [ { name="Aron Radics", email="aron.radics.jozsef@gmail.com" } diff --git a/tests/manual.py b/tests/manual.py index b3580c6..3817d56 100644 --- a/tests/manual.py +++ b/tests/manual.py @@ -55,22 +55,18 @@ def pretty_print(torrent): print_category("Most seeded torrents/category") for t_type in SearchParamType: - torrent = client.search( - pattern="", type=t_type, number=1, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING - )[0] - pretty_print(torrent) + result = client.search(pattern="", type=t_type, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING) + pretty_print(result.torrents[0]) - print("") - print("Donwnload torrent") - torrent = client.search( - pattern="Forrest gump", + print_category("Donwnload torrent") + result = client.search( + pattern="Forrest", type=SearchParamType.HD_HUN, - number=1, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING, - )[0] - - client.download(torrent, "/tmp", override=True) + ) + pretty_print(result.torrents[0]) + client.download(result.torrents[0], "/tmp", override=True) print_category("List by rss") torrents = client.get_by_rss(args.rss_feed) diff --git a/tests/manual_async.py b/tests/manual_async.py index 95f6d72..07f0a95 100644 --- a/tests/manual_async.py +++ b/tests/manual_async.py @@ -56,7 +56,7 @@ async def main(): print_category("Most seeded torrents/category") for t_type in SearchParamType: torrents = await client.search( - pattern="", type=t_type, number=1, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING + pattern="", type=t_type, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING ) pretty_print(torrents[0]) @@ -65,7 +65,6 @@ async def main(): torrents = await client.search( pattern="Forrest gump", type=SearchParamType.HD_HUN, - number=1, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING, ) diff --git a/tests/test_module/test.py b/tests/test_module/test.py index 86c8617..c57753c 100644 --- a/tests/test_module/test.py +++ b/tests/test_module/test.py @@ -1,5 +1,5 @@ -import pytest import os +import pytest from ncoreparser import Client, SearchParamType from ncoreparser.torrent import Torrent @@ -15,12 +15,9 @@ def client(self): return c def test_search(self, client): - torrents = client.search(pattern="forrest gump", type=SearchParamType.HD, number=1) + torrents = client.search(pattern="forrest gump", type=SearchParamType.HD) - assert len(torrents) == 1 - torrent = torrents[0] - - assert isinstance(torrent, Torrent) + assert isinstance(torrents[0], Torrent) def test_rss(self, client): rss_url = os.environ["RSS_URL"] @@ -42,7 +39,7 @@ def test_recommended(self, client): assert torrent["type"] == SearchParamType.HDSER_HUN def test_download(self, client): - torrent = client.search(pattern="forrest gump", type=SearchParamType.HD_HUN, number=1)[0] + torrent = client.search(pattern="forrest gump", type=SearchParamType.HD_HUN)[0] client.download(torrent, ".", override=True)