Support pagination for search

.env.example

@@ -0,0 +1,3 @@
+NCORE_USERNAME="<username>"
+NCORE_PASSWORD="<password>"
+RSS_URL="<rss_url>"

.gitignore

@@ -7,5 +7,6 @@ ncoreparser.egg-info/
 .venv
 .vscode
 .env.sh
+.env
 *.torrent
 .tox*

Makefile

@@ -1,5 +1,9 @@
 ACTIVATE = . .venv/bin/activate
 
+define LOAD_ENV
+    @set -a; [ -f .env ] && source .env; set +a;
+endef
+
 .venv:
 	rm -rf .venv
 	python3 -m venv .venv
@@ -21,10 +25,10 @@ test:
 	$(ACTIVATE) && tox
 
 module-test:
-	$(ACTIVATE) && cd ./tests/test_module && NCORE_USERNAME="${NCORE_USERNAME}" NCORE_PASSWORD="${NCORE_PASSWORD}" RSS_URL="${RSS_URL}" tox
+	$(LOAD_ENV) $(ACTIVATE) && cd ./tests/test_module && NCORE_USERNAME="${NCORE_USERNAME}" NCORE_PASSWORD="${NCORE_PASSWORD}" RSS_URL="${RSS_URL}" tox
 
 manual-test:
-	$(ACTIVATE) && python -m tests.manual --user ${NCORE_USERNAME} \
+	$(LOAD_ENV) $(ACTIVATE) && python -m tests.manual --user ${NCORE_USERNAME} \
 		--passw "${NCORE_PASSWORD}" --rss-feed "${RSS_URL}"
 
 git-tag:

Readme.md

@@ -31,8 +31,12 @@ if __name__ == "__main__":
     client.login("<username>", "<password>")
 
     for t_type in SearchParamType:
-        torrent = client.search(pattern="", type=t_type, number=1,
-                                sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING)[0]
+        torrent = client.search(
+            pattern="",
+            type=t_type,
+            sort_by=ParamSort.SEEDERS,
+            sort_order=ParamSeq.DECREASING
+        )[0]
         print(torrent['title'], torrent['type'], torrent['size'], torrent['id'])
 
     client.logout()
@@ -50,8 +54,12 @@ if __name__ == "__main__":
     client.login("<username>", "<password>")
 
 
-    torrent = client.search(pattern="Forrest gump", type=SearchParamType.HD_HUN, number=1,
-                            sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING)[0]
+    torrent = client.search(
+        pattern="Forrest gump",
+        type=SearchParamType.HD_HUN,
+        sort_by=ParamSort.SEEDERS,
+        sort_order=ParamSeq.DECREASING
+    )[0]
 
     client.download(torrent, "/tmp")
     client.logout()
@@ -88,8 +96,14 @@ if __name__ == "__main__":
 
     torrents = client.get_by_activity()
     for torrent in torrents:
-        print(torrent['title'], torrent['type'], torrent['size'],
-              torrent['id'], torrent['rate'], torrent['remaining'])
+        print(
+            torrent['title'],
+            torrent['type'],
+            torrent['size'],
+            torrent['id'],
+            torrent['rate'],
+            torrent['remaining']
+        )
 
     client.logout()
 ```
@@ -125,8 +139,12 @@ async def main():
     await client.login("<username>", "<password>")
 
     for t_type in SearchParamType:
-        torrent = await client.search(pattern="", type=t_type, number=1,
-                                      sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING)[0]
+        torrent = await client.search(
+            pattern="",
+            type=t_type,
+            sort_by=ParamSort.SEEDERS,
+            sort_order=ParamSeq.DECREASING
+        )[0]
         print(torrent['title'], torrent['type'], torrent['size'], torrent['id'])
 
     await client.logout()

ncoreparser/__init__.py

@@ -5,3 +5,4 @@
 from .error import NcoreDownloadError, NcoreParserError, NcoreCredentialError, NcoreConnectionError
 from .util import Size
 from .torrent import Torrent
+from .types import SearchResult

ncoreparser/client.py

@@ -1,11 +1,12 @@
 import os
 import httpx
-from typing_extensions import Any, Generator, Union
+from typing_extensions import Any, Generator, Union  # pylint: disable=no-name-in-module
 from ncoreparser.data import URLs, SearchParamType, SearchParamWhere, ParamSort, ParamSeq
 from ncoreparser.error import NcoreConnectionError, NcoreCredentialError, NcoreDownloadError
 from ncoreparser.parser import TorrentsPageParser, TorrenDetailParser, RssParser, ActivityParser, RecommendedParser
 from ncoreparser.util import Size, check_login
 from ncoreparser.torrent import Torrent
+from ncoreparser.types import SearchResult
 
 
 class Client:
@@ -40,29 +41,23 @@ def search(
         where: SearchParamWhere = SearchParamWhere.NAME,
         sort_by: ParamSort = ParamSort.UPLOAD,
         sort_order: ParamSeq = ParamSeq.DECREASING,
-        number: Union[int, None] = None,
-    ) -> list[Torrent]:
-        page_count = 1
-        torrents: list[Torrent] = []
-        while number is None or len(torrents) < number:
-            url = URLs.DOWNLOAD_PATTERN.value.format(
-                page=page_count,
-                t_type=type.value,
-                sort=sort_by.value,
-                seq=sort_order.value,
-                pattern=pattern,
-                where=where.value,
-            )
-            try:
-                request = self._client.get(url)
-            except Exception as e:
-                raise NcoreConnectionError(f"Error while searhing torrents. {e}") from e
-            new_torrents = [Torrent(**params) for params in self._page_parser.get_items(request.text)]
-            torrents.extend(new_torrents)
-            if number is None or len(new_torrents) == 0:
-                return torrents
-            page_count += 1
-        return torrents[:number]
+        page: int = 1,
+    ) -> SearchResult:
+        url = URLs.DOWNLOAD_PATTERN.value.format(
+            page=page,
+            t_type=type.value,
+            sort=sort_by.value,
+            seq=sort_order.value,
+            pattern=pattern,
+            where=where.value,
+        )
+        try:
+            request = self._client.get(url)
+        except Exception as e:
+            raise NcoreConnectionError(f"Error while searhing torrents. {e}") from e
+        torrents = [Torrent(**params) for params in self._page_parser.get_items(request.text)]
+        num_of_pages = self._page_parser.get_num_of_pages(request.text)
+        return SearchResult(torrents=torrents, num_of_pages=num_of_pages)
 
     @check_login
     def get_torrent(self, id: str, **ext_params: Any) -> Torrent:

ncoreparser/client_async.py

@@ -2,12 +2,13 @@
 
 import os
 import httpx
-from typing_extensions import Any, AsyncGenerator, Union
+from typing_extensions import Any, AsyncGenerator, Union  # pylint: disable=no-name-in-module
 from ncoreparser.data import URLs, SearchParamType, SearchParamWhere, ParamSort, ParamSeq
 from ncoreparser.error import NcoreConnectionError, NcoreCredentialError, NcoreDownloadError
 from ncoreparser.parser import TorrentsPageParser, TorrenDetailParser, RssParser, ActivityParser, RecommendedParser
 from ncoreparser.util import Size, check_login
 from ncoreparser.torrent import Torrent
+from ncoreparser.types import SearchResult
 
 
 class AsyncClient:
@@ -42,29 +43,23 @@ async def search(
         where: SearchParamWhere = SearchParamWhere.NAME,
         sort_by: ParamSort = ParamSort.UPLOAD,
         sort_order: ParamSeq = ParamSeq.DECREASING,
-        number: Union[int, None] = None,
-    ) -> list[Torrent]:
-        page_count = 1
-        torrents: list[Torrent] = []
-        while number is None or len(torrents) < number:
-            url = URLs.DOWNLOAD_PATTERN.value.format(
-                page=page_count,
-                t_type=type.value,
-                sort=sort_by.value,
-                seq=sort_order.value,
-                pattern=pattern,
-                where=where.value,
-            )
-            try:
-                request = await self._client.get(url)
-            except Exception as e:
-                raise NcoreConnectionError(f"Error while searhing torrents. {e}") from e
-            new_torrents = [Torrent(**params) for params in self._page_parser.get_items(request.text)]
-            torrents.extend(new_torrents)
-            if number is None or len(new_torrents) == 0:
-                return torrents
-            page_count += 1
-        return torrents[:number]
+        page: int = 1,
+    ) -> SearchResult:
+        url = URLs.DOWNLOAD_PATTERN.value.format(
+            page=page,
+            t_type=type.value,
+            sort=sort_by.value,
+            seq=sort_order.value,
+            pattern=pattern,
+            where=where.value,
+        )
+        try:
+            request = await self._client.get(url)
+        except Exception as e:
+            raise NcoreConnectionError(f"Error while searhing torrents. {e}") from e
+        torrents = [Torrent(**params) for params in self._page_parser.get_items(request.text)]
+        num_of_pages = self._page_parser.get_num_of_pages(request.text)
+        return SearchResult(torrents=torrents, num_of_pages=num_of_pages)
 
     @check_login
     async def get_torrent(self, id: str, **ext_params: Any) -> Torrent:

ncoreparser/parser.py

@@ -1,12 +1,14 @@
+import math
 import re
 import datetime
-from typing_extensions import Generator, Any, Union
+from typing_extensions import Generator, Any, Union  # pylint: disable=no-name-in-module
 from ncoreparser.error import NcoreParserError
 from ncoreparser.util import parse_datetime, Size
 from ncoreparser.data import SearchParamType, get_detailed_param
 
 
 class TorrentsPageParser:
+    # pylint: disable=too-many-instance-attributes
     def __init__(self) -> None:
         self.type_pattern = re.compile(
             r'<a href=".*\/torrents\.php\?tipus=(.*?)"><img src=".*" class="categ_link" alt=".*" title=".*">'
@@ -19,6 +21,8 @@ def __init__(self) -> None:
         self.not_found_pattern = re.compile(r'<div class="lista_mini_error">Nincs találat!</div>')
         self.seeders_pattern = re.compile(r'<div class="box_s2"><a class="torrent" href=".*">([0-9]+)</a></div>')
         self.leechers_pattern = re.compile(r'<div class="box_l2"><a class="torrent" href=".*">([0-9]+)</a></div>')
+        self.current_page_pattern = re.compile(r'<span class="active_link"><strong>(\d+).*?</strong></span>')
+        self.last_page_pattern = re.compile(r'<a href="/torrents\.php\?oldal=(\d+)[^>]*><strong>Utolsó</strong></a>')
 
     @staticmethod
     def get_key(data: str) -> Union[str, None]:
@@ -57,6 +61,19 @@ def get_items(self, data: str) -> Generator[dict, None, None]:
                 "leech": leech[i],
             }
 
+    def get_num_of_pages(self, data: str) -> int:
+        current_num_of_items_found = self.current_page_pattern.search(data)
+        last_page_found = self.last_page_pattern.search(data)
+
+        num_of_pages = 0
+        if current_num_of_items_found:
+            current_num_of_items = int(current_num_of_items_found.group(1))
+            num_of_pages = math.ceil(current_num_of_items / 25)
+        if last_page_found:
+            last_page = int(last_page_found.group(1))
+            num_of_pages = max(num_of_pages, last_page)
+        return num_of_pages
+
 
 class TorrenDetailParser:
     def __init__(self) -> None:

ncoreparser/types.py

@@ -0,0 +1,8 @@
+from dataclasses import dataclass
+from ncoreparser.torrent import Torrent
+
+
+@dataclass
+class SearchResult:
+    torrents: list[Torrent]
+    num_of_pages: int

ncoreparser/util.py

@@ -1,6 +1,6 @@
 import datetime
 import functools
-from typing_extensions import Self, Callable, Any, Union
+from typing_extensions import Self, Callable, Any, Union  # pylint: disable=no-name-in-module
 from ncoreparser.error import NcoreConnectionError
 
 

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ncoreparser"
-version = "3.2.0"
+version = "4.0.0"
 description = "Package to download from ncore.pro"
 authors = [
   { name="Aron Radics", email="aron.radics.jozsef@gmail.com" }

tests/manual.py

@@ -55,22 +55,18 @@ def pretty_print(torrent):
 
     print_category("Most seeded torrents/category")
     for t_type in SearchParamType:
-        torrent = client.search(
-            pattern="", type=t_type, number=1, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING
-        )[0]
-        pretty_print(torrent)
+        result = client.search(pattern="", type=t_type, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING)
+        pretty_print(result.torrents[0])
 
-    print("")
-    print("Donwnload torrent")
-    torrent = client.search(
-        pattern="Forrest gump",
+    print_category("Donwnload torrent")
+    result = client.search(
+        pattern="Forrest",
         type=SearchParamType.HD_HUN,
-        number=1,
         sort_by=ParamSort.SEEDERS,
         sort_order=ParamSeq.DECREASING,
-    )[0]
-
-    client.download(torrent, "/tmp", override=True)
+    )
+    pretty_print(result.torrents[0])
+    client.download(result.torrents[0], "/tmp", override=True)
 
     print_category("List by rss")
     torrents = client.get_by_rss(args.rss_feed)

tests/manual_async.py

@@ -56,7 +56,7 @@ async def main():
     print_category("Most seeded torrents/category")
     for t_type in SearchParamType:
         torrents = await client.search(
-            pattern="", type=t_type, number=1, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING
+            pattern="", type=t_type, sort_by=ParamSort.SEEDERS, sort_order=ParamSeq.DECREASING
         )
         pretty_print(torrents[0])
 
@@ -65,7 +65,6 @@ async def main():
     torrents = await client.search(
         pattern="Forrest gump",
         type=SearchParamType.HD_HUN,
-        number=1,
         sort_by=ParamSort.SEEDERS,
         sort_order=ParamSeq.DECREASING,
     )

tests/test_module/test.py

@@ -1,5 +1,5 @@
-import pytest
 import os
+import pytest
 from ncoreparser import Client, SearchParamType
 from ncoreparser.torrent import Torrent
 
@@ -15,12 +15,9 @@ def client(self):
         return c
 
     def test_search(self, client):
-        torrents = client.search(pattern="forrest gump", type=SearchParamType.HD, number=1)
+        torrents = client.search(pattern="forrest gump", type=SearchParamType.HD)
 
-        assert len(torrents) == 1
-        torrent = torrents[0]
-
-        assert isinstance(torrent, Torrent)
+        assert isinstance(torrents[0], Torrent)
 
     def test_rss(self, client):
         rss_url = os.environ["RSS_URL"]
@@ -42,7 +39,7 @@ def test_recommended(self, client):
             assert torrent["type"] == SearchParamType.HDSER_HUN
 
     def test_download(self, client):
-        torrent = client.search(pattern="forrest gump", type=SearchParamType.HD_HUN, number=1)[0]
+        torrent = client.search(pattern="forrest gump", type=SearchParamType.HD_HUN)[0]
 
         client.download(torrent, ".", override=True)