From ea4e281d12cb563189f001e52bc123665f5dc5f3 Mon Sep 17 00:00:00 2001 From: tamas6 <81620050+tamas6@users.noreply.github.com> Date: Wed, 3 Jan 2024 15:13:14 +0100 Subject: [PATCH 1/4] updated session timeout check & upload --- swarmsync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarmsync.py b/swarmsync.py index 79269d9..037e7f0 100644 --- a/swarmsync.py +++ b/swarmsync.py @@ -554,7 +554,7 @@ async def directupload(file: FileManager, url: str, session: aiohttp.ClientSessi async def async_check(scheduled, url: str): global display,args sem = asyncio.Semaphore(args.count) - session_timeout=aiohttp.ClientTimeout(total=14400) + session_timeout=aiohttp.ClientTimeout(total=(3*14400)) async with sem, aiohttp.ClientSession(timeout=session_timeout) as session: res = await asyncio.gather(*[aioget(ref, url, session, sem) for ref in scheduled]) display.close() @@ -566,7 +566,7 @@ async def async_upload(scheduled, urll): l_url = list(islice(cycle(urll), len(scheduled))) scheduled = [FileManager(file) for file in scheduled] sem = asyncio.Semaphore(args.count) - session_timeout=aiohttp.ClientTimeout(total=14400) + session_timeout=aiohttp.ClientTimeout(total=(3*14400)) async with sem, aiohttp.ClientSession(timeout=session_timeout) as session: res = await asyncio.gather(*[aioupload(file, url, session, sem) for file, url in zip(scheduled, l_url)]) print(f'\nitems uploaded ({len(res)})') From b7765c264f496f6801a46543dbe224176c4ce1fb Mon Sep 17 00:00:00 2001 From: tamas6 <81620050+tamas6@users.noreply.github.com> Date: Thu, 23 May 2024 14:19:29 +0200 Subject: [PATCH 2/4] feat: html generator, docs --- README.md | 69 ++++++++++++++----- generate_html.py | 171 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+), 15 deletions(-) create mode 100644 generate_html.py diff --git a/README.md b/README.md index 32b95de..104874f 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Swarmsync is a Python script that provides a set of tools for managing files on - Create and manage tags for uploaded files - Encrypt and pin files during upload - Monitor and send Prometheus statistics +- Generate html file from .json containing download references ## Getting Started @@ -35,6 +36,14 @@ Swarmsync is a Python script that provides a set of tools for managing files on ### Usage +```bash +python3 +``` + +```bash +python3 +``` + #### Show You can use the `show` command to display information about the files you have uploaded and their status. @@ -44,6 +53,7 @@ python swarmsync.py show [options] ``` Options: + - `-s, --saved-tag`: Check the existing/stored tag UID. - `-t, --tag`: Enter a tag UID to fetch information about a specific tag. @@ -56,6 +66,7 @@ python swarmsync.py download [options] ``` Options: + - `-c, --count`: Number of concurrent tasks for downloading. - `-u, --beeurl`: Bee node URL(s) (comma-separated) to connect to. @@ -68,6 +79,7 @@ python swarmsync.py check [options] ``` Options: + - `-c, --count`: Number of concurrent tasks for checking. - `-u, --beeurl`: Bee node URL to connect to. @@ -80,6 +92,7 @@ python swarmsync.py upload [options] ``` Options: + - `-p, --path`: Path to the folder to be uploaded. - `-s, --search`: Search parameter (e.g., `*`, `*.jpg`, `somename.txt`). - `-P, --pin`: Should files be pinned during upload. @@ -99,6 +112,7 @@ python swarmsync.py mantaray [options] ``` Options: + - `-c, --count`: Number of concurrent tasks for uploading the Mantaray index. - `-u, --beeurl`: Bee node URL(s) (comma-separated) to connect to. @@ -106,38 +120,63 @@ Options: - To upload a folder with files and subfolders: - ```bash - python swarmsync.py upload -p /path/to/folder - ``` + ```bash + python swarmsync.py upload -p /path/to/folder + ``` - To download files from the Ethereum Swarm network: - ```bash - python swarmsync.py download -c 5 -u http://yourbeeurl:1633 - ``` + ```bash + python swarmsync.py download -c 5 -u http://yourbeeurl:1633 + ``` - To check the status of uploaded files: - ```bash - python swarmsync.py check -c 10 -u http://yourbeeurl:1633 - ``` + ```bash + python swarmsync.py check -c 10 -u http://yourbeeurl:1633 + ``` - To display information about uploaded files and their status: - ```bash - python swarmsync.py show responses - ``` + ```bash + python swarmsync.py show responses + ``` - To create and manage a Mantaray index: - ```bash - python swarmsync.py mantaray -u http://yourbeeurl:1633 - ``` + ```bash + python swarmsync.py mantaray -u http://yourbeeurl:1633 + ``` ### Additional Options Swarmsync allows you to configure additional options by modifying the script. These options include the Prometheus statistics endpoint, xBee header usage, and more. Please refer to the script's source code for these advanced configurations. +## Generate HTML + +If you want to show your set of data available on swarm in which inculdes several files, this tool helps you organize it in a comprehensible way. + +```bash +python swarmsync.py [xyz.json] [footer.txt] +``` + +### Features + +- `footer.txt`: after adding the .json fifle, any txt format file can be added to be the footer
+- `thumbnail.png`: in the working directory if there's a thumbnail with this specific name, it will be added to the html as full size header + +- It will print the exact total size of all the data in megabytes, which can be downloaded. + +- The output file name is the same to the .json it was created from + +### Example + +```bash +➜ python generate_html.py Stack-Exchange-Kiwix.bzz.json +Stack-Exchange-Kiwix.bzz.html file has been generated successfully. +Total size of files: 330160.76 MB +``` + ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/generate_html.py b/generate_html.py new file mode 100644 index 0000000..8c04b35 --- /dev/null +++ b/generate_html.py @@ -0,0 +1,171 @@ +import sys +import re +import json +import os.path +import base64 + +def convert_bytes_to_mb(size_in_bytes): + return round(size_in_bytes / (1024 * 1024), 2) + +def generate_html(json_file, page_title="File Names and References", footer_file=None): + # Extract the filename from the JSON file path + output_filename = os.path.splitext(os.path.basename(json_file))[0] + ".html" + + # Read the JSON file + with open(json_file, 'r') as f: + json_data = f.read() + + # Parse the JSON data + data = json.loads(json_data) + + # Extract the filenames, references, and sizes + table_rows = '' + total_size_mb = 0 # New variable to hold the total size in MB + for obj in data: + file_value = obj.get('file', '') + filename = os.path.basename(file_value) + reference = obj.get('reference', 'Unknown') + size = obj.get('size', 0) + total_size_mb += size # Accumulate the size for total calculation + + link = f'https://pubee.datafund.io/bzz/{reference}' + table_rows += f'{filename}{reference}{convert_bytes_to_mb(size)}' + + # Embed the thumbnail image into base64 format + thumbnail_image = "" + if os.path.exists("thumbnail.png"): + with open("thumbnail.png", "rb") as image_file: + encoded_image = base64.b64encode(image_file.read()).decode('utf-8') + thumbnail_image = f'Thumbnail Image' + + # Read the content from the footer file + footer_content = "" + if footer_file: + with open(footer_file, 'r') as f: + footer_content = f.read() + + # Generate the HTML content + html_content = f''' + + + {page_title} + + + + + {thumbnail_image} +

{page_title}

+ + + + + + + + {table_rows} +
File NameSwarm Reference HashSize in MB
+
+ {footer_content} +
+ + + ''' + + # Write the HTML content to the output file named after the JSON file + with open(output_filename, 'w') as f: + f.write(html_content) + + # Print the HTML generation message along with the total size in MB + print(f"{output_filename} file has been generated successfully.") + print(f"Total size of files: {convert_bytes_to_mb(total_size_mb)} MB") + +# Check if the JSON file location is provided as an argument +if len(sys.argv) < 2: + print("Please provide the location of the JSON file as an argument.") +else: + json_file_location = sys.argv[1] + page_title = sys.argv[2] if len(sys.argv) >= 3 else "File Names and References" + footer_file = sys.argv[3] if len(sys.argv) >= 4 else None + generate_html(json_file_location, page_title, footer_file) From 9aac17f61aa40680bf56b50b5d08a62cf372c3ed Mon Sep 17 00:00:00 2001 From: tamas6 <81620050+tamas6@users.noreply.github.com> Date: Thu, 23 May 2024 14:54:44 +0200 Subject: [PATCH 3/4] fix: docs --- README.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 104874f..3eaca68 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Swarmsync is a Python script that provides a set of tools for managing files on - Create and manage tags for uploaded files - Encrypt and pin files during upload - Monitor and send Prometheus statistics -- Generate html file from .json containing download references +- Generate HTML files from JSON containing download references ## Getting Started @@ -36,10 +36,20 @@ Swarmsync is a Python script that provides a set of tools for managing files on ### Usage +For interacting with Swarm network + ```bash python3 ``` +In case you have multiple files uploaded to Swarm, and you would like to show them together. For example: + +- a dapp, +- earth geospatial data, +- royalty free videos,
+ + this is a convenient way to handle. Upload the created HTML via swarmsync. + ```bash python3 ``` @@ -154,7 +164,7 @@ Swarmsync allows you to configure additional options by modifying the script. Th ## Generate HTML -If you want to show your set of data available on swarm in which inculdes several files, this tool helps you organize it in a comprehensible way. +If you want to show your set of data available on Swarm, this tool helps you organize it in a comprehensible way. ```bash python swarmsync.py [xyz.json] [footer.txt] @@ -162,17 +172,18 @@ python swarmsync.py [xyz.json] [footer.txt] ### Features -- `footer.txt`: after adding the .json fifle, any txt format file can be added to be the footer
-- `thumbnail.png`: in the working directory if there's a thumbnail with this specific name, it will be added to the html as full size header +- `footer.txt`: After adding the JSON file, any text file can be added as the footer.
+- `thumbnail.png`: If a file named thumbnail.png is in the working directory, it will be added to the HTML as a full-size header. - It will print the exact total size of all the data in megabytes, which can be downloaded. -- The output file name is the same to the .json it was created from +- The output file name matches the JSON file it was created from. ### Example ```bash -➜ python generate_html.py Stack-Exchange-Kiwix.bzz.json +python3 generate_html.py Stack-Exchange-Kiwix.bzz.json footer.txt + Stack-Exchange-Kiwix.bzz.html file has been generated successfully. Total size of files: 330160.76 MB ``` From 2b8afb6b461ff875cf4815a06a901b5554c40072 Mon Sep 17 00:00:00 2001 From: tamas6 <81620050+tamas6@users.noreply.github.com> Date: Mon, 27 May 2024 12:47:33 +0200 Subject: [PATCH 4/4] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3eaca68..8fd3b08 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ Swarmsync allows you to configure additional options by modifying the script. Th If you want to show your set of data available on Swarm, this tool helps you organize it in a comprehensible way. ```bash -python swarmsync.py [xyz.json] [footer.txt] +python generate_html.py [anything.json] [footer.txt] ``` ### Features