From a46060a53fe440890d738063a6a2858f7597e5ba Mon Sep 17 00:00:00 2001 From: Peter Marheine Date: Wed, 20 Mar 2019 11:20:11 +1100 Subject: [PATCH 1/2] Parallelize compressors Compressors can be slow, so it's easy to get a significant speedup by running compressors in parallel with a thread pool. --- optimage.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/optimage.py b/optimage.py index 7e904a9..2a77c80 100644 --- a/optimage.py +++ b/optimage.py @@ -16,6 +16,7 @@ import collections import contextlib import logging +import multiprocessing.pool import os import os.path import shutil @@ -173,9 +174,15 @@ def _compress_with(input_filename, output_filename, compressors): input. """ with _temporary_filenames(len(compressors)) as temp_filenames: - results = [] - for compressor, temp_filename in zip(compressors, temp_filenames): - results.append(_process(compressor, input_filename, temp_filename)) + process_args = [ + (compressor, input_filename, temp_filename) + for compressor, temp_filename in zip(compressors, temp_filenames) + ] + process_unpack = lambda args: _process(*args) + results = multiprocessing.pool.ThreadPool().map(process_unpack, + process_args, + chunksize=1) + best_result = min(results) os.rename(best_result.filename, output_filename) From eece150034c2cdf7b16b39db570225454223116b Mon Sep 17 00:00:00 2001 From: Peter Marheine Date: Wed, 20 Mar 2019 15:21:38 +1100 Subject: [PATCH 2/2] Stop using pytest-catchlog Since pytest 3.3 the caplog fixture is provided by pytest, and it conflicts with pytest-catchlog. We can simply remove the catchlog plugin requirement and require pytest >= 3.3. --- setup.py | 2 +- test_optimage_e2e.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/setup.py b/setup.py index bb11f12..4b04c14 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ scripts=['scripts/optimage'], setup_requires=['pytest-runner'], install_requires=['Pillow'], - tests_require=['pytest', 'pytest-cov', 'pytest-catchlog'], + tests_require=['pytest>=3.3', 'pytest-cov'], classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', diff --git a/test_optimage_e2e.py b/test_optimage_e2e.py index e3bd388..5442e5e 100644 --- a/test_optimage_e2e.py +++ b/test_optimage_e2e.py @@ -3,11 +3,6 @@ import subprocess import pytest -try: - import pytest_catchlog - catchlog_available = True -except ImportError: - catchlog_available = False import optimage @@ -147,8 +142,6 @@ def mock_check_output(args, stderr=None): assert 'Output:\ncustom error' in err -@pytest.mark.skipif(not catchlog_available, - reason='pytest_catchlog not available') @pytest.mark.parametrize('filename, compressor', [ # Do not specify compressor as it depends on the version of the commands # installed. In Mac I get that pngcrush is better than zopflipng, but not in