From ec21f83b83564be873aab28c48711e4fa0eb60fa Mon Sep 17 00:00:00 2001 From: Killian Blais Date: Sun, 17 Aug 2025 13:42:30 +0530 Subject: [PATCH] feat: dual install modes (modern Py>=3.6 via pyproject, legacy via setup.py); drop lib2to3 on Py3; update README, tox, classifiers; bump to 0.6.0 --- .gitignore | 2 ++ .vscode/settings.json | 5 +++++ README.rst | 27 +++++++++++++++++++++++++++ poetry.lock | 17 +++++++++++++++++ pyproject.toml | 39 +++++++++++++++++++++++++++++++++++++++ setup.py | 1 + tox.ini | 7 +++++-- unify.py | 34 ++++++++++++++++++++++++++++++---- 8 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 poetry.lock create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index d55efed..805b92d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ unify.egg-info/ untokenize-0.1-py3.3.egg .idea/ .python-version +.continue +environment.yml diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..06fee29 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [] +} diff --git a/README.rst b/README.rst index 07e4d10..6000200 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,33 @@ unify Modifies strings to all use the same quote where possible. +Installation +============ + +There are two installation modes: + +- Modern (Python 3.6+): + + Uses the default package which removes the deprecated ``lib2to3`` dependency + and supports Python 3.6 and newer. + + - From PyPI: + + ``pip install unify`` + + - From source (builds a wheel via PEP 517): + + ``pip install .`` + +- Legacy (Python 2.6/2.7): + + For legacy environments, install from source using ``setup.py``. This path + retains Python 2.x compatibility and uses ``lib2to3`` only on Python 2 for + encoding detection. + + ``python2 setup.py install`` + + Example ======= diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..e3ffc74 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,17 @@ +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. + +[[package]] +name = "untokenize" +version = "0.1.1" +description = "Transforms tokens into original source code (while preserving whitespace)." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "untokenize-0.1.1.tar.gz", hash = "sha256:3865dbbbb8efb4bb5eaa72f1be7f3e0be00ea8b7f125c69cbd1f5fda926f37a2"}, +] + +[metadata] +lock-version = "2.1" +python-versions = ">=2.6, <4.0" +content-hash = "263afe7136c891a28c2a32d7b36884f259efe93fbda488089af9332deed9c1d1" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b4ecd27 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[project] +name = "unify" +version = "0.6.0" +description = "Modifies strings to all use the same quote where possible." +authors = [{ name = "Steven Myint", email = "git@stevenmyint.com" }] +license = { text = "MIT License" } +readme = "README.rst" +requires-python = ">=3.6" +dependencies = ["untokenize (>=0.1.1,<0.2.0)"] + +classifiers = [ + "Intended Audience :: Developers", + "Environment :: Console", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", +] + + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project.urls] +Homepage = "https://github.com/myint/unify" +Repository = "https://github.com/myint/unify" + +[project.scripts] +unify = "unify:main" + +[tool.setuptools] +py-modules = ["unify"] diff --git a/setup.py b/setup.py index 51a48e7..b722ae9 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ def version(): description='Modifies strings to all use the same ' '(single/double) quote where possible.', long_description=readme.read(), + long_description_content_type='text/x-rst', license='Expat License', author='Steven Myint', url='https://github.com/myint/unify', diff --git a/tox.ini b/tox.ini index c49dc8f..1b72753 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,8 @@ [tox] -envlist = py2.6,py2.7,py3.4,py3.5,py3.6 +envlist = py36,py37,py38,py39,py310,py311,py312 [testenv] -commands=python setup.py test +deps = + pytest +commands = + pytest -q diff --git a/unify.py b/unify.py index 15f0a56..409defb 100755 --- a/unify.py +++ b/unify.py @@ -36,7 +36,7 @@ import untokenize -__version__ = '0.5' +__version__ = '0.6.0' try: @@ -67,7 +67,7 @@ def _format_code(source, preferred_quote): end, line) in tokenize.generate_tokens(sio.readline): - if token_type == tokenize.STRING: + if token_type == tokenize.STRING or _looks_like_string_literal(token_string): token_string = unify_quotes(token_string, preferred_quote=preferred_quote) @@ -112,6 +112,28 @@ def unify_quotes(token_string, preferred_quote): ) +def _looks_like_string_literal(token_string): + """Return True if token_string appears to be a string literal. + + This is a heuristic to handle Python 3.12+ f-string tokenization changes + where f-strings may not always be returned as tokenize.STRING. + """ + if not token_string or len(token_string) < 2: + return False + + # Accept optional single-letter prefixes and triple-quoted strings + prefixes = ("", "f", "r", "u", "b") + quotes = ("'", '"') + + for prefix in prefixes: + for quote in quotes: + start = prefix + quote + triple_start = prefix + quote * 3 + if token_string.startswith(start) or token_string.startswith(triple_start): + return True + return False + + def open_with_encoding(filename, encoding, mode='r'): """Return opened file with a specific encoding.""" return io.open(filename, mode=mode, encoding=encoding, @@ -122,8 +144,12 @@ def detect_encoding(filename): """Return file encoding.""" try: with open(filename, 'rb') as input_file: - from lib2to3.pgen2 import tokenize as lib2to3_tokenize - encoding = lib2to3_tokenize.detect_encoding(input_file.readline)[0] + if sys.version_info[0] < 3: + from lib2to3.pgen2 import tokenize as lib2to3_tokenize + encoding = lib2to3_tokenize.detect_encoding(input_file.readline)[0] + else: + # Use the standard library's tokenize in modern Python. + encoding = tokenize.detect_encoding(input_file.readline)[0] # Check for correctness of encoding. with open_with_encoding(filename, encoding) as input_file: