Add support for Linux and macOS

This is an MVP implementation of Linux and macOS support. Everything is
functional but there are two caveats that should certainly be resolved
in future versions:
 * The standard library is not pre-compiled and zipped so it takes up
   more space than the Windows variant.
 * The environment is not as locked down as the Windows variant:
   `pip` is still accessible in the final package.

On Windows, we can simply download a pre-built binary embedded package,
but on macOS and Linux, we need to build from source and lock down the
environment ourselves. This means that the two points above require a
bit more effort. We can tackle this later. For now, this implementation
is certainly good enough to get going.

Author: dean0x7d

.github/workflows/test_package.yml

@@ -4,10 +4,11 @@ on: [push]
 
 jobs:
   test_package:
-    runs-on: windows-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        embedded-py: [3.7.7, 3.8.3]
+        os: [windows-latest, ubuntu-latest, macos-latest]
+        embedded-py: [3.7.7, 3.8.10]
     env:
       create_pck: conan create . lumicks/testing -o embedded_python:version=${{ matrix.embedded-py }}
     steps:
@@ -18,7 +19,7 @@ jobs:
         python-version: 3.7
     - name: Install Conan
       run: | 
-        python -m pip install conan==1.25.2
+        python -m pip install conan==1.36.0
         conan profile new default --detect
     - name: Test baseline
       run: ${{ env.create_pck }}

changelog.md

@@ -1,7 +1,10 @@
 # Changelog
 
-## v1.3.0 | In development
+## v1.3.0 | 2021-06-03
 
+- Added support for Linux and macOS with a couple of caveats to be resolved later:
+  * The standard library is not pre-compiled and zipped so it takes up more space than the Windows variant
+  * The environment is not as locked down as the Windows variant: `pip` is still accessible in the final package
 - The `packages` option now accepts the full contents of a `requirements.txt` file.
   Previously, the contents needed to be converted into a space-separated list (`.replace("\n", " ")`) and stripped of comments and markers.
 - CMake will now automatically call `find_package(Python)` and ensure that the embedded distribution is found instead of a system-installed Python.

conanfile.py

@@ -10,12 +10,42 @@ class EmbeddedPython(ConanFile):
     description = "Embedded distribution of Python"
     url = "https://www.python.org/"
     license = "PSFL"
-    settings = {"os": ["Windows"]}
-    options = {"version": "ANY", "packages": "ANY"}
-    default_options = "packages=None"
+    settings = "os", "compiler", "build_type", "arch"
+    options = {
+        "version": "ANY", 
+        "packages": "ANY", 
+        "openssl_variant": ["lowercase", "uppercase"]  # see explanation in `build_requirements()`
+    }
+    default_options = {
+        "packages": None,
+        "openssl_variant": "lowercase"
+    }
     exports = "embedded_python_tools.py", "embedded_python.cmake"
     short_paths = True  # some of the pip packages go over the 260 char path limit on Windows
 
+    def config_options(self):
+        """On Windows, we download a binary so these options have no effect"""
+        if self.settings.os == "Windows":
+            del self.settings.compiler
+            del self.settings.build_type
+
+    def build_requirements(self):
+        """On Windows, we download a binary so we don't need anything else"""
+        if self.settings.os == "Windows":
+            return
+
+        self.build_requires("sqlite3/3.35.5")
+        self.build_requires("bzip2/1.0.8")
+
+        # The pre-conan-center-index version of `openssl` was capitalized as `OpenSSL`.
+        # Both versions can't live in the same Conan cache so we need this compatibility
+        # option to pick the available version. The cache case-sensitivity issue should
+        # be solved in Conan 2.0, but we need this for now.
+        if self.options.openssl_variant == "lowercase":
+            self.build_requires("openssl/1.1.1k")
+        else:
+            self.build_requires("OpenSSL/1.1.1f")
+
     @property
     def pyversion(self):
         """Full Python version that we want to package"""
@@ -64,7 +94,10 @@ def _gather_licenses(self, bootstrap):
 
     def build(self):
         prefix = pathlib.Path(self.build_folder) / "embedded_python"
-        build_helper = WindowsBuildHelper(self, prefix)
+        if self.settings.os == "Windows":
+            build_helper = WindowsBuildHelper(self, prefix)
+        else:
+            build_helper = UnixLikeBuildHelper(self, prefix)
         build_helper.build_embedded()
 
         if not self.options.packages:
@@ -142,3 +175,72 @@ def build_bootstrap(self):
 
         return python_exe
 
+
+class UnixLikeBuildHelper:
+    def __init__(self, conanfile, prefix_dir):
+        self.conanfile = conanfile
+        self.prefix_dir = prefix_dir
+
+    def _get_source(self):
+        url = f"https://github.com/python/cpython/archive/v{self.conanfile.pyversion}.tar.gz"
+        tools.get(url)
+        os.rename(f"cpython-{self.conanfile.pyversion}", "src")
+
+    @property
+    def _openssl_path(self):
+        if self.conanfile.options.openssl_variant == "lowercase":
+            pck = "openssl"
+        else:
+            pck = "OpenSSL"
+        return self.conanfile.deps_cpp_info[pck].rootpath
+
+    def _build(self, dest_dir):
+        from conans import AutoToolsBuildEnvironment
+
+        autotools = AutoToolsBuildEnvironment(self.conanfile)
+        env_vars = autotools.vars
+
+        # On Linux, we need to set RPATH so that `root/bin/python3` can correctly find the `.so`
+        # file in `root/lib` no matter where `root` is. We need it to be portable. We explicitly
+        # set `--disable-new-dtags` to use RPATH instead of RUNPATH. RUNPATH can be overridden by
+        # the LD_LIBRARY_PATH env variable which is not at all what we want for this self-contained
+        # package. Unlike RUNPATH, RPATH takes precedence over LD_LIBRARY_PATH.
+        if self.conanfile.settings.os == "Linux":
+            env_vars["LDFLAGS"] += " -Wl,-rpath,'$$ORIGIN/../lib' -Wl,--disable-new-dtags"
+        
+        config_args = " ".join([
+            "--enable-shared",
+            f"--prefix={dest_dir}",
+            f"--with-openssl={self._openssl_path}",
+        ])
+
+        tools.mkdir("./build")
+        with tools.chdir("./build"), tools.environment_append(env_vars):
+            self.conanfile.run(f"../src/configure {config_args}")
+            self.conanfile.run("make -j8")
+            self.conanfile.run("make install -j8")
+
+        ver = ".".join(self.conanfile.pyversion.split(".")[:2])
+        exe = str(dest_dir / f"bin/python{ver}")
+        self.conanfile.run(f"{exe} -m pip install -U pip==21.1.1")
+
+        # Move the license file to match the Windows layout
+        lib_dir = dest_dir / "lib"
+        os.rename(lib_dir / f"python{ver}/LICENSE.txt", dest_dir / "LICENSE.txt")
+
+        # Give write permissions, otherwise end-user projects won't be able to re-import
+        # the shared libraries (re-import happens on subsequent `conan install` runs).
+        for file in lib_dir.glob("libpython*"):
+            self.conanfile.run(f"chmod 777 {file}")
+
+    def build_embedded(self):
+        self._get_source()
+        self._build(self.prefix_dir)
+
+    def enable_site_packages(self):
+        """These are enabled by default when building from source"""
+        pass
+
+    def build_bootstrap(self):
+        """For now, as a shortcut, we'll let the Unix-like builds bootstrap themselves"""
+        return self.prefix_dir / "bin/python3"

embedded_python_tools.py

@@ -3,7 +3,17 @@
 import pathlib
 
 
-def symlink_import(self, dst="bin/python/interpreter", bin="bin"):
+def _symlink_compat(conanfile, src, dst):
+    """On Windows, symlinks require admin privileges, so we use a directory junction instead"""
+    if conanfile.settings.os == "Windows":
+        import _winapi
+
+        _winapi.CreateJunction(str(src), str(dst))
+    else:
+        os.symlink(src, dst)
+
+
+def symlink_import(conanfile, dst="bin/python/interpreter", bin="bin"):
     """Copying the entire embedded Python environment is extremely slow, so we just symlink it
 
     Usage:
@@ -13,8 +23,7 @@ def imports(self):
         embedded_python_tools.symlink_import(self, dst="bin/python/interpreter")
     ```
 
-    On Windows, symlinks require admin privileges, so we use a directory junction instead.
-    It points to the Conan package location. We still want to copy in `python*.dll` and
+    The symlink points to the Conan package location. We still want to copy in `python*.dll` and
     `python*.zip` right next to the executable so that they can be found, but the rest of
     the Python environment is in a subfolder:
 
@@ -27,24 +36,21 @@ def imports(self):
     |- python*.zip
     \- ...
     """
-    if self.settings.os != "Windows":
-        return
-
-    import _winapi
-
     dst = pathlib.Path(dst).absolute()
     if not dst.parent.exists():
         dst.parent.mkdir(parents=True)
 
     if dst.exists():
-        try:  # to remove any existing junction
+        try:  # to remove any existing junction/symlink
             os.remove(dst)
         except:  # this seems to be the only way to find out this is not a junction 
             shutil.rmtree(dst)
 
-    src = pathlib.Path(self.deps_cpp_info["embedded_python"].rootpath) / "embedded_python"
-    _winapi.CreateJunction(str(src), str(dst))
+    src = pathlib.Path(conanfile.deps_cpp_info["embedded_python"].rootpath) / "embedded_python"
+    _symlink_compat(conanfile, src, dst)
 
     bin = pathlib.Path(bin).absolute()
-    self.copy("python*.dll", dst=bin, src="embedded_python", keep_path=False)
-    self.copy("python*.zip", dst=bin, src="embedded_python", keep_path=False)
+    conanfile.copy("python*.dll", dst=bin, src="embedded_python", keep_path=False)
+    conanfile.copy("libpython*.so*", dst=bin, src="embedded_python/lib", keep_path=False)
+    conanfile.copy("libpython*.dylib", dst=bin, src="embedded_python/lib", keep_path=False)
+    conanfile.copy("python*.zip", dst=bin, src="embedded_python", keep_path=False)

test_package/conanfile.py

@@ -41,7 +41,10 @@ def _test_env(self):
         if self.options.env:
             script += f"import {name}; print('Found {name}');"
 
-        python_exe = str(pathlib.Path("./bin/python/python").resolve())
+        if self.settings.os == "Windows":
+            python_exe = str(pathlib.Path("./bin/python/python").resolve())
+        else:
+            python_exe = str(pathlib.Path("./bin/python/bin/python3").resolve())
         self.run([python_exe, "-c", script])
 
         if self.options.env:

test_package/envs/baseline_test.py

@@ -1,6 +1,8 @@
+import bz2
+import lzma
+import ssl
 import sqlite3
-import _ssl
-import _decimal
 import zlib
+import _decimal
 
 print("All optional Python features are importable")

test_package/envs/nbconvert.txt

@@ -22,7 +22,7 @@ pygments==2.7.4
 pyparsing==2.4.7
 pyrsistent==0.17.3
 python-dateutil==2.8.1
-pywin32==300
+pywin32==300; sys_platform == "win32"
 pyzmq==22.0.2
 six==1.15.0
 testpath==0.4.4