From c55ea8e0d7a7cb1049c359f41d5f2c330aa54c6a Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Wed, 11 Nov 2026 20:31:23 +0000 Subject: [PATCH 1/4] Load libpython of a Python outside the system library directories before jep (#54) On Linux, libjep.so failed to load with "libpython3.X.so.1.0: cannot open shared object file" for a Python installed by uv, pyenv, conda or in a custom prefix, unless its lib directory was on LD_LIBRARY_PATH at JVM start. The environment probe now reports sysconfig LIBDIR and INSTSONAME; before Jep loads its native library, PythonEnvironment System.loads LIBDIR/INSTSONAME by its path (Linux only, shared library only), so the loader resolves libjep's dependency by soname. As the JVM loads it RTLD_LOCAL, and Jep's own RTLD_GLOBAL promotion looks it up by file name, which the loader cannot find, the first interpreter makes its symbols global by loading it as an extension module with RTLD_GLOBAL dlopen flags, so that C extensions not linked to libpython (numpy, _ctypes of most builds) still import. If loading still fails, the error names the directory to put on LD_LIBRARY_PATH. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FJmHQ3jHRbVdpwUiNUkApG --- DEVELOPMENT.md | 11 +- README.md | 10 ++ src/main/kotlin/Python.kt | 5 +- src/main/kotlin/PythonEnvironment.kt | 132 +++++++++++++++-- src/test/kotlin/ForkedJvm.kt | 8 +- src/test/kotlin/PythonEnvironmentTest.kt | 177 ++++++++++++++++++++++- src/test/kotlin/PythonLibraryTest.kt | 54 +++++++ 7 files changed, 371 insertions(+), 26 deletions(-) create mode 100644 src/test/kotlin/PythonLibraryTest.kt diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b861595..5cc36ac 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -18,12 +18,13 @@ or with the `ORG_GRADLE_PROJECT_basePython` environment variable, e.g. on CI: ``` It has to be Python 3.10 or newer, otherwise the build fails before creating the environment. -On Linux, the tests fail to load `libjep.so` when it cannot find `libpython3.*.so` ([#54](https://github.com/xemantic/xemantic-kotlin-script-python/issues/54)). -A Python outside the system library directories, e.g. installed by `uv`, -needs its `lib` directory in `LD_LIBRARY_PATH`, -which `actions/setup-python` sets on CI. +On Linux, a Python outside the system library directories, e.g. installed by `uv`, +does not need its `lib` directory in `LD_LIBRARY_PATH`, +as `PythonEnvironment` loads its `libpython` before `libjep.so` ([#54](https://github.com/xemantic/xemantic-kotlin-script-python/issues/54)). +`actions/setup-python` sets `LD_LIBRARY_PATH` on CI anyway, +so `PythonLibraryTest` removes it in the JVM it starts, to test the loading with such a base Python. `pyenv` builds Python without a shared `libpython` by default, -build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"` (and then set `LD_LIBRARY_PATH` as well). +build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`. An existing `.venv` is judged by its own Python: it is kept when this Python runs, is 3.10 or newer, and has `pip`, diff --git a/README.md b/README.md index 0c56628..4998d2d 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,16 @@ The Python executable is, in this order: The Python code imports the modules from the same paths as this executable, e.g. the `site-packages` of a virtual environment, but not the ones of its base Python. +The native library of `jep` links to the shared `libpython` of the Python, e.g. `libpython3.13.so.1.0`, +which on Linux the dynamic loader finds only in the system library directories, or in the ones of `LD_LIBRARY_PATH`. +For a Python installed elsewhere, e.g. by `uv python install`, pyenv, conda, or in a custom prefix, +the library loads `libpython` from the `LIBDIR` of the Python's `sysconfig` itself, +so that `LD_LIBRARY_PATH` is not needed. +If `libpython` is still not found, e.g. on macOS, where this is not done, +put its directory, which the error message names, on `LD_LIBRARY_PATH` (`DYLD_LIBRARY_PATH` on macOS), +set before the JVM starts, as the loader reads it only then. +pyenv builds Python without a shared `libpython` by default, build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`. + The embedded Python, like every Python process started by this library, e.g. `pip`, runs in the [UTF-8 mode](https://peps.python.org/pep-0540/), whatever the locale, e.g. cp1252 on Windows, or `C` on Linux: diff --git a/src/main/kotlin/Python.kt b/src/main/kotlin/Python.kt index 15193d1..3f9afa3 100644 --- a/src/main/kotlin/Python.kt +++ b/src/main/kotlin/Python.kt @@ -301,7 +301,10 @@ public class Python private constructor( // assigned only once checked, so that a later call does not use an interpreter failing the check PythonEnvironment.starter.createInterpreter( create = { SharedInterpreter() }, - check = { PythonEnvironment.checkUtf8Mode(it) } + check = { + PythonEnvironment.checkUtf8Mode(it) + PythonEnvironment.makePythonLibraryGlobal(it) + } ).also { interpreter = it } } interpreter.block() diff --git a/src/main/kotlin/PythonEnvironment.kt b/src/main/kotlin/PythonEnvironment.kt index af69f86..f36cbd5 100644 --- a/src/main/kotlin/PythonEnvironment.kt +++ b/src/main/kotlin/PythonEnvironment.kt @@ -36,6 +36,7 @@ import kotlin.io.path.Path import kotlin.io.path.createTempFile import kotlin.io.path.deleteIfExists import kotlin.io.path.exists +import kotlin.io.path.isRegularFile import kotlin.io.path.listDirectoryEntries import kotlin.io.path.name import kotlin.io.path.readText @@ -102,8 +103,9 @@ public object PythonEnvironment { val library = jepLibrary(Path(jepLocation), python, probe.externallyManaged) MainInterpreter.setJepLibraryPath(library.toString()) MainInterpreter.setInitParams(embeddedPythonConfig(executable)) + val preloadFailure = preloadPythonLibrary(pythonLibrary(probe.libraryDirectory, probe.librarySoname)) this.executable = executable - starter = EmbeddedPythonStarter(executable, probe.error) + starter = EmbeddedPythonStarter(executable, probe.error, probe.libraryDirectory, preloadFailure) if (dependencyDirectory == null) { dependencyDirectory = System.getProperty(DEPENDENCY_DIRECTORY_PROPERTY)?.let { Path(it) } } @@ -119,6 +121,70 @@ public object PythonEnvironment { initialized = true } + /** The shared `libpython` loaded by [preloadPythonLibrary], whose symbols are not global yet. */ + @Volatile + private var preloadedLibrary: Path? = null + + /** + * Loads the shared [library] of Python, if any, before Jep loads its native library, which depends on it. + * + * On Linux, the dynamic loader finds the `libpython3.X.so.1.0` of the native library of Jep by its soname + * only in the system library directories, or the ones of `LD_LIBRARY_PATH`, read when the JVM starts, + * so not the one of a Python installed elsewhere, e.g. by uv, pyenv, conda, or `actions/setup-python`; + * once loaded by its path, the loader takes the library of the same soname instead, + * also over another Python of the same version in the system library directories. + * + * @return the failure to load the [library], e.g. of another architecture, + * kept to tell why Jep cannot load its native library, if it cannot, or `null`. + */ + private fun preloadPythonLibrary(library: Path?): UnsatisfiedLinkError? { + if (library == null) return null + return try { + System.load(library.toString()) + preloadedLibrary = library + null + } catch (e: UnsatisfiedLinkError) { + e + } + } + + /** + * Makes the symbols of the shared `libpython` loaded by [preloadPythonLibrary], if any, global, + * run in the [interpreter], the first one, before any code of the user. + * + * The JVM loads a native library with `RTLD_LOCAL`, so the symbols of the preloaded `libpython` + * are not global, while a C extension module on Linux is not linked to `libpython`, + * e.g. the ones of numpy, or of the standard library of most Pythons, `_ctypes` among them, + * and would fail to import with `undefined symbol: PyExc_ValueError`. + * Jep makes `libpython` global itself, but finds it by its file name, + * which the loader cannot find, like its soname. + * The JVM of Java 17 cannot load a library with `RTLD_GLOBAL`, but CPython can, + * loading an extension module with the flags of `sys.setdlopenflags`: + * loading the already loaded `libpython` as one makes it global, + * before it fails as an extension module without its initialization function. + */ + internal fun makePythonLibraryGlobal(interpreter: Interpreter) { + val library = preloadedLibrary ?: return + interpreter.exec( + """ + |def _xemantic_make_python_library_global(): + | import _imp, os, sys + | from importlib.machinery import ModuleSpec + | flags = sys.getdlopenflags() + | sys.setdlopenflags(flags | os.RTLD_GLOBAL) + | try: + | _imp.create_dynamic(ModuleSpec('_xemantic_libpython', None, origin=${pythonPath(library.toString())})) + | except ImportError: + | pass + | finally: + | sys.setdlopenflags(flags) + |_xemantic_make_python_library_global() + |del _xemantic_make_python_library_global + |""".trimMargin() + ) + preloadedLibrary = null + } + /** * The version of the `org.ninia:jep` library, which the `jep` package must match, * read from the `pom.properties` of its jar, as its manifest has no version, @@ -266,10 +332,14 @@ public object PythonEnvironment { val begin = lines.lastIndexOf("\n$PROBE_BEGIN\n").takeIf { it != -1 }?.let { it + PROBE_BEGIN.length + 2 } val end = begin?.let { lines.indexOf("\n$PROBE_END", it) }?.takeIf { it != -1 } val values = if (begin == null || end == null) null else lines.substring(begin, end).split(' ') - if (values == null || values.size != 5) return null - val (executable, cacheTag, prefix, jepLocation, externallyManaged) = - values.map { it.hexToPath() ?: return null } - return PythonProbe(executable, cacheTag, prefix, jepLocation, externallyManaged.isNotEmpty()) + if (values == null || values.size != 7) return null + val paths = values.map { it.hexToPath() ?: return null } + val (executable, cacheTag, prefix, jepLocation, externallyManaged) = paths + return PythonProbe( + executable, cacheTag, prefix, jepLocation, externallyManaged.isNotEmpty(), + libraryDirectory = paths[5], + librarySoname = paths[6] + ) } /** @@ -281,9 +351,11 @@ public object PythonEnvironment { * as text, a path which is not valid UTF-8, e.g. `/home/josé` in ISO-8859-1, * would be surrogate-escaped by Python in UTF-8 mode, and could not be encoded to UTF-8, * while the JVM would decode a path of UTF-8 bytes to another one, in another encoding of its locale. - * The last value tells whether the Python is externally managed (PEP 668), `1` if it is, empty if not: + * The fifth value tells whether the Python is externally managed (PEP 668), `1` if it is, empty if not: * a base Python, not a virtual environment, with an `EXTERNALLY-MANAGED` file in its standard library, * into which pip refuses to install. + * The values after it are the directory and the soname of the `libpython` of the base Python, + * `LIBDIR` and `INSTSONAME` of `sysconfig`, empty if unknown, e.g. on Windows, see [pythonLibrary]. * Without the `jep` package, it exits with the [PROBE_JEP_MISSING_EXIT_CODE], * or the [PROBE_JEP_MISSING_EXTERNALLY_MANAGED_EXIT_CODE] if the Python is externally managed, * writing to the standard error the file of a `jep` module, if any, e.g. a `jep.py` shadowing the package. @@ -312,6 +384,8 @@ public object PythonEnvironment { | sys.prefix, | jep_spec.submodule_search_locations[0], | '1' if externally_managed else '', + | sysconfig.get_config_var('LIBDIR'), + | sysconfig.get_config_var('INSTSONAME'), |] |payload = ' '.join('' if value is None else os.fsencode(str(value)).hex() for value in values) |sys.stdout.flush() @@ -485,13 +559,39 @@ private fun String.hexToPath(): String? { .toString() } +/** + * The shared `libpython` of the [soname] in the library [directory] of a Python, see [PythonEnvironment.probeCommand], + * to be loaded before the native library of Jep, see [PythonEnvironment.preloadPythonLibrary], + * or `null` if there is none, e.g. of a Python linked to `libpython` statically, with `libpython3.X.a` as its soname. + * + * Not on Windows, where the native library of Jep finds the DLL of Python next to the `python.exe`, + * nor on macOS, where it refers to `libpython` by its path, e.g. in a framework, + * and the same library loaded by another path, e.g. of a symbolic link, could be loaded twice. + */ +internal fun pythonLibrary( + directory: String, + soname: String, + osName: String = System.getProperty("os.name") +): Path? { + if (osName.startsWith("Windows") || osName.startsWith("Mac")) return null + if (directory.isEmpty() || soname.isEmpty() || soname.endsWith(".a")) return null + return Path(directory, soname).takeIf { it.isRegularFile() } +} + /** * Creates the interpreters of the embedded CPython of the Python [executable], * telling a failure of its startup with what the probe of the [executable] wrote to the standard error, if anything, * as a warning of a successful probe, e.g. `Could not find platform libraries` of a wrong `PYTHONHOME`, * might tell why the embedded CPython, configured the same way, fails. + * When the native library of Jep cannot load `libpython`, it tells the [libraryDirectory] of `libpython`, + * and keeps the [libraryPreloadFailure] of [PythonEnvironment.preloadPythonLibrary], if any, as suppressed. */ -internal class EmbeddedPythonStarter(private val executable: String, private val probeError: String) { +internal class EmbeddedPythonStarter( + private val executable: String, + private val probeError: String, + private val libraryDirectory: String = "", + private val libraryPreloadFailure: UnsatisfiedLinkError? = null +) { /** Whether an interpreter has been created, so that the embedded CPython has started. */ @Volatile @@ -535,12 +635,22 @@ internal class EmbeddedPythonStarter(private val executable: String, private val private fun startupFailure(cause: Throwable): IllegalStateException { // Jep wraps the failure of initializing CPython in an Error, whose message is its toString() val reason = cause.cause?.takeIf { cause.message == it.toString() } ?: cause - val message = "Cannot initialize the embedded Python $executable: ${reason.message ?: reason}" + val headline = "Cannot initialize the embedded Python $executable: ${reason.message ?: reason}" + // e.g. libjep.so: libpython3.13.so.1.0: cannot open shared object file + val libraryAdvice = if ( + libraryDirectory.isNotEmpty() && reason is UnsatisfiedLinkError && reason.message?.contains("libpython") == true + ) { + "\nThe dynamic loader cannot find the libpython of this Python in $libraryDirectory, " + + "put this directory on LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on macOS) before the JVM starts." + } else "" + val message = headline + libraryAdvice return IllegalStateException( if (probeError.isEmpty()) message else "$message\nThe probe of this Python wrote to the standard error, which might tell why:\n$probeError", cause - ) + ).apply { + libraryPreloadFailure?.let { addSuppressed(it) } + } } } @@ -553,6 +663,8 @@ internal class EmbeddedPythonStarter(private val executable: String, private val * @property prefix the `sys.prefix`, the virtual environment, if any. * @property jepLocation the directory of the `jep` package. * @property externallyManaged whether pip refuses to install into it (PEP 668), see [PythonEnvironment.probeCommand]. + * @property libraryDirectory the directory of the `libpython` of the base Python, `LIBDIR`, empty if unknown. + * @property librarySoname the soname of the `libpython`, `INSTSONAME`, empty if unknown, see [pythonLibrary]. * @property error the standard error of the probe, e.g. a warning of Python, empty if none. */ internal data class PythonProbe( @@ -561,6 +673,8 @@ internal data class PythonProbe( val prefix: String, val jepLocation: String, val externallyManaged: Boolean = false, + val libraryDirectory: String = "", + val librarySoname: String = "", val error: String = "" ) diff --git a/src/test/kotlin/ForkedJvm.kt b/src/test/kotlin/ForkedJvm.kt index cddecbc..47f623a 100644 --- a/src/test/kotlin/ForkedJvm.kt +++ b/src/test/kotlin/ForkedJvm.kt @@ -26,7 +26,7 @@ import kotlin.io.path.readText /** * Runs the [mainClass] of the test classpath in a new JVM, with the [jvmArguments], - * and with the [environment] variables added to the inherited ones, + * and with the [environment] variables added to the inherited ones, or removed from them, when `null`, * unlike [runProcess], which would put Python in UTF-8 mode, * writing the [input] to its standard input, as UTF-8. * @@ -38,7 +38,7 @@ import kotlin.io.path.readText internal fun runInJvm( mainClass: String, input: String, - environment: Map, + environment: Map, jvmArguments: List = emptyList() ): Pair { val java = Path(System.getProperty("java.home"), "bin", "java").toString() @@ -49,7 +49,9 @@ internal fun runInJvm( listOf(java, "--enable-native-access=ALL-UNNAMED") + jvmArguments + listOf("-cp", System.getProperty("java.class.path"), mainClass) ).redirectErrorStream(true).redirectOutput(outputFile.toFile()) - builder.environment().putAll(environment) + builder.environment().apply { + environment.forEach { (name, value) -> if (value == null) remove(name) else put(name, value) } + } val process = builder.start() try { process.outputStream.use { it.write(input.toByteArray(Charsets.UTF_8)) } diff --git a/src/test/kotlin/PythonEnvironmentTest.kt b/src/test/kotlin/PythonEnvironmentTest.kt index ee2592a..7e57467 100644 --- a/src/test/kotlin/PythonEnvironmentTest.kt +++ b/src/test/kotlin/PythonEnvironmentTest.kt @@ -28,6 +28,7 @@ import kotlin.concurrent.thread import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.Path import kotlin.io.path.createDirectory +import kotlin.io.path.createFile import kotlin.io.path.createTempDirectory import kotlin.io.path.createTempFile import kotlin.io.path.deleteIfExists @@ -252,7 +253,7 @@ class PythonEnvironmentTest { @Test fun `should parse probe output with output before and after values`() { // given - val output = "noise\n${probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "")}\nmore noise" + val output = "noise\n${probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "")}\nmore noise" // when val probe = PythonEnvironment.parseProbeOutput(output) @@ -266,7 +267,7 @@ class PythonEnvironmentTest { // given val output = "${PythonEnvironment.PROBE_BEGIN}\nfake\n${PythonEnvironment.PROBE_END}\n" + "${PythonEnvironment.PROBE_BEGIN}\n" + - probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "") + probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "") // when val probe = PythonEnvironment.parseProbeOutput(output) @@ -278,7 +279,7 @@ class PythonEnvironmentTest { @Test fun `should parse probe output with empty first value`() { // given - val output = probePayload("", "cpython-313", "/venv", "/venv/jep", "") + val output = probePayload("", "cpython-313", "/venv", "/venv/jep", "", "", "") // when val probe = PythonEnvironment.parseProbeOutput(output) @@ -290,7 +291,7 @@ class PythonEnvironmentTest { @Test fun `should parse probe output with values containing spaces and non-ASCII`() { // given - val output = probePayload("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep", "") + val output = probePayload("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep", "", "", "") // when val probe = PythonEnvironment.parseProbeOutput(output) @@ -299,6 +300,47 @@ class PythonEnvironmentTest { assert(probe == PythonProbe("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep")) } + @Test + fun `should parse probe output with library directory and soname of libpython`() { + // given + val output = probePayload( + "/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "1", "/opt/python/lib", "libpython3.13.so.1.0" + ) + + // when + val probe = PythonEnvironment.parseProbeOutput(output) + + // then + assert( + probe == PythonProbe( + "/venv/bin/python", "cpython-313", "/venv", "/venv/jep", + externallyManaged = true, + libraryDirectory = "/opt/python/lib", + librarySoname = "libpython3.13.so.1.0" + ) + ) + } + + @Test + fun `should probe library directory and soname of libpython of Python`() { + // given + // the values of the base Python, as a virtual environment has no libpython of its own + val expected = runProcess( + pythonCommand( + venvPython, + "import sysconfig\n" + + "print(sysconfig.get_config_var('LIBDIR') or '', sysconfig.get_config_var('INSTSONAME') or '', sep='|')" + ), + mergeErrorStream = false + ).output + + // when + val probe = PythonEnvironment.probePython(venvPython) + + // then + assert("${probe.libraryDirectory}|${probe.librarySoname}" == expected) + } + @Test fun `should not parse probe output without values`() { // when @@ -311,7 +353,7 @@ class PythonEnvironmentTest { @Test fun `should not parse probe output with begin marker but without end marker`() { // given - val output = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "") + val output = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "") .removeSuffix("\n${PythonEnvironment.PROBE_END}") // when @@ -322,10 +364,10 @@ class PythonEnvironmentTest { } @Test - fun `should not parse probe output without exactly five values`() { + fun `should not parse probe output without exactly seven values`() { // given - val tooFew = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep") - val tooMany = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "extra") + val tooFew = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "") + val tooMany = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "", "extra") // when val probes = listOf(tooFew, tooMany).map { PythonEnvironment.parseProbeOutput(it) } @@ -742,6 +784,125 @@ class PythonEnvironmentTest { } } + @OptIn(ExperimentalPathApi::class) + @Test + fun `should find shared libpython in its library directory on Linux`() { + // given + val directory = createTempDirectory("python-library") + val library = directory.resolve("libpython3.13.so.1.0").createFile() + + try { + // when + val found = pythonLibrary(directory.toString(), "libpython3.13.so.1.0", osName = "Linux") + + // then + assert(found == library) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should not find static libpython`() { + // given + // a Python built without --enable-shared, e.g. by pyenv by default, has its libpython linked statically + val directory = createTempDirectory("python-library") + directory.resolve("libpython3.13.a").createFile() + + try { + // when + val found = pythonLibrary(directory.toString(), "libpython3.13.a", osName = "Linux") + + // then + assert(found == null) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should not find libpython missing from its library directory`() { + // given + val directory = createTempDirectory("python-library") + + try { + // when + val missing = pythonLibrary(directory.toString(), "libpython3.13.so.1.0", osName = "Linux") + val unknownSoname = pythonLibrary(directory.toString(), "", osName = "Linux") + val unknownDirectory = pythonLibrary("", "libpython3.13.so.1.0", osName = "Linux") + + // then + assert(missing == null && unknownSoname == null && unknownDirectory == null) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should not find libpython on macOS and Windows`() { + // given + val directory = createTempDirectory("python-library") + directory.resolve("libpython3.13.dylib").createFile() + directory.resolve("python313.dll").createFile() + + try { + // when + val macOs = pythonLibrary(directory.toString(), "libpython3.13.dylib", osName = "Mac OS X") + val windows = pythonLibrary(directory.toString(), "python313.dll", osName = "Windows 11") + + // then + assert(macOs == null && windows == null) + } finally { + directory.deleteRecursively() + } + } + + @Test + fun `should advise library directory of libpython when native library of Jep cannot load it`() { + // given + val preloadFailure = UnsatisfiedLinkError("/opt/python/lib/libpython3.13.so.1.0: wrong ELF class: ELFCLASS32") + val starter = EmbeddedPythonStarter( + "/venv/bin/python", "", libraryDirectory = "/opt/python/lib", libraryPreloadFailure = preloadFailure + ) + val cause = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.13.so.1.0: cannot open shared object file: No such file or directory" + ) + + // when + val exception = assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + } + + // then + val message = exception.message!! + assert( + exception.cause === cause && + exception.suppressed.toList() == listOf(preloadFailure) && + message.startsWith("Cannot initialize the embedded Python /venv/bin/python: ${cause.message}\n") && + "/opt/python/lib" in message && + "LD_LIBRARY_PATH" in message + ) + } + + @Test + fun `should not advise library directory of libpython on other failure of embedded Python startup`() { + // given + val starter = EmbeddedPythonStarter("/venv/bin/python", "", libraryDirectory = "/opt/python/lib") + val cause = UnsatisfiedLinkError("/venv/lib/python3.13/site-packages/jep/libjep.so: wrong ELF class: ELFCLASS32") + + // when + val exception = assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + } + + // then + assert(exception.message == "Cannot initialize the embedded Python /venv/bin/python: ${cause.message}") + } + @Test fun `should report stderr of probe on Error of embedded Python startup`() { // given diff --git a/src/test/kotlin/PythonLibraryTest.kt b/src/test/kotlin/PythonLibraryTest.kt new file mode 100644 index 0000000..aa7eb48 --- /dev/null +++ b/src/test/kotlin/PythonLibraryTest.kt @@ -0,0 +1,54 @@ +/* + * xemantic-kotlin-script-python - Stateful Python, embedded with Jep, in Kotlin scripts + * Copyright (C) 2026 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.xemantic.kotlin.script.python + +import com.xemantic.kotlin.test.assert +import org.junit.jupiter.api.Assumptions.assumeTrue +import kotlin.test.Test + +/** + * The embedded CPython loading the shared `libpython` of its Python without `LD_LIBRARY_PATH`, + * in a JVM of its own, as the environment of the dynamic loader is read when a process starts. + * + * It tests the loading only with a Python outside the system library directories, + * e.g. installed by uv or by `actions/setup-python`, which sets `LD_LIBRARY_PATH` itself, + * as the test venv of such a base Python, see `-PbasePython` in DEVELOPMENT.md, + * while with a Python of the system it passes anyway. + */ +class PythonLibraryTest { + + @Test + fun `should start embedded Python with symbols of libpython global without LD_LIBRARY_PATH`() { + assumeTrue(System.getProperty("os.name").startsWith("Linux"), "LD_LIBRARY_PATH is the one of Linux") + + // when + val (exitCode, output) = runInJvm( + EmbeddedPythonMain::class.java.name, + // the symbols of the global scope, which a C extension not linked to libpython needs, e.g. of numpy; + // importing ctypes fails without them, unless it is a built-in module, like in a Python of uv + input = "import ctypes\nprint(hasattr(ctypes.pythonapi, 'Py_IsInitialized'), flush=True)", + environment = mapOf("LD_LIBRARY_PATH" to null) + ) + + // then + assert(output == "True") + assert(exitCode == 0) + } + +} From 26e419851a0838b0c1fc4ca113124bed62db5b2b Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Wed, 11 Nov 2026 20:52:49 +0000 Subject: [PATCH 2/4] Load libpython with global symbols before Python starts, on Java 22+ (#54) Making libpython global only in the first interpreter came after the startup of CPython, so a C extension imported by a .pth file or sitecustomize, e.g. numpy or coverage, still failed with an undefined symbol. On Java 22 or newer, libpython is now loaded with dlopen(RTLD_GLOBAL) through the foreign linker (via reflection, the library targets Java 17) before Jep starts CPython. On Java 17 to 21 the System.load + make-global-in-the-first- interpreter fallback stays, now best-effort (logged, never failing startup), run by EmbeddedPythonStarter; README tells its .pth limitation. The LD_LIBRARY_PATH advice now walks the cause chain, also of Jep's retry Error, and is given only for the soname of the shared libpython found. PythonLibraryTest compiles a C extension not linked to libpython, imported by the code, by sitecustomize at startup, and without the foreign linker. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FJmHQ3jHRbVdpwUiNUkApG --- CLAUDE.md | 4 +- DEVELOPMENT.md | 3 +- README.md | 10 +- src/main/kotlin/Python.kt | 5 +- src/main/kotlin/PythonEnvironment.kt | 168 ++++++++++------------- src/main/kotlin/PythonLibrary.kt | 130 ++++++++++++++++++ src/test/kotlin/PythonEnvironmentTest.kt | 109 ++++++++++----- src/test/kotlin/PythonLibraryTest.kt | 142 +++++++++++++++++-- 8 files changed, 417 insertions(+), 154 deletions(-) create mode 100644 src/main/kotlin/PythonLibrary.kt diff --git a/CLAUDE.md b/CLAUDE.md index 5b3d0e6..bf7a115 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -70,8 +70,8 @@ see [DEVELOPMENT.md](DEVELOPMENT.md#markdown-soft-wrapping-in-the-ide) for how t - The embedded CPython does not detect a virtual environment by itself — without the venv's `python` as its program name (`PyConfig.setProgramName`), it ignores `pyvenv.cfg` and also imports from the global and user `site-packages` of the base Python. When a module is unexpectedly importable, check `sys.path` of the embedded interpreter, not of `.venv/bin/python`. -- A running Gradle daemon does not pass a changed `LD_LIBRARY_PATH` on to the test JVM, - so use `--no-daemon` when trying a `-PbasePython` which needs it (see [DEVELOPMENT.md](DEVELOPMENT.md#the-python-of-the-tests)). +- A running Gradle daemon does not pass a changed `LD_LIBRARY_PATH` on to the test JVM, so an `LD_LIBRARY_PATH` of your shell can be absent, or a stale one present, in the tests — + with a `-PbasePython` outside the system library directories, e.g. of uv, run `--no-daemon` without `LD_LIBRARY_PATH` to test that `libpython` loads without it, as on CI `PythonLibraryTest` does. - Jep passes the `PyConfig` strings to CPython as JNI *modified* UTF-8 (`GetStringUTFChars`), not plain UTF-8 — a character outside the BMP becomes a CESU-8 surrogate pair, which CPython does not decode as that character. For a path under a non-UTF-8 locale, see `pythonPath`. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5cc36ac..8c75e3c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -22,7 +22,8 @@ On Linux, a Python outside the system library directories, e.g. installed by `uv does not need its `lib` directory in `LD_LIBRARY_PATH`, as `PythonEnvironment` loads its `libpython` before `libjep.so` ([#54](https://github.com/xemantic/xemantic-kotlin-script-python/issues/54)). `actions/setup-python` sets `LD_LIBRARY_PATH` on CI anyway, -so `PythonLibraryTest` removes it in the JVM it starts, to test the loading with such a base Python. +so `PythonLibraryTest` removes it in the JVM it starts, to test the loading with such a base Python, +importing a C extension it compiles with `cc`, which tests nothing with a Python of the system library directories. `pyenv` builds Python without a shared `libpython` by default, build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`. diff --git a/README.md b/README.md index 4998d2d..35abdea 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,15 @@ e.g. the `site-packages` of a virtual environment, but not the ones of its base The native library of `jep` links to the shared `libpython` of the Python, e.g. `libpython3.13.so.1.0`, which on Linux the dynamic loader finds only in the system library directories, or in the ones of `LD_LIBRARY_PATH`. For a Python installed elsewhere, e.g. by `uv python install`, pyenv, conda, or in a custom prefix, -the library loads `libpython` from the `LIBDIR` of the Python's `sysconfig` itself, +the library loads `libpython` from the `LIBDIR` of the Python's `sysconfig` itself, with its symbols global, +as C extension modules, e.g. of numpy, need them, so that `LD_LIBRARY_PATH` is not needed. -If `libpython` is still not found, e.g. on macOS, where this is not done, +On Java 22 or newer, it does so with the foreign linker, a restricted method, +which, like loading the native library of `jep` on Java 24 or newer, prints a warning without `--enable-native-access=ALL-UNNAMED`. +On Java 17 to 21, the symbols become global only after the startup of Python, +so a C extension imported at the startup, by a `.pth` file of the `site-packages`, e.g. of coverage, or by `sitecustomize`, +fails with an `undefined symbol` and still needs `LD_LIBRARY_PATH`. +If `libpython` is not found, e.g. on macOS, where it is not loaded so, put its directory, which the error message names, on `LD_LIBRARY_PATH` (`DYLD_LIBRARY_PATH` on macOS), set before the JVM starts, as the loader reads it only then. pyenv builds Python without a shared `libpython` by default, build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`. diff --git a/src/main/kotlin/Python.kt b/src/main/kotlin/Python.kt index 3f9afa3..15193d1 100644 --- a/src/main/kotlin/Python.kt +++ b/src/main/kotlin/Python.kt @@ -301,10 +301,7 @@ public class Python private constructor( // assigned only once checked, so that a later call does not use an interpreter failing the check PythonEnvironment.starter.createInterpreter( create = { SharedInterpreter() }, - check = { - PythonEnvironment.checkUtf8Mode(it) - PythonEnvironment.makePythonLibraryGlobal(it) - } + check = { PythonEnvironment.checkUtf8Mode(it) } ).also { interpreter = it } } interpreter.block() diff --git a/src/main/kotlin/PythonEnvironment.kt b/src/main/kotlin/PythonEnvironment.kt index f36cbd5..788531f 100644 --- a/src/main/kotlin/PythonEnvironment.kt +++ b/src/main/kotlin/PythonEnvironment.kt @@ -36,7 +36,6 @@ import kotlin.io.path.Path import kotlin.io.path.createTempFile import kotlin.io.path.deleteIfExists import kotlin.io.path.exists -import kotlin.io.path.isRegularFile import kotlin.io.path.listDirectoryEntries import kotlin.io.path.name import kotlin.io.path.readText @@ -103,9 +102,14 @@ public object PythonEnvironment { val library = jepLibrary(Path(jepLocation), python, probe.externallyManaged) MainInterpreter.setJepLibraryPath(library.toString()) MainInterpreter.setInitParams(embeddedPythonConfig(executable)) - val preloadFailure = preloadPythonLibrary(pythonLibrary(probe.libraryDirectory, probe.librarySoname)) + val pythonLibrary = pythonLibrary(probe.libraryDirectory, probe.librarySoname) + val loading = if (pythonLibrary != null && System.getProperty("os.name").startsWith("Linux")) { + loadPythonLibrary(pythonLibrary) + } else PythonLibraryLoading() this.executable = executable - starter = EmbeddedPythonStarter(executable, probe.error, probe.libraryDirectory, preloadFailure) + starter = EmbeddedPythonStarter( + executable, probe.error, pythonLibrary, loading.failure, loading.libraryToMakeGlobal + ) if (dependencyDirectory == null) { dependencyDirectory = System.getProperty(DEPENDENCY_DIRECTORY_PROPERTY)?.let { Path(it) } } @@ -121,70 +125,6 @@ public object PythonEnvironment { initialized = true } - /** The shared `libpython` loaded by [preloadPythonLibrary], whose symbols are not global yet. */ - @Volatile - private var preloadedLibrary: Path? = null - - /** - * Loads the shared [library] of Python, if any, before Jep loads its native library, which depends on it. - * - * On Linux, the dynamic loader finds the `libpython3.X.so.1.0` of the native library of Jep by its soname - * only in the system library directories, or the ones of `LD_LIBRARY_PATH`, read when the JVM starts, - * so not the one of a Python installed elsewhere, e.g. by uv, pyenv, conda, or `actions/setup-python`; - * once loaded by its path, the loader takes the library of the same soname instead, - * also over another Python of the same version in the system library directories. - * - * @return the failure to load the [library], e.g. of another architecture, - * kept to tell why Jep cannot load its native library, if it cannot, or `null`. - */ - private fun preloadPythonLibrary(library: Path?): UnsatisfiedLinkError? { - if (library == null) return null - return try { - System.load(library.toString()) - preloadedLibrary = library - null - } catch (e: UnsatisfiedLinkError) { - e - } - } - - /** - * Makes the symbols of the shared `libpython` loaded by [preloadPythonLibrary], if any, global, - * run in the [interpreter], the first one, before any code of the user. - * - * The JVM loads a native library with `RTLD_LOCAL`, so the symbols of the preloaded `libpython` - * are not global, while a C extension module on Linux is not linked to `libpython`, - * e.g. the ones of numpy, or of the standard library of most Pythons, `_ctypes` among them, - * and would fail to import with `undefined symbol: PyExc_ValueError`. - * Jep makes `libpython` global itself, but finds it by its file name, - * which the loader cannot find, like its soname. - * The JVM of Java 17 cannot load a library with `RTLD_GLOBAL`, but CPython can, - * loading an extension module with the flags of `sys.setdlopenflags`: - * loading the already loaded `libpython` as one makes it global, - * before it fails as an extension module without its initialization function. - */ - internal fun makePythonLibraryGlobal(interpreter: Interpreter) { - val library = preloadedLibrary ?: return - interpreter.exec( - """ - |def _xemantic_make_python_library_global(): - | import _imp, os, sys - | from importlib.machinery import ModuleSpec - | flags = sys.getdlopenflags() - | sys.setdlopenflags(flags | os.RTLD_GLOBAL) - | try: - | _imp.create_dynamic(ModuleSpec('_xemantic_libpython', None, origin=${pythonPath(library.toString())})) - | except ImportError: - | pass - | finally: - | sys.setdlopenflags(flags) - |_xemantic_make_python_library_global() - |del _xemantic_make_python_library_global - |""".trimMargin() - ) - preloadedLibrary = null - } - /** * The version of the `org.ninia:jep` library, which the `jep` package must match, * read from the `pom.properties` of its jar, as its manifest has no version, @@ -559,40 +499,30 @@ private fun String.hexToPath(): String? { .toString() } -/** - * The shared `libpython` of the [soname] in the library [directory] of a Python, see [PythonEnvironment.probeCommand], - * to be loaded before the native library of Jep, see [PythonEnvironment.preloadPythonLibrary], - * or `null` if there is none, e.g. of a Python linked to `libpython` statically, with `libpython3.X.a` as its soname. - * - * Not on Windows, where the native library of Jep finds the DLL of Python next to the `python.exe`, - * nor on macOS, where it refers to `libpython` by its path, e.g. in a framework, - * and the same library loaded by another path, e.g. of a symbolic link, could be loaded twice. - */ -internal fun pythonLibrary( - directory: String, - soname: String, - osName: String = System.getProperty("os.name") -): Path? { - if (osName.startsWith("Windows") || osName.startsWith("Mac")) return null - if (directory.isEmpty() || soname.isEmpty() || soname.endsWith(".a")) return null - return Path(directory, soname).takeIf { it.isRegularFile() } -} - /** * Creates the interpreters of the embedded CPython of the Python [executable], * telling a failure of its startup with what the probe of the [executable] wrote to the standard error, if anything, * as a warning of a successful probe, e.g. `Could not find platform libraries` of a wrong `PYTHONHOME`, * might tell why the embedded CPython, configured the same way, fails. - * When the native library of Jep cannot load `libpython`, it tells the [libraryDirectory] of `libpython`, - * and keeps the [libraryPreloadFailure] of [PythonEnvironment.preloadPythonLibrary], if any, as suppressed. + * When the native library of Jep cannot load the shared [pythonLibrary] of the Python, see [pythonLibrary], + * it tells to put its directory on `LD_LIBRARY_PATH`, + * and keeps the [libraryPreloadFailure] of [loadPythonLibrary], if any, as suppressed. + * The symbols of the `libpython` loaded locally by [loadPythonLibrary], if any, + * are made global in the first interpreter, see [makePythonLibraryGlobal]. */ internal class EmbeddedPythonStarter( private val executable: String, private val probeError: String, - private val libraryDirectory: String = "", - private val libraryPreloadFailure: UnsatisfiedLinkError? = null + private val pythonLibrary: Path? = null, + private val libraryPreloadFailure: Throwable? = null, + libraryToMakeGlobal: Path? = null ) { + /** The `libpython` to make global in the first interpreter, see [makePythonLibraryGlobal]. */ + @Volatile + private var libraryToMakeGlobal: Path? = libraryToMakeGlobal + + /** Whether an interpreter has been created, so that the embedded CPython has started. */ @Volatile private var started = false @@ -620,6 +550,7 @@ internal class EmbeddedPythonStarter( } started = true try { + if (interpreter is Interpreter) makePythonLibraryGlobal(interpreter) check(interpreter) } catch (e: Throwable) { try { @@ -632,16 +563,65 @@ internal class EmbeddedPythonStarter( return interpreter } + /** + * Makes the symbols of the `libpython` loaded locally by [loadPythonLibrary], if any, global, + * in the first [interpreter], before any code of the user, but after the startup of Python, + * see [loadPythonLibrary]. + * + * CPython can load a library with `RTLD_GLOBAL`, + * loading an extension module with the flags of `sys.setdlopenflags`: + * loading the already loaded `libpython` as one makes it global, + * before it fails as an extension module without its initialization function. + * It is best-effort: a failure is only logged, as the code of the user might not need any C extension. + */ + private fun makePythonLibraryGlobal(interpreter: Interpreter) { + val library = libraryToMakeGlobal ?: return + libraryToMakeGlobal = null + try { + interpreter.exec( + """ + |def _xemantic_make_python_library_global(): + | import _imp, os, sys + | from importlib.machinery import ModuleSpec + | flags = sys.getdlopenflags() + | sys.setdlopenflags(flags | os.RTLD_GLOBAL) + | try: + | _imp.create_dynamic(ModuleSpec('_xemantic_libpython', None, origin=${pythonPath(library.toString())})) + | except ImportError: + | pass + | finally: + | sys.setdlopenflags(flags) + |try: + | _xemantic_make_python_library_global() + |finally: + | del _xemantic_make_python_library_global + |""".trimMargin() + ) + } catch (e: Exception) { + System.getLogger(EmbeddedPythonStarter::class.java.name).log( + System.Logger.Level.WARNING, + "Cannot make the symbols of $library global, " + + "a C extension module might fail to import with an undefined symbol, " + + "put ${library.parent} on LD_LIBRARY_PATH before the JVM starts", + e + ) + } + } + private fun startupFailure(cause: Throwable): IllegalStateException { // Jep wraps the failure of initializing CPython in an Error, whose message is its toString() val reason = cause.cause?.takeIf { cause.message == it.toString() } ?: cause val headline = "Cannot initialize the embedded Python $executable: ${reason.message ?: reason}" - // e.g. libjep.so: libpython3.13.so.1.0: cannot open shared object file + // e.g. libjep.so: libpython3.13.so.1.0: cannot open shared object file, + // also as the cause of the Error of Jep retrying to initialize CPython + val library = pythonLibrary val libraryAdvice = if ( - libraryDirectory.isNotEmpty() && reason is UnsatisfiedLinkError && reason.message?.contains("libpython") == true + library != null && generateSequence(cause) { it.cause }.any { + it is UnsatisfiedLinkError && it.message?.contains(library.fileName.toString()) == true + } ) { - "\nThe dynamic loader cannot find the libpython of this Python in $libraryDirectory, " + - "put this directory on LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on macOS) before the JVM starts." + "\nThe dynamic loader cannot load ${library.fileName} of this Python, " + + "put its directory ${library.parent} on LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on macOS) before the JVM starts." } else "" val message = headline + libraryAdvice return IllegalStateException( diff --git a/src/main/kotlin/PythonLibrary.kt b/src/main/kotlin/PythonLibrary.kt new file mode 100644 index 0000000..27529ef --- /dev/null +++ b/src/main/kotlin/PythonLibrary.kt @@ -0,0 +1,130 @@ +/* + * xemantic-kotlin-script-python - Stateful Python, embedded with Jep, in Kotlin scripts + * Copyright (C) 2026 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.xemantic.kotlin.script.python + +import java.lang.invoke.MethodHandle +import java.nio.file.Path +import java.util.Optional +import kotlin.io.path.Path +import kotlin.io.path.isRegularFile + +/** + * The system property which, set, keeps [loadPythonLibrary] from using the foreign linker of Java 22, + * a test seam of loading `libpython` like on Java 17 to 21. + */ +internal const val NO_FOREIGN_LINKER_PROPERTY = "xemantic.kotlin.script.python.noForeignLinker" + +/** + * The shared `libpython` of the [soname] in the library [directory] of a Python, see [PythonEnvironment.probeCommand], + * or `null` if there is none, e.g. of a Python linked to `libpython` statically, with `libpython3.X.a` as its soname. + */ +internal fun pythonLibrary(directory: String, soname: String): Path? { + if (directory.isEmpty() || soname.isEmpty() || soname.endsWith(".a")) return null + return Path(directory, soname).takeIf { it.isRegularFile() } +} + +/** + * The result of [loadPythonLibrary]. + * + * @property failure the failure to load the library, e.g. of another architecture, + * kept to tell why Jep cannot load its native library, if it cannot. + * @property libraryToMakeGlobal the library loaded with local symbols, to be made global once Python runs, + * see [EmbeddedPythonStarter]. + */ +internal class PythonLibraryLoading( + val failure: Throwable? = null, + val libraryToMakeGlobal: Path? = null +) + +/** + * Loads the shared `libpython` [library] of the Python on Linux, before Jep loads its native library, which depends on it. + * + * The dynamic loader finds the `libpython3.X.so.1.0` of the native library of Jep by its soname + * only in the system library directories, or the ones of `LD_LIBRARY_PATH`, read when the JVM starts, + * so not the one of a Python installed elsewhere, e.g. by uv, pyenv, conda, or `actions/setup-python`; + * once loaded by its path, the loader takes the library of the same soname instead. + * + * Its symbols must be global as well, as a C extension module on Linux is not linked to `libpython`, + * e.g. the ones of numpy, or of the standard library of most Pythons, `_ctypes` among them, + * and would fail to import with `undefined symbol: PyExc_ValueError`. + * Jep makes `libpython` global itself, but finds it by its file name, which the loader cannot find, like its soname. + * The JVM loads a native library with local symbols (`RTLD_LOCAL`), + * so on Java 22 or newer, the library is loaded with `dlopen` and `RTLD_GLOBAL`, called with the foreign linker, + * before CPython starts, also for a C extension imported at its startup, e.g. by a `.pth` file of the `site-packages`. + * On Java 17 to 21, it is loaded with [System.load], and made global in the first interpreter, + * so only after the startup of CPython. + */ +internal fun loadPythonLibrary(library: Path): PythonLibraryLoading { + if (System.getProperty(NO_FOREIGN_LINKER_PROPERTY) == null && loadGlobally(library)) return PythonLibraryLoading() + return try { + System.load(library.toString()) + PythonLibraryLoading(libraryToMakeGlobal = library) + } catch (e: UnsatisfiedLinkError) { + PythonLibraryLoading(failure = e) + } +} + +/** `RTLD_LAZY | RTLD_GLOBAL` of glibc and musl. */ +private const val RTLD_LAZY_GLOBAL = 0x101 + +/** + * Loads the [library] with `dlopen` and `RTLD_GLOBAL`, called with the foreign linker of Java 22 or newer, + * through reflection, as the library is compiled for Java 17. + * + * @return `true` if loaded, `false` if the foreign linker is not available, or `dlopen` fails. + */ +private fun loadGlobally(library: Path): Boolean = try { + val linkerClass = Class.forName("java.lang.foreign.Linker") + val arenaClass = Class.forName("java.lang.foreign.Arena") + val segmentClass = Class.forName("java.lang.foreign.MemorySegment") + val layoutClass = Class.forName("java.lang.foreign.MemoryLayout") + val valueLayoutClass = Class.forName("java.lang.foreign.ValueLayout") + val descriptorClass = Class.forName("java.lang.foreign.FunctionDescriptor") + val optionClass = Class.forName("java.lang.foreign.Linker\$Option") + val linker = linkerClass.getMethod("nativeLinker").invoke(null) + val lookup = linkerClass.getMethod("defaultLookup").invoke(linker) + val dlopen = (Class.forName("java.lang.foreign.SymbolLookup").getMethod("find", String::class.java) + .invoke(lookup, "dlopen") as Optional<*>).orElse(null) + if (dlopen == null) { + false + } else { + val address = valueLayoutClass.getField("ADDRESS").get(null) + val int = valueLayoutClass.getField("JAVA_INT").get(null) + val arguments = java.lang.reflect.Array.newInstance(layoutClass, 2).also { + java.lang.reflect.Array.set(it, 0, address) + java.lang.reflect.Array.set(it, 1, int) + } + val descriptor = descriptorClass.getMethod("of", layoutClass, arguments.javaClass) + .invoke(null, address, arguments) + val noOptions = java.lang.reflect.Array.newInstance(optionClass, 0) + val handle = linkerClass.getMethod("downcallHandle", segmentClass, descriptorClass, noOptions.javaClass) + .invoke(linker, dlopen, descriptor, noOptions) as MethodHandle + // allocateFrom is of Java 22, the foreign linker of Java 21 is a preview + val allocateFrom = arenaClass.getMethod("allocateFrom", String::class.java) + (arenaClass.getMethod("ofConfined").invoke(null) as AutoCloseable).use { arena -> + val path = allocateFrom.invoke(arena, library.toString()) + val result = handle.invokeWithArguments(path, RTLD_LAZY_GLOBAL) + segmentClass.getMethod("address").invoke(result) != 0L + } + } +} catch (_: ReflectiveOperationException) { + false +} catch (_: RuntimeException) { + false +} diff --git a/src/test/kotlin/PythonEnvironmentTest.kt b/src/test/kotlin/PythonEnvironmentTest.kt index 7e57467..524c83b 100644 --- a/src/test/kotlin/PythonEnvironmentTest.kt +++ b/src/test/kotlin/PythonEnvironmentTest.kt @@ -34,6 +34,7 @@ import kotlin.io.path.createTempFile import kotlin.io.path.deleteIfExists import kotlin.io.path.deleteRecursively import kotlin.io.path.exists +import kotlin.io.path.isRegularFile import kotlin.io.path.readText import kotlin.io.path.setPosixFilePermissions import kotlin.io.path.writeText @@ -322,23 +323,20 @@ class PythonEnvironmentTest { } @Test - fun `should probe library directory and soname of libpython of Python`() { + fun `should probe shared libpython of Python built with it`() { // given - // the values of the base Python, as a virtual environment has no libpython of its own - val expected = runProcess( - pythonCommand( - venvPython, - "import sysconfig\n" + - "print(sysconfig.get_config_var('LIBDIR') or '', sysconfig.get_config_var('INSTSONAME') or '', sep='|')" - ), + val shared = runProcess( + pythonCommand(venvPython, "import sysconfig\nprint(sysconfig.get_config_var('Py_ENABLE_SHARED'))"), mergeErrorStream = false ).output + assumeTrue(shared == "1", "the Python is not built with a shared libpython") // when val probe = PythonEnvironment.probePython(venvPython) // then - assert("${probe.libraryDirectory}|${probe.librarySoname}" == expected) + val library = pythonLibrary(probe.libraryDirectory, probe.librarySoname) + assert(library != null && library.isRegularFile()) } @Test @@ -793,7 +791,7 @@ class PythonEnvironmentTest { try { // when - val found = pythonLibrary(directory.toString(), "libpython3.13.so.1.0", osName = "Linux") + val found = pythonLibrary(directory.toString(), "libpython3.13.so.1.0") // then assert(found == library) @@ -812,7 +810,7 @@ class PythonEnvironmentTest { try { // when - val found = pythonLibrary(directory.toString(), "libpython3.13.a", osName = "Linux") + val found = pythonLibrary(directory.toString(), "libpython3.13.a") // then assert(found == null) @@ -829,9 +827,9 @@ class PythonEnvironmentTest { try { // when - val missing = pythonLibrary(directory.toString(), "libpython3.13.so.1.0", osName = "Linux") - val unknownSoname = pythonLibrary(directory.toString(), "", osName = "Linux") - val unknownDirectory = pythonLibrary("", "libpython3.13.so.1.0", osName = "Linux") + val missing = pythonLibrary(directory.toString(), "libpython3.13.so.1.0") + val unknownSoname = pythonLibrary(directory.toString(), "") + val unknownDirectory = pythonLibrary("", "libpython3.13.so.1.0") // then assert(missing == null && unknownSoname == null && unknownDirectory == null) @@ -840,32 +838,14 @@ class PythonEnvironmentTest { } } - @OptIn(ExperimentalPathApi::class) - @Test - fun `should not find libpython on macOS and Windows`() { - // given - val directory = createTempDirectory("python-library") - directory.resolve("libpython3.13.dylib").createFile() - directory.resolve("python313.dll").createFile() - - try { - // when - val macOs = pythonLibrary(directory.toString(), "libpython3.13.dylib", osName = "Mac OS X") - val windows = pythonLibrary(directory.toString(), "python313.dll", osName = "Windows 11") - - // then - assert(macOs == null && windows == null) - } finally { - directory.deleteRecursively() - } - } - @Test fun `should advise library directory of libpython when native library of Jep cannot load it`() { // given val preloadFailure = UnsatisfiedLinkError("/opt/python/lib/libpython3.13.so.1.0: wrong ELF class: ELFCLASS32") val starter = EmbeddedPythonStarter( - "/venv/bin/python", "", libraryDirectory = "/opt/python/lib", libraryPreloadFailure = preloadFailure + "/venv/bin/python", "", + pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0"), + libraryPreloadFailure = preloadFailure ) val cause = UnsatisfiedLinkError( "/venv/lib/python3.13/site-packages/jep/libjep.so: " + @@ -888,11 +868,66 @@ class PythonEnvironmentTest { ) } + @Test + fun `should advise library directory of libpython when initialization of embedded Python is retried`() { + // given + val starter = EmbeddedPythonStarter( + "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0") + ) + val first = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.13.so.1.0: cannot open shared object file: No such file or directory" + ) + // like jep's MainInterpreter, rethrowing the failure of the first attempt + val cause = Error("The main Python interpreter previously failed to initialize.", first) + + // when + val exception = assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + } + + // then + val message = exception.message!! + assert(exception.cause === cause && "/opt/python/lib" in message && "LD_LIBRARY_PATH" in message) + } + @Test fun `should not advise library directory of libpython on other failure of embedded Python startup`() { // given - val starter = EmbeddedPythonStarter("/venv/bin/python", "", libraryDirectory = "/opt/python/lib") - val cause = UnsatisfiedLinkError("/venv/lib/python3.13/site-packages/jep/libjep.so: wrong ELF class: ELFCLASS32") + val starter = EmbeddedPythonStarter( + "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0") + ) + val wrongClass = UnsatisfiedLinkError("/venv/lib/python3.13/site-packages/jep/libjep.so: wrong ELF class: ELFCLASS32") + // of jep built for another Python + val anotherVersion = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.12.so.1.0: cannot open shared object file: No such file or directory" + ) + + // when + val messages = listOf(wrongClass, anotherVersion).map { cause -> + assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + }.message + } + + // then + assert( + messages == listOf( + "Cannot initialize the embedded Python /venv/bin/python: ${wrongClass.message}", + "Cannot initialize the embedded Python /venv/bin/python: ${anotherVersion.message}" + ) + ) + } + + @Test + fun `should not advise library directory of Python without shared libpython`() { + // given + val starter = EmbeddedPythonStarter("/venv/bin/python", "") + val cause = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.13.so.1.0: cannot open shared object file: No such file or directory" + ) // when val exception = assertFailsWith { diff --git a/src/test/kotlin/PythonLibraryTest.kt b/src/test/kotlin/PythonLibraryTest.kt index aa7eb48..3b63fd5 100644 --- a/src/test/kotlin/PythonLibraryTest.kt +++ b/src/test/kotlin/PythonLibraryTest.kt @@ -20,10 +20,20 @@ package com.xemantic.kotlin.script.python import com.xemantic.kotlin.test.assert import org.junit.jupiter.api.Assumptions.assumeTrue +import java.io.IOException +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.Path +import kotlin.io.path.createTempDirectory +import kotlin.io.path.deleteRecursively +import kotlin.io.path.exists +import kotlin.io.path.writeText import kotlin.test.Test /** * The embedded CPython loading the shared `libpython` of its Python without `LD_LIBRARY_PATH`, + * and importing a C extension module which, like the ones of numpy, is not linked to `libpython`, + * but needs its symbols to be global, * in a JVM of its own, as the environment of the dynamic loader is read when a process starts. * * It tests the loading only with a Python outside the system library directories, @@ -33,22 +43,126 @@ import kotlin.test.Test */ class PythonLibraryTest { + private val venvPython = Path(System.getenv("VIRTUAL_ENV"), "bin", "python").toString() + + @OptIn(ExperimentalPathApi::class) @Test - fun `should start embedded Python with symbols of libpython global without LD_LIBRARY_PATH`() { - assumeTrue(System.getProperty("os.name").startsWith("Linux"), "LD_LIBRARY_PATH is the one of Linux") - - // when - val (exitCode, output) = runInJvm( - EmbeddedPythonMain::class.java.name, - // the symbols of the global scope, which a C extension not linked to libpython needs, e.g. of numpy; - // importing ctypes fails without them, unless it is a built-in module, like in a Python of uv - input = "import ctypes\nprint(hasattr(ctypes.pythonapi, 'Py_IsInitialized'), flush=True)", - environment = mapOf("LD_LIBRARY_PATH" to null) - ) + fun `should import C extension not linked to libpython without LD_LIBRARY_PATH`() { + // given + val directory = createTempDirectory("python-library") + try { + compileExtension(directory) + + // when + val (exitCode, output) = runInJvm( + EmbeddedPythonMain::class.java.name, + input = "import xemantic_extension\nprint(xemantic_extension.VALUE, flush=True)", + environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString()) + ) + + // then + assert(output == "imported") + assert(exitCode == 0) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should import C extension not linked to libpython at startup of Python without LD_LIBRARY_PATH`() { + // given + val directory = createTempDirectory("python-library") + try { + compileExtension(directory) + // imported by the site module at the startup, like a .pth file of the site-packages does, e.g. of coverage + directory.resolve("sitecustomize.py").writeText( + """ + |import sys + |try: + | import xemantic_extension + | sys.xemantic_startup_import = xemantic_extension.VALUE + |except ImportError as e: + | sys.xemantic_startup_import = repr(e) + |""".trimMargin() + ) + + // when + val (exitCode, output) = runInJvm( + EmbeddedPythonMain::class.java.name, + input = "import sys\nprint(sys.xemantic_startup_import, flush=True)", + environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString()) + ) - // then - assert(output == "True") - assert(exitCode == 0) + // then + assert(output == "imported") + assert(exitCode == 0) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should import C extension not linked to libpython without LD_LIBRARY_PATH and foreign linker`() { + // given + val directory = createTempDirectory("python-library") + try { + compileExtension(directory) + + // when + val (exitCode, output) = runInJvm( + EmbeddedPythonMain::class.java.name, + input = "import xemantic_extension\nprint(xemantic_extension.VALUE, flush=True)", + environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString()), + // like on Java 17 to 21, without the foreign linker + jvmArguments = listOf("-D${NO_FOREIGN_LINKER_PROPERTY}=true") + ) + + // then + assert(output == "imported") + assert(exitCode == 0) + } finally { + directory.deleteRecursively() + } + } + + /** + * Compiles the `xemantic_extension` C extension module of the test venv Python into the [directory], + * not linked to `libpython`, like the C extensions on Linux are, + * skipping the test if not on Linux, or without a C compiler, or the Python headers. + */ + private fun compileExtension(directory: Path) { + assumeTrue(System.getProperty("os.name").startsWith("Linux"), "the libpython is loaded so only on Linux") + val (include, suffix) = runProcess( + pythonCommand( + venvPython, + "import sysconfig\nprint(sysconfig.get_paths()['include'])\nprint(sysconfig.get_config_var('EXT_SUFFIX'))" + ), + mergeErrorStream = false + ).output.lines() + assumeTrue(Path(include, "Python.h").exists(), "no Python headers in $include") + val source = directory.resolve("xemantic_extension.c") + source.writeText( + """ + |#include + | + |static struct PyModuleDef definition = {PyModuleDef_HEAD_INIT, "xemantic_extension", NULL, -1, NULL}; + | + |PyMODINIT_FUNC PyInit_xemantic_extension(void) { + | PyObject *module = PyModule_Create(&definition); + | if (module != NULL) PyModule_AddStringConstant(module, "VALUE", "imported"); + | return module; + |} + |""".trimMargin() + ) + val extension = directory.resolve("xemantic_extension$suffix") + val compiled = try { + runProcess(listOf("cc", "-shared", "-fPIC", "-I$include", source.toString(), "-o", extension.toString())) + } catch (_: IOException) { + null + } + assumeTrue(compiled?.exitCode == 0, "cannot compile a C extension with cc: ${compiled?.output}") } } From 31e0fc88f410513bc55c95070ad77d4641366587 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Wed, 11 Nov 2026 21:09:07 +0000 Subject: [PATCH 3/4] Load libpython only when the dynamic loader would not find it (#54) A system Python, whose libpython the loader finds by its soname, is left alone again, so it gets no new restricted method warning on Java 22+: loaderFinds checks LD_LIBRARY_PATH, /etc/ld.so.cache and the system directories for the same file, like the loader does. On Java 21, the foreign linker is no longer touched (allocateFrom is looked up before any restricted call), so it prints no warning either. The startup-import test of PythonLibraryTest is assumed on Java 22+ only, as on Java 17-21 libpython becomes global after the startup, as documented. Making libpython global on Java 17-21 is now synchronized, so that a second interpreter waits for it; the LD_LIBRARY_PATH advice is given only for "cannot open shared object file" / "Library not loaded" of the soname. README no longer lists conda as handled (its libjep has a RUNPATH), mentions RPATH, and DEVELOPMENT.md the pyenv LD_LIBRARY_PATH need on 17-21. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FJmHQ3jHRbVdpwUiNUkApG --- DEVELOPMENT.md | 1 + README.md | 7 +- src/main/kotlin/PythonEnvironment.kt | 33 +++++---- src/main/kotlin/PythonLibrary.kt | 77 ++++++++++++++++++--- src/test/kotlin/PythonEnvironmentTest.kt | 86 +++++++++++++++++++++++- src/test/kotlin/PythonLibraryTest.kt | 2 + 6 files changed, 180 insertions(+), 26 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 8c75e3c..be7368e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -26,6 +26,7 @@ so `PythonLibraryTest` removes it in the JVM it starts, to test the loading with importing a C extension it compiles with `cc`, which tests nothing with a Python of the system library directories. `pyenv` builds Python without a shared `libpython` by default, build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`. +On Java 17 to 21, also set `LD_LIBRARY_PATH` then, for a C extension imported at the startup of Python, see the README. An existing `.venv` is judged by its own Python: it is kept when this Python runs, is 3.10 or newer, and has `pip`, diff --git a/README.md b/README.md index 35abdea..0773804 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,9 @@ The Python code imports the modules from the same paths as this executable, e.g. the `site-packages` of a virtual environment, but not the ones of its base Python. The native library of `jep` links to the shared `libpython` of the Python, e.g. `libpython3.13.so.1.0`, -which on Linux the dynamic loader finds only in the system library directories, or in the ones of `LD_LIBRARY_PATH`. -For a Python installed elsewhere, e.g. by `uv python install`, pyenv, conda, or in a custom prefix, +which on Linux the dynamic loader finds by its soname in the directories of `LD_LIBRARY_PATH`, of `ld.so.cache`, and the system ones, +unless the native library of `jep` has a `RUNPATH` of it, e.g. in conda. +For a Python installed elsewhere, e.g. by `uv python install`, pyenv, or in a custom prefix, the library loads `libpython` from the `LIBDIR` of the Python's `sysconfig` itself, with its symbols global, as C extension modules, e.g. of numpy, need them, so that `LD_LIBRARY_PATH` is not needed. @@ -84,7 +85,7 @@ On Java 17 to 21, the symbols become global only after the startup of Python, so a C extension imported at the startup, by a `.pth` file of the `site-packages`, e.g. of coverage, or by `sitecustomize`, fails with an `undefined symbol` and still needs `LD_LIBRARY_PATH`. If `libpython` is not found, e.g. on macOS, where it is not loaded so, -put its directory, which the error message names, on `LD_LIBRARY_PATH` (`DYLD_LIBRARY_PATH` on macOS), +put its directory, the `LIBDIR` of `sysconfig`, on `LD_LIBRARY_PATH` (`DYLD_LIBRARY_PATH` on macOS), set before the JVM starts, as the loader reads it only then. pyenv builds Python without a shared `libpython` by default, build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`. diff --git a/src/main/kotlin/PythonEnvironment.kt b/src/main/kotlin/PythonEnvironment.kt index 788531f..3545fa6 100644 --- a/src/main/kotlin/PythonEnvironment.kt +++ b/src/main/kotlin/PythonEnvironment.kt @@ -103,12 +103,15 @@ public object PythonEnvironment { MainInterpreter.setJepLibraryPath(library.toString()) MainInterpreter.setInitParams(embeddedPythonConfig(executable)) val pythonLibrary = pythonLibrary(probe.libraryDirectory, probe.librarySoname) - val loading = if (pythonLibrary != null && System.getProperty("os.name").startsWith("Linux")) { + // only when needed, as the foreign linker prints a warning without --enable-native-access + val loading = if ( + pythonLibrary != null && System.getProperty("os.name").startsWith("Linux") && !loaderFinds(pythonLibrary) + ) { loadPythonLibrary(pythonLibrary) } else PythonLibraryLoading() this.executable = executable starter = EmbeddedPythonStarter( - executable, probe.error, pythonLibrary, loading.failure, loading.libraryToMakeGlobal + executable, probe.error, pythonLibrary, loading.failure, loading.localSymbols ) if (dependencyDirectory == null) { dependencyDirectory = System.getProperty(DEPENDENCY_DIRECTORY_PROPERTY)?.let { Path(it) } @@ -507,7 +510,7 @@ private fun String.hexToPath(): String? { * When the native library of Jep cannot load the shared [pythonLibrary] of the Python, see [pythonLibrary], * it tells to put its directory on `LD_LIBRARY_PATH`, * and keeps the [libraryPreloadFailure] of [loadPythonLibrary], if any, as suppressed. - * The symbols of the `libpython` loaded locally by [loadPythonLibrary], if any, + * The symbols of the [pythonLibrary] loaded by [loadPythonLibrary], if [libraryLocalSymbols], * are made global in the first interpreter, see [makePythonLibraryGlobal]. */ internal class EmbeddedPythonStarter( @@ -515,13 +518,11 @@ internal class EmbeddedPythonStarter( private val probeError: String, private val pythonLibrary: Path? = null, private val libraryPreloadFailure: Throwable? = null, - libraryToMakeGlobal: Path? = null + libraryLocalSymbols: Boolean = false ) { - /** The `libpython` to make global in the first interpreter, see [makePythonLibraryGlobal]. */ - @Volatile - private var libraryToMakeGlobal: Path? = libraryToMakeGlobal - + /** Whether the symbols of the [pythonLibrary] are still to be made global, see [makePythonLibraryGlobal]. */ + private var libraryLocalSymbols = libraryLocalSymbols /** Whether an interpreter has been created, so that the embedded CPython has started. */ @Volatile @@ -564,9 +565,10 @@ internal class EmbeddedPythonStarter( } /** - * Makes the symbols of the `libpython` loaded locally by [loadPythonLibrary], if any, global, + * Makes the symbols of the [pythonLibrary] loaded by [loadPythonLibrary] global, if [libraryLocalSymbols], * in the first [interpreter], before any code of the user, but after the startup of Python, * see [loadPythonLibrary]. + * The interpreters created meanwhile on other threads wait for it, so that no code runs before. * * CPython can load a library with `RTLD_GLOBAL`, * loading an extension module with the flags of `sys.setdlopenflags`: @@ -574,9 +576,12 @@ internal class EmbeddedPythonStarter( * before it fails as an extension module without its initialization function. * It is best-effort: a failure is only logged, as the code of the user might not need any C extension. */ + @Synchronized private fun makePythonLibraryGlobal(interpreter: Interpreter) { - val library = libraryToMakeGlobal ?: return - libraryToMakeGlobal = null + if (!libraryLocalSymbols) return + val library = pythonLibrary ?: return + // also on a failure, which would only repeat + libraryLocalSymbols = false try { interpreter.exec( """ @@ -612,12 +617,14 @@ internal class EmbeddedPythonStarter( // Jep wraps the failure of initializing CPython in an Error, whose message is its toString() val reason = cause.cause?.takeIf { cause.message == it.toString() } ?: cause val headline = "Cannot initialize the embedded Python $executable: ${reason.message ?: reason}" - // e.g. libjep.so: libpython3.13.so.1.0: cannot open shared object file, + // e.g. libjep.so: libpython3.13.so.1.0: cannot open shared object file, or Library not loaded on macOS, // also as the cause of the Error of Jep retrying to initialize CPython val library = pythonLibrary val libraryAdvice = if ( library != null && generateSequence(cause) { it.cause }.any { - it is UnsatisfiedLinkError && it.message?.contains(library.fileName.toString()) == true + val message = it.message ?: "" + it is UnsatisfiedLinkError && library.fileName.toString() in message && + ("cannot open shared object file" in message || "Library not loaded" in message) } ) { "\nThe dynamic loader cannot load ${library.fileName} of this Python, " + diff --git a/src/main/kotlin/PythonLibrary.kt b/src/main/kotlin/PythonLibrary.kt index 27529ef..496587c 100644 --- a/src/main/kotlin/PythonLibrary.kt +++ b/src/main/kotlin/PythonLibrary.kt @@ -18,11 +18,14 @@ package com.xemantic.kotlin.script.python +import java.io.IOException import java.lang.invoke.MethodHandle import java.nio.file.Path import java.util.Optional import kotlin.io.path.Path import kotlin.io.path.isRegularFile +import kotlin.io.path.isSameFileAs +import kotlin.io.path.readBytes /** * The system property which, set, keeps [loadPythonLibrary] from using the foreign linker of Java 22, @@ -44,20 +47,21 @@ internal fun pythonLibrary(directory: String, soname: String): Path? { * * @property failure the failure to load the library, e.g. of another architecture, * kept to tell why Jep cannot load its native library, if it cannot. - * @property libraryToMakeGlobal the library loaded with local symbols, to be made global once Python runs, + * @property localSymbols whether the library is loaded with local symbols, to be made global once Python runs, * see [EmbeddedPythonStarter]. */ internal class PythonLibraryLoading( val failure: Throwable? = null, - val libraryToMakeGlobal: Path? = null + val localSymbols: Boolean = false ) /** - * Loads the shared `libpython` [library] of the Python on Linux, before Jep loads its native library, which depends on it. + * Loads the shared `libpython` [library] of the Python on Linux, before Jep loads its native library, which depends on it, + * when the dynamic loader would not find it, see [loaderFinds]. * * The dynamic loader finds the `libpython3.X.so.1.0` of the native library of Jep by its soname - * only in the system library directories, or the ones of `LD_LIBRARY_PATH`, read when the JVM starts, - * so not the one of a Python installed elsewhere, e.g. by uv, pyenv, conda, or `actions/setup-python`; + * in the directories of `LD_LIBRARY_PATH`, read when the JVM starts, of `ld.so.cache`, and the system ones, + * so not the one of a Python installed elsewhere, e.g. by uv, pyenv, or `actions/setup-python`; * once loaded by its path, the loader takes the library of the same soname instead. * * Its symbols must be global as well, as a C extension module on Linux is not linked to `libpython`, @@ -74,12 +78,66 @@ internal fun loadPythonLibrary(library: Path): PythonLibraryLoading { if (System.getProperty(NO_FOREIGN_LINKER_PROPERTY) == null && loadGlobally(library)) return PythonLibraryLoading() return try { System.load(library.toString()) - PythonLibraryLoading(libraryToMakeGlobal = library) + PythonLibraryLoading(localSymbols = true) } catch (e: UnsatisfiedLinkError) { PythonLibraryLoading(failure = e) } } +/** + * Whether the dynamic loader finds the [library] by its soname, its file name, as the same file, + * so that it needs not be loaded by its path, see [loadPythonLibrary], e.g. of a Python of the system. + * + * Like the loader, it takes the first file of the soname in the directories of the [libraryPath], `LD_LIBRARY_PATH`, + * then in the [cache] of the loader, `/etc/ld.so.cache`, which lists the full paths of the libraries, + * of the directories configured in `/etc/ld.so.conf`, and then in the system directories. + * It does not know the `RUNPATH` of the native library of Jep, e.g. of conda, + * with which the library would be loaded when not needed. + */ +internal fun loaderFinds( + library: Path, + libraryPath: String? = System.getenv("LD_LIBRARY_PATH"), + cache: ByteArray? = readLoaderCache() +): Boolean { + val soname = library.fileName.toString() + val candidates = libraryPath.orEmpty().split(':').filter { it.isNotEmpty() }.map { Path(it, soname) } + + cachedLibraries(cache, soname) + + SYSTEM_LIBRARY_DIRECTORIES.map { Path(it, soname) } + val found = candidates.firstOrNull { it.isRegularFile() } ?: return false + return try { + found.isSameFileAs(library) + } catch (_: IOException) { + false + } +} + +/** The directories which the dynamic loader of glibc searches last, also without `ld.so.cache`. */ +private val SYSTEM_LIBRARY_DIRECTORIES = listOf("/lib64", "/usr/lib64", "/lib", "/usr/lib") + +/** The `/etc/ld.so.cache` of the dynamic loader, or `null` if it cannot be read, e.g. of musl, which has none. */ +private fun readLoaderCache(): ByteArray? = try { + Path("/etc/ld.so.cache").takeIf { it.isRegularFile() }?.readBytes() +} catch (_: IOException) { + null +} + +/** The paths of the libraries of the [soname] in the [cache] of the dynamic loader, its null-terminated strings. */ +private fun cachedLibraries(cache: ByteArray?, soname: String): List { + if (cache == null) return emptyList() + // ISO-8859-1 maps each byte to a char, so that the indexes are the ones of the bytes + val text = String(cache, Charsets.ISO_8859_1) + val suffix = "/" + String(soname.toByteArray(filesystemCharset), Charsets.ISO_8859_1) + "\u0000" + val paths = mutableListOf() + var end = text.indexOf(suffix) + while (end != -1) { + val start = text.lastIndexOf('\u0000', end) + 1 + val bytes = text.substring(start, end + suffix.length - 1).toByteArray(Charsets.ISO_8859_1) + String(bytes, filesystemCharset).takeIf { it.startsWith("/") }?.let { paths.add(Path(it)) } + end = text.indexOf(suffix, end + 1) + } + return paths +} + /** `RTLD_LAZY | RTLD_GLOBAL` of glibc and musl. */ private const val RTLD_LAZY_GLOBAL = 0x101 @@ -90,8 +148,11 @@ private const val RTLD_LAZY_GLOBAL = 0x101 * @return `true` if loaded, `false` if the foreign linker is not available, or `dlopen` fails. */ private fun loadGlobally(library: Path): Boolean = try { - val linkerClass = Class.forName("java.lang.foreign.Linker") val arenaClass = Class.forName("java.lang.foreign.Arena") + // of Java 22, as the foreign linker of Java 21 is a preview, + // looked up before calling a restricted method, which would print a warning without --enable-native-access + val allocateFrom = arenaClass.getMethod("allocateFrom", String::class.java) + val linkerClass = Class.forName("java.lang.foreign.Linker") val segmentClass = Class.forName("java.lang.foreign.MemorySegment") val layoutClass = Class.forName("java.lang.foreign.MemoryLayout") val valueLayoutClass = Class.forName("java.lang.foreign.ValueLayout") @@ -115,8 +176,6 @@ private fun loadGlobally(library: Path): Boolean = try { val noOptions = java.lang.reflect.Array.newInstance(optionClass, 0) val handle = linkerClass.getMethod("downcallHandle", segmentClass, descriptorClass, noOptions.javaClass) .invoke(linker, dlopen, descriptor, noOptions) as MethodHandle - // allocateFrom is of Java 22, the foreign linker of Java 21 is a preview - val allocateFrom = arenaClass.getMethod("allocateFrom", String::class.java) (arenaClass.getMethod("ofConfined").invoke(null) as AutoCloseable).use { arena -> val path = allocateFrom.invoke(arena, library.toString()) val result = handle.invokeWithArguments(path, RTLD_LAZY_GLOBAL) diff --git a/src/test/kotlin/PythonEnvironmentTest.kt b/src/test/kotlin/PythonEnvironmentTest.kt index 524c83b..f0d0b9a 100644 --- a/src/test/kotlin/PythonEnvironmentTest.kt +++ b/src/test/kotlin/PythonEnvironmentTest.kt @@ -838,6 +838,85 @@ class PythonEnvironmentTest { } } + @OptIn(ExperimentalPathApi::class) + @Test + fun `should tell that dynamic loader finds libpython in directory of LD_LIBRARY_PATH`() { + // given + val directory = createTempDirectory("python-library") + val other = directory.resolve("other").createDirectory() + val library = directory.resolve("libxemantic-test.so.1").createFile() + + try { + // when + val found = loaderFinds(library, libraryPath = "$other:$directory", cache = null) + + // then + assert(found) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should tell that dynamic loader finds libpython listed in its cache`() { + // given + val directory = createTempDirectory("python-library") + val library = directory.resolve("libxemantic-test.so.1").createFile() + val cache = "ld.so-1.7.0\u0000libxemantic-test.so.1\u0000$library\u0000/usr/lib/libc.so.6\u0000" + .toByteArray(filesystemCharset) + + try { + // when + val found = loaderFinds(library, libraryPath = null, cache = cache) + + // then + assert(found) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should tell that dynamic loader does not find libpython`() { + // given + val directory = createTempDirectory("python-library") + val library = directory.resolve("libxemantic-test.so.1").createFile() + val cache = "ld.so-1.7.0\u0000libc.so.6\u0000/usr/lib/libc.so.6\u0000".toByteArray(filesystemCharset) + + try { + // when + val found = loaderFinds(library, libraryPath = "", cache = cache) + + // then + assert(!found) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should tell that dynamic loader finds another libpython of same soname first`() { + // given + // e.g. of another installation of the same Python version + val directory = createTempDirectory("python-library") + val other = directory.resolve("other").createDirectory() + other.resolve("libxemantic-test.so.1").createFile() + val library = directory.resolve("libxemantic-test.so.1").createFile() + + try { + // when + val found = loaderFinds(library, libraryPath = "$other:$directory", cache = null) + + // then + assert(!found) + } finally { + directory.deleteRecursively() + } + } + @Test fun `should advise library directory of libpython when native library of Jep cannot load it`() { // given @@ -898,6 +977,10 @@ class PythonEnvironmentTest { "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0") ) val wrongClass = UnsatisfiedLinkError("/venv/lib/python3.13/site-packages/jep/libjep.so: wrong ELF class: ELFCLASS32") + // of libpython found, but of another Python + val undefinedSymbol = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: undefined symbol: PyFoo, version libpython3.13.so.1.0" + ) // of jep built for another Python val anotherVersion = UnsatisfiedLinkError( "/venv/lib/python3.13/site-packages/jep/libjep.so: " + @@ -905,7 +988,7 @@ class PythonEnvironmentTest { ) // when - val messages = listOf(wrongClass, anotherVersion).map { cause -> + val messages = listOf(wrongClass, undefinedSymbol, anotherVersion).map { cause -> assertFailsWith { starter.createInterpreter(create = { throw cause }, check = {}) }.message @@ -915,6 +998,7 @@ class PythonEnvironmentTest { assert( messages == listOf( "Cannot initialize the embedded Python /venv/bin/python: ${wrongClass.message}", + "Cannot initialize the embedded Python /venv/bin/python: ${undefinedSymbol.message}", "Cannot initialize the embedded Python /venv/bin/python: ${anotherVersion.message}" ) ) diff --git a/src/test/kotlin/PythonLibraryTest.kt b/src/test/kotlin/PythonLibraryTest.kt index 3b63fd5..c5c4e86 100644 --- a/src/test/kotlin/PythonLibraryTest.kt +++ b/src/test/kotlin/PythonLibraryTest.kt @@ -72,6 +72,8 @@ class PythonLibraryTest { @Test fun `should import C extension not linked to libpython at startup of Python without LD_LIBRARY_PATH`() { // given + // on Java 17 to 21, the symbols of libpython become global only after the startup, see loadPythonLibrary + assumeTrue(Runtime.version().feature() >= 22, "the foreign linker is of Java 22") val directory = createTempDirectory("python-library") try { compileExtension(directory) From 7567ba727c2077fb56c98ebdb3c1a438132c78b2 Mon Sep 17 00:00:00 2001 From: Kazik Pogoda Date: Wed, 11 Nov 2026 21:28:40 +0000 Subject: [PATCH 4/4] Keep the startup failure shape when the JVM denies loading libpython (#54) - A RuntimeException of System.load of libpython, e.g. IllegalCallerException with --illegal-native-access=deny, is kept as the preload failure instead of escaping initialize() unwrapped. - A failed preload is named in the startup failure message, instead of the LD_LIBRARY_PATH advice, which would not help then. - The noForeignLinker test seam is parsed as a boolean. - /etc/ld.so.cache is read only when no LD_LIBRARY_PATH entry decides. - The CLAUDE.md entry is trimmed to the non-obvious lesson. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FJmHQ3jHRbVdpwUiNUkApG --- CLAUDE.md | 4 +- src/main/kotlin/PythonEnvironment.kt | 7 +- src/main/kotlin/PythonLibrary.kt | 31 +++++--- src/test/kotlin/PythonEnvironmentTest.kt | 94 ++++++++++++++++++++++-- 4 files changed, 115 insertions(+), 21 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index bf7a115..2ba2ce2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -70,8 +70,8 @@ see [DEVELOPMENT.md](DEVELOPMENT.md#markdown-soft-wrapping-in-the-ide) for how t - The embedded CPython does not detect a virtual environment by itself — without the venv's `python` as its program name (`PyConfig.setProgramName`), it ignores `pyvenv.cfg` and also imports from the global and user `site-packages` of the base Python. When a module is unexpectedly importable, check `sys.path` of the embedded interpreter, not of `.venv/bin/python`. -- A running Gradle daemon does not pass a changed `LD_LIBRARY_PATH` on to the test JVM, so an `LD_LIBRARY_PATH` of your shell can be absent, or a stale one present, in the tests — - with a `-PbasePython` outside the system library directories, e.g. of uv, run `--no-daemon` without `LD_LIBRARY_PATH` to test that `libpython` loads without it, as on CI `PythonLibraryTest` does. +- A running Gradle daemon does not pass a changed `LD_LIBRARY_PATH` on to the test JVM, so a stale one can hide a failure to load `libpython` — + run a `-PbasePython` of uv with `--no-daemon`, without `LD_LIBRARY_PATH`. - Jep passes the `PyConfig` strings to CPython as JNI *modified* UTF-8 (`GetStringUTFChars`), not plain UTF-8 — a character outside the BMP becomes a CESU-8 surrogate pair, which CPython does not decode as that character. For a path under a non-UTF-8 locale, see `pythonPath`. diff --git a/src/main/kotlin/PythonEnvironment.kt b/src/main/kotlin/PythonEnvironment.kt index 3545fa6..75bc861 100644 --- a/src/main/kotlin/PythonEnvironment.kt +++ b/src/main/kotlin/PythonEnvironment.kt @@ -509,7 +509,7 @@ private fun String.hexToPath(): String? { * might tell why the embedded CPython, configured the same way, fails. * When the native library of Jep cannot load the shared [pythonLibrary] of the Python, see [pythonLibrary], * it tells to put its directory on `LD_LIBRARY_PATH`, - * and keeps the [libraryPreloadFailure] of [loadPythonLibrary], if any, as suppressed. + * or the [libraryPreloadFailure] of [loadPythonLibrary], if any, also kept as suppressed. * The symbols of the [pythonLibrary] loaded by [loadPythonLibrary], if [libraryLocalSymbols], * are made global in the first interpreter, see [makePythonLibraryGlobal]. */ @@ -620,7 +620,10 @@ internal class EmbeddedPythonStarter( // e.g. libjep.so: libpython3.13.so.1.0: cannot open shared object file, or Library not loaded on macOS, // also as the cause of the Error of Jep retrying to initialize CPython val library = pythonLibrary - val libraryAdvice = if ( + val libraryAdvice = if (libraryPreloadFailure != null) { + // LD_LIBRARY_PATH would not help, as the loader would fail to load it the same way, e.g. of another architecture + "\nCannot load the libpython of this Python: ${libraryPreloadFailure.message ?: libraryPreloadFailure}" + } else if ( library != null && generateSequence(cause) { it.cause }.any { val message = it.message ?: "" it is UnsatisfiedLinkError && library.fileName.toString() in message && diff --git a/src/main/kotlin/PythonLibrary.kt b/src/main/kotlin/PythonLibrary.kt index 496587c..6bf7b79 100644 --- a/src/main/kotlin/PythonLibrary.kt +++ b/src/main/kotlin/PythonLibrary.kt @@ -28,11 +28,14 @@ import kotlin.io.path.isSameFileAs import kotlin.io.path.readBytes /** - * The system property which, set, keeps [loadPythonLibrary] from using the foreign linker of Java 22, + * The system property which, set to `true`, keeps [loadPythonLibrary] from using the foreign linker of Java 22, * a test seam of loading `libpython` like on Java 17 to 21. */ internal const val NO_FOREIGN_LINKER_PROPERTY = "xemantic.kotlin.script.python.noForeignLinker" +/** Whether [loadPythonLibrary] may use the foreign linker, unless the [NO_FOREIGN_LINKER_PROPERTY] is `true`. */ +internal fun foreignLinkerEnabled(): Boolean = !java.lang.Boolean.getBoolean(NO_FOREIGN_LINKER_PROPERTY) + /** * The shared `libpython` of the [soname] in the library [directory] of a Python, see [PythonEnvironment.probeCommand], * or `null` if there is none, e.g. of a Python linked to `libpython` statically, with `libpython3.X.a` as its soname. @@ -45,7 +48,7 @@ internal fun pythonLibrary(directory: String, soname: String): Path? { /** * The result of [loadPythonLibrary]. * - * @property failure the failure to load the library, e.g. of another architecture, + * @property failure the failure to load the library, e.g. of another architecture, or denied by the JVM, * kept to tell why Jep cannot load its native library, if it cannot. * @property localSymbols whether the library is loaded with local symbols, to be made global once Python runs, * see [EmbeddedPythonStarter]. @@ -73,14 +76,22 @@ internal class PythonLibraryLoading( * before CPython starts, also for a C extension imported at its startup, e.g. by a `.pth` file of the `site-packages`. * On Java 17 to 21, it is loaded with [System.load], and made global in the first interpreter, * so only after the startup of CPython. + * The [load] of the library is a test seam of [System.load]. */ -internal fun loadPythonLibrary(library: Path): PythonLibraryLoading { - if (System.getProperty(NO_FOREIGN_LINKER_PROPERTY) == null && loadGlobally(library)) return PythonLibraryLoading() +internal fun loadPythonLibrary( + library: Path, + load: (String) -> Unit = System::load +): PythonLibraryLoading { + if (foreignLinkerEnabled() && loadGlobally(library)) return PythonLibraryLoading() return try { - System.load(library.toString()) + load(library.toString()) PythonLibraryLoading(localSymbols = true) } catch (e: UnsatisfiedLinkError) { PythonLibraryLoading(failure = e) + } catch (e: RuntimeException) { + // e.g. an IllegalCallerException with --illegal-native-access=deny, + // which then fails loading the native library of Jep as well, reported as a failure of the startup + PythonLibraryLoading(failure = e) } } @@ -90,19 +101,19 @@ internal fun loadPythonLibrary(library: Path): PythonLibraryLoading { * * Like the loader, it takes the first file of the soname in the directories of the [libraryPath], `LD_LIBRARY_PATH`, * then in the [cache] of the loader, `/etc/ld.so.cache`, which lists the full paths of the libraries, - * of the directories configured in `/etc/ld.so.conf`, and then in the system directories. + * of the directories configured in `/etc/ld.so.conf`, read only if needed, and then in the system directories. * It does not know the `RUNPATH` of the native library of Jep, e.g. of conda, * with which the library would be loaded when not needed. */ internal fun loaderFinds( library: Path, libraryPath: String? = System.getenv("LD_LIBRARY_PATH"), - cache: ByteArray? = readLoaderCache() + cache: () -> ByteArray? = ::readLoaderCache ): Boolean { val soname = library.fileName.toString() - val candidates = libraryPath.orEmpty().split(':').filter { it.isNotEmpty() }.map { Path(it, soname) } + - cachedLibraries(cache, soname) + - SYSTEM_LIBRARY_DIRECTORIES.map { Path(it, soname) } + val candidates = libraryPath.orEmpty().split(':').asSequence().filter { it.isNotEmpty() }.map { Path(it, soname) } + + sequence { yieldAll(cachedLibraries(cache(), soname)) } + + SYSTEM_LIBRARY_DIRECTORIES.asSequence().map { Path(it, soname) } val found = candidates.firstOrNull { it.isRegularFile() } ?: return false return try { found.isSameFileAs(library) diff --git a/src/test/kotlin/PythonEnvironmentTest.kt b/src/test/kotlin/PythonEnvironmentTest.kt index f0d0b9a..82e767e 100644 --- a/src/test/kotlin/PythonEnvironmentTest.kt +++ b/src/test/kotlin/PythonEnvironmentTest.kt @@ -848,7 +848,7 @@ class PythonEnvironmentTest { try { // when - val found = loaderFinds(library, libraryPath = "$other:$directory", cache = null) + val found = loaderFinds(library, libraryPath = "$other:$directory", cache = { null }) // then assert(found) @@ -868,7 +868,7 @@ class PythonEnvironmentTest { try { // when - val found = loaderFinds(library, libraryPath = null, cache = cache) + val found = loaderFinds(library, libraryPath = null, cache = { cache }) // then assert(found) @@ -877,6 +877,50 @@ class PythonEnvironmentTest { } } + @OptIn(ExperimentalPathApi::class) + @Test + fun `should not read cache of dynamic loader when libpython is found in directory of LD_LIBRARY_PATH`() { + // given + val directory = createTempDirectory("python-library") + val library = directory.resolve("libxemantic-test.so.1").createFile() + + try { + // when + val found = loaderFinds(library, libraryPath = "$directory", cache = { error("cache read") }) + + // then + assert(found) + } finally { + directory.deleteRecursively() + } + } + + @Test + fun `should keep failure of loading libpython denied by JVM`() { + // given + // like System.load with --illegal-native-access=deny + val denied = IllegalCallerException("Illegal native access from: unnamed module") + + // when + val loading = withSystemProperty(NO_FOREIGN_LINKER_PROPERTY, "true") { + loadPythonLibrary(Path("/opt/python/lib/libpython3.13.so.1.0"), load = { throw denied }) + } + + // then + assert(loading.failure === denied && !loading.localSymbols) + } + + @Test + fun `should use foreign linker unless disabled by system property set to true`() { + // when + val usage = listOf(null, "false", "true", "TRUE").map { value -> + withSystemProperty(NO_FOREIGN_LINKER_PROPERTY, value) { foreignLinkerEnabled() } + } + + // then + assert(usage == listOf(true, true, false, false)) + } + @OptIn(ExperimentalPathApi::class) @Test fun `should tell that dynamic loader does not find libpython`() { @@ -887,7 +931,7 @@ class PythonEnvironmentTest { try { // when - val found = loaderFinds(library, libraryPath = "", cache = cache) + val found = loaderFinds(library, libraryPath = "", cache = { cache }) // then assert(!found) @@ -908,7 +952,7 @@ class PythonEnvironmentTest { try { // when - val found = loaderFinds(library, libraryPath = "$other:$directory", cache = null) + val found = loaderFinds(library, libraryPath = "$other:$directory", cache = { null }) // then assert(!found) @@ -920,6 +964,33 @@ class PythonEnvironmentTest { @Test fun `should advise library directory of libpython when native library of Jep cannot load it`() { // given + val starter = EmbeddedPythonStarter( + "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0") + ) + val cause = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.13.so.1.0: cannot open shared object file: No such file or directory" + ) + + // when + val exception = assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + } + + // then + val message = exception.message!! + assert( + exception.cause === cause && + message.startsWith("Cannot initialize the embedded Python /venv/bin/python: ${cause.message}\n") && + "/opt/python/lib" in message && + "LD_LIBRARY_PATH" in message + ) + } + + @Test + fun `should tell failure to load libpython instead of advising its directory`() { + // given + // LD_LIBRARY_PATH would not help, as the loader would fail to load the library the same way val preloadFailure = UnsatisfiedLinkError("/opt/python/lib/libpython3.13.so.1.0: wrong ELF class: ELFCLASS32") val starter = EmbeddedPythonStarter( "/venv/bin/python", "", @@ -941,9 +1012,8 @@ class PythonEnvironmentTest { assert( exception.cause === cause && exception.suppressed.toList() == listOf(preloadFailure) && - message.startsWith("Cannot initialize the embedded Python /venv/bin/python: ${cause.message}\n") && - "/opt/python/lib" in message && - "LD_LIBRARY_PATH" in message + message == "Cannot initialize the embedded Python /venv/bin/python: ${cause.message}\n" + + "Cannot load the libpython of this Python: ${preloadFailure.message}" ) } @@ -1122,6 +1192,16 @@ class PythonEnvironmentTest { } /** The output of [PythonEnvironment.probeCommand] writing the [values], trimmed as by [runProcess]. */ + private fun withSystemProperty(name: String, value: String?, block: () -> T): T { + val previous = System.getProperty(name) + if (value == null) System.clearProperty(name) else System.setProperty(name, value) + try { + return block() + } finally { + if (previous == null) System.clearProperty(name) else System.setProperty(name, previous) + } + } + private fun probePayload(vararg values: String): String { val hexValues = values.joinToString(" ") { it.toByteArray(filesystemCharset).toHexString() } return "${PythonEnvironment.PROBE_BEGIN}\n$hexValues\n${PythonEnvironment.PROBE_END}"