diff --git a/CLAUDE.md b/CLAUDE.md index 5b3d0e6..2ba2ce2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -70,8 +70,8 @@ see [DEVELOPMENT.md](DEVELOPMENT.md#markdown-soft-wrapping-in-the-ide) for how t - The embedded CPython does not detect a virtual environment by itself — without the venv's `python` as its program name (`PyConfig.setProgramName`), it ignores `pyvenv.cfg` and also imports from the global and user `site-packages` of the base Python. When a module is unexpectedly importable, check `sys.path` of the embedded interpreter, not of `.venv/bin/python`. -- A running Gradle daemon does not pass a changed `LD_LIBRARY_PATH` on to the test JVM, - so use `--no-daemon` when trying a `-PbasePython` which needs it (see [DEVELOPMENT.md](DEVELOPMENT.md#the-python-of-the-tests)). +- A running Gradle daemon does not pass a changed `LD_LIBRARY_PATH` on to the test JVM, so a stale one can hide a failure to load `libpython` — + run a `-PbasePython` of uv with `--no-daemon`, without `LD_LIBRARY_PATH`. - Jep passes the `PyConfig` strings to CPython as JNI *modified* UTF-8 (`GetStringUTFChars`), not plain UTF-8 — a character outside the BMP becomes a CESU-8 surrogate pair, which CPython does not decode as that character. For a path under a non-UTF-8 locale, see `pythonPath`. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b861595..be7368e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -18,12 +18,15 @@ or with the `ORG_GRADLE_PROJECT_basePython` environment variable, e.g. on CI: ``` It has to be Python 3.10 or newer, otherwise the build fails before creating the environment. -On Linux, the tests fail to load `libjep.so` when it cannot find `libpython3.*.so` ([#54](https://github.com/xemantic/xemantic-kotlin-script-python/issues/54)). -A Python outside the system library directories, e.g. installed by `uv`, -needs its `lib` directory in `LD_LIBRARY_PATH`, -which `actions/setup-python` sets on CI. +On Linux, a Python outside the system library directories, e.g. installed by `uv`, +does not need its `lib` directory in `LD_LIBRARY_PATH`, +as `PythonEnvironment` loads its `libpython` before `libjep.so` ([#54](https://github.com/xemantic/xemantic-kotlin-script-python/issues/54)). +`actions/setup-python` sets `LD_LIBRARY_PATH` on CI anyway, +so `PythonLibraryTest` removes it in the JVM it starts, to test the loading with such a base Python, +importing a C extension it compiles with `cc`, which tests nothing with a Python of the system library directories. `pyenv` builds Python without a shared `libpython` by default, -build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"` (and then set `LD_LIBRARY_PATH` as well). +build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`. +On Java 17 to 21, also set `LD_LIBRARY_PATH` then, for a C extension imported at the startup of Python, see the README. An existing `.venv` is judged by its own Python: it is kept when this Python runs, is 3.10 or newer, and has `pip`, diff --git a/README.md b/README.md index 0c56628..0773804 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,23 @@ The Python executable is, in this order: The Python code imports the modules from the same paths as this executable, e.g. the `site-packages` of a virtual environment, but not the ones of its base Python. +The native library of `jep` links to the shared `libpython` of the Python, e.g. `libpython3.13.so.1.0`, +which on Linux the dynamic loader finds by its soname in the directories of `LD_LIBRARY_PATH`, of `ld.so.cache`, and the system ones, +unless the native library of `jep` has a `RUNPATH` of it, e.g. in conda. +For a Python installed elsewhere, e.g. by `uv python install`, pyenv, or in a custom prefix, +the library loads `libpython` from the `LIBDIR` of the Python's `sysconfig` itself, with its symbols global, +as C extension modules, e.g. of numpy, need them, +so that `LD_LIBRARY_PATH` is not needed. +On Java 22 or newer, it does so with the foreign linker, a restricted method, +which, like loading the native library of `jep` on Java 24 or newer, prints a warning without `--enable-native-access=ALL-UNNAMED`. +On Java 17 to 21, the symbols become global only after the startup of Python, +so a C extension imported at the startup, by a `.pth` file of the `site-packages`, e.g. of coverage, or by `sitecustomize`, +fails with an `undefined symbol` and still needs `LD_LIBRARY_PATH`. +If `libpython` is not found, e.g. on macOS, where it is not loaded so, +put its directory, the `LIBDIR` of `sysconfig`, on `LD_LIBRARY_PATH` (`DYLD_LIBRARY_PATH` on macOS), +set before the JVM starts, as the loader reads it only then. +pyenv builds Python without a shared `libpython` by default, build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`. + The embedded Python, like every Python process started by this library, e.g. `pip`, runs in the [UTF-8 mode](https://peps.python.org/pep-0540/), whatever the locale, e.g. cp1252 on Windows, or `C` on Linux: diff --git a/src/main/kotlin/PythonEnvironment.kt b/src/main/kotlin/PythonEnvironment.kt index af69f86..75bc861 100644 --- a/src/main/kotlin/PythonEnvironment.kt +++ b/src/main/kotlin/PythonEnvironment.kt @@ -102,8 +102,17 @@ public object PythonEnvironment { val library = jepLibrary(Path(jepLocation), python, probe.externallyManaged) MainInterpreter.setJepLibraryPath(library.toString()) MainInterpreter.setInitParams(embeddedPythonConfig(executable)) + val pythonLibrary = pythonLibrary(probe.libraryDirectory, probe.librarySoname) + // only when needed, as the foreign linker prints a warning without --enable-native-access + val loading = if ( + pythonLibrary != null && System.getProperty("os.name").startsWith("Linux") && !loaderFinds(pythonLibrary) + ) { + loadPythonLibrary(pythonLibrary) + } else PythonLibraryLoading() this.executable = executable - starter = EmbeddedPythonStarter(executable, probe.error) + starter = EmbeddedPythonStarter( + executable, probe.error, pythonLibrary, loading.failure, loading.localSymbols + ) if (dependencyDirectory == null) { dependencyDirectory = System.getProperty(DEPENDENCY_DIRECTORY_PROPERTY)?.let { Path(it) } } @@ -266,10 +275,14 @@ public object PythonEnvironment { val begin = lines.lastIndexOf("\n$PROBE_BEGIN\n").takeIf { it != -1 }?.let { it + PROBE_BEGIN.length + 2 } val end = begin?.let { lines.indexOf("\n$PROBE_END", it) }?.takeIf { it != -1 } val values = if (begin == null || end == null) null else lines.substring(begin, end).split(' ') - if (values == null || values.size != 5) return null - val (executable, cacheTag, prefix, jepLocation, externallyManaged) = - values.map { it.hexToPath() ?: return null } - return PythonProbe(executable, cacheTag, prefix, jepLocation, externallyManaged.isNotEmpty()) + if (values == null || values.size != 7) return null + val paths = values.map { it.hexToPath() ?: return null } + val (executable, cacheTag, prefix, jepLocation, externallyManaged) = paths + return PythonProbe( + executable, cacheTag, prefix, jepLocation, externallyManaged.isNotEmpty(), + libraryDirectory = paths[5], + librarySoname = paths[6] + ) } /** @@ -281,9 +294,11 @@ public object PythonEnvironment { * as text, a path which is not valid UTF-8, e.g. `/home/josé` in ISO-8859-1, * would be surrogate-escaped by Python in UTF-8 mode, and could not be encoded to UTF-8, * while the JVM would decode a path of UTF-8 bytes to another one, in another encoding of its locale. - * The last value tells whether the Python is externally managed (PEP 668), `1` if it is, empty if not: + * The fifth value tells whether the Python is externally managed (PEP 668), `1` if it is, empty if not: * a base Python, not a virtual environment, with an `EXTERNALLY-MANAGED` file in its standard library, * into which pip refuses to install. + * The values after it are the directory and the soname of the `libpython` of the base Python, + * `LIBDIR` and `INSTSONAME` of `sysconfig`, empty if unknown, e.g. on Windows, see [pythonLibrary]. * Without the `jep` package, it exits with the [PROBE_JEP_MISSING_EXIT_CODE], * or the [PROBE_JEP_MISSING_EXTERNALLY_MANAGED_EXIT_CODE] if the Python is externally managed, * writing to the standard error the file of a `jep` module, if any, e.g. a `jep.py` shadowing the package. @@ -312,6 +327,8 @@ public object PythonEnvironment { | sys.prefix, | jep_spec.submodule_search_locations[0], | '1' if externally_managed else '', + | sysconfig.get_config_var('LIBDIR'), + | sysconfig.get_config_var('INSTSONAME'), |] |payload = ' '.join('' if value is None else os.fsencode(str(value)).hex() for value in values) |sys.stdout.flush() @@ -490,8 +507,22 @@ private fun String.hexToPath(): String? { * telling a failure of its startup with what the probe of the [executable] wrote to the standard error, if anything, * as a warning of a successful probe, e.g. `Could not find platform libraries` of a wrong `PYTHONHOME`, * might tell why the embedded CPython, configured the same way, fails. + * When the native library of Jep cannot load the shared [pythonLibrary] of the Python, see [pythonLibrary], + * it tells to put its directory on `LD_LIBRARY_PATH`, + * or the [libraryPreloadFailure] of [loadPythonLibrary], if any, also kept as suppressed. + * The symbols of the [pythonLibrary] loaded by [loadPythonLibrary], if [libraryLocalSymbols], + * are made global in the first interpreter, see [makePythonLibraryGlobal]. */ -internal class EmbeddedPythonStarter(private val executable: String, private val probeError: String) { +internal class EmbeddedPythonStarter( + private val executable: String, + private val probeError: String, + private val pythonLibrary: Path? = null, + private val libraryPreloadFailure: Throwable? = null, + libraryLocalSymbols: Boolean = false +) { + + /** Whether the symbols of the [pythonLibrary] are still to be made global, see [makePythonLibraryGlobal]. */ + private var libraryLocalSymbols = libraryLocalSymbols /** Whether an interpreter has been created, so that the embedded CPython has started. */ @Volatile @@ -520,6 +551,7 @@ internal class EmbeddedPythonStarter(private val executable: String, private val } started = true try { + if (interpreter is Interpreter) makePythonLibraryGlobal(interpreter) check(interpreter) } catch (e: Throwable) { try { @@ -532,15 +564,83 @@ internal class EmbeddedPythonStarter(private val executable: String, private val return interpreter } + /** + * Makes the symbols of the [pythonLibrary] loaded by [loadPythonLibrary] global, if [libraryLocalSymbols], + * in the first [interpreter], before any code of the user, but after the startup of Python, + * see [loadPythonLibrary]. + * The interpreters created meanwhile on other threads wait for it, so that no code runs before. + * + * CPython can load a library with `RTLD_GLOBAL`, + * loading an extension module with the flags of `sys.setdlopenflags`: + * loading the already loaded `libpython` as one makes it global, + * before it fails as an extension module without its initialization function. + * It is best-effort: a failure is only logged, as the code of the user might not need any C extension. + */ + @Synchronized + private fun makePythonLibraryGlobal(interpreter: Interpreter) { + if (!libraryLocalSymbols) return + val library = pythonLibrary ?: return + // also on a failure, which would only repeat + libraryLocalSymbols = false + try { + interpreter.exec( + """ + |def _xemantic_make_python_library_global(): + | import _imp, os, sys + | from importlib.machinery import ModuleSpec + | flags = sys.getdlopenflags() + | sys.setdlopenflags(flags | os.RTLD_GLOBAL) + | try: + | _imp.create_dynamic(ModuleSpec('_xemantic_libpython', None, origin=${pythonPath(library.toString())})) + | except ImportError: + | pass + | finally: + | sys.setdlopenflags(flags) + |try: + | _xemantic_make_python_library_global() + |finally: + | del _xemantic_make_python_library_global + |""".trimMargin() + ) + } catch (e: Exception) { + System.getLogger(EmbeddedPythonStarter::class.java.name).log( + System.Logger.Level.WARNING, + "Cannot make the symbols of $library global, " + + "a C extension module might fail to import with an undefined symbol, " + + "put ${library.parent} on LD_LIBRARY_PATH before the JVM starts", + e + ) + } + } + private fun startupFailure(cause: Throwable): IllegalStateException { // Jep wraps the failure of initializing CPython in an Error, whose message is its toString() val reason = cause.cause?.takeIf { cause.message == it.toString() } ?: cause - val message = "Cannot initialize the embedded Python $executable: ${reason.message ?: reason}" + val headline = "Cannot initialize the embedded Python $executable: ${reason.message ?: reason}" + // e.g. libjep.so: libpython3.13.so.1.0: cannot open shared object file, or Library not loaded on macOS, + // also as the cause of the Error of Jep retrying to initialize CPython + val library = pythonLibrary + val libraryAdvice = if (libraryPreloadFailure != null) { + // LD_LIBRARY_PATH would not help, as the loader would fail to load it the same way, e.g. of another architecture + "\nCannot load the libpython of this Python: ${libraryPreloadFailure.message ?: libraryPreloadFailure}" + } else if ( + library != null && generateSequence(cause) { it.cause }.any { + val message = it.message ?: "" + it is UnsatisfiedLinkError && library.fileName.toString() in message && + ("cannot open shared object file" in message || "Library not loaded" in message) + } + ) { + "\nThe dynamic loader cannot load ${library.fileName} of this Python, " + + "put its directory ${library.parent} on LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on macOS) before the JVM starts." + } else "" + val message = headline + libraryAdvice return IllegalStateException( if (probeError.isEmpty()) message else "$message\nThe probe of this Python wrote to the standard error, which might tell why:\n$probeError", cause - ) + ).apply { + libraryPreloadFailure?.let { addSuppressed(it) } + } } } @@ -553,6 +653,8 @@ internal class EmbeddedPythonStarter(private val executable: String, private val * @property prefix the `sys.prefix`, the virtual environment, if any. * @property jepLocation the directory of the `jep` package. * @property externallyManaged whether pip refuses to install into it (PEP 668), see [PythonEnvironment.probeCommand]. + * @property libraryDirectory the directory of the `libpython` of the base Python, `LIBDIR`, empty if unknown. + * @property librarySoname the soname of the `libpython`, `INSTSONAME`, empty if unknown, see [pythonLibrary]. * @property error the standard error of the probe, e.g. a warning of Python, empty if none. */ internal data class PythonProbe( @@ -561,6 +663,8 @@ internal data class PythonProbe( val prefix: String, val jepLocation: String, val externallyManaged: Boolean = false, + val libraryDirectory: String = "", + val librarySoname: String = "", val error: String = "" ) diff --git a/src/main/kotlin/PythonLibrary.kt b/src/main/kotlin/PythonLibrary.kt new file mode 100644 index 0000000..6bf7b79 --- /dev/null +++ b/src/main/kotlin/PythonLibrary.kt @@ -0,0 +1,200 @@ +/* + * xemantic-kotlin-script-python - Stateful Python, embedded with Jep, in Kotlin scripts + * Copyright (C) 2026 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.xemantic.kotlin.script.python + +import java.io.IOException +import java.lang.invoke.MethodHandle +import java.nio.file.Path +import java.util.Optional +import kotlin.io.path.Path +import kotlin.io.path.isRegularFile +import kotlin.io.path.isSameFileAs +import kotlin.io.path.readBytes + +/** + * The system property which, set to `true`, keeps [loadPythonLibrary] from using the foreign linker of Java 22, + * a test seam of loading `libpython` like on Java 17 to 21. + */ +internal const val NO_FOREIGN_LINKER_PROPERTY = "xemantic.kotlin.script.python.noForeignLinker" + +/** Whether [loadPythonLibrary] may use the foreign linker, unless the [NO_FOREIGN_LINKER_PROPERTY] is `true`. */ +internal fun foreignLinkerEnabled(): Boolean = !java.lang.Boolean.getBoolean(NO_FOREIGN_LINKER_PROPERTY) + +/** + * The shared `libpython` of the [soname] in the library [directory] of a Python, see [PythonEnvironment.probeCommand], + * or `null` if there is none, e.g. of a Python linked to `libpython` statically, with `libpython3.X.a` as its soname. + */ +internal fun pythonLibrary(directory: String, soname: String): Path? { + if (directory.isEmpty() || soname.isEmpty() || soname.endsWith(".a")) return null + return Path(directory, soname).takeIf { it.isRegularFile() } +} + +/** + * The result of [loadPythonLibrary]. + * + * @property failure the failure to load the library, e.g. of another architecture, or denied by the JVM, + * kept to tell why Jep cannot load its native library, if it cannot. + * @property localSymbols whether the library is loaded with local symbols, to be made global once Python runs, + * see [EmbeddedPythonStarter]. + */ +internal class PythonLibraryLoading( + val failure: Throwable? = null, + val localSymbols: Boolean = false +) + +/** + * Loads the shared `libpython` [library] of the Python on Linux, before Jep loads its native library, which depends on it, + * when the dynamic loader would not find it, see [loaderFinds]. + * + * The dynamic loader finds the `libpython3.X.so.1.0` of the native library of Jep by its soname + * in the directories of `LD_LIBRARY_PATH`, read when the JVM starts, of `ld.so.cache`, and the system ones, + * so not the one of a Python installed elsewhere, e.g. by uv, pyenv, or `actions/setup-python`; + * once loaded by its path, the loader takes the library of the same soname instead. + * + * Its symbols must be global as well, as a C extension module on Linux is not linked to `libpython`, + * e.g. the ones of numpy, or of the standard library of most Pythons, `_ctypes` among them, + * and would fail to import with `undefined symbol: PyExc_ValueError`. + * Jep makes `libpython` global itself, but finds it by its file name, which the loader cannot find, like its soname. + * The JVM loads a native library with local symbols (`RTLD_LOCAL`), + * so on Java 22 or newer, the library is loaded with `dlopen` and `RTLD_GLOBAL`, called with the foreign linker, + * before CPython starts, also for a C extension imported at its startup, e.g. by a `.pth` file of the `site-packages`. + * On Java 17 to 21, it is loaded with [System.load], and made global in the first interpreter, + * so only after the startup of CPython. + * The [load] of the library is a test seam of [System.load]. + */ +internal fun loadPythonLibrary( + library: Path, + load: (String) -> Unit = System::load +): PythonLibraryLoading { + if (foreignLinkerEnabled() && loadGlobally(library)) return PythonLibraryLoading() + return try { + load(library.toString()) + PythonLibraryLoading(localSymbols = true) + } catch (e: UnsatisfiedLinkError) { + PythonLibraryLoading(failure = e) + } catch (e: RuntimeException) { + // e.g. an IllegalCallerException with --illegal-native-access=deny, + // which then fails loading the native library of Jep as well, reported as a failure of the startup + PythonLibraryLoading(failure = e) + } +} + +/** + * Whether the dynamic loader finds the [library] by its soname, its file name, as the same file, + * so that it needs not be loaded by its path, see [loadPythonLibrary], e.g. of a Python of the system. + * + * Like the loader, it takes the first file of the soname in the directories of the [libraryPath], `LD_LIBRARY_PATH`, + * then in the [cache] of the loader, `/etc/ld.so.cache`, which lists the full paths of the libraries, + * of the directories configured in `/etc/ld.so.conf`, read only if needed, and then in the system directories. + * It does not know the `RUNPATH` of the native library of Jep, e.g. of conda, + * with which the library would be loaded when not needed. + */ +internal fun loaderFinds( + library: Path, + libraryPath: String? = System.getenv("LD_LIBRARY_PATH"), + cache: () -> ByteArray? = ::readLoaderCache +): Boolean { + val soname = library.fileName.toString() + val candidates = libraryPath.orEmpty().split(':').asSequence().filter { it.isNotEmpty() }.map { Path(it, soname) } + + sequence { yieldAll(cachedLibraries(cache(), soname)) } + + SYSTEM_LIBRARY_DIRECTORIES.asSequence().map { Path(it, soname) } + val found = candidates.firstOrNull { it.isRegularFile() } ?: return false + return try { + found.isSameFileAs(library) + } catch (_: IOException) { + false + } +} + +/** The directories which the dynamic loader of glibc searches last, also without `ld.so.cache`. */ +private val SYSTEM_LIBRARY_DIRECTORIES = listOf("/lib64", "/usr/lib64", "/lib", "/usr/lib") + +/** The `/etc/ld.so.cache` of the dynamic loader, or `null` if it cannot be read, e.g. of musl, which has none. */ +private fun readLoaderCache(): ByteArray? = try { + Path("/etc/ld.so.cache").takeIf { it.isRegularFile() }?.readBytes() +} catch (_: IOException) { + null +} + +/** The paths of the libraries of the [soname] in the [cache] of the dynamic loader, its null-terminated strings. */ +private fun cachedLibraries(cache: ByteArray?, soname: String): List { + if (cache == null) return emptyList() + // ISO-8859-1 maps each byte to a char, so that the indexes are the ones of the bytes + val text = String(cache, Charsets.ISO_8859_1) + val suffix = "/" + String(soname.toByteArray(filesystemCharset), Charsets.ISO_8859_1) + "\u0000" + val paths = mutableListOf() + var end = text.indexOf(suffix) + while (end != -1) { + val start = text.lastIndexOf('\u0000', end) + 1 + val bytes = text.substring(start, end + suffix.length - 1).toByteArray(Charsets.ISO_8859_1) + String(bytes, filesystemCharset).takeIf { it.startsWith("/") }?.let { paths.add(Path(it)) } + end = text.indexOf(suffix, end + 1) + } + return paths +} + +/** `RTLD_LAZY | RTLD_GLOBAL` of glibc and musl. */ +private const val RTLD_LAZY_GLOBAL = 0x101 + +/** + * Loads the [library] with `dlopen` and `RTLD_GLOBAL`, called with the foreign linker of Java 22 or newer, + * through reflection, as the library is compiled for Java 17. + * + * @return `true` if loaded, `false` if the foreign linker is not available, or `dlopen` fails. + */ +private fun loadGlobally(library: Path): Boolean = try { + val arenaClass = Class.forName("java.lang.foreign.Arena") + // of Java 22, as the foreign linker of Java 21 is a preview, + // looked up before calling a restricted method, which would print a warning without --enable-native-access + val allocateFrom = arenaClass.getMethod("allocateFrom", String::class.java) + val linkerClass = Class.forName("java.lang.foreign.Linker") + val segmentClass = Class.forName("java.lang.foreign.MemorySegment") + val layoutClass = Class.forName("java.lang.foreign.MemoryLayout") + val valueLayoutClass = Class.forName("java.lang.foreign.ValueLayout") + val descriptorClass = Class.forName("java.lang.foreign.FunctionDescriptor") + val optionClass = Class.forName("java.lang.foreign.Linker\$Option") + val linker = linkerClass.getMethod("nativeLinker").invoke(null) + val lookup = linkerClass.getMethod("defaultLookup").invoke(linker) + val dlopen = (Class.forName("java.lang.foreign.SymbolLookup").getMethod("find", String::class.java) + .invoke(lookup, "dlopen") as Optional<*>).orElse(null) + if (dlopen == null) { + false + } else { + val address = valueLayoutClass.getField("ADDRESS").get(null) + val int = valueLayoutClass.getField("JAVA_INT").get(null) + val arguments = java.lang.reflect.Array.newInstance(layoutClass, 2).also { + java.lang.reflect.Array.set(it, 0, address) + java.lang.reflect.Array.set(it, 1, int) + } + val descriptor = descriptorClass.getMethod("of", layoutClass, arguments.javaClass) + .invoke(null, address, arguments) + val noOptions = java.lang.reflect.Array.newInstance(optionClass, 0) + val handle = linkerClass.getMethod("downcallHandle", segmentClass, descriptorClass, noOptions.javaClass) + .invoke(linker, dlopen, descriptor, noOptions) as MethodHandle + (arenaClass.getMethod("ofConfined").invoke(null) as AutoCloseable).use { arena -> + val path = allocateFrom.invoke(arena, library.toString()) + val result = handle.invokeWithArguments(path, RTLD_LAZY_GLOBAL) + segmentClass.getMethod("address").invoke(result) != 0L + } + } +} catch (_: ReflectiveOperationException) { + false +} catch (_: RuntimeException) { + false +} diff --git a/src/test/kotlin/ForkedJvm.kt b/src/test/kotlin/ForkedJvm.kt index cddecbc..47f623a 100644 --- a/src/test/kotlin/ForkedJvm.kt +++ b/src/test/kotlin/ForkedJvm.kt @@ -26,7 +26,7 @@ import kotlin.io.path.readText /** * Runs the [mainClass] of the test classpath in a new JVM, with the [jvmArguments], - * and with the [environment] variables added to the inherited ones, + * and with the [environment] variables added to the inherited ones, or removed from them, when `null`, * unlike [runProcess], which would put Python in UTF-8 mode, * writing the [input] to its standard input, as UTF-8. * @@ -38,7 +38,7 @@ import kotlin.io.path.readText internal fun runInJvm( mainClass: String, input: String, - environment: Map, + environment: Map, jvmArguments: List = emptyList() ): Pair { val java = Path(System.getProperty("java.home"), "bin", "java").toString() @@ -49,7 +49,9 @@ internal fun runInJvm( listOf(java, "--enable-native-access=ALL-UNNAMED") + jvmArguments + listOf("-cp", System.getProperty("java.class.path"), mainClass) ).redirectErrorStream(true).redirectOutput(outputFile.toFile()) - builder.environment().putAll(environment) + builder.environment().apply { + environment.forEach { (name, value) -> if (value == null) remove(name) else put(name, value) } + } val process = builder.start() try { process.outputStream.use { it.write(input.toByteArray(Charsets.UTF_8)) } diff --git a/src/test/kotlin/PythonEnvironmentTest.kt b/src/test/kotlin/PythonEnvironmentTest.kt index ee2592a..82e767e 100644 --- a/src/test/kotlin/PythonEnvironmentTest.kt +++ b/src/test/kotlin/PythonEnvironmentTest.kt @@ -28,11 +28,13 @@ import kotlin.concurrent.thread import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.Path import kotlin.io.path.createDirectory +import kotlin.io.path.createFile import kotlin.io.path.createTempDirectory import kotlin.io.path.createTempFile import kotlin.io.path.deleteIfExists import kotlin.io.path.deleteRecursively import kotlin.io.path.exists +import kotlin.io.path.isRegularFile import kotlin.io.path.readText import kotlin.io.path.setPosixFilePermissions import kotlin.io.path.writeText @@ -252,7 +254,7 @@ class PythonEnvironmentTest { @Test fun `should parse probe output with output before and after values`() { // given - val output = "noise\n${probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "")}\nmore noise" + val output = "noise\n${probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "")}\nmore noise" // when val probe = PythonEnvironment.parseProbeOutput(output) @@ -266,7 +268,7 @@ class PythonEnvironmentTest { // given val output = "${PythonEnvironment.PROBE_BEGIN}\nfake\n${PythonEnvironment.PROBE_END}\n" + "${PythonEnvironment.PROBE_BEGIN}\n" + - probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "") + probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "") // when val probe = PythonEnvironment.parseProbeOutput(output) @@ -278,7 +280,7 @@ class PythonEnvironmentTest { @Test fun `should parse probe output with empty first value`() { // given - val output = probePayload("", "cpython-313", "/venv", "/venv/jep", "") + val output = probePayload("", "cpython-313", "/venv", "/venv/jep", "", "", "") // when val probe = PythonEnvironment.parseProbeOutput(output) @@ -290,7 +292,7 @@ class PythonEnvironmentTest { @Test fun `should parse probe output with values containing spaces and non-ASCII`() { // given - val output = probePayload("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep", "") + val output = probePayload("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep", "", "", "") // when val probe = PythonEnvironment.parseProbeOutput(output) @@ -299,6 +301,44 @@ class PythonEnvironmentTest { assert(probe == PythonProbe("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep")) } + @Test + fun `should parse probe output with library directory and soname of libpython`() { + // given + val output = probePayload( + "/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "1", "/opt/python/lib", "libpython3.13.so.1.0" + ) + + // when + val probe = PythonEnvironment.parseProbeOutput(output) + + // then + assert( + probe == PythonProbe( + "/venv/bin/python", "cpython-313", "/venv", "/venv/jep", + externallyManaged = true, + libraryDirectory = "/opt/python/lib", + librarySoname = "libpython3.13.so.1.0" + ) + ) + } + + @Test + fun `should probe shared libpython of Python built with it`() { + // given + val shared = runProcess( + pythonCommand(venvPython, "import sysconfig\nprint(sysconfig.get_config_var('Py_ENABLE_SHARED'))"), + mergeErrorStream = false + ).output + assumeTrue(shared == "1", "the Python is not built with a shared libpython") + + // when + val probe = PythonEnvironment.probePython(venvPython) + + // then + val library = pythonLibrary(probe.libraryDirectory, probe.librarySoname) + assert(library != null && library.isRegularFile()) + } + @Test fun `should not parse probe output without values`() { // when @@ -311,7 +351,7 @@ class PythonEnvironmentTest { @Test fun `should not parse probe output with begin marker but without end marker`() { // given - val output = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "") + val output = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "") .removeSuffix("\n${PythonEnvironment.PROBE_END}") // when @@ -322,10 +362,10 @@ class PythonEnvironmentTest { } @Test - fun `should not parse probe output without exactly five values`() { + fun `should not parse probe output without exactly seven values`() { // given - val tooFew = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep") - val tooMany = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "extra") + val tooFew = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "") + val tooMany = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "", "extra") // when val probes = listOf(tooFew, tooMany).map { PythonEnvironment.parseProbeOutput(it) } @@ -742,6 +782,316 @@ class PythonEnvironmentTest { } } + @OptIn(ExperimentalPathApi::class) + @Test + fun `should find shared libpython in its library directory on Linux`() { + // given + val directory = createTempDirectory("python-library") + val library = directory.resolve("libpython3.13.so.1.0").createFile() + + try { + // when + val found = pythonLibrary(directory.toString(), "libpython3.13.so.1.0") + + // then + assert(found == library) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should not find static libpython`() { + // given + // a Python built without --enable-shared, e.g. by pyenv by default, has its libpython linked statically + val directory = createTempDirectory("python-library") + directory.resolve("libpython3.13.a").createFile() + + try { + // when + val found = pythonLibrary(directory.toString(), "libpython3.13.a") + + // then + assert(found == null) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should not find libpython missing from its library directory`() { + // given + val directory = createTempDirectory("python-library") + + try { + // when + val missing = pythonLibrary(directory.toString(), "libpython3.13.so.1.0") + val unknownSoname = pythonLibrary(directory.toString(), "") + val unknownDirectory = pythonLibrary("", "libpython3.13.so.1.0") + + // then + assert(missing == null && unknownSoname == null && unknownDirectory == null) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should tell that dynamic loader finds libpython in directory of LD_LIBRARY_PATH`() { + // given + val directory = createTempDirectory("python-library") + val other = directory.resolve("other").createDirectory() + val library = directory.resolve("libxemantic-test.so.1").createFile() + + try { + // when + val found = loaderFinds(library, libraryPath = "$other:$directory", cache = { null }) + + // then + assert(found) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should tell that dynamic loader finds libpython listed in its cache`() { + // given + val directory = createTempDirectory("python-library") + val library = directory.resolve("libxemantic-test.so.1").createFile() + val cache = "ld.so-1.7.0\u0000libxemantic-test.so.1\u0000$library\u0000/usr/lib/libc.so.6\u0000" + .toByteArray(filesystemCharset) + + try { + // when + val found = loaderFinds(library, libraryPath = null, cache = { cache }) + + // then + assert(found) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should not read cache of dynamic loader when libpython is found in directory of LD_LIBRARY_PATH`() { + // given + val directory = createTempDirectory("python-library") + val library = directory.resolve("libxemantic-test.so.1").createFile() + + try { + // when + val found = loaderFinds(library, libraryPath = "$directory", cache = { error("cache read") }) + + // then + assert(found) + } finally { + directory.deleteRecursively() + } + } + + @Test + fun `should keep failure of loading libpython denied by JVM`() { + // given + // like System.load with --illegal-native-access=deny + val denied = IllegalCallerException("Illegal native access from: unnamed module") + + // when + val loading = withSystemProperty(NO_FOREIGN_LINKER_PROPERTY, "true") { + loadPythonLibrary(Path("/opt/python/lib/libpython3.13.so.1.0"), load = { throw denied }) + } + + // then + assert(loading.failure === denied && !loading.localSymbols) + } + + @Test + fun `should use foreign linker unless disabled by system property set to true`() { + // when + val usage = listOf(null, "false", "true", "TRUE").map { value -> + withSystemProperty(NO_FOREIGN_LINKER_PROPERTY, value) { foreignLinkerEnabled() } + } + + // then + assert(usage == listOf(true, true, false, false)) + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should tell that dynamic loader does not find libpython`() { + // given + val directory = createTempDirectory("python-library") + val library = directory.resolve("libxemantic-test.so.1").createFile() + val cache = "ld.so-1.7.0\u0000libc.so.6\u0000/usr/lib/libc.so.6\u0000".toByteArray(filesystemCharset) + + try { + // when + val found = loaderFinds(library, libraryPath = "", cache = { cache }) + + // then + assert(!found) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should tell that dynamic loader finds another libpython of same soname first`() { + // given + // e.g. of another installation of the same Python version + val directory = createTempDirectory("python-library") + val other = directory.resolve("other").createDirectory() + other.resolve("libxemantic-test.so.1").createFile() + val library = directory.resolve("libxemantic-test.so.1").createFile() + + try { + // when + val found = loaderFinds(library, libraryPath = "$other:$directory", cache = { null }) + + // then + assert(!found) + } finally { + directory.deleteRecursively() + } + } + + @Test + fun `should advise library directory of libpython when native library of Jep cannot load it`() { + // given + val starter = EmbeddedPythonStarter( + "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0") + ) + val cause = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.13.so.1.0: cannot open shared object file: No such file or directory" + ) + + // when + val exception = assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + } + + // then + val message = exception.message!! + assert( + exception.cause === cause && + message.startsWith("Cannot initialize the embedded Python /venv/bin/python: ${cause.message}\n") && + "/opt/python/lib" in message && + "LD_LIBRARY_PATH" in message + ) + } + + @Test + fun `should tell failure to load libpython instead of advising its directory`() { + // given + // LD_LIBRARY_PATH would not help, as the loader would fail to load the library the same way + val preloadFailure = UnsatisfiedLinkError("/opt/python/lib/libpython3.13.so.1.0: wrong ELF class: ELFCLASS32") + val starter = EmbeddedPythonStarter( + "/venv/bin/python", "", + pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0"), + libraryPreloadFailure = preloadFailure + ) + val cause = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.13.so.1.0: cannot open shared object file: No such file or directory" + ) + + // when + val exception = assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + } + + // then + val message = exception.message!! + assert( + exception.cause === cause && + exception.suppressed.toList() == listOf(preloadFailure) && + message == "Cannot initialize the embedded Python /venv/bin/python: ${cause.message}\n" + + "Cannot load the libpython of this Python: ${preloadFailure.message}" + ) + } + + @Test + fun `should advise library directory of libpython when initialization of embedded Python is retried`() { + // given + val starter = EmbeddedPythonStarter( + "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0") + ) + val first = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.13.so.1.0: cannot open shared object file: No such file or directory" + ) + // like jep's MainInterpreter, rethrowing the failure of the first attempt + val cause = Error("The main Python interpreter previously failed to initialize.", first) + + // when + val exception = assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + } + + // then + val message = exception.message!! + assert(exception.cause === cause && "/opt/python/lib" in message && "LD_LIBRARY_PATH" in message) + } + + @Test + fun `should not advise library directory of libpython on other failure of embedded Python startup`() { + // given + val starter = EmbeddedPythonStarter( + "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0") + ) + val wrongClass = UnsatisfiedLinkError("/venv/lib/python3.13/site-packages/jep/libjep.so: wrong ELF class: ELFCLASS32") + // of libpython found, but of another Python + val undefinedSymbol = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: undefined symbol: PyFoo, version libpython3.13.so.1.0" + ) + // of jep built for another Python + val anotherVersion = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.12.so.1.0: cannot open shared object file: No such file or directory" + ) + + // when + val messages = listOf(wrongClass, undefinedSymbol, anotherVersion).map { cause -> + assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + }.message + } + + // then + assert( + messages == listOf( + "Cannot initialize the embedded Python /venv/bin/python: ${wrongClass.message}", + "Cannot initialize the embedded Python /venv/bin/python: ${undefinedSymbol.message}", + "Cannot initialize the embedded Python /venv/bin/python: ${anotherVersion.message}" + ) + ) + } + + @Test + fun `should not advise library directory of Python without shared libpython`() { + // given + val starter = EmbeddedPythonStarter("/venv/bin/python", "") + val cause = UnsatisfiedLinkError( + "/venv/lib/python3.13/site-packages/jep/libjep.so: " + + "libpython3.13.so.1.0: cannot open shared object file: No such file or directory" + ) + + // when + val exception = assertFailsWith { + starter.createInterpreter(create = { throw cause }, check = {}) + } + + // then + assert(exception.message == "Cannot initialize the embedded Python /venv/bin/python: ${cause.message}") + } + @Test fun `should report stderr of probe on Error of embedded Python startup`() { // given @@ -842,6 +1192,16 @@ class PythonEnvironmentTest { } /** The output of [PythonEnvironment.probeCommand] writing the [values], trimmed as by [runProcess]. */ + private fun withSystemProperty(name: String, value: String?, block: () -> T): T { + val previous = System.getProperty(name) + if (value == null) System.clearProperty(name) else System.setProperty(name, value) + try { + return block() + } finally { + if (previous == null) System.clearProperty(name) else System.setProperty(name, previous) + } + } + private fun probePayload(vararg values: String): String { val hexValues = values.joinToString(" ") { it.toByteArray(filesystemCharset).toHexString() } return "${PythonEnvironment.PROBE_BEGIN}\n$hexValues\n${PythonEnvironment.PROBE_END}" diff --git a/src/test/kotlin/PythonLibraryTest.kt b/src/test/kotlin/PythonLibraryTest.kt new file mode 100644 index 0000000..c5c4e86 --- /dev/null +++ b/src/test/kotlin/PythonLibraryTest.kt @@ -0,0 +1,170 @@ +/* + * xemantic-kotlin-script-python - Stateful Python, embedded with Jep, in Kotlin scripts + * Copyright (C) 2026 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.xemantic.kotlin.script.python + +import com.xemantic.kotlin.test.assert +import org.junit.jupiter.api.Assumptions.assumeTrue +import java.io.IOException +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.Path +import kotlin.io.path.createTempDirectory +import kotlin.io.path.deleteRecursively +import kotlin.io.path.exists +import kotlin.io.path.writeText +import kotlin.test.Test + +/** + * The embedded CPython loading the shared `libpython` of its Python without `LD_LIBRARY_PATH`, + * and importing a C extension module which, like the ones of numpy, is not linked to `libpython`, + * but needs its symbols to be global, + * in a JVM of its own, as the environment of the dynamic loader is read when a process starts. + * + * It tests the loading only with a Python outside the system library directories, + * e.g. installed by uv or by `actions/setup-python`, which sets `LD_LIBRARY_PATH` itself, + * as the test venv of such a base Python, see `-PbasePython` in DEVELOPMENT.md, + * while with a Python of the system it passes anyway. + */ +class PythonLibraryTest { + + private val venvPython = Path(System.getenv("VIRTUAL_ENV"), "bin", "python").toString() + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should import C extension not linked to libpython without LD_LIBRARY_PATH`() { + // given + val directory = createTempDirectory("python-library") + try { + compileExtension(directory) + + // when + val (exitCode, output) = runInJvm( + EmbeddedPythonMain::class.java.name, + input = "import xemantic_extension\nprint(xemantic_extension.VALUE, flush=True)", + environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString()) + ) + + // then + assert(output == "imported") + assert(exitCode == 0) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should import C extension not linked to libpython at startup of Python without LD_LIBRARY_PATH`() { + // given + // on Java 17 to 21, the symbols of libpython become global only after the startup, see loadPythonLibrary + assumeTrue(Runtime.version().feature() >= 22, "the foreign linker is of Java 22") + val directory = createTempDirectory("python-library") + try { + compileExtension(directory) + // imported by the site module at the startup, like a .pth file of the site-packages does, e.g. of coverage + directory.resolve("sitecustomize.py").writeText( + """ + |import sys + |try: + | import xemantic_extension + | sys.xemantic_startup_import = xemantic_extension.VALUE + |except ImportError as e: + | sys.xemantic_startup_import = repr(e) + |""".trimMargin() + ) + + // when + val (exitCode, output) = runInJvm( + EmbeddedPythonMain::class.java.name, + input = "import sys\nprint(sys.xemantic_startup_import, flush=True)", + environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString()) + ) + + // then + assert(output == "imported") + assert(exitCode == 0) + } finally { + directory.deleteRecursively() + } + } + + @OptIn(ExperimentalPathApi::class) + @Test + fun `should import C extension not linked to libpython without LD_LIBRARY_PATH and foreign linker`() { + // given + val directory = createTempDirectory("python-library") + try { + compileExtension(directory) + + // when + val (exitCode, output) = runInJvm( + EmbeddedPythonMain::class.java.name, + input = "import xemantic_extension\nprint(xemantic_extension.VALUE, flush=True)", + environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString()), + // like on Java 17 to 21, without the foreign linker + jvmArguments = listOf("-D${NO_FOREIGN_LINKER_PROPERTY}=true") + ) + + // then + assert(output == "imported") + assert(exitCode == 0) + } finally { + directory.deleteRecursively() + } + } + + /** + * Compiles the `xemantic_extension` C extension module of the test venv Python into the [directory], + * not linked to `libpython`, like the C extensions on Linux are, + * skipping the test if not on Linux, or without a C compiler, or the Python headers. + */ + private fun compileExtension(directory: Path) { + assumeTrue(System.getProperty("os.name").startsWith("Linux"), "the libpython is loaded so only on Linux") + val (include, suffix) = runProcess( + pythonCommand( + venvPython, + "import sysconfig\nprint(sysconfig.get_paths()['include'])\nprint(sysconfig.get_config_var('EXT_SUFFIX'))" + ), + mergeErrorStream = false + ).output.lines() + assumeTrue(Path(include, "Python.h").exists(), "no Python headers in $include") + val source = directory.resolve("xemantic_extension.c") + source.writeText( + """ + |#include + | + |static struct PyModuleDef definition = {PyModuleDef_HEAD_INIT, "xemantic_extension", NULL, -1, NULL}; + | + |PyMODINIT_FUNC PyInit_xemantic_extension(void) { + | PyObject *module = PyModule_Create(&definition); + | if (module != NULL) PyModule_AddStringConstant(module, "VALUE", "imported"); + | return module; + |} + |""".trimMargin() + ) + val extension = directory.resolve("xemantic_extension$suffix") + val compiled = try { + runProcess(listOf("cc", "-shared", "-fPIC", "-I$include", source.toString(), "-o", extension.toString())) + } catch (_: IOException) { + null + } + assumeTrue(compiled?.exitCode == 0, "cannot compile a C extension with cc: ${compiled?.output}") + } + +}