diff --git a/CLAUDE.md b/CLAUDE.md
index 5b3d0e6..2ba2ce2 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -70,8 +70,8 @@ see [DEVELOPMENT.md](DEVELOPMENT.md#markdown-soft-wrapping-in-the-ide) for how t
- The embedded CPython does not detect a virtual environment by itself —
without the venv's `python` as its program name (`PyConfig.setProgramName`), it ignores `pyvenv.cfg` and also imports from the global and user `site-packages` of the base Python.
When a module is unexpectedly importable, check `sys.path` of the embedded interpreter, not of `.venv/bin/python`.
-- A running Gradle daemon does not pass a changed `LD_LIBRARY_PATH` on to the test JVM,
- so use `--no-daemon` when trying a `-PbasePython` which needs it (see [DEVELOPMENT.md](DEVELOPMENT.md#the-python-of-the-tests)).
+- A running Gradle daemon does not pass a changed `LD_LIBRARY_PATH` on to the test JVM, so a stale one can hide a failure to load `libpython` —
+ run a `-PbasePython` of uv with `--no-daemon`, without `LD_LIBRARY_PATH`.
- Jep passes the `PyConfig` strings to CPython as JNI *modified* UTF-8 (`GetStringUTFChars`), not plain UTF-8 —
a character outside the BMP becomes a CESU-8 surrogate pair, which CPython does not decode as that character.
For a path under a non-UTF-8 locale, see `pythonPath`.
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index b861595..be7368e 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -18,12 +18,15 @@ or with the `ORG_GRADLE_PROJECT_basePython` environment variable, e.g. on CI:
```
It has to be Python 3.10 or newer, otherwise the build fails before creating the environment.
-On Linux, the tests fail to load `libjep.so` when it cannot find `libpython3.*.so` ([#54](https://github.com/xemantic/xemantic-kotlin-script-python/issues/54)).
-A Python outside the system library directories, e.g. installed by `uv`,
-needs its `lib` directory in `LD_LIBRARY_PATH`,
-which `actions/setup-python` sets on CI.
+On Linux, a Python outside the system library directories, e.g. installed by `uv`,
+does not need its `lib` directory in `LD_LIBRARY_PATH`,
+as `PythonEnvironment` loads its `libpython` before `libjep.so` ([#54](https://github.com/xemantic/xemantic-kotlin-script-python/issues/54)).
+`actions/setup-python` sets `LD_LIBRARY_PATH` on CI anyway,
+so `PythonLibraryTest` removes it in the JVM it starts, to test the loading with such a base Python,
+importing a C extension it compiles with `cc`, which tests nothing with a Python of the system library directories.
`pyenv` builds Python without a shared `libpython` by default,
-build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"` (and then set `LD_LIBRARY_PATH` as well).
+build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`.
+On Java 17 to 21, also set `LD_LIBRARY_PATH` then, for a C extension imported at the startup of Python, see the README.
An existing `.venv` is judged by its own Python:
it is kept when this Python runs, is 3.10 or newer, and has `pip`,
diff --git a/README.md b/README.md
index 0c56628..0773804 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,23 @@ The Python executable is, in this order:
The Python code imports the modules from the same paths as this executable,
e.g. the `site-packages` of a virtual environment, but not the ones of its base Python.
+The native library of `jep` links to the shared `libpython` of the Python, e.g. `libpython3.13.so.1.0`,
+which on Linux the dynamic loader finds by its soname in the directories of `LD_LIBRARY_PATH`, of `ld.so.cache`, and the system ones,
+unless the native library of `jep` has a `RUNPATH` of it, e.g. in conda.
+For a Python installed elsewhere, e.g. by `uv python install`, pyenv, or in a custom prefix,
+the library loads `libpython` from the `LIBDIR` of the Python's `sysconfig` itself, with its symbols global,
+as C extension modules, e.g. of numpy, need them,
+so that `LD_LIBRARY_PATH` is not needed.
+On Java 22 or newer, it does so with the foreign linker, a restricted method,
+which, like loading the native library of `jep` on Java 24 or newer, prints a warning without `--enable-native-access=ALL-UNNAMED`.
+On Java 17 to 21, the symbols become global only after the startup of Python,
+so a C extension imported at the startup, by a `.pth` file of the `site-packages`, e.g. of coverage, or by `sitecustomize`,
+fails with an `undefined symbol` and still needs `LD_LIBRARY_PATH`.
+If `libpython` is not found, e.g. on macOS, where it is not loaded so,
+put its directory, the `LIBDIR` of `sysconfig`, on `LD_LIBRARY_PATH` (`DYLD_LIBRARY_PATH` on macOS),
+set before the JVM starts, as the loader reads it only then.
+pyenv builds Python without a shared `libpython` by default, build it with `PYTHON_CONFIGURE_OPTS="--enable-shared"`.
+
The embedded Python, like every Python process started by this library, e.g. `pip`,
runs in the [UTF-8 mode](https://peps.python.org/pep-0540/),
whatever the locale, e.g. cp1252 on Windows, or `C` on Linux:
diff --git a/src/main/kotlin/PythonEnvironment.kt b/src/main/kotlin/PythonEnvironment.kt
index af69f86..75bc861 100644
--- a/src/main/kotlin/PythonEnvironment.kt
+++ b/src/main/kotlin/PythonEnvironment.kt
@@ -102,8 +102,17 @@ public object PythonEnvironment {
val library = jepLibrary(Path(jepLocation), python, probe.externallyManaged)
MainInterpreter.setJepLibraryPath(library.toString())
MainInterpreter.setInitParams(embeddedPythonConfig(executable))
+ val pythonLibrary = pythonLibrary(probe.libraryDirectory, probe.librarySoname)
+ // only when needed, as the foreign linker prints a warning without --enable-native-access
+ val loading = if (
+ pythonLibrary != null && System.getProperty("os.name").startsWith("Linux") && !loaderFinds(pythonLibrary)
+ ) {
+ loadPythonLibrary(pythonLibrary)
+ } else PythonLibraryLoading()
this.executable = executable
- starter = EmbeddedPythonStarter(executable, probe.error)
+ starter = EmbeddedPythonStarter(
+ executable, probe.error, pythonLibrary, loading.failure, loading.localSymbols
+ )
if (dependencyDirectory == null) {
dependencyDirectory = System.getProperty(DEPENDENCY_DIRECTORY_PROPERTY)?.let { Path(it) }
}
@@ -266,10 +275,14 @@ public object PythonEnvironment {
val begin = lines.lastIndexOf("\n$PROBE_BEGIN\n").takeIf { it != -1 }?.let { it + PROBE_BEGIN.length + 2 }
val end = begin?.let { lines.indexOf("\n$PROBE_END", it) }?.takeIf { it != -1 }
val values = if (begin == null || end == null) null else lines.substring(begin, end).split(' ')
- if (values == null || values.size != 5) return null
- val (executable, cacheTag, prefix, jepLocation, externallyManaged) =
- values.map { it.hexToPath() ?: return null }
- return PythonProbe(executable, cacheTag, prefix, jepLocation, externallyManaged.isNotEmpty())
+ if (values == null || values.size != 7) return null
+ val paths = values.map { it.hexToPath() ?: return null }
+ val (executable, cacheTag, prefix, jepLocation, externallyManaged) = paths
+ return PythonProbe(
+ executable, cacheTag, prefix, jepLocation, externallyManaged.isNotEmpty(),
+ libraryDirectory = paths[5],
+ librarySoname = paths[6]
+ )
}
/**
@@ -281,9 +294,11 @@ public object PythonEnvironment {
* as text, a path which is not valid UTF-8, e.g. `/home/josé` in ISO-8859-1,
* would be surrogate-escaped by Python in UTF-8 mode, and could not be encoded to UTF-8,
* while the JVM would decode a path of UTF-8 bytes to another one, in another encoding of its locale.
- * The last value tells whether the Python is externally managed (PEP 668), `1` if it is, empty if not:
+ * The fifth value tells whether the Python is externally managed (PEP 668), `1` if it is, empty if not:
* a base Python, not a virtual environment, with an `EXTERNALLY-MANAGED` file in its standard library,
* into which pip refuses to install.
+ * The values after it are the directory and the soname of the `libpython` of the base Python,
+ * `LIBDIR` and `INSTSONAME` of `sysconfig`, empty if unknown, e.g. on Windows, see [pythonLibrary].
* Without the `jep` package, it exits with the [PROBE_JEP_MISSING_EXIT_CODE],
* or the [PROBE_JEP_MISSING_EXTERNALLY_MANAGED_EXIT_CODE] if the Python is externally managed,
* writing to the standard error the file of a `jep` module, if any, e.g. a `jep.py` shadowing the package.
@@ -312,6 +327,8 @@ public object PythonEnvironment {
| sys.prefix,
| jep_spec.submodule_search_locations[0],
| '1' if externally_managed else '',
+ | sysconfig.get_config_var('LIBDIR'),
+ | sysconfig.get_config_var('INSTSONAME'),
|]
|payload = ' '.join('' if value is None else os.fsencode(str(value)).hex() for value in values)
|sys.stdout.flush()
@@ -490,8 +507,22 @@ private fun String.hexToPath(): String? {
* telling a failure of its startup with what the probe of the [executable] wrote to the standard error, if anything,
* as a warning of a successful probe, e.g. `Could not find platform libraries` of a wrong `PYTHONHOME`,
* might tell why the embedded CPython, configured the same way, fails.
+ * When the native library of Jep cannot load the shared [pythonLibrary] of the Python, see [pythonLibrary],
+ * it tells to put its directory on `LD_LIBRARY_PATH`,
+ * or the [libraryPreloadFailure] of [loadPythonLibrary], if any, also kept as suppressed.
+ * The symbols of the [pythonLibrary] loaded by [loadPythonLibrary], if [libraryLocalSymbols],
+ * are made global in the first interpreter, see [makePythonLibraryGlobal].
*/
-internal class EmbeddedPythonStarter(private val executable: String, private val probeError: String) {
+internal class EmbeddedPythonStarter(
+ private val executable: String,
+ private val probeError: String,
+ private val pythonLibrary: Path? = null,
+ private val libraryPreloadFailure: Throwable? = null,
+ libraryLocalSymbols: Boolean = false
+) {
+
+ /** Whether the symbols of the [pythonLibrary] are still to be made global, see [makePythonLibraryGlobal]. */
+ private var libraryLocalSymbols = libraryLocalSymbols
/** Whether an interpreter has been created, so that the embedded CPython has started. */
@Volatile
@@ -520,6 +551,7 @@ internal class EmbeddedPythonStarter(private val executable: String, private val
}
started = true
try {
+ if (interpreter is Interpreter) makePythonLibraryGlobal(interpreter)
check(interpreter)
} catch (e: Throwable) {
try {
@@ -532,15 +564,83 @@ internal class EmbeddedPythonStarter(private val executable: String, private val
return interpreter
}
+ /**
+ * Makes the symbols of the [pythonLibrary] loaded by [loadPythonLibrary] global, if [libraryLocalSymbols],
+ * in the first [interpreter], before any code of the user, but after the startup of Python,
+ * see [loadPythonLibrary].
+ * The interpreters created meanwhile on other threads wait for it, so that no code runs before.
+ *
+ * CPython can load a library with `RTLD_GLOBAL`,
+ * loading an extension module with the flags of `sys.setdlopenflags`:
+ * loading the already loaded `libpython` as one makes it global,
+ * before it fails as an extension module without its initialization function.
+ * It is best-effort: a failure is only logged, as the code of the user might not need any C extension.
+ */
+ @Synchronized
+ private fun makePythonLibraryGlobal(interpreter: Interpreter) {
+ if (!libraryLocalSymbols) return
+ val library = pythonLibrary ?: return
+ // also on a failure, which would only repeat
+ libraryLocalSymbols = false
+ try {
+ interpreter.exec(
+ """
+ |def _xemantic_make_python_library_global():
+ | import _imp, os, sys
+ | from importlib.machinery import ModuleSpec
+ | flags = sys.getdlopenflags()
+ | sys.setdlopenflags(flags | os.RTLD_GLOBAL)
+ | try:
+ | _imp.create_dynamic(ModuleSpec('_xemantic_libpython', None, origin=${pythonPath(library.toString())}))
+ | except ImportError:
+ | pass
+ | finally:
+ | sys.setdlopenflags(flags)
+ |try:
+ | _xemantic_make_python_library_global()
+ |finally:
+ | del _xemantic_make_python_library_global
+ |""".trimMargin()
+ )
+ } catch (e: Exception) {
+ System.getLogger(EmbeddedPythonStarter::class.java.name).log(
+ System.Logger.Level.WARNING,
+ "Cannot make the symbols of $library global, " +
+ "a C extension module might fail to import with an undefined symbol, " +
+ "put ${library.parent} on LD_LIBRARY_PATH before the JVM starts",
+ e
+ )
+ }
+ }
+
private fun startupFailure(cause: Throwable): IllegalStateException {
// Jep wraps the failure of initializing CPython in an Error, whose message is its toString()
val reason = cause.cause?.takeIf { cause.message == it.toString() } ?: cause
- val message = "Cannot initialize the embedded Python $executable: ${reason.message ?: reason}"
+ val headline = "Cannot initialize the embedded Python $executable: ${reason.message ?: reason}"
+ // e.g. libjep.so: libpython3.13.so.1.0: cannot open shared object file, or Library not loaded on macOS,
+ // also as the cause of the Error of Jep retrying to initialize CPython
+ val library = pythonLibrary
+ val libraryAdvice = if (libraryPreloadFailure != null) {
+ // LD_LIBRARY_PATH would not help, as the loader would fail to load it the same way, e.g. of another architecture
+ "\nCannot load the libpython of this Python: ${libraryPreloadFailure.message ?: libraryPreloadFailure}"
+ } else if (
+ library != null && generateSequence(cause) { it.cause }.any {
+ val message = it.message ?: ""
+ it is UnsatisfiedLinkError && library.fileName.toString() in message &&
+ ("cannot open shared object file" in message || "Library not loaded" in message)
+ }
+ ) {
+ "\nThe dynamic loader cannot load ${library.fileName} of this Python, " +
+ "put its directory ${library.parent} on LD_LIBRARY_PATH (DYLD_LIBRARY_PATH on macOS) before the JVM starts."
+ } else ""
+ val message = headline + libraryAdvice
return IllegalStateException(
if (probeError.isEmpty()) message
else "$message\nThe probe of this Python wrote to the standard error, which might tell why:\n$probeError",
cause
- )
+ ).apply {
+ libraryPreloadFailure?.let { addSuppressed(it) }
+ }
}
}
@@ -553,6 +653,8 @@ internal class EmbeddedPythonStarter(private val executable: String, private val
* @property prefix the `sys.prefix`, the virtual environment, if any.
* @property jepLocation the directory of the `jep` package.
* @property externallyManaged whether pip refuses to install into it (PEP 668), see [PythonEnvironment.probeCommand].
+ * @property libraryDirectory the directory of the `libpython` of the base Python, `LIBDIR`, empty if unknown.
+ * @property librarySoname the soname of the `libpython`, `INSTSONAME`, empty if unknown, see [pythonLibrary].
* @property error the standard error of the probe, e.g. a warning of Python, empty if none.
*/
internal data class PythonProbe(
@@ -561,6 +663,8 @@ internal data class PythonProbe(
val prefix: String,
val jepLocation: String,
val externallyManaged: Boolean = false,
+ val libraryDirectory: String = "",
+ val librarySoname: String = "",
val error: String = ""
)
diff --git a/src/main/kotlin/PythonLibrary.kt b/src/main/kotlin/PythonLibrary.kt
new file mode 100644
index 0000000..6bf7b79
--- /dev/null
+++ b/src/main/kotlin/PythonLibrary.kt
@@ -0,0 +1,200 @@
+/*
+ * xemantic-kotlin-script-python - Stateful Python, embedded with Jep, in Kotlin scripts
+ * Copyright (C) 2026 Kazimierz Pogoda / Xemantic
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+package com.xemantic.kotlin.script.python
+
+import java.io.IOException
+import java.lang.invoke.MethodHandle
+import java.nio.file.Path
+import java.util.Optional
+import kotlin.io.path.Path
+import kotlin.io.path.isRegularFile
+import kotlin.io.path.isSameFileAs
+import kotlin.io.path.readBytes
+
+/**
+ * The system property which, set to `true`, keeps [loadPythonLibrary] from using the foreign linker of Java 22,
+ * a test seam of loading `libpython` like on Java 17 to 21.
+ */
+internal const val NO_FOREIGN_LINKER_PROPERTY = "xemantic.kotlin.script.python.noForeignLinker"
+
+/** Whether [loadPythonLibrary] may use the foreign linker, unless the [NO_FOREIGN_LINKER_PROPERTY] is `true`. */
+internal fun foreignLinkerEnabled(): Boolean = !java.lang.Boolean.getBoolean(NO_FOREIGN_LINKER_PROPERTY)
+
+/**
+ * The shared `libpython` of the [soname] in the library [directory] of a Python, see [PythonEnvironment.probeCommand],
+ * or `null` if there is none, e.g. of a Python linked to `libpython` statically, with `libpython3.X.a` as its soname.
+ */
+internal fun pythonLibrary(directory: String, soname: String): Path? {
+ if (directory.isEmpty() || soname.isEmpty() || soname.endsWith(".a")) return null
+ return Path(directory, soname).takeIf { it.isRegularFile() }
+}
+
+/**
+ * The result of [loadPythonLibrary].
+ *
+ * @property failure the failure to load the library, e.g. of another architecture, or denied by the JVM,
+ * kept to tell why Jep cannot load its native library, if it cannot.
+ * @property localSymbols whether the library is loaded with local symbols, to be made global once Python runs,
+ * see [EmbeddedPythonStarter].
+ */
+internal class PythonLibraryLoading(
+ val failure: Throwable? = null,
+ val localSymbols: Boolean = false
+)
+
+/**
+ * Loads the shared `libpython` [library] of the Python on Linux, before Jep loads its native library, which depends on it,
+ * when the dynamic loader would not find it, see [loaderFinds].
+ *
+ * The dynamic loader finds the `libpython3.X.so.1.0` of the native library of Jep by its soname
+ * in the directories of `LD_LIBRARY_PATH`, read when the JVM starts, of `ld.so.cache`, and the system ones,
+ * so not the one of a Python installed elsewhere, e.g. by uv, pyenv, or `actions/setup-python`;
+ * once loaded by its path, the loader takes the library of the same soname instead.
+ *
+ * Its symbols must be global as well, as a C extension module on Linux is not linked to `libpython`,
+ * e.g. the ones of numpy, or of the standard library of most Pythons, `_ctypes` among them,
+ * and would fail to import with `undefined symbol: PyExc_ValueError`.
+ * Jep makes `libpython` global itself, but finds it by its file name, which the loader cannot find, like its soname.
+ * The JVM loads a native library with local symbols (`RTLD_LOCAL`),
+ * so on Java 22 or newer, the library is loaded with `dlopen` and `RTLD_GLOBAL`, called with the foreign linker,
+ * before CPython starts, also for a C extension imported at its startup, e.g. by a `.pth` file of the `site-packages`.
+ * On Java 17 to 21, it is loaded with [System.load], and made global in the first interpreter,
+ * so only after the startup of CPython.
+ * The [load] of the library is a test seam of [System.load].
+ */
+internal fun loadPythonLibrary(
+ library: Path,
+ load: (String) -> Unit = System::load
+): PythonLibraryLoading {
+ if (foreignLinkerEnabled() && loadGlobally(library)) return PythonLibraryLoading()
+ return try {
+ load(library.toString())
+ PythonLibraryLoading(localSymbols = true)
+ } catch (e: UnsatisfiedLinkError) {
+ PythonLibraryLoading(failure = e)
+ } catch (e: RuntimeException) {
+ // e.g. an IllegalCallerException with --illegal-native-access=deny,
+ // which then fails loading the native library of Jep as well, reported as a failure of the startup
+ PythonLibraryLoading(failure = e)
+ }
+}
+
+/**
+ * Whether the dynamic loader finds the [library] by its soname, its file name, as the same file,
+ * so that it needs not be loaded by its path, see [loadPythonLibrary], e.g. of a Python of the system.
+ *
+ * Like the loader, it takes the first file of the soname in the directories of the [libraryPath], `LD_LIBRARY_PATH`,
+ * then in the [cache] of the loader, `/etc/ld.so.cache`, which lists the full paths of the libraries,
+ * of the directories configured in `/etc/ld.so.conf`, read only if needed, and then in the system directories.
+ * It does not know the `RUNPATH` of the native library of Jep, e.g. of conda,
+ * with which the library would be loaded when not needed.
+ */
+internal fun loaderFinds(
+ library: Path,
+ libraryPath: String? = System.getenv("LD_LIBRARY_PATH"),
+ cache: () -> ByteArray? = ::readLoaderCache
+): Boolean {
+ val soname = library.fileName.toString()
+ val candidates = libraryPath.orEmpty().split(':').asSequence().filter { it.isNotEmpty() }.map { Path(it, soname) } +
+ sequence { yieldAll(cachedLibraries(cache(), soname)) } +
+ SYSTEM_LIBRARY_DIRECTORIES.asSequence().map { Path(it, soname) }
+ val found = candidates.firstOrNull { it.isRegularFile() } ?: return false
+ return try {
+ found.isSameFileAs(library)
+ } catch (_: IOException) {
+ false
+ }
+}
+
+/** The directories which the dynamic loader of glibc searches last, also without `ld.so.cache`. */
+private val SYSTEM_LIBRARY_DIRECTORIES = listOf("/lib64", "/usr/lib64", "/lib", "/usr/lib")
+
+/** The `/etc/ld.so.cache` of the dynamic loader, or `null` if it cannot be read, e.g. of musl, which has none. */
+private fun readLoaderCache(): ByteArray? = try {
+ Path("/etc/ld.so.cache").takeIf { it.isRegularFile() }?.readBytes()
+} catch (_: IOException) {
+ null
+}
+
+/** The paths of the libraries of the [soname] in the [cache] of the dynamic loader, its null-terminated strings. */
+private fun cachedLibraries(cache: ByteArray?, soname: String): List {
+ if (cache == null) return emptyList()
+ // ISO-8859-1 maps each byte to a char, so that the indexes are the ones of the bytes
+ val text = String(cache, Charsets.ISO_8859_1)
+ val suffix = "/" + String(soname.toByteArray(filesystemCharset), Charsets.ISO_8859_1) + "\u0000"
+ val paths = mutableListOf()
+ var end = text.indexOf(suffix)
+ while (end != -1) {
+ val start = text.lastIndexOf('\u0000', end) + 1
+ val bytes = text.substring(start, end + suffix.length - 1).toByteArray(Charsets.ISO_8859_1)
+ String(bytes, filesystemCharset).takeIf { it.startsWith("/") }?.let { paths.add(Path(it)) }
+ end = text.indexOf(suffix, end + 1)
+ }
+ return paths
+}
+
+/** `RTLD_LAZY | RTLD_GLOBAL` of glibc and musl. */
+private const val RTLD_LAZY_GLOBAL = 0x101
+
+/**
+ * Loads the [library] with `dlopen` and `RTLD_GLOBAL`, called with the foreign linker of Java 22 or newer,
+ * through reflection, as the library is compiled for Java 17.
+ *
+ * @return `true` if loaded, `false` if the foreign linker is not available, or `dlopen` fails.
+ */
+private fun loadGlobally(library: Path): Boolean = try {
+ val arenaClass = Class.forName("java.lang.foreign.Arena")
+ // of Java 22, as the foreign linker of Java 21 is a preview,
+ // looked up before calling a restricted method, which would print a warning without --enable-native-access
+ val allocateFrom = arenaClass.getMethod("allocateFrom", String::class.java)
+ val linkerClass = Class.forName("java.lang.foreign.Linker")
+ val segmentClass = Class.forName("java.lang.foreign.MemorySegment")
+ val layoutClass = Class.forName("java.lang.foreign.MemoryLayout")
+ val valueLayoutClass = Class.forName("java.lang.foreign.ValueLayout")
+ val descriptorClass = Class.forName("java.lang.foreign.FunctionDescriptor")
+ val optionClass = Class.forName("java.lang.foreign.Linker\$Option")
+ val linker = linkerClass.getMethod("nativeLinker").invoke(null)
+ val lookup = linkerClass.getMethod("defaultLookup").invoke(linker)
+ val dlopen = (Class.forName("java.lang.foreign.SymbolLookup").getMethod("find", String::class.java)
+ .invoke(lookup, "dlopen") as Optional<*>).orElse(null)
+ if (dlopen == null) {
+ false
+ } else {
+ val address = valueLayoutClass.getField("ADDRESS").get(null)
+ val int = valueLayoutClass.getField("JAVA_INT").get(null)
+ val arguments = java.lang.reflect.Array.newInstance(layoutClass, 2).also {
+ java.lang.reflect.Array.set(it, 0, address)
+ java.lang.reflect.Array.set(it, 1, int)
+ }
+ val descriptor = descriptorClass.getMethod("of", layoutClass, arguments.javaClass)
+ .invoke(null, address, arguments)
+ val noOptions = java.lang.reflect.Array.newInstance(optionClass, 0)
+ val handle = linkerClass.getMethod("downcallHandle", segmentClass, descriptorClass, noOptions.javaClass)
+ .invoke(linker, dlopen, descriptor, noOptions) as MethodHandle
+ (arenaClass.getMethod("ofConfined").invoke(null) as AutoCloseable).use { arena ->
+ val path = allocateFrom.invoke(arena, library.toString())
+ val result = handle.invokeWithArguments(path, RTLD_LAZY_GLOBAL)
+ segmentClass.getMethod("address").invoke(result) != 0L
+ }
+ }
+} catch (_: ReflectiveOperationException) {
+ false
+} catch (_: RuntimeException) {
+ false
+}
diff --git a/src/test/kotlin/ForkedJvm.kt b/src/test/kotlin/ForkedJvm.kt
index cddecbc..47f623a 100644
--- a/src/test/kotlin/ForkedJvm.kt
+++ b/src/test/kotlin/ForkedJvm.kt
@@ -26,7 +26,7 @@ import kotlin.io.path.readText
/**
* Runs the [mainClass] of the test classpath in a new JVM, with the [jvmArguments],
- * and with the [environment] variables added to the inherited ones,
+ * and with the [environment] variables added to the inherited ones, or removed from them, when `null`,
* unlike [runProcess], which would put Python in UTF-8 mode,
* writing the [input] to its standard input, as UTF-8.
*
@@ -38,7 +38,7 @@ import kotlin.io.path.readText
internal fun runInJvm(
mainClass: String,
input: String,
- environment: Map,
+ environment: Map,
jvmArguments: List = emptyList()
): Pair {
val java = Path(System.getProperty("java.home"), "bin", "java").toString()
@@ -49,7 +49,9 @@ internal fun runInJvm(
listOf(java, "--enable-native-access=ALL-UNNAMED") + jvmArguments +
listOf("-cp", System.getProperty("java.class.path"), mainClass)
).redirectErrorStream(true).redirectOutput(outputFile.toFile())
- builder.environment().putAll(environment)
+ builder.environment().apply {
+ environment.forEach { (name, value) -> if (value == null) remove(name) else put(name, value) }
+ }
val process = builder.start()
try {
process.outputStream.use { it.write(input.toByteArray(Charsets.UTF_8)) }
diff --git a/src/test/kotlin/PythonEnvironmentTest.kt b/src/test/kotlin/PythonEnvironmentTest.kt
index ee2592a..82e767e 100644
--- a/src/test/kotlin/PythonEnvironmentTest.kt
+++ b/src/test/kotlin/PythonEnvironmentTest.kt
@@ -28,11 +28,13 @@ import kotlin.concurrent.thread
import kotlin.io.path.ExperimentalPathApi
import kotlin.io.path.Path
import kotlin.io.path.createDirectory
+import kotlin.io.path.createFile
import kotlin.io.path.createTempDirectory
import kotlin.io.path.createTempFile
import kotlin.io.path.deleteIfExists
import kotlin.io.path.deleteRecursively
import kotlin.io.path.exists
+import kotlin.io.path.isRegularFile
import kotlin.io.path.readText
import kotlin.io.path.setPosixFilePermissions
import kotlin.io.path.writeText
@@ -252,7 +254,7 @@ class PythonEnvironmentTest {
@Test
fun `should parse probe output with output before and after values`() {
// given
- val output = "noise\n${probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "")}\nmore noise"
+ val output = "noise\n${probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "")}\nmore noise"
// when
val probe = PythonEnvironment.parseProbeOutput(output)
@@ -266,7 +268,7 @@ class PythonEnvironmentTest {
// given
val output = "${PythonEnvironment.PROBE_BEGIN}\nfake\n${PythonEnvironment.PROBE_END}\n" +
"${PythonEnvironment.PROBE_BEGIN}\n" +
- probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "")
+ probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "")
// when
val probe = PythonEnvironment.parseProbeOutput(output)
@@ -278,7 +280,7 @@ class PythonEnvironmentTest {
@Test
fun `should parse probe output with empty first value`() {
// given
- val output = probePayload("", "cpython-313", "/venv", "/venv/jep", "")
+ val output = probePayload("", "cpython-313", "/venv", "/venv/jep", "", "", "")
// when
val probe = PythonEnvironment.parseProbeOutput(output)
@@ -290,7 +292,7 @@ class PythonEnvironmentTest {
@Test
fun `should parse probe output with values containing spaces and non-ASCII`() {
// given
- val output = probePayload("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep", "")
+ val output = probePayload("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep", "", "", "")
// when
val probe = PythonEnvironment.parseProbeOutput(output)
@@ -299,6 +301,44 @@ class PythonEnvironmentTest {
assert(probe == PythonProbe("/My Zażółć/bin/python", "cpython-313", "/My Zażółć", "/My Zażółć/jep"))
}
+ @Test
+ fun `should parse probe output with library directory and soname of libpython`() {
+ // given
+ val output = probePayload(
+ "/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "1", "/opt/python/lib", "libpython3.13.so.1.0"
+ )
+
+ // when
+ val probe = PythonEnvironment.parseProbeOutput(output)
+
+ // then
+ assert(
+ probe == PythonProbe(
+ "/venv/bin/python", "cpython-313", "/venv", "/venv/jep",
+ externallyManaged = true,
+ libraryDirectory = "/opt/python/lib",
+ librarySoname = "libpython3.13.so.1.0"
+ )
+ )
+ }
+
+ @Test
+ fun `should probe shared libpython of Python built with it`() {
+ // given
+ val shared = runProcess(
+ pythonCommand(venvPython, "import sysconfig\nprint(sysconfig.get_config_var('Py_ENABLE_SHARED'))"),
+ mergeErrorStream = false
+ ).output
+ assumeTrue(shared == "1", "the Python is not built with a shared libpython")
+
+ // when
+ val probe = PythonEnvironment.probePython(venvPython)
+
+ // then
+ val library = pythonLibrary(probe.libraryDirectory, probe.librarySoname)
+ assert(library != null && library.isRegularFile())
+ }
+
@Test
fun `should not parse probe output without values`() {
// when
@@ -311,7 +351,7 @@ class PythonEnvironmentTest {
@Test
fun `should not parse probe output with begin marker but without end marker`() {
// given
- val output = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "")
+ val output = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "")
.removeSuffix("\n${PythonEnvironment.PROBE_END}")
// when
@@ -322,10 +362,10 @@ class PythonEnvironmentTest {
}
@Test
- fun `should not parse probe output without exactly five values`() {
+ fun `should not parse probe output without exactly seven values`() {
// given
- val tooFew = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep")
- val tooMany = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "extra")
+ val tooFew = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "")
+ val tooMany = probePayload("/venv/bin/python", "cpython-313", "/venv", "/venv/jep", "", "", "", "extra")
// when
val probes = listOf(tooFew, tooMany).map { PythonEnvironment.parseProbeOutput(it) }
@@ -742,6 +782,316 @@ class PythonEnvironmentTest {
}
}
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should find shared libpython in its library directory on Linux`() {
+ // given
+ val directory = createTempDirectory("python-library")
+ val library = directory.resolve("libpython3.13.so.1.0").createFile()
+
+ try {
+ // when
+ val found = pythonLibrary(directory.toString(), "libpython3.13.so.1.0")
+
+ // then
+ assert(found == library)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should not find static libpython`() {
+ // given
+ // a Python built without --enable-shared, e.g. by pyenv by default, has its libpython linked statically
+ val directory = createTempDirectory("python-library")
+ directory.resolve("libpython3.13.a").createFile()
+
+ try {
+ // when
+ val found = pythonLibrary(directory.toString(), "libpython3.13.a")
+
+ // then
+ assert(found == null)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should not find libpython missing from its library directory`() {
+ // given
+ val directory = createTempDirectory("python-library")
+
+ try {
+ // when
+ val missing = pythonLibrary(directory.toString(), "libpython3.13.so.1.0")
+ val unknownSoname = pythonLibrary(directory.toString(), "")
+ val unknownDirectory = pythonLibrary("", "libpython3.13.so.1.0")
+
+ // then
+ assert(missing == null && unknownSoname == null && unknownDirectory == null)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should tell that dynamic loader finds libpython in directory of LD_LIBRARY_PATH`() {
+ // given
+ val directory = createTempDirectory("python-library")
+ val other = directory.resolve("other").createDirectory()
+ val library = directory.resolve("libxemantic-test.so.1").createFile()
+
+ try {
+ // when
+ val found = loaderFinds(library, libraryPath = "$other:$directory", cache = { null })
+
+ // then
+ assert(found)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should tell that dynamic loader finds libpython listed in its cache`() {
+ // given
+ val directory = createTempDirectory("python-library")
+ val library = directory.resolve("libxemantic-test.so.1").createFile()
+ val cache = "ld.so-1.7.0\u0000libxemantic-test.so.1\u0000$library\u0000/usr/lib/libc.so.6\u0000"
+ .toByteArray(filesystemCharset)
+
+ try {
+ // when
+ val found = loaderFinds(library, libraryPath = null, cache = { cache })
+
+ // then
+ assert(found)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should not read cache of dynamic loader when libpython is found in directory of LD_LIBRARY_PATH`() {
+ // given
+ val directory = createTempDirectory("python-library")
+ val library = directory.resolve("libxemantic-test.so.1").createFile()
+
+ try {
+ // when
+ val found = loaderFinds(library, libraryPath = "$directory", cache = { error("cache read") })
+
+ // then
+ assert(found)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @Test
+ fun `should keep failure of loading libpython denied by JVM`() {
+ // given
+ // like System.load with --illegal-native-access=deny
+ val denied = IllegalCallerException("Illegal native access from: unnamed module")
+
+ // when
+ val loading = withSystemProperty(NO_FOREIGN_LINKER_PROPERTY, "true") {
+ loadPythonLibrary(Path("/opt/python/lib/libpython3.13.so.1.0"), load = { throw denied })
+ }
+
+ // then
+ assert(loading.failure === denied && !loading.localSymbols)
+ }
+
+ @Test
+ fun `should use foreign linker unless disabled by system property set to true`() {
+ // when
+ val usage = listOf(null, "false", "true", "TRUE").map { value ->
+ withSystemProperty(NO_FOREIGN_LINKER_PROPERTY, value) { foreignLinkerEnabled() }
+ }
+
+ // then
+ assert(usage == listOf(true, true, false, false))
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should tell that dynamic loader does not find libpython`() {
+ // given
+ val directory = createTempDirectory("python-library")
+ val library = directory.resolve("libxemantic-test.so.1").createFile()
+ val cache = "ld.so-1.7.0\u0000libc.so.6\u0000/usr/lib/libc.so.6\u0000".toByteArray(filesystemCharset)
+
+ try {
+ // when
+ val found = loaderFinds(library, libraryPath = "", cache = { cache })
+
+ // then
+ assert(!found)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should tell that dynamic loader finds another libpython of same soname first`() {
+ // given
+ // e.g. of another installation of the same Python version
+ val directory = createTempDirectory("python-library")
+ val other = directory.resolve("other").createDirectory()
+ other.resolve("libxemantic-test.so.1").createFile()
+ val library = directory.resolve("libxemantic-test.so.1").createFile()
+
+ try {
+ // when
+ val found = loaderFinds(library, libraryPath = "$other:$directory", cache = { null })
+
+ // then
+ assert(!found)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @Test
+ fun `should advise library directory of libpython when native library of Jep cannot load it`() {
+ // given
+ val starter = EmbeddedPythonStarter(
+ "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0")
+ )
+ val cause = UnsatisfiedLinkError(
+ "/venv/lib/python3.13/site-packages/jep/libjep.so: " +
+ "libpython3.13.so.1.0: cannot open shared object file: No such file or directory"
+ )
+
+ // when
+ val exception = assertFailsWith {
+ starter.createInterpreter(create = { throw cause }, check = {})
+ }
+
+ // then
+ val message = exception.message!!
+ assert(
+ exception.cause === cause &&
+ message.startsWith("Cannot initialize the embedded Python /venv/bin/python: ${cause.message}\n") &&
+ "/opt/python/lib" in message &&
+ "LD_LIBRARY_PATH" in message
+ )
+ }
+
+ @Test
+ fun `should tell failure to load libpython instead of advising its directory`() {
+ // given
+ // LD_LIBRARY_PATH would not help, as the loader would fail to load the library the same way
+ val preloadFailure = UnsatisfiedLinkError("/opt/python/lib/libpython3.13.so.1.0: wrong ELF class: ELFCLASS32")
+ val starter = EmbeddedPythonStarter(
+ "/venv/bin/python", "",
+ pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0"),
+ libraryPreloadFailure = preloadFailure
+ )
+ val cause = UnsatisfiedLinkError(
+ "/venv/lib/python3.13/site-packages/jep/libjep.so: " +
+ "libpython3.13.so.1.0: cannot open shared object file: No such file or directory"
+ )
+
+ // when
+ val exception = assertFailsWith {
+ starter.createInterpreter(create = { throw cause }, check = {})
+ }
+
+ // then
+ val message = exception.message!!
+ assert(
+ exception.cause === cause &&
+ exception.suppressed.toList() == listOf(preloadFailure) &&
+ message == "Cannot initialize the embedded Python /venv/bin/python: ${cause.message}\n" +
+ "Cannot load the libpython of this Python: ${preloadFailure.message}"
+ )
+ }
+
+ @Test
+ fun `should advise library directory of libpython when initialization of embedded Python is retried`() {
+ // given
+ val starter = EmbeddedPythonStarter(
+ "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0")
+ )
+ val first = UnsatisfiedLinkError(
+ "/venv/lib/python3.13/site-packages/jep/libjep.so: " +
+ "libpython3.13.so.1.0: cannot open shared object file: No such file or directory"
+ )
+ // like jep's MainInterpreter, rethrowing the failure of the first attempt
+ val cause = Error("The main Python interpreter previously failed to initialize.", first)
+
+ // when
+ val exception = assertFailsWith {
+ starter.createInterpreter(create = { throw cause }, check = {})
+ }
+
+ // then
+ val message = exception.message!!
+ assert(exception.cause === cause && "/opt/python/lib" in message && "LD_LIBRARY_PATH" in message)
+ }
+
+ @Test
+ fun `should not advise library directory of libpython on other failure of embedded Python startup`() {
+ // given
+ val starter = EmbeddedPythonStarter(
+ "/venv/bin/python", "", pythonLibrary = Path("/opt/python/lib/libpython3.13.so.1.0")
+ )
+ val wrongClass = UnsatisfiedLinkError("/venv/lib/python3.13/site-packages/jep/libjep.so: wrong ELF class: ELFCLASS32")
+ // of libpython found, but of another Python
+ val undefinedSymbol = UnsatisfiedLinkError(
+ "/venv/lib/python3.13/site-packages/jep/libjep.so: undefined symbol: PyFoo, version libpython3.13.so.1.0"
+ )
+ // of jep built for another Python
+ val anotherVersion = UnsatisfiedLinkError(
+ "/venv/lib/python3.13/site-packages/jep/libjep.so: " +
+ "libpython3.12.so.1.0: cannot open shared object file: No such file or directory"
+ )
+
+ // when
+ val messages = listOf(wrongClass, undefinedSymbol, anotherVersion).map { cause ->
+ assertFailsWith {
+ starter.createInterpreter(create = { throw cause }, check = {})
+ }.message
+ }
+
+ // then
+ assert(
+ messages == listOf(
+ "Cannot initialize the embedded Python /venv/bin/python: ${wrongClass.message}",
+ "Cannot initialize the embedded Python /venv/bin/python: ${undefinedSymbol.message}",
+ "Cannot initialize the embedded Python /venv/bin/python: ${anotherVersion.message}"
+ )
+ )
+ }
+
+ @Test
+ fun `should not advise library directory of Python without shared libpython`() {
+ // given
+ val starter = EmbeddedPythonStarter("/venv/bin/python", "")
+ val cause = UnsatisfiedLinkError(
+ "/venv/lib/python3.13/site-packages/jep/libjep.so: " +
+ "libpython3.13.so.1.0: cannot open shared object file: No such file or directory"
+ )
+
+ // when
+ val exception = assertFailsWith {
+ starter.createInterpreter(create = { throw cause }, check = {})
+ }
+
+ // then
+ assert(exception.message == "Cannot initialize the embedded Python /venv/bin/python: ${cause.message}")
+ }
+
@Test
fun `should report stderr of probe on Error of embedded Python startup`() {
// given
@@ -842,6 +1192,16 @@ class PythonEnvironmentTest {
}
/** The output of [PythonEnvironment.probeCommand] writing the [values], trimmed as by [runProcess]. */
+ private fun withSystemProperty(name: String, value: String?, block: () -> T): T {
+ val previous = System.getProperty(name)
+ if (value == null) System.clearProperty(name) else System.setProperty(name, value)
+ try {
+ return block()
+ } finally {
+ if (previous == null) System.clearProperty(name) else System.setProperty(name, previous)
+ }
+ }
+
private fun probePayload(vararg values: String): String {
val hexValues = values.joinToString(" ") { it.toByteArray(filesystemCharset).toHexString() }
return "${PythonEnvironment.PROBE_BEGIN}\n$hexValues\n${PythonEnvironment.PROBE_END}"
diff --git a/src/test/kotlin/PythonLibraryTest.kt b/src/test/kotlin/PythonLibraryTest.kt
new file mode 100644
index 0000000..c5c4e86
--- /dev/null
+++ b/src/test/kotlin/PythonLibraryTest.kt
@@ -0,0 +1,170 @@
+/*
+ * xemantic-kotlin-script-python - Stateful Python, embedded with Jep, in Kotlin scripts
+ * Copyright (C) 2026 Kazimierz Pogoda / Xemantic
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+package com.xemantic.kotlin.script.python
+
+import com.xemantic.kotlin.test.assert
+import org.junit.jupiter.api.Assumptions.assumeTrue
+import java.io.IOException
+import java.nio.file.Path
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.Path
+import kotlin.io.path.createTempDirectory
+import kotlin.io.path.deleteRecursively
+import kotlin.io.path.exists
+import kotlin.io.path.writeText
+import kotlin.test.Test
+
+/**
+ * The embedded CPython loading the shared `libpython` of its Python without `LD_LIBRARY_PATH`,
+ * and importing a C extension module which, like the ones of numpy, is not linked to `libpython`,
+ * but needs its symbols to be global,
+ * in a JVM of its own, as the environment of the dynamic loader is read when a process starts.
+ *
+ * It tests the loading only with a Python outside the system library directories,
+ * e.g. installed by uv or by `actions/setup-python`, which sets `LD_LIBRARY_PATH` itself,
+ * as the test venv of such a base Python, see `-PbasePython` in DEVELOPMENT.md,
+ * while with a Python of the system it passes anyway.
+ */
+class PythonLibraryTest {
+
+ private val venvPython = Path(System.getenv("VIRTUAL_ENV"), "bin", "python").toString()
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should import C extension not linked to libpython without LD_LIBRARY_PATH`() {
+ // given
+ val directory = createTempDirectory("python-library")
+ try {
+ compileExtension(directory)
+
+ // when
+ val (exitCode, output) = runInJvm(
+ EmbeddedPythonMain::class.java.name,
+ input = "import xemantic_extension\nprint(xemantic_extension.VALUE, flush=True)",
+ environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString())
+ )
+
+ // then
+ assert(output == "imported")
+ assert(exitCode == 0)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should import C extension not linked to libpython at startup of Python without LD_LIBRARY_PATH`() {
+ // given
+ // on Java 17 to 21, the symbols of libpython become global only after the startup, see loadPythonLibrary
+ assumeTrue(Runtime.version().feature() >= 22, "the foreign linker is of Java 22")
+ val directory = createTempDirectory("python-library")
+ try {
+ compileExtension(directory)
+ // imported by the site module at the startup, like a .pth file of the site-packages does, e.g. of coverage
+ directory.resolve("sitecustomize.py").writeText(
+ """
+ |import sys
+ |try:
+ | import xemantic_extension
+ | sys.xemantic_startup_import = xemantic_extension.VALUE
+ |except ImportError as e:
+ | sys.xemantic_startup_import = repr(e)
+ |""".trimMargin()
+ )
+
+ // when
+ val (exitCode, output) = runInJvm(
+ EmbeddedPythonMain::class.java.name,
+ input = "import sys\nprint(sys.xemantic_startup_import, flush=True)",
+ environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString())
+ )
+
+ // then
+ assert(output == "imported")
+ assert(exitCode == 0)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ @OptIn(ExperimentalPathApi::class)
+ @Test
+ fun `should import C extension not linked to libpython without LD_LIBRARY_PATH and foreign linker`() {
+ // given
+ val directory = createTempDirectory("python-library")
+ try {
+ compileExtension(directory)
+
+ // when
+ val (exitCode, output) = runInJvm(
+ EmbeddedPythonMain::class.java.name,
+ input = "import xemantic_extension\nprint(xemantic_extension.VALUE, flush=True)",
+ environment = mapOf("LD_LIBRARY_PATH" to null, "PYTHONPATH" to directory.toString()),
+ // like on Java 17 to 21, without the foreign linker
+ jvmArguments = listOf("-D${NO_FOREIGN_LINKER_PROPERTY}=true")
+ )
+
+ // then
+ assert(output == "imported")
+ assert(exitCode == 0)
+ } finally {
+ directory.deleteRecursively()
+ }
+ }
+
+ /**
+ * Compiles the `xemantic_extension` C extension module of the test venv Python into the [directory],
+ * not linked to `libpython`, like the C extensions on Linux are,
+ * skipping the test if not on Linux, or without a C compiler, or the Python headers.
+ */
+ private fun compileExtension(directory: Path) {
+ assumeTrue(System.getProperty("os.name").startsWith("Linux"), "the libpython is loaded so only on Linux")
+ val (include, suffix) = runProcess(
+ pythonCommand(
+ venvPython,
+ "import sysconfig\nprint(sysconfig.get_paths()['include'])\nprint(sysconfig.get_config_var('EXT_SUFFIX'))"
+ ),
+ mergeErrorStream = false
+ ).output.lines()
+ assumeTrue(Path(include, "Python.h").exists(), "no Python headers in $include")
+ val source = directory.resolve("xemantic_extension.c")
+ source.writeText(
+ """
+ |#include
+ |
+ |static struct PyModuleDef definition = {PyModuleDef_HEAD_INIT, "xemantic_extension", NULL, -1, NULL};
+ |
+ |PyMODINIT_FUNC PyInit_xemantic_extension(void) {
+ | PyObject *module = PyModule_Create(&definition);
+ | if (module != NULL) PyModule_AddStringConstant(module, "VALUE", "imported");
+ | return module;
+ |}
+ |""".trimMargin()
+ )
+ val extension = directory.resolve("xemantic_extension$suffix")
+ val compiled = try {
+ runProcess(listOf("cc", "-shared", "-fPIC", "-I$include", source.toString(), "-o", extension.toString()))
+ } catch (_: IOException) {
+ null
+ }
+ assumeTrue(compiled?.exitCode == 0, "cannot compile a C extension with cc: ${compiled?.output}")
+ }
+
+}