diff --git a/src/main/kotlin/PythonDependencies.kt b/src/main/kotlin/PythonDependencies.kt index a342d87..6b2e737 100644 --- a/src/main/kotlin/PythonDependencies.kt +++ b/src/main/kotlin/PythonDependencies.kt @@ -413,7 +413,7 @@ internal object PythonDependencies { # while the ones of DIST_INFOS above, of the standard library, cannot request = json.loads(sys.stdin.read()) - import base64, csv, hashlib, importlib, importlib.metadata as metadata + import base64, csv, hashlib, importlib, importlib.machinery, importlib.metadata as metadata from pathlib import Path from urllib.parse import parse_qsl, urlsplit, urlunsplit @@ -485,6 +485,48 @@ internal object PythonDependencies { return None return distribution + # the suffixes of the module files, the longest first, like .cpython-310-x86_64-linux-gnu.so before .so + module_suffixes = sorted(importlib.machinery.all_suffixes(), key=len, reverse=True) + + def top_level_name(path, vendored=False): + # the top-level module of a path recorded in RECORD, + # like demo of demo/__init__.py, demo.py or demo.cpython-310-x86_64-linux-gnu.so, + # also of demo.libs/ of auditwheel, if vendored, as the demo module loads the shared libraries in it, + # or None of e.g. ../../bin/ of pip install --target, the dist-info, the data, __pycache__ or a .pth file + first, separator, _ = path.partition("/") + if separator: + name = first.partition(".")[0] if vendored else first + else: + name = next((first[:-len(suffix)] for suffix in module_suffixes if first.endswith(suffix)), "") + return name if name.isidentifier() and name != "__pycache__" else None + + def top_level(distribution): + # the top-level modules of the distribution, told by top_level.txt, otherwise by the recorded files, + # as e.g. hatchling and flit do not write it: + # packages_distributions of importlib.metadata tells them only by top_level.txt on Python 3.10, + # only of the .py files on 3.11, and Distribution.files fails on a malformed RECORD, + # e.g. of a row with a blank line, an extra column or a size which is not a number, + # so RECORD is read as it is, like in imported_files + try: + names = set((distribution.read_text("top_level.txt") or "").split()) + except (OSError, ValueError): + names = set() + if names: + return names + try: + rows = csv.reader((distribution.read_text("RECORD") or "").splitlines()) + return {top_level_name(row[0]) for row in rows if row} - {None} + except (OSError, ValueError, csv.Error): + return set() + + def is_imported(name, distribution): + # whether the distribution in the dependency directory, or another dist-info of its name there, + # tells an imported top-level module + candidates = [distribution] + [ + metadata.PathDistribution(Path(path)) for path in dependency_dist_infos.get(normalized_name(name), []) + ] + return any(not imported.isdisjoint(top_level(candidate)) for candidate in candidates) + def imported_files(distribution): # the number of the recorded files of the imported top-level modules of the distribution, # e.g. demo/__init__.py or demo.py of the imported demo, if they are all on disk with their recorded hashes; @@ -495,7 +537,7 @@ internal object PythonDependencies { (row[0].split("/"), row[1].partition("=")) for row in csv.reader((distribution.read_text("RECORD") or "").splitlines()) if len(row) > 1 and row[1] and not row[0].endswith(".pyc") - and row[0].split("/")[0].partition(".")[0] in imported + and top_level_name(row[0], vendored=True) in imported ] for parts, (algorithm, _, value) in files: if algorithm not in hashlib.algorithms_guaranteed or algorithm.startswith("shake_"): @@ -636,13 +678,11 @@ internal object PythonDependencies { # pip resolves the missing ones without the dependency directory, so it would install a distribution # required by them from the index over the one installed from a URL, also in the same version missing = [dependency for dependency in dependencies if dependency in missing or dependency in url_dependencies] - distributions_of = metadata.packages_distributions() - in_use = {name for module in imported for name in distributions_of.get(module, [])} constraints = {} for distribution in metadata.distributions(): name = distribution.metadata["Name"] in_dependencies = os.path.realpath(distribution.locate_file("")) == dependency_location - if name and name not in constraints and (not in_dependencies or name in in_use): + if name and name not in constraints and (not in_dependencies or is_imported(name, distribution)): if in_dependencies and len(dependency_dist_infos.get(normalized_name(name), [])) > 1: # not the version of a dist-info left by a replaced version distribution = imported_distribution(name, distribution) diff --git a/src/test/kotlin/PythonDependenciesTest.kt b/src/test/kotlin/PythonDependenciesTest.kt index 0fb6178..969cab0 100644 --- a/src/test/kotlin/PythonDependenciesTest.kt +++ b/src/test/kotlin/PythonDependenciesTest.kt @@ -57,6 +57,7 @@ import kotlin.io.path.outputStream import kotlin.io.path.readBytes import kotlin.io.path.readText import kotlin.io.path.setLastModifiedTime +import kotlin.io.path.writeBytes import kotlin.io.path.writeText import kotlin.test.Test import kotlin.test.assertFailsWith @@ -1244,6 +1245,190 @@ class PythonDependenciesTest { } } + @Test + fun `should constrain imported package of distribution without top_level_txt`() { + // given + // like a wheel built by hatchling or flit, which Python 3.10 tells the modules of only by top_level.txt + withDistInfos { directory -> + val modules = mapOf( + "replaced_probe/__init__.py" to "X = 1\n", + "../../bin/replaced-probe" to "" + ) + directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n") + writeDistInfo(directory, "1.0", modules, topLevel = null) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints == listOf("replaced-probe==1.0")) + } + } + + @Test + fun `should constrain imported single-file module of distribution without top_level_txt`() { + // given + withDistInfos { directory -> + val modules = mapOf("replaced_probe.py" to "X = 1\n") + modules.forEach { (path, content) -> directory.resolve(path).writeText(content) } + writeDistInfo(directory, "1.0", modules, topLevel = null) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints == listOf("replaced-probe==1.0")) + } + } + + @Test + fun `should not constrain distribution without top_level_txt of modules not imported`() { + // given + // its .pth file, scripts and data, and files outside the directory are no modules + withDistInfos { directory -> + val files = mapOf( + "other_probe/__init__.py" to "X = 1\n", + "replaced_probe.pth" to "", + "replaced_probe-1.0.data/scripts/replaced_probe" to "", + "../../bin/replaced_probe" to "" + ) + directory.resolve("other_probe/__init__.py").createParentDirectories().writeText("X = 1\n") + writeDistInfo(directory, "1.0", files, topLevel = null) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints.isEmpty()) + } + } + + @Test + fun `should constrain imported extension module of distribution without top_level_txt`() { + // given + // like replaced_probe.cpython-310-x86_64-linux-gnu.so, which ends also with the shorter suffix .so + withDistInfos { directory -> + val modules = mapOf("replaced_probe${extensionSuffix()}" to "") + modules.forEach { (path, content) -> directory.resolve(path).writeText(content) } + writeDistInfo(directory, "1.0", modules, topLevel = null) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints == listOf("replaced-probe==1.0")) + } + } + + @Test + fun `should constrain imported package of distribution with empty top_level_txt`() { + // given + // like Python 3.11+, which tells the modules of such a distribution by its recorded files + withDistInfos { directory -> + val modules = mapOf("replaced_probe/__init__.py" to "X = 1\n") + directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n") + writeDistInfo(directory, "1.0", modules, topLevel = "") + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints == listOf("replaced-probe==1.0")) + } + } + + @Test + fun `should constrain imported package of distribution without top_level_txt of RECORD with non-integer size`() { + // given + // Distribution.files fails on it with ValueError + withDistInfos { directory -> + directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n") + writeDistInfo( + directory, "1.0", topLevel = null, + record = "replaced_probe/__init__.py,sha256=x,notanint\nreplaced_probe-1.0.dist-info/RECORD,,\n" + ) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints == listOf("replaced-probe==1.0")) + } + } + + @Test + fun `should constrain imported package of distribution without top_level_txt of RECORD with extra column`() { + // given + // Distribution.files fails on it with TypeError + withDistInfos { directory -> + directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n") + writeDistInfo( + directory, "1.0", topLevel = null, + record = "replaced_probe/__init__.py,sha256=x,2,extra\nreplaced_probe-1.0.dist-info/RECORD,,\n" + ) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints == listOf("replaced-probe==1.0")) + } + } + + @Test + fun `should constrain imported package of distribution without top_level_txt of RECORD with blank line`() { + // given + // Distribution.files fails on it with TypeError + withDistInfos { directory -> + directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n") + writeDistInfo( + directory, "1.0", topLevel = null, + record = "replaced_probe/__init__.py,,\n\nreplaced_probe-1.0.dist-info/RECORD,,\n" + ) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints == listOf("replaced-probe==1.0")) + } + } + + @Test + fun `should constrain imported package of distribution with top_level_txt not in UTF-8`() { + // given + // told by its recorded files then + withDistInfos { directory -> + val modules = mapOf("replaced_probe/__init__.py" to "X = 1\n") + directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n") + writeDistInfo(directory, "1.0", modules, topLevel = null) + directory.resolve("replaced_probe-1.0.dist-info/top_level.txt").writeBytes(byteArrayOf(0xff.toByte(), 0xfe.toByte())) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints == listOf("replaced-probe==1.0")) + } + } + + @Test + fun `should not constrain distribution without top_level_txt of dotted directory named like imported module`() { + // given + // like replaced_probe.libs/ of auditwheel, which is no module + withDistInfos { directory -> + val files = mapOf("replaced_probe.libs/libprobe.so" to "", "replaced_probe.b/__init__.py" to "") + files.forEach { (path, content) -> directory.resolve(path).createParentDirectories().writeText(content) } + writeDistInfo(directory, "1.0", files, topLevel = null) + + // when + val constraints = importedConstraints(directory) + + // then + assert(constraints.isEmpty()) + } + } + @Test fun `should remove dist-info of other version by name when identities are unknown`() { withDistInfos { directory -> @@ -1478,22 +1663,35 @@ class PythonDependenciesTest { /** * Writes the dist-info directory of the [name] [version] into the [directory], - * with a `RECORD` of the [files] with the hashes of the given content, whatever the [directory] holds. + * with a `RECORD` of the [files] with the hashes of the given content, whatever the [directory] holds, + * unless the [record] is given, and with the [topLevel] content of `top_level.txt`, unless it is `null`. */ private fun writeDistInfo( directory: Path, version: String, files: Map = emptyMap(), - name: String = "replaced-probe" + name: String = "replaced-probe", + // as importlib.metadata.packages_distributions of Python 3.10 tells the modules only by it + topLevel: String? = "${name.replace('-', '_')}\n", + record: String? = null ) { val distInfo = "${name.replace('-', '_')}-$version.dist-info" val metadata = mapOf( - "$distInfo/METADATA" to "Metadata-Version: 2.1\nName: $name\nVersion: $version\n", - // as importlib.metadata.packages_distributions of Python 3.10 tells the modules only by it - "$distInfo/top_level.txt" to "${name.replace('-', '_')}\n" - ) + "$distInfo/METADATA" to "Metadata-Version: 2.1\nName: $name\nVersion: $version\n" + ) + if (topLevel != null) mapOf("$distInfo/top_level.txt" to topLevel) else emptyMap() metadata.forEach { (path, content) -> directory.resolve(path).createParentDirectories().writeText(content) } - directory.resolve("$distInfo/RECORD").writeText(record(files + metadata, distInfo)) + directory.resolve("$distInfo/RECORD").writeText(record ?: record(files + metadata, distInfo)) + } + + /** The suffix of the extension modules of the Python of the tests, like `.cpython-310-x86_64-linux-gnu.so`. */ + private fun extensionSuffix(): String { + val process = ProcessBuilder( + PythonEnvironment.executable, "-c", "import importlib.machinery; print(importlib.machinery.EXTENSION_SUFFIXES[0])" + ).redirectError(ProcessBuilder.Redirect.INHERIT).start() + process.outputStream.close() + val suffix = process.inputStream.bufferedReader().readLine()?.trim()?.ifEmpty { null } + check(process.waitFor(5, TimeUnit.MINUTES) && process.exitValue() == 0) { "The Python did not tell the extension suffix" } + return checkNotNull(suffix) { "No extension suffix of the Python" } } private fun Path.distInfoNames(): List =