Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 45 additions & 5 deletions src/main/kotlin/PythonDependencies.kt
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ internal object PythonDependencies {
# while the ones of DIST_INFOS above, of the standard library, cannot
request = json.loads(sys.stdin.read())

import base64, csv, hashlib, importlib, importlib.metadata as metadata
import base64, csv, hashlib, importlib, importlib.machinery, importlib.metadata as metadata
from pathlib import Path
from urllib.parse import parse_qsl, urlsplit, urlunsplit

Expand Down Expand Up @@ -485,6 +485,48 @@ internal object PythonDependencies {
return None
return distribution

# the suffixes of the module files, the longest first, like .cpython-310-x86_64-linux-gnu.so before .so
module_suffixes = sorted(importlib.machinery.all_suffixes(), key=len, reverse=True)

def top_level_name(path, vendored=False):
# the top-level module of a path recorded in RECORD,
# like demo of demo/__init__.py, demo.py or demo.cpython-310-x86_64-linux-gnu.so,
# also of demo.libs/ of auditwheel, if vendored, as the demo module loads the shared libraries in it,
# or None of e.g. ../../bin/ of pip install --target, the dist-info, the data, __pycache__ or a .pth file
first, separator, _ = path.partition("/")
if separator:
name = first.partition(".")[0] if vendored else first
else:
name = next((first[:-len(suffix)] for suffix in module_suffixes if first.endswith(suffix)), "")
return name if name.isidentifier() and name != "__pycache__" else None

def top_level(distribution):
# the top-level modules of the distribution, told by top_level.txt, otherwise by the recorded files,
# as e.g. hatchling and flit do not write it:
# packages_distributions of importlib.metadata tells them only by top_level.txt on Python 3.10,
# only of the .py files on 3.11, and Distribution.files fails on a malformed RECORD,
# e.g. of a row with a blank line, an extra column or a size which is not a number,
# so RECORD is read as it is, like in imported_files
try:
names = set((distribution.read_text("top_level.txt") or "").split())
except (OSError, ValueError):
names = set()
if names:
return names
try:
rows = csv.reader((distribution.read_text("RECORD") or "").splitlines())
return {top_level_name(row[0]) for row in rows if row} - {None}
except (OSError, ValueError, csv.Error):
return set()

def is_imported(name, distribution):
# whether the distribution in the dependency directory, or another dist-info of its name there,
# tells an imported top-level module
candidates = [distribution] + [
metadata.PathDistribution(Path(path)) for path in dependency_dist_infos.get(normalized_name(name), [])
]
return any(not imported.isdisjoint(top_level(candidate)) for candidate in candidates)

def imported_files(distribution):
# the number of the recorded files of the imported top-level modules of the distribution,
# e.g. demo/__init__.py or demo.py of the imported demo, if they are all on disk with their recorded hashes;
Expand All @@ -495,7 +537,7 @@ internal object PythonDependencies {
(row[0].split("/"), row[1].partition("="))
for row in csv.reader((distribution.read_text("RECORD") or "").splitlines())
if len(row) > 1 and row[1] and not row[0].endswith(".pyc")
and row[0].split("/")[0].partition(".")[0] in imported
and top_level_name(row[0], vendored=True) in imported
]
for parts, (algorithm, _, value) in files:
if algorithm not in hashlib.algorithms_guaranteed or algorithm.startswith("shake_"):
Expand Down Expand Up @@ -636,13 +678,11 @@ internal object PythonDependencies {
# pip resolves the missing ones without the dependency directory, so it would install a distribution
# required by them from the index over the one installed from a URL, also in the same version
missing = [dependency for dependency in dependencies if dependency in missing or dependency in url_dependencies]
distributions_of = metadata.packages_distributions()
in_use = {name for module in imported for name in distributions_of.get(module, [])}
constraints = {}
for distribution in metadata.distributions():
name = distribution.metadata["Name"]
in_dependencies = os.path.realpath(distribution.locate_file("")) == dependency_location
if name and name not in constraints and (not in_dependencies or name in in_use):
if name and name not in constraints and (not in_dependencies or is_imported(name, distribution)):
if in_dependencies and len(dependency_dist_infos.get(normalized_name(name), [])) > 1:
# not the version of a dist-info left by a replaced version
distribution = imported_distribution(name, distribution)
Expand Down
212 changes: 205 additions & 7 deletions src/test/kotlin/PythonDependenciesTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import kotlin.io.path.outputStream
import kotlin.io.path.readBytes
import kotlin.io.path.readText
import kotlin.io.path.setLastModifiedTime
import kotlin.io.path.writeBytes
import kotlin.io.path.writeText
import kotlin.test.Test
import kotlin.test.assertFailsWith
Expand Down Expand Up @@ -1244,6 +1245,190 @@ class PythonDependenciesTest {
}
}

@Test
fun `should constrain imported package of distribution without top_level_txt`() {
// given
// like a wheel built by hatchling or flit, which Python 3.10 tells the modules of only by top_level.txt
withDistInfos { directory ->
val modules = mapOf(
"replaced_probe/__init__.py" to "X = 1\n",
"../../bin/replaced-probe" to ""
)
directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n")
writeDistInfo(directory, "1.0", modules, topLevel = null)

// when
val constraints = importedConstraints(directory)

// then
assert(constraints == listOf("replaced-probe==1.0"))
}
}

@Test
fun `should constrain imported single-file module of distribution without top_level_txt`() {
// given
withDistInfos { directory ->
val modules = mapOf("replaced_probe.py" to "X = 1\n")
modules.forEach { (path, content) -> directory.resolve(path).writeText(content) }
writeDistInfo(directory, "1.0", modules, topLevel = null)

// when
val constraints = importedConstraints(directory)

// then
assert(constraints == listOf("replaced-probe==1.0"))
}
}

@Test
fun `should not constrain distribution without top_level_txt of modules not imported`() {
// given
// its .pth file, scripts and data, and files outside the directory are no modules
withDistInfos { directory ->
val files = mapOf(
"other_probe/__init__.py" to "X = 1\n",
"replaced_probe.pth" to "",
"replaced_probe-1.0.data/scripts/replaced_probe" to "",
"../../bin/replaced_probe" to ""
)
directory.resolve("other_probe/__init__.py").createParentDirectories().writeText("X = 1\n")
writeDistInfo(directory, "1.0", files, topLevel = null)

// when
val constraints = importedConstraints(directory)

// then
assert(constraints.isEmpty())
}
}

@Test
fun `should constrain imported extension module of distribution without top_level_txt`() {
// given
// like replaced_probe.cpython-310-x86_64-linux-gnu.so, which ends also with the shorter suffix .so
withDistInfos { directory ->
val modules = mapOf("replaced_probe${extensionSuffix()}" to "")
modules.forEach { (path, content) -> directory.resolve(path).writeText(content) }
writeDistInfo(directory, "1.0", modules, topLevel = null)

// when
val constraints = importedConstraints(directory)

// then
assert(constraints == listOf("replaced-probe==1.0"))
}
}

@Test
fun `should constrain imported package of distribution with empty top_level_txt`() {
// given
// like Python 3.11+, which tells the modules of such a distribution by its recorded files
withDistInfos { directory ->
val modules = mapOf("replaced_probe/__init__.py" to "X = 1\n")
directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n")
writeDistInfo(directory, "1.0", modules, topLevel = "")

// when
val constraints = importedConstraints(directory)

// then
assert(constraints == listOf("replaced-probe==1.0"))
}
}

@Test
fun `should constrain imported package of distribution without top_level_txt of RECORD with non-integer size`() {
// given
// Distribution.files fails on it with ValueError
withDistInfos { directory ->
directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n")
writeDistInfo(
directory, "1.0", topLevel = null,
record = "replaced_probe/__init__.py,sha256=x,notanint\nreplaced_probe-1.0.dist-info/RECORD,,\n"
)

// when
val constraints = importedConstraints(directory)

// then
assert(constraints == listOf("replaced-probe==1.0"))
}
}

@Test
fun `should constrain imported package of distribution without top_level_txt of RECORD with extra column`() {
// given
// Distribution.files fails on it with TypeError
withDistInfos { directory ->
directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n")
writeDistInfo(
directory, "1.0", topLevel = null,
record = "replaced_probe/__init__.py,sha256=x,2,extra\nreplaced_probe-1.0.dist-info/RECORD,,\n"
)

// when
val constraints = importedConstraints(directory)

// then
assert(constraints == listOf("replaced-probe==1.0"))
}
}

@Test
fun `should constrain imported package of distribution without top_level_txt of RECORD with blank line`() {
// given
// Distribution.files fails on it with TypeError
withDistInfos { directory ->
directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n")
writeDistInfo(
directory, "1.0", topLevel = null,
record = "replaced_probe/__init__.py,,\n\nreplaced_probe-1.0.dist-info/RECORD,,\n"
)

// when
val constraints = importedConstraints(directory)

// then
assert(constraints == listOf("replaced-probe==1.0"))
}
}

@Test
fun `should constrain imported package of distribution with top_level_txt not in UTF-8`() {
// given
// told by its recorded files then
withDistInfos { directory ->
val modules = mapOf("replaced_probe/__init__.py" to "X = 1\n")
directory.resolve("replaced_probe/__init__.py").createParentDirectories().writeText("X = 1\n")
writeDistInfo(directory, "1.0", modules, topLevel = null)
directory.resolve("replaced_probe-1.0.dist-info/top_level.txt").writeBytes(byteArrayOf(0xff.toByte(), 0xfe.toByte()))

// when
val constraints = importedConstraints(directory)

// then
assert(constraints == listOf("replaced-probe==1.0"))
}
}

@Test
fun `should not constrain distribution without top_level_txt of dotted directory named like imported module`() {
// given
// like replaced_probe.libs/ of auditwheel, which is no module
withDistInfos { directory ->
val files = mapOf("replaced_probe.libs/libprobe.so" to "", "replaced_probe.b/__init__.py" to "")
files.forEach { (path, content) -> directory.resolve(path).createParentDirectories().writeText(content) }
writeDistInfo(directory, "1.0", files, topLevel = null)

// when
val constraints = importedConstraints(directory)

// then
assert(constraints.isEmpty())
}
}

@Test
fun `should remove dist-info of other version by name when identities are unknown`() {
withDistInfos { directory ->
Expand Down Expand Up @@ -1478,22 +1663,35 @@ class PythonDependenciesTest {

/**
* Writes the dist-info directory of the [name] [version] into the [directory],
* with a `RECORD` of the [files] with the hashes of the given content, whatever the [directory] holds.
* with a `RECORD` of the [files] with the hashes of the given content, whatever the [directory] holds,
* unless the [record] is given, and with the [topLevel] content of `top_level.txt`, unless it is `null`.
*/
private fun writeDistInfo(
directory: Path,
version: String,
files: Map<String, String> = emptyMap(),
name: String = "replaced-probe"
name: String = "replaced-probe",
// as importlib.metadata.packages_distributions of Python 3.10 tells the modules only by it
topLevel: String? = "${name.replace('-', '_')}\n",
record: String? = null
) {
val distInfo = "${name.replace('-', '_')}-$version.dist-info"
val metadata = mapOf(
"$distInfo/METADATA" to "Metadata-Version: 2.1\nName: $name\nVersion: $version\n",
// as importlib.metadata.packages_distributions of Python 3.10 tells the modules only by it
"$distInfo/top_level.txt" to "${name.replace('-', '_')}\n"
)
"$distInfo/METADATA" to "Metadata-Version: 2.1\nName: $name\nVersion: $version\n"
) + if (topLevel != null) mapOf("$distInfo/top_level.txt" to topLevel) else emptyMap()
metadata.forEach { (path, content) -> directory.resolve(path).createParentDirectories().writeText(content) }
directory.resolve("$distInfo/RECORD").writeText(record(files + metadata, distInfo))
directory.resolve("$distInfo/RECORD").writeText(record ?: record(files + metadata, distInfo))
}

/** The suffix of the extension modules of the Python of the tests, like `.cpython-310-x86_64-linux-gnu.so`. */
private fun extensionSuffix(): String {
val process = ProcessBuilder(
PythonEnvironment.executable, "-c", "import importlib.machinery; print(importlib.machinery.EXTENSION_SUFFIXES[0])"
).redirectError(ProcessBuilder.Redirect.INHERIT).start()
process.outputStream.close()
val suffix = process.inputStream.bufferedReader().readLine()?.trim()?.ifEmpty { null }
check(process.waitFor(5, TimeUnit.MINUTES) && process.exitValue() == 0) { "The Python did not tell the extension suffix" }
return checkNotNull(suffix) { "No extension suffix of the Python" }
}

private fun Path.distInfoNames(): List<String> =
Expand Down
Loading