From a15030054d9ac062a18ad7aa01abd377f30644aa Mon Sep 17 00:00:00 2001 From: Martin Thoma Date: Sat, 14 Nov 2026 10:23:39 +0100 Subject: [PATCH 1/5] PI: Decode PDFDocEncoding strings with a C-speed translation instead of per-character conccatenation --- pypdf/generic/_utils.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pypdf/generic/_utils.py b/pypdf/generic/_utils.py index d150c18346..deb2eb508f 100644 --- a/pypdf/generic/_utils.py +++ b/pypdf/generic/_utils.py @@ -7,6 +7,7 @@ from ..errors import STREAM_TRUNCATED_PREMATURELY, PdfStreamError from ._base import ByteStringObject, TextStringObject +_PDFDOC_DECODE_MAP = dict(enumerate(_pdfdoc_encoding)) def hex_to_rgb(value: str) -> tuple[float, float, float]: return tuple(int(value.lstrip("#")[i : i + 2], 16) / 255.0 for i in (0, 2, 4)) # type: ignore[return-value] @@ -240,18 +241,20 @@ def _create_unicode_string_object(data: bytes) -> TextStringObject | None: def decode_pdfdocencoding(byte_array: bytes) -> str: - retval = "" - for b in byte_array: - c = _pdfdoc_encoding[b] - if c == "\u0000": - raise UnicodeDecodeError( - "pdfdocencoding", - bytearray(b), - -1, - -1, - "does not exist in translation table", - ) - retval += c + # Be optimistic and decode using the PDFDocEncoding map, as + # in almost all valid PDF files, unassigned bytes (\u0000) do not occur. + retval = bytes(byte_array).decode("latin1").translate(_PDFDOC_DECODE_MAP) + + if "\u0000" in retval: + for b in byte_array: + if _pdfdoc_encoding[b] == "\u0000": + raise UnicodeDecodeError( + "pdfdocencoding", + bytearray(b), + -1, + -1, + "does not exist in translation table", + ) return retval From e82c995106cedccb4978f35a115745c51e1b190f Mon Sep 17 00:00:00 2001 From: Martin Thoma Date: Sat, 14 Nov 2026 11:30:25 +0100 Subject: [PATCH 2/5] Add performance test --- tests/test_generic.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_generic.py b/tests/test_generic.py index 8e15397517..9d263aa143 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -8,6 +8,7 @@ from base64 import a85encode from copy import deepcopy from io import BytesIO +from types import CodeType import pytest @@ -36,6 +37,7 @@ TextStringObject, TreeObject, create_string_object, + decode_pdfdocencoding, encode_pdfdocencoding, is_null_or_none, read_hex_string_from_stream, @@ -1797,3 +1799,44 @@ def test_content_stream_replicate__raw_data(): replica = content.replicate(writer) assert replica.get_data() == b"q 1 0 0 1 10 10 cm Q" + + + + + +def test_decode_pdfdocencoding_performance(): + """ + Ensure the performance of decode_pdfdocencoding is within acceptable limits. + + This test counts executed Python bytecode instructions using sys.monitoring. + """ + if not hasattr(sys, "monitoring"): + pytest.skip("sys.monitoring requires Python 3.12+") + + # 10KB payload is large enough to show a massive bytecode divergence + test_payload = b"A" * 10_000 + instruction_count = 0 + + def instruction_callback(code: CodeType, instruction_offset: int) -> None: + nonlocal instruction_count + instruction_count += 1 + + tool_id = 0 + try: + sys.monitoring.use_tool_id(tool_id, "InstructionCounter") + sys.monitoring.register_callback( + tool_id, sys.monitoring.events.INSTRUCTION, instruction_callback + ) + sys.monitoring.set_events(tool_id, sys.monitoring.events.INSTRUCTION) + + decode_pdfdocencoding(test_payload) + finally: + sys.monitoring.set_events(tool_id, sys.monitoring.events.NO_EVENTS) + sys.monitoring.free_tool_id(tool_id) + + # We expect a tiny, fixed number of setup/C-call instructions (~500 total). + max_allowed_instructions = 2_000 + + assert instruction_count < max_allowed_instructions, ( + f"Instruction budget exceeded: executed {instruction_count:,} bytecode instructions. " + ) From 418e81bd59d2d03104175b520e800427a0586ebd Mon Sep 17 00:00:00 2001 From: Martin Thoma Date: Sat, 14 Nov 2026 11:45:20 +0100 Subject: [PATCH 3/5] Better error reporting; test happy path + exception --- pypdf/generic/_utils.py | 9 +++++---- tests/test_generic.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pypdf/generic/_utils.py b/pypdf/generic/_utils.py index deb2eb508f..ba9d042af0 100644 --- a/pypdf/generic/_utils.py +++ b/pypdf/generic/_utils.py @@ -246,13 +246,14 @@ def decode_pdfdocencoding(byte_array: bytes) -> str: retval = bytes(byte_array).decode("latin1").translate(_PDFDOC_DECODE_MAP) if "\u0000" in retval: - for b in byte_array: + # Find the first invalid byte and report its exact position + for idx, b in enumerate(byte_array): if _pdfdoc_encoding[b] == "\u0000": raise UnicodeDecodeError( "pdfdocencoding", - bytearray(b), - -1, - -1, + bytearray([b]), + idx, # Exact start index + idx + 1, # Exact end index "does not exist in translation table", ) return retval diff --git a/tests/test_generic.py b/tests/test_generic.py index 9d263aa143..88a3c82a26 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -1801,6 +1801,41 @@ def test_content_stream_replicate__raw_data(): assert replica.get_data() == b"q 1 0 0 1 10 10 cm Q" +@pytest.mark.parametrize( + ("input_bytes", "expected_output"), + [ + (b"Hello", "Hello"), + (b"", ""), + (b"PDF 1.7 Specification", "PDF 1.7 Specification"), + (b"Test 123", "Test 123"), + ], +) +def test_decode_pdfdocencoding_success(input_bytes, expected_output): + """Test that valid byte sequences decode correctly.""" + assert decode_pdfdocencoding(input_bytes) == expected_output + + +@pytest.mark.parametrize( + ("input_bytes", "expected_start", "expected_end", "expected_hex"), + [ + (b"He\x7flo", 2, 3, "0x7f"), + (b"\x7f", 0, 1, "0x7f"), + (b"Start\x7fEnd", 5, 6, "0x7f"), + (b"ABC\x00XYZ", 3, 4, "0x00"), + ], +) +def test_decode_pdfdocencoding_raises_error( + input_bytes, expected_start, expected_end, expected_hex +): + """Test that unassigned bytes raise a precise UnicodeDecodeError at the correct position.""" + with pytest.raises(UnicodeDecodeError) as exc_info: + decode_pdfdocencoding(input_bytes) + + err = exc_info.value + assert err.encoding == "pdfdocencoding" + assert "does not exist" in err.reason + assert (err.start, err.end) == (expected_start, expected_end) + From 11ecb1545ad6d86ce5b5e016332b8c261757665f Mon Sep 17 00:00:00 2001 From: Martin Thoma Date: Sat, 14 Nov 2026 14:28:09 +0100 Subject: [PATCH 4/5] Fix coverage --- pypdf/generic/_utils.py | 19 ++++++++++--------- tests/test_generic.py | 2 -- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pypdf/generic/_utils.py b/pypdf/generic/_utils.py index ba9d042af0..e74fd7f3d2 100644 --- a/pypdf/generic/_utils.py +++ b/pypdf/generic/_utils.py @@ -9,6 +9,7 @@ _PDFDOC_DECODE_MAP = dict(enumerate(_pdfdoc_encoding)) + def hex_to_rgb(value: str) -> tuple[float, float, float]: return tuple(int(value.lstrip("#")[i : i + 2], 16) / 255.0 for i in (0, 2, 4)) # type: ignore[return-value] @@ -247,15 +248,15 @@ def decode_pdfdocencoding(byte_array: bytes) -> str: if "\u0000" in retval: # Find the first invalid byte and report its exact position - for idx, b in enumerate(byte_array): - if _pdfdoc_encoding[b] == "\u0000": - raise UnicodeDecodeError( - "pdfdocencoding", - bytearray([b]), - idx, # Exact start index - idx + 1, # Exact end index - "does not exist in translation table", - ) + idx = retval.index("\u0000") + b = byte_array[idx] + raise UnicodeDecodeError( + "pdfdocencoding", + bytearray([b]), + idx, # Exact start index + idx + 1, # Exact end index + "does not exist in translation table", + ) return retval diff --git a/tests/test_generic.py b/tests/test_generic.py index 88a3c82a26..f61b452278 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -1837,8 +1837,6 @@ def test_decode_pdfdocencoding_raises_error( assert (err.start, err.end) == (expected_start, expected_end) - - def test_decode_pdfdocencoding_performance(): """ Ensure the performance of decode_pdfdocencoding is within acceptable limits. From e12c89eca5770e962f9b3cfc934faa9f7b86ef2f Mon Sep 17 00:00:00 2001 From: Martin Thoma Date: Sat, 14 Nov 2026 18:06:49 +0100 Subject: [PATCH 5/5] Test improvement; error message --- pypdf/generic/_utils.py | 7 +++---- tests/test_generic.py | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pypdf/generic/_utils.py b/pypdf/generic/_utils.py index e74fd7f3d2..796b041f7b 100644 --- a/pypdf/generic/_utils.py +++ b/pypdf/generic/_utils.py @@ -249,12 +249,11 @@ def decode_pdfdocencoding(byte_array: bytes) -> str: if "\u0000" in retval: # Find the first invalid byte and report its exact position idx = retval.index("\u0000") - b = byte_array[idx] raise UnicodeDecodeError( "pdfdocencoding", - bytearray([b]), - idx, # Exact start index - idx + 1, # Exact end index + bytes(byte_array), + idx, # start + idx + 1, # end "does not exist in translation table", ) return retval diff --git a/tests/test_generic.py b/tests/test_generic.py index f61b452278..5439955435 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -1808,6 +1808,7 @@ def test_content_stream_replicate__raw_data(): (b"", ""), (b"PDF 1.7 Specification", "PDF 1.7 Specification"), (b"Test 123", "Test 123"), + (b"\x18\x19\x80\x84\xa0", "\u02d8\u02c7\u2022\u2014\u20ac"), ], ) def test_decode_pdfdocencoding_success(input_bytes, expected_output):