From 5c1eeee7126fd1df8c563e84ea7de3262cb57483 Mon Sep 17 00:00:00 2001 From: Carlos Lopez <27969022+lopezca@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:39:09 -0600 Subject: [PATCH 1/2] test: atEnd, position and UTF-8 boundaries around peeked elements - atEnd before an ignored byte order mark that ends the input - position after peek of a multi-byte character - UTF-8 encoded byte count and overlong form of U+FFFF --- ...testAtEndBeforeAByteOrderMarkThatIsKept.st | 10 +++++++++ ...eAnIgnoredByteOrderMarkThatEndsTheInput.st | 19 ++++++++++++++++ .../instance/testUTF8Boundaries.st | 3 ++- .../instance/testUTF8Overlong.st | 3 +++ .../instance/testPositionAfterPeekAndAtEnd.st | 22 +++++++++++++++++++ 5 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAByteOrderMarkThatIsKept.st create mode 100644 repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput.st create mode 100644 repository/Zinc-Character-Encoding-Tests.package/ZnCharacterStreamTest.class/instance/testPositionAfterPeekAndAtEnd.st diff --git a/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAByteOrderMarkThatIsKept.st b/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAByteOrderMarkThatIsKept.st new file mode 100644 index 000000000..481e80230 --- /dev/null +++ b/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAByteOrderMarkThatIsKept.st @@ -0,0 +1,10 @@ +testing +testAtEndBeforeAByteOrderMarkThatIsKept + self byteOrderMarkEncoders do: [ :encoder | | bytes stream | + encoder ignoreByteOrderMark: false. + bytes := encoder encodeString: (self withByteOrderMarks: 'a<'). + stream := ZnCharacterReadStream on: bytes readStream encoding: encoder. + self assert: stream next equals: $a. + self deny: stream atEnd. + self assert: stream next equals: (Character value: 16rFEFF). + self assert: stream atEnd ] \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput.st b/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput.st new file mode 100644 index 000000000..a3d9a6fb1 --- /dev/null +++ b/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput.st @@ -0,0 +1,19 @@ +testing +testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput + "A mark that ends the input is read as nothing, so the stream is at its end before it" + + self byteOrderMarkEncoders do: [ :encoder | | bytes | + #( 'a<' 'a<<' ' Date: Wed, 11 Nov 2026 18:39:10 -0600 Subject: [PATCH 2/2] fix: atEnd before a trailing byte order mark, peek size in position, U+FFFF - atEnd decodes one element ahead, without sending peek - position subtracts the encoded size of the peeked element - UTF-8 counts U+FFFF as 3 bytes and rejects its 4-byte form - upToEnd temporary no longer shadows the encoder variable --- .../ZnCharacterReadStream.class/instance/upToEnd.st | 8 ++++---- .../ZnEncodedReadStream.class/instance/atEnd.st | 8 +++++++- .../ZnEncodedReadStream.class/instance/position.st | 3 ++- .../instance/encodedByteCountForCodePoint..st | 2 +- .../instance/nextCodePointFromStream..st | 2 +- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st b/repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st index bfdcadd78..57e08a707 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st @@ -4,12 +4,12 @@ upToEnd An ignored mark is only answered at the end of the input, see ZnCharacterEncoder>>#isIgnoredCodePoint:. It is not written, so a ByteString is not widened for it." - ^ self collectionSpecies streamContents: [ :out | | encoder character | + ^ self collectionSpecies streamContents: [ :out | | characterEncoder character | peeked ifNotNil: [ out nextPut: peeked. peeked := nil ]. - encoder := self encoder. + characterEncoder := self encoder. [ stream atEnd ] whileFalse: [ - character := encoder nextFromStream: stream. - (character asInteger == 16rFEFF and: [ encoder isIgnoredCodePoint: 16rFEFF ]) + character := characterEncoder nextFromStream: stream. + (character asInteger == 16rFEFF and: [ characterEncoder isIgnoredCodePoint: 16rFEFF ]) ifFalse: [ out nextPut: character ] ] ] \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st b/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st index a7c90e6d1..de47c30bb 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st @@ -1,3 +1,9 @@ testing atEnd - ^ peeked isNil and: [ stream atEnd ] \ No newline at end of file + "Decode one element ahead when bytes remain: an ignored byte order mark that ends the input reads as nothing, + see ZnCharacterEncoder>>#isIgnoredCodePoint:. So #next answers an element whenever I am not at the end. + Written out instead of sending #peek, to keep the [ atEnd ] whileFalse: [ next ] loop as fast as without the look-ahead." + + peeked ifNotNil: [ ^ false ]. + stream atEnd ifTrue: [ ^ true ]. + ^ (peeked := self nextElement) isNil \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st b/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st index d358ccb90..98bbde5fd 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st @@ -1,4 +1,5 @@ accessing position + "Answer the byte position in the wrapped stream, before a peeked element that is not read yet" - ^ super position - (peeked ifNil: [ 0 ] ifNotNil: [ 1 ]) \ No newline at end of file + ^ super position - (peeked ifNil: [ 0 ] ifNotNil: [ self encoder encodedByteCountForCodePoint: peeked asInteger ]) \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st b/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st index 43a782cf8..244e715de 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st @@ -4,6 +4,6 @@ encodedByteCountForCodePoint: codePoint codePoint < 128 ifTrue: [ ^ 1 ]. codePoint < 2048 ifTrue: [ ^ 2 ]. - codePoint < 65535 ifTrue: [ ^ 3 ]. + codePoint < 65536 ifTrue: [ ^ 3 ]. codePoint <= self maximumUTFCode ifTrue: [ ^ 4 ]. ^ self errorOutsideRangeByteCount: codePoint \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st b/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st index cfd93e836..ae56a4470 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st @@ -34,7 +34,7 @@ nextCodePointFromStream: stream ((next := stream next ifNil: [ ^ self errorIncompleteFrom: stream ]) bitAnd: 2r11000000) == 2r10000000 ifTrue: [ code := (code bitShift: 6) + (next bitAnd: 2r00111111) ] ifFalse: [ ^ self errorIllegalContinuationByte: next from: stream ] ]. - code < 65535 ifTrue: [ ^ self errorOverlong: code from: stream ]. + code < 65536 ifTrue: [ ^ self errorOverlong: code from: stream ]. code > self maximumUTFCode ifTrue: [ ^ self errorOutsideRange: code from: stream ]. ^ code ]. ^ self errorIllegalLeadingByte: byte from: stream ] ] repeat \ No newline at end of file