diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st b/repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st index bfdcadd7..57e08a70 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st @@ -4,12 +4,12 @@ upToEnd An ignored mark is only answered at the end of the input, see ZnCharacterEncoder>>#isIgnoredCodePoint:. It is not written, so a ByteString is not widened for it." - ^ self collectionSpecies streamContents: [ :out | | encoder character | + ^ self collectionSpecies streamContents: [ :out | | characterEncoder character | peeked ifNotNil: [ out nextPut: peeked. peeked := nil ]. - encoder := self encoder. + characterEncoder := self encoder. [ stream atEnd ] whileFalse: [ - character := encoder nextFromStream: stream. - (character asInteger == 16rFEFF and: [ encoder isIgnoredCodePoint: 16rFEFF ]) + character := characterEncoder nextFromStream: stream. + (character asInteger == 16rFEFF and: [ characterEncoder isIgnoredCodePoint: 16rFEFF ]) ifFalse: [ out nextPut: character ] ] ] \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st b/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st index a7c90e6d..de47c30b 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st @@ -1,3 +1,9 @@ testing atEnd - ^ peeked isNil and: [ stream atEnd ] \ No newline at end of file + "Decode one element ahead when bytes remain: an ignored byte order mark that ends the input reads as nothing, + see ZnCharacterEncoder>>#isIgnoredCodePoint:. So #next answers an element whenever I am not at the end. + Written out instead of sending #peek, to keep the [ atEnd ] whileFalse: [ next ] loop as fast as without the look-ahead." + + peeked ifNotNil: [ ^ false ]. + stream atEnd ifTrue: [ ^ true ]. + ^ (peeked := self nextElement) isNil \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st b/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st index d358ccb9..98bbde5f 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st @@ -1,4 +1,5 @@ accessing position + "Answer the byte position in the wrapped stream, before a peeked element that is not read yet" - ^ super position - (peeked ifNil: [ 0 ] ifNotNil: [ 1 ]) \ No newline at end of file + ^ super position - (peeked ifNil: [ 0 ] ifNotNil: [ self encoder encodedByteCountForCodePoint: peeked asInteger ]) \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st b/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st index 43a782cf..244e715d 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st @@ -4,6 +4,6 @@ encodedByteCountForCodePoint: codePoint codePoint < 128 ifTrue: [ ^ 1 ]. codePoint < 2048 ifTrue: [ ^ 2 ]. - codePoint < 65535 ifTrue: [ ^ 3 ]. + codePoint < 65536 ifTrue: [ ^ 3 ]. codePoint <= self maximumUTFCode ifTrue: [ ^ 4 ]. ^ self errorOutsideRangeByteCount: codePoint \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st b/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st index cfd93e83..ae56a447 100644 --- a/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st +++ b/repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st @@ -34,7 +34,7 @@ nextCodePointFromStream: stream ((next := stream next ifNil: [ ^ self errorIncompleteFrom: stream ]) bitAnd: 2r11000000) == 2r10000000 ifTrue: [ code := (code bitShift: 6) + (next bitAnd: 2r00111111) ] ifFalse: [ ^ self errorIllegalContinuationByte: next from: stream ] ]. - code < 65535 ifTrue: [ ^ self errorOverlong: code from: stream ]. + code < 65536 ifTrue: [ ^ self errorOverlong: code from: stream ]. code > self maximumUTFCode ifTrue: [ ^ self errorOutsideRange: code from: stream ]. ^ code ]. ^ self errorIllegalLeadingByte: byte from: stream ] ] repeat \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAByteOrderMarkThatIsKept.st b/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAByteOrderMarkThatIsKept.st new file mode 100644 index 00000000..481e8023 --- /dev/null +++ b/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAByteOrderMarkThatIsKept.st @@ -0,0 +1,10 @@ +testing +testAtEndBeforeAByteOrderMarkThatIsKept + self byteOrderMarkEncoders do: [ :encoder | | bytes stream | + encoder ignoreByteOrderMark: false. + bytes := encoder encodeString: (self withByteOrderMarks: 'a<'). + stream := ZnCharacterReadStream on: bytes readStream encoding: encoder. + self assert: stream next equals: $a. + self deny: stream atEnd. + self assert: stream next equals: (Character value: 16rFEFF). + self assert: stream atEnd ] \ No newline at end of file diff --git a/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput.st b/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput.st new file mode 100644 index 00000000..a3d9a6fb --- /dev/null +++ b/repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput.st @@ -0,0 +1,19 @@ +testing +testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput + "A mark that ends the input is read as nothing, so the stream is at its end before it" + + self byteOrderMarkEncoders do: [ :encoder | | bytes | + #( 'a<' 'a<<' '