Fix atEnd edge case

repository/Zinc-Character-Encoding-Core.package/ZnCharacterReadStream.class/instance/upToEnd.st

@@ -4,12 +4,12 @@ upToEnd
 	An ignored mark is only answered at the end of the input, see ZnCharacterEncoder>>#isIgnoredCodePoint:.
 	It is not written, so a ByteString is not widened for it."
 
-	^ self collectionSpecies streamContents: [ :out | | encoder character |
+	^ self collectionSpecies streamContents: [ :out | | characterEncoder character |
 		peeked ifNotNil: [
 			out nextPut: peeked.
 			peeked := nil ].
-		encoder := self encoder.
+		characterEncoder := self encoder.
 		[ stream atEnd ] whileFalse: [
-			character := encoder nextFromStream: stream.
-			(character asInteger == 16rFEFF and: [ encoder isIgnoredCodePoint: 16rFEFF ])
+			character := characterEncoder nextFromStream: stream.
+			(character asInteger == 16rFEFF and: [ characterEncoder isIgnoredCodePoint: 16rFEFF ])
 				ifFalse: [ out nextPut: character ] ] ]

repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/atEnd.st

@@ -1,3 +1,9 @@
 testing
 atEnd
-	^ peeked isNil and: [ stream atEnd ]
+	"Decode one element ahead when bytes remain: an ignored byte order mark that ends the input reads as nothing,
+	see ZnCharacterEncoder>>#isIgnoredCodePoint:. So #next answers an element whenever I am not at the end.
+	Written out instead of sending #peek, to keep the [ atEnd ] whileFalse: [ next ] loop as fast as without the look-ahead."
+
+	peeked ifNotNil: [ ^ false ].
+	stream atEnd ifTrue: [ ^ true ].
+	^ (peeked := self nextElement) isNil

repository/Zinc-Character-Encoding-Core.package/ZnEncodedReadStream.class/instance/position.st

@@ -1,4 +1,5 @@
 accessing
 position
+	"Answer the byte position in the wrapped stream, before a peeked element that is not read yet"
 
-	^ super position - (peeked ifNil: [ 0 ] ifNotNil: [ 1 ])
+	^ super position - (peeked ifNil: [ 0 ] ifNotNil: [ self encoder encodedByteCountForCodePoint: peeked asInteger ])

repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/encodedByteCountForCodePoint..st

@@ -4,6 +4,6 @@ encodedByteCountForCodePoint: codePoint
 
 	codePoint < 128 ifTrue: [ ^ 1 ].
 	codePoint < 2048 ifTrue: [ ^ 2 ].
-	codePoint < 65535 ifTrue: [ ^ 3 ].
+	codePoint < 65536 ifTrue: [ ^ 3 ].
 	codePoint <= self maximumUTFCode ifTrue: [ ^ 4 ].
 	^ self errorOutsideRangeByteCount: codePoint

repository/Zinc-Character-Encoding-Core.package/ZnUTF8Encoder.class/instance/nextCodePointFromStream..st

@@ -34,7 +34,7 @@ nextCodePointFromStream: stream
 							((next := stream next ifNil: [ ^ self errorIncompleteFrom: stream ]) bitAnd: 2r11000000) == 2r10000000
 								ifTrue: [ code := (code bitShift: 6) + (next bitAnd: 2r00111111) ]
 								ifFalse: [ ^ self errorIllegalContinuationByte: next from: stream ] ].
-						code < 65535 ifTrue: [ ^ self errorOverlong: code from: stream ].
+						code < 65536 ifTrue: [ ^ self errorOverlong: code from: stream ].
 						code > self maximumUTFCode ifTrue: [ ^ self errorOutsideRange: code from: stream ].
 						^ code ].
 				^ self errorIllegalLeadingByte: byte from: stream ] ] repeat

repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAByteOrderMarkThatIsKept.st

@@ -0,0 +1,10 @@
+testing
+testAtEndBeforeAByteOrderMarkThatIsKept
+	self byteOrderMarkEncoders do: [ :encoder | | bytes stream |
+		encoder ignoreByteOrderMark: false.
+		bytes := encoder encodeString: (self withByteOrderMarks: 'a<').
+		stream := ZnCharacterReadStream on: bytes readStream encoding: encoder.
+		self assert: stream next equals: $a.
+		self deny: stream atEnd.
+		self assert: stream next equals: (Character value: 16rFEFF).
+		self assert: stream atEnd ]

repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput.st

@@ -0,0 +1,19 @@
+testing
+testAtEndBeforeAnIgnoredByteOrderMarkThatEndsTheInput
+	"A mark that ends the input is read as nothing, so the stream is at its end before it"
+
+	self byteOrderMarkEncoders do: [ :encoder | | bytes |
+		#( 'a<' 'a<<' '<a<' ) do: [ :pattern | | stream |
+			bytes := encoder encodeString: (self withByteOrderMarks: pattern).
+			stream := ZnCharacterReadStream on: bytes readStream encoding: encoder.
+			self assert: (String streamContents: [ :out | [ stream atEnd ] whileFalse: [ out nextPut: stream next ] ]) equals: 'a'.
+			stream := ZnCodePointReadStream on: bytes readStream encoding: encoder.
+			self assert: (Array streamContents: [ :out | [ stream atEnd ] whileFalse: [ out nextPut: stream next ] ]) equals: #( 97 ).
+			stream := ZnCharacterReadStream on: bytes readStream encoding: encoder.
+			stream skip: 1.
+			self assert: stream atEnd.
+			self assert: stream next isNil.
+			self assert: stream peek isNil ].
+		bytes := encoder encodeString: (self withByteOrderMarks: '<').
+		self assert: (ZnCharacterReadStream on: bytes readStream encoding: encoder) atEnd.
+		self assert: (ZnCodePointReadStream on: bytes readStream encoding: encoder) atEnd ]

repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testUTF8Boundaries.st

@@ -14,4 +14,5 @@ testUTF8Boundaries
 			string := String with: each asCharacter.
 			encoded := utf8Encoder encodeString: string.
 			self assert: (utf8Encoder decodeBytes: encoded) equals: string.
-			self assert: encoded size equals: byteCount ] ]
+			self assert: encoded size equals: byteCount.
+			self assert: (utf8Encoder encodedByteCountFor: each asCharacter) equals: byteCount ] ]

repository/Zinc-Character-Encoding-Tests.package/ZnCharacterEncoderTest.class/instance/testUTF8Overlong.st

@@ -6,6 +6,9 @@ testUTF8Overlong
 	self
 		should: [ ZnUTF8Encoder new decodeBytes: #[ 16rF0 16r82 16r82 16rAC ] ]
 		raise: ZnInvalidUTF8.
+	self
+		should: [ ZnUTF8Encoder new decodeBytes: #[ 16rF0 16r8F 16rBF 16rBF ] ]
+		raise: ZnInvalidUTF8.
 	self
 		should: [ ZnUTF8Encoder new decodeBytes: #(193 129 193 130 193 131 ) ]
 		raise: ZnInvalidUTF8

repository/Zinc-Character-Encoding-Tests.package/ZnCharacterStreamTest.class/instance/testPositionAfterPeekAndAtEnd.st

@@ -0,0 +1,22 @@
+tests
+testPositionAfterPeekAndAtEnd
+	"A peeked character is not read yet: the byte position stays before it, whatever its encoded size"
+
+	| euro text stream |
+	euro := Character value: 16r20AC.
+	text := 'a' , (String with: euro) , 'b'.
+	#( 'utf8' 'utf16' 'utf32' ) do: [ :name | | encoder bytes |
+		encoder := ZnCharacterEncoder newForEncoding: name.
+		bytes := encoder encodeString: text.
+		stream := ZnCharacterReadStream on: bytes readStream encoding: name.
+		self assert: stream next equals: $a.
+		self assert: stream position equals: (encoder encodedByteCountFor: $a).
+		self assert: stream peek equals: euro.
+		self assert: stream position equals: (encoder encodedByteCountFor: $a).
+		self deny: stream atEnd.
+		self assert: stream position equals: (encoder encodedByteCountFor: $a).
+		self assert: stream next equals: euro.
+		self assert: stream position equals: bytes size - (encoder encodedByteCountFor: $b).
+		stream := ZnCodePointReadStream on: bytes readStream encoding: name.
+		stream next; peek.
+		self assert: stream position equals: (encoder encodedByteCountFor: $a) ]