'From Cuis 5.0 [latest update: #4889] on 27 September 2021 at 3:55:55 pm'!

!XMLTokenizer methodsFor: 'tokenizing' stamp: 'jmv 9/27/2021 15:52:14'!
checkAndExpandReference: parsingContext
	| referenceString nextChar |
	nextChar _ self peek.
	self validating
		ifFalse: [^nil].
	nextChar == $&
		ifTrue: [
			self next.
			self peek == $# ifTrue: [
				^self pushStream: (ReadStream on: self nextCharReference asString)].
			referenceString _ self nextLiteral.
			self next == $;
				ifFalse: [self errorExpected: ';'].
			self handleEntity: referenceString in: parsingContext ]
		ifFalse: [
			((nextChar == $%
				and: [self parsingMarkup])
				and: [parsingContext == #entityValue])
				ifTrue: [
					self skipSeparators.
					referenceString _ self nextLiteral.
					self handleEntity: referenceString in: parsingContext]].

	self atEnd ifTrue: [self errorExpected: 'Character expected.']! !

!XMLTokenizer methodsFor: 'tokenizing' stamp: 'jmv 9/27/2021 15:53:46'!
nextAttributeValue
	| delimiterChar attributeValueStream nextChar nextPeek referenceString entity entityValue c |
	delimiterChar _ self next.
	(delimiterChar == $"
		or: [delimiterChar == $'])
		ifFalse: [self errorExpected: 'Attribute value delimiter expected.'].
	attributeValueStream _ attributeBuffer reset.
	[
		nextPeek _ nextChar _ self next.
		nextChar ifNil: [self errorExpected: 'Character expected.'].
		nextChar == $&
			ifTrue: [
				self peek == $#
					ifTrue: [
						nextPeek _ nil.
						nextChar _ self nextCharReference]
					ifFalse: [
						referenceString _ self nextLiteral.
						c _ self next.
						c == $;
							ifFalse: [
								"self errorExpected: ';'" "Just ignore errors. consider badformed xml as a simple literal string"
								attributeValueStream
									nextPut: nextChar;
									nextPutAll: referenceString;	"mhhhhh"
									nextPut: c.
								nextChar _ nil
								]
							ifTrue: [
								entity _ self entity: referenceString.
								entityValue _ entity valueForContext: #content.
								(self class isCharEscape: entityValue)
									ifTrue: [
										nextPeek _ nil.
										nextChar _ entityValue]
									ifFalse: [
										entityValue _ entityValue asString.
										entityValue isEmpty
											ifTrue: [nextPeek _ nextChar _ nil]
											ifFalse: [
												self pushStream: (ReadStream on: entityValue asString).
												nextPeek _ nextChar _ self next]]]]].
		nextPeek == delimiterChar
	]
		whileFalse: [
			nextChar ifNotNil: [
				nextChar isString
					ifFalse: [attributeValueStream nextPut: nextChar]
					ifTrue: [attributeValueStream nextPutAll: nextChar]]].
	^self fastStreamStringContents: attributeValueStream
"	^attributeValueStream contents"! !

!XMLTokenizer methodsFor: 'tokenizing' stamp: 'jmv 9/27/2021 15:39:57'!
nextCharReference
	"See http://en.wikipedia.org/wiki/Numeric_character_reference
	Note: If the NCR can not be represented as a Cuis Character, answer a String with an NCR."
	| base asciiCodeOrCodePoint |

	self next == $#
		ifFalse: [self errorExpected: 'character reference'].
	base _ self peek == $x
		ifTrue: [
			self next.
			16]
		ifFalse: [10].

	asciiCodeOrCodePoint _ [self readNumberBase: base] on: Error do: [:ex | self errorExpected: 'Number.'].
	self next == $;
		ifFalse: [self errorExpected: '";"'].

	^(Character codePoint: asciiCodeOrCodePoint trySimilar: true) ifNil: [
		String streamContents: [ :strm | String addUnicodeCodePoint: asciiCodeOrCodePoint to: strm hex: (base=16) ]].! !

!XMLTokenizer methodsFor: 'tokenizing' stamp: 'jmv 9/27/2021 15:54:29'!
nextEntityValue
	| delimiterChar entityValueStream nextChar nextPeek referenceString entity entityValue |
	delimiterChar _ self next.
	(delimiterChar == $"
		or: [delimiterChar == $'])
		ifFalse: [self errorExpected: 'Entity value delimiter expected.'].

	entityValueStream _ WriteStream on: (String new).
	[
	nextPeek _ nextChar _ self peek.
	nextChar ifNil: [self errorExpected: 'Character expected.'].
	nextChar == $&
		ifTrue: [
			self next.
			self peek == $#
				ifTrue: [
					nextPeek _ nil.
					nextChar _ self nextCharReference]
				ifFalse: [
					referenceString _ self nextLiteral.
					self next == $;
						ifFalse: [self errorExpected: ';'].
					entity _ self entity: referenceString.
					entityValue _ entity valueForContext: #entityValue.
					self pushStream: (ReadStream on: entityValue asString).
					nextPeek _ nextChar _ self next]]
		ifFalse: [
			nextChar == $%
				ifTrue: [
					self skipSeparators.
					referenceString _ self nextLiteral.
					nextChar _ self handleEntity: referenceString in: #entityValue.
					nextPeek _ nextChar _ self next]
				ifFalse: [self next]].
	nextPeek == delimiterChar]
		whileFalse: [
			nextChar ifNotNil: [
				nextChar isString
					ifFalse: [entityValueStream nextPut: nextChar]
					ifTrue: [entityValueStream nextPutAll: nextChar]]].
	^entityValueStream contents! !

!XMLTokenizer methodsFor: 'tokenizing' stamp: 'jmv 9/27/2021 15:55:03'!
nextLiteral
	| resultStream nextChar resultString |
	resultStream _ (String new: 10) writeStream.
	((nextChar _ self peek) isLetter
		or: [nextChar == $_])
		ifFalse: [self errorExpected: 'Name literal.'].
	[nextChar _ self peek.
	(LiteralChars at: nextChar asciiValue+1)
		ifTrue: [
			nextChar == $&
				ifTrue: [
					nextChar _ self next.
					self peek == $#
						ifTrue: [resultStream nextPutAll: self nextCharReference asString]
						ifFalse: [^resultStream contents]]
				ifFalse: [
					resultStream nextPut: self next]]
		ifFalse: [resultString _ resultStream contents.
			resultString isEmpty
				ifTrue: [self errorExpected: 'Name literal']
				ifFalse: [^resultString]]] repeat! !

!XMLTokenizer methodsFor: 'tokenizing' stamp: 'jmv 9/27/2021 15:50:20'!
nextPCData
	| resultStream nextChar referenceString entity entityValue nextPeek p |
	resultStream _ (String new: 10) writeStream.
	self validating
		ifFalse: [
			[
				p _ self peek.
				p isNil or: [p == $<]
				]
				whileFalse: [resultStream nextPut: self next].
			^self handlePCData: resultStream contents].

	[
	nextPeek _ nextChar _ self peek.
	nextChar ifNil: [self errorExpected: 'Character expected.'].
	nextChar == $&
		ifTrue: [
			self next.
			self peek == $#
				ifTrue: [
					nextPeek _ nil.
					nextChar _ self nextCharReference]
				ifFalse: [
					referenceString _ self nextLiteral.
					self next == $;
						ifFalse: [self errorExpected: ';'].
					entity _ self entity: referenceString.
					entityValue _ entity valueForContext: #content.
					(self class isCharEscape: entityValue)
						ifTrue: [
							nextPeek _ nil.
							nextChar _ entityValue]
						ifFalse: [
							entityValue _ entityValue asString.
							entityValue isEmpty
								ifTrue: [nextPeek _ nextChar _ nil]
								ifFalse: [
									self pushStream: (ReadStream on: entityValue asString).
									nextPeek _ nextChar _ self peek]]]]
		ifFalse: [nextPeek == $< ifFalse: [self next]].
	nextPeek == $<]
		whileFalse: [
			nextChar ifNotNil: [
				nextChar isString
					ifFalse: [resultStream nextPut: nextChar]
					ifTrue: [resultStream nextPutAll: nextChar]]].
	self handlePCData: resultStream contents! !

