Skip to content

fix: Make the JSON tokenizer and writer RFC 8259 compliant #48

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
keelerm84 merged 1 commit into from
Jul 29, 2026
Merged

fix: Make the JSON tokenizer and writer RFC 8259 compliant #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
202 changes: 140 additions & 62 deletions jreader/token_reader_default.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"io"
"strconv"
"unicode"
"unicode/utf16"
"unicode/utf8"
)

Expand Down Expand Up @@ -367,7 +368,9 @@ func (r *tokenReader) skipWhitespaceAndReadByte() (byte, bool) {
if !ok {
return 0, false
}
if !unicode.IsSpace(rune(ch)) {
// JSON permits only these four whitespace characters between tokens. Any other
// character (including other Unicode spaces) is the start of a token.
if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' {
r.lastPos = r.pos - 1
return ch, true
}
Expand All @@ -390,67 +393,96 @@ func (r *tokenReader) consumeASCIILowercaseAlphabeticChars() int {
return n
}

func (r *tokenReader) readNumber(_ byte) (float64, bool) {
startPos := r.lastPos
func isDigit(b byte) bool {
return b >= '0' && b <= '9'
}

// consumeDigits advances past any decimal digits in data starting at index i, returning the
// index of the first non-digit (or the end of the input).
func consumeDigits(data []byte, i, n int) int {
for i < n && isDigit(data[i]) {
i++
}
return i
}

func (r *tokenReader) readNumber(first byte) (float64, bool) {
// The grammar is: [ - ] int [ frac ] [ exp ], where int is a single 0 or a 1-9 digit
// followed by more digits, frac is '.' followed by at least one digit, and exp is
// [eE][-+]? followed by at least one digit. We scan the input directly by index rather
// than through readByte/unreadByte, then leave r.pos pointing just past the number.
start := r.lastPos
data, n := r.data, r.len
i := start + 1 // the first byte has already been read
isFloat := false
var ch byte
var ok bool
for {
ch, ok = r.readByte()
if !ok {
break
}
if (ch < '0' || ch > '9') && (ch != '.' || isFloat) {
break
}
if ch == '.' {
isFloat = true

// Optional minus sign, then the first digit of the integer part.
if first == '-' {
if i >= n || !isDigit(data[i]) {
return 0, false
}
first = data[i]
i++
}
hasExponent := false
if ch == 'e' || ch == 'E' {
// exponent must match this regex: [eE][-+]?[0-9]+
ch, ok = r.readByte()
if !ok {
return 0, false

// Integer part.
if first == '0' {
if i < n && isDigit(data[i]) {
return 0, false // a leading zero cannot be followed by another digit
}
if ch == '+' || ch == '-' { //nolint:gocritic,revive
} else if ch >= '0' && ch <= '9' {
r.unreadByte()
} else {
} else {
i = consumeDigits(data, i, n)
}

// Fractional part: a decimal point must be followed by at least one digit.
if i < n && data[i] == '.' {
isFloat = true
i++
if i >= n || !isDigit(data[i]) {
return 0, false
}
for {
ch, ok = r.readByte()
if !ok {
break
}
if ch < '0' || ch > '9' {
r.unreadByte()
break
}
hasExponent = true
i = consumeDigits(data, i, n)
}

// Exponent part: [eE][-+]? followed by at least one digit.
if i < n && (data[i] == 'e' || data[i] == 'E') {
isFloat = true
i++
if i < n && (data[i] == '+' || data[i] == '-') {
i++
}
if !hasExponent {
if i >= n || !isDigit(data[i]) {
return 0, false
}
isFloat = true
} else { //nolint:gocritic
if ok {
r.unreadByte()
i = consumeDigits(data, i, n)
}

r.pos = i
chars := data[start:i]
if !isFloat {
if num, ok := parseIntFromBytes(chars); ok {
return float64(num), true
}
// The integer literal overflows int64. Fall through to float parsing, which yields the
// same value encoding/json produces for magnitudes within float64 range (and rejects
// those beyond it, consistent with out-of-range float literals).
}
chars := r.data[startPos:r.pos]
if isFloat {
// Unfortunately, strconv.ParseFloat requires a string - there is no []byte equivalent. This means we can't
// avoid a heap allocation here. Easyjson works around this by creating an unsafe string that points directly
// at the existing bytes, but in our default implementation we can't use unsafe.
n, err := strconv.ParseFloat(string(chars), 64)
return n, err == nil
} else { //nolint:revive
n, ok := parseIntFromBytes(chars)
return float64(n), ok
// Unfortunately, strconv.ParseFloat requires a string - there is no []byte equivalent. This means we can't
// avoid a heap allocation here. Easyjson works around this by creating an unsafe string that points directly
// at the existing bytes, but in our default implementation we can't use unsafe.
num, err := strconv.ParseFloat(string(chars), 64)
return num, err == nil
}

// beginEscapedCopy switches readString off its zero-copy fast path by copying the literal
// prefix [startPos:endPos) that has been validated so far into a fresh buffer with headroom
// for the decoded remainder.
func beginEscapedCopy(data []byte, startPos, endPos int) []byte {
buf := make([]byte, endPos-startPos, endPos-startPos+20)
if endPos > startPos {
copy(buf, data[startPos:endPos])
}
return buf
}

func (r *tokenReader) readString() ([]byte, error) {
Expand All @@ -462,25 +494,33 @@ func (r *tokenReader) readString() ([]byte, error) {
_, _ = reader.Seek(int64(r.pos), io.SeekStart)

for {
ch, _, err := reader.ReadRune()
ch, size, err := reader.ReadRune()
if err != nil {
return nil, r.syntaxErrorOnLastToken(errMsgInvalidString)
}
if ch == '"' {
break
}
if ch < 0x20 {
// Control characters must be escaped inside a JSON string.
return nil, r.syntaxErrorOnLastToken(errMsgInvalidString)
}
if ch != '\\' {
if haveEscaped {
if ch == utf8.RuneError && size == 1 {
// An invalid UTF-8 byte. encoding/json substitutes the Unicode replacement
// character for these; do the same, which forces us off the zero-copy path.
if !haveEscaped {
chars = beginEscapedCopy(r.data, startPos, (r.len-reader.Len())-1)
haveEscaped = true
}
chars = appendRune(chars, utf8.RuneError)
} else if haveEscaped {
chars = appendRune(chars, ch)
}
continue
}
if !haveEscaped {
pos := (r.len - reader.Len()) - 1 // don't include the backslash we just read
chars = make([]byte, pos-startPos, pos-startPos+20)
if pos > startPos {
copy(chars, r.data[startPos:pos])
}
chars = beginEscapedCopy(r.data, startPos, (r.len-reader.Len())-1) // exclude the backslash just read
haveEscaped = true
}
ch, _, err = reader.ReadRune()
Expand All @@ -501,10 +541,9 @@ func (r *tokenReader) readString() ([]byte, error) {
case 't':
chars = appendRune(chars, '\t')
case 'u':
if ch, ok := readHexChar(&reader); ok {
chars = appendRune(chars, ch)
} else {
return nil, r.syntaxErrorOnLastToken(errMsgInvalidString)
chars, err = r.readUnicodeEscape(&reader, chars)
if err != nil {
return nil, err
}
default:
return nil, r.syntaxErrorOnLastToken(errMsgInvalidString)
Expand All @@ -525,6 +564,35 @@ func (r *tokenReader) readString() ([]byte, error) {
}
}

// readUnicodeEscape decodes a \u escape (the leading "\u" has already been consumed), combining
// a UTF-16 surrogate pair into a single code point when the escape is a surrogate followed by a
// valid pairing escape. A lone or invalid surrogate becomes the Unicode replacement character
// with the following bytes left to be parsed normally, matching encoding/json.
func (r *tokenReader) readUnicodeEscape(reader *bytes.Reader, chars []byte) ([]byte, error) {
decoded, ok := readHexChar(reader)
if !ok {
return nil, r.syntaxErrorOnLastToken(errMsgInvalidString)
}
if !utf16.IsSurrogate(decoded) {
return appendRune(chars, decoded), nil
}
mark := r.len - reader.Len()
combined := unicode.ReplacementChar
if b1, e1 := reader.ReadByte(); e1 == nil && b1 == '\\' {
if b2, e2 := reader.ReadByte(); e2 == nil && b2 == 'u' {
if low, lowOK := readHexChar(reader); lowOK {
if pair := utf16.DecodeRune(decoded, low); pair != unicode.ReplacementChar {
combined = pair
}
}
}
}
if combined == unicode.ReplacementChar {
_, _ = reader.Seek(int64(mark), io.SeekStart)
}
return appendRune(chars, combined), nil
}

func readHexChar(reader *bytes.Reader) (rune, bool) {
var digits [4]byte
for i := 0; i < 4; i++ {
Expand All @@ -551,6 +619,10 @@ func (r *tokenReader) syntaxErrorOnNextToken(msg string) error {
}

// This is faster than creating a string to pass to strconv.Atoi.
//
// It returns ok == false when the literal's magnitude exceeds MaxInt64 (including int64's own
// minimum, -2^63, whose magnitude is 2^63) so the caller can fall back to float parsing rather
// than receive a silently wrapped value.
func parseIntFromBytes(chars []byte) (int64, bool) {
negate := false
p := 0
Expand All @@ -565,8 +637,14 @@ func parseIntFromBytes(chars []byte) (int64, bool) {
return 0, false
}
}
const maxInt64 = 1<<63 - 1
for p < len(chars) {
ret = ret*10 + int64(chars[p]-'0')
d := int64(chars[p] - '0')
// Signal overflow rather than silently wrapping; the caller falls back to float parsing.
if ret > (maxInt64-d)/10 {
return 0, false
}
ret = ret*10 + d
p++
}
if negate {
Expand Down
Loading
Loading