fix(go.sum): update ResolveSpec dependency to v1.0.87
CI / build-and-test (push) Failing after 1s
Release / release (push) Failing after 19m26s

This commit is contained in:
Hein
2026-06-23 13:17:16 +02:00
parent 0227912325
commit 1adf50e3db
2436 changed files with 1078758 additions and 114 deletions
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 Segment.io, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+40
View File
@@ -0,0 +1,40 @@
//go:generate go run equal_fold_asm.go -out equal_fold_amd64.s -stubs equal_fold_amd64.go
package ascii
import (
"github.com/segmentio/asm/ascii"
)
// EqualFold is a version of bytes.EqualFold designed to work on ASCII input
// instead of UTF-8.
//
// When the program has guarantees that the input is composed of ASCII
// characters only, it allows for greater optimizations.
func EqualFold(a, b []byte) bool {
return ascii.EqualFold(a, b)
}
func HasPrefixFold(s, prefix []byte) bool {
return ascii.HasPrefixFold(s, prefix)
}
func HasSuffixFold(s, suffix []byte) bool {
return ascii.HasSuffixFold(s, suffix)
}
// EqualFoldString is a version of strings.EqualFold designed to work on ASCII
// input instead of UTF-8.
//
// When the program has guarantees that the input is composed of ASCII
// characters only, it allows for greater optimizations.
func EqualFoldString(a, b string) bool {
return ascii.EqualFoldString(a, b)
}
func HasPrefixFoldString(s, prefix string) bool {
return ascii.HasPrefixFoldString(s, prefix)
}
func HasSuffixFoldString(s, suffix string) bool {
return ascii.HasSuffixFoldString(s, suffix)
}
+26
View File
@@ -0,0 +1,26 @@
//go:generate go run valid_asm.go -out valid_amd64.s -stubs valid_amd64.go
package ascii
import (
"github.com/segmentio/asm/ascii"
)
// Valid returns true if b contains only ASCII characters.
func Valid(b []byte) bool {
return ascii.Valid(b)
}
// ValidBytes returns true if b is an ASCII character.
func ValidByte(b byte) bool {
return ascii.ValidByte(b)
}
// ValidBytes returns true if b is an ASCII character.
func ValidRune(r rune) bool {
return ascii.ValidRune(r)
}
// ValidString returns true if s contains only ASCII characters.
func ValidString(s string) bool {
return ascii.ValidString(s)
}
+26
View File
@@ -0,0 +1,26 @@
//go:generate go run valid_print_asm.go -out valid_print_amd64.s -stubs valid_print_amd64.go
package ascii
import (
"github.com/segmentio/asm/ascii"
)
// Valid returns true if b contains only printable ASCII characters.
func ValidPrint(b []byte) bool {
return ascii.ValidPrint(b)
}
// ValidBytes returns true if b is an ASCII character.
func ValidPrintByte(b byte) bool {
return ascii.ValidPrintByte(b)
}
// ValidBytes returns true if b is an ASCII character.
func ValidPrintRune(r rune) bool {
return ascii.ValidPrintRune(r)
}
// ValidString returns true if s contains only printable ASCII characters.
func ValidPrintString(s string) bool {
return ascii.ValidPrintString(s)
}
+185
View File
@@ -0,0 +1,185 @@
package iso8601
import (
"encoding/binary"
"errors"
"time"
"unsafe"
)
var (
errInvalidTimestamp = errors.New("invalid ISO8601 timestamp")
errMonthOutOfRange = errors.New("month out of range")
errDayOutOfRange = errors.New("day out of range")
errHourOutOfRange = errors.New("hour out of range")
errMinuteOutOfRange = errors.New("minute out of range")
errSecondOutOfRange = errors.New("second out of range")
)
// Parse parses an ISO8601 timestamp, e.g. "2021-03-25T21:36:12Z".
func Parse(input string) (time.Time, error) {
b := unsafeStringToBytes(input)
if len(b) >= 20 && len(b) <= 30 && b[len(b)-1] == 'Z' {
if len(b) == 21 || (len(b) > 21 && b[19] != '.') {
return time.Time{}, errInvalidTimestamp
}
t1 := binary.LittleEndian.Uint64(b)
t2 := binary.LittleEndian.Uint64(b[8:16])
t3 := uint64(b[16]) | uint64(b[17])<<8 | uint64(b[18])<<16 | uint64('Z')<<24
// Check for valid separators by masking input with " - - T : : Z".
// If separators are all valid, replace them with a '0' (0x30) byte and
// check all bytes are now numeric.
if !match(t1, mask1) || !match(t2, mask2) || !match(t3, mask3) {
return time.Time{}, errInvalidTimestamp
}
t1 ^= replace1
t2 ^= replace2
t3 ^= replace3
if (nonNumeric(t1) | nonNumeric(t2) | nonNumeric(t3)) != 0 {
return time.Time{}, errInvalidTimestamp
}
t1 -= zero
t2 -= zero
t3 -= zero
year := (t1&0xF)*1000 + (t1>>8&0xF)*100 + (t1>>16&0xF)*10 + (t1 >> 24 & 0xF)
month := (t1>>40&0xF)*10 + (t1 >> 48 & 0xF)
day := (t2&0xF)*10 + (t2 >> 8 & 0xF)
hour := (t2>>24&0xF)*10 + (t2 >> 32 & 0xF)
minute := (t2>>48&0xF)*10 + (t2 >> 56)
second := (t3>>8&0xF)*10 + (t3 >> 16)
nanos := int64(0)
if len(b) > 20 {
for _, c := range b[20 : len(b)-1] {
if c < '0' || c > '9' {
return time.Time{}, errInvalidTimestamp
}
nanos = (nanos * 10) + int64(c-'0')
}
nanos *= pow10[30-len(b)]
}
if err := validate(year, month, day, hour, minute, second); err != nil {
return time.Time{}, err
}
unixSeconds := int64(daysSinceEpoch(year, month, day))*86400 + int64(hour*3600+minute*60+second)
return time.Unix(unixSeconds, nanos).UTC(), nil
}
// Fallback to using time.Parse().
t, err := time.Parse(time.RFC3339Nano, input)
if err != nil {
// Override (and don't wrap) the error here. The error returned by
// time.Parse() is dynamic, and includes a reference to the input
// string. By overriding the error, we guarantee that the input string
// doesn't escape.
return time.Time{}, errInvalidTimestamp
}
return t, nil
}
var pow10 = []int64{1, 10, 100, 1000, 1e4, 1e5, 1e6, 1e7, 1e8}
const (
mask1 = 0x2d00002d00000000 // YYYY-MM-
mask2 = 0x00003a0000540000 // DDTHH:MM
mask3 = 0x000000005a00003a // :SSZ____
// Generate masks that replace the separators with a numeric byte.
// The input must have valid separators. XOR with the separator bytes
// to zero them out and then XOR with 0x30 to replace them with '0'.
replace1 = mask1 ^ 0x3000003000000000
replace2 = mask2 ^ 0x0000300000300000
replace3 = mask3 ^ 0x3030303030000030
lsb = ^uint64(0) / 255
msb = lsb * 0x80
zero = lsb * '0'
nine = lsb * '9'
)
func validate(year, month, day, hour, minute, second uint64) error {
if day == 0 || day > 31 {
return errDayOutOfRange
}
if month == 0 || month > 12 {
return errMonthOutOfRange
}
if hour >= 24 {
return errHourOutOfRange
}
if minute >= 60 {
return errMinuteOutOfRange
}
if second >= 60 {
return errSecondOutOfRange
}
if month == 2 && (day > 29 || (day == 29 && !isLeapYear(year))) {
return errDayOutOfRange
}
if day == 31 {
switch month {
case 4, 6, 9, 11:
return errDayOutOfRange
}
}
return nil
}
func match(u, mask uint64) bool {
return (u & mask) == mask
}
func nonNumeric(u uint64) uint64 {
// Derived from https://graphics.stanford.edu/~seander/bithacks.html#HasLessInWord.
// Subtract '0' (0x30) from each byte so that the MSB is set in each byte
// if there's a byte less than '0' (0x30). Add 0x46 (0x7F-'9') so that the
// MSB is set if there's a byte greater than '9' (0x39). To handle overflow
// when adding 0x46, include the MSB from the input bytes in the final mask.
// Remove all but the MSBs and then you're left with a mask where each
// non-numeric byte from the input has its MSB set in the output.
return ((u - zero) | (u + (^msb - nine)) | u) & msb
}
func daysSinceEpoch(year, month, day uint64) uint64 {
// Derived from https://blog.reverberate.org/2020/05/12/optimizing-date-algorithms.html.
monthAdjusted := month - 3
var carry uint64
if monthAdjusted > month {
carry = 1
}
var adjust uint64
if carry == 1 {
adjust = 12
}
yearAdjusted := year + 4800 - carry
monthDays := ((monthAdjusted+adjust)*62719 + 769) / 2048
leapDays := yearAdjusted/4 - yearAdjusted/100 + yearAdjusted/400
return yearAdjusted*365 + leapDays + monthDays + (day - 1) - 2472632
}
func isLeapYear(y uint64) bool {
return (y%4) == 0 && ((y%100) != 0 || (y%400) == 0)
}
func unsafeStringToBytes(s string) []byte {
return *(*[]byte)(unsafe.Pointer(&sliceHeader{
Data: *(*unsafe.Pointer)(unsafe.Pointer(&s)),
Len: len(s),
Cap: len(s),
}))
}
// sliceHeader is like reflect.SliceHeader but the Data field is a
// unsafe.Pointer instead of being a uintptr to avoid invalid
// conversions from uintptr to unsafe.Pointer.
type sliceHeader struct {
Data unsafe.Pointer
Len int
Cap int
}
+179
View File
@@ -0,0 +1,179 @@
package iso8601
// ValidFlags is a bitset type used to configure the behavior of the Valid
// function.
type ValidFlags int
const (
// Strict is a validation flag used to represent a string iso8601 validation
// (this is the default).
Strict ValidFlags = 0
// AllowSpaceSeparator allows the presence of a space instead of a 'T' as
// separator between the date and time.
AllowSpaceSeparator ValidFlags = 1 << iota
// AllowMissingTime allows the value to contain only a date.
AllowMissingTime
// AllowMissingSubsecond allows the value to contain only a date and time.
AllowMissingSubsecond
// AllowMissingTimezone allows the value to be missing the timezone
// information.
AllowMissingTimezone
// AllowNumericTimezone allows the value to represent timezones in their
// numeric form.
AllowNumericTimezone
// Flexible is a combination of all validation flag that allow for
// non-strict checking of the input value.
Flexible = AllowSpaceSeparator | AllowMissingTime | AllowMissingSubsecond | AllowMissingTimezone | AllowNumericTimezone
)
// Valid check value to verify whether or not it is a valid iso8601 time
// representation.
func Valid(value string, flags ValidFlags) bool {
var ok bool
// year
if value, ok = readDigits(value, 4, 4); !ok {
return false
}
if value, ok = readByte(value, '-'); !ok {
return false
}
// month
if value, ok = readDigits(value, 2, 2); !ok {
return false
}
if value, ok = readByte(value, '-'); !ok {
return false
}
// day
if value, ok = readDigits(value, 2, 2); !ok {
return false
}
if len(value) == 0 && (flags&AllowMissingTime) != 0 {
return true // date only
}
// separator
if value, ok = readByte(value, 'T'); !ok {
if (flags & AllowSpaceSeparator) == 0 {
return false
}
if value, ok = readByte(value, ' '); !ok {
return false
}
}
// hour
if value, ok = readDigits(value, 2, 2); !ok {
return false
}
if value, ok = readByte(value, ':'); !ok {
return false
}
// minute
if value, ok = readDigits(value, 2, 2); !ok {
return false
}
if value, ok = readByte(value, ':'); !ok {
return false
}
// second
if value, ok = readDigits(value, 2, 2); !ok {
return false
}
// microsecond
if value, ok = readByte(value, '.'); !ok {
if (flags & AllowMissingSubsecond) == 0 {
return false
}
} else {
if value, ok = readDigits(value, 1, 9); !ok {
return false
}
}
if len(value) == 0 && (flags&AllowMissingTimezone) != 0 {
return true // date and time
}
// timezone
if value, ok = readByte(value, 'Z'); ok {
return len(value) == 0
}
if (flags & AllowSpaceSeparator) != 0 {
value, _ = readByte(value, ' ')
}
if value, ok = readByte(value, '+'); !ok {
if value, ok = readByte(value, '-'); !ok {
return false
}
}
// timezone hour
if value, ok = readDigits(value, 2, 2); !ok {
return false
}
if value, ok = readByte(value, ':'); !ok {
if (flags & AllowNumericTimezone) == 0 {
return false
}
}
// timezone minute
if value, ok = readDigits(value, 2, 2); !ok {
return false
}
return len(value) == 0
}
func readDigits(value string, min, max int) (string, bool) {
if len(value) < min {
return value, false
}
i := 0
for i < max && i < len(value) && isDigit(value[i]) {
i++
}
if i < max && i < min {
return value, false
}
return value[i:], true
}
func readByte(value string, c byte) (string, bool) {
if len(value) == 0 {
return value, false
}
if value[0] != c {
return value, false
}
return value[1:], true
}
func isDigit(c byte) bool {
return '0' <= c && c <= '9'
}
+76
View File
@@ -0,0 +1,76 @@
# encoding/json [![GoDoc](https://godoc.org/github.com/segmentio/encoding/json?status.svg)](https://godoc.org/github.com/segmentio/encoding/json)
Go package offering a replacement implementation of the standard library's
[`encoding/json`](https://golang.org/pkg/encoding/json/) package, with much
better performance.
## Usage
The exported API of this package mirrors the standard library's
[`encoding/json`](https://golang.org/pkg/encoding/json/) package, the only
change needed to take advantage of the performance improvements is the import
path of the `json` package, from:
```go
import (
"encoding/json"
)
```
to
```go
import (
"github.com/segmentio/encoding/json"
)
```
One way to gain higher encoding throughput is to disable HTML escaping.
It allows the string encoding to use a much more efficient code path which
does not require parsing UTF-8 runes most of the time.
## Performance Improvements
The internal implementation uses a fair amount of unsafe operations (untyped
code, pointer arithmetic, etc...) to avoid using reflection as much as possible,
which is often the reason why serialization code has a large CPU and memory
footprint.
The package aims for zero unnecessary dynamic memory allocations and hot code
paths that are mostly free from calls into the reflect package.
## Compatibility with encoding/json
This package aims to be a drop-in replacement, therefore it is tested to behave
exactly like the standard library's package. However, there are still a few
missing features that have not been ported yet:
- Streaming decoder, currently the `Decoder` implementation offered by the
package does not support progressively reading values from a JSON array (unlike
the standard library). In our experience this is a very rare use-case, if you
need it you're better off sticking to the standard library, or spend a bit of
time implementing it in here ;)
Note that none of those features should result in performance degradations if
they were implemented in the package, and we welcome contributions!
## Trade-offs
As one would expect, we had to make a couple of trade-offs to achieve greater
performance than the standard library, but there were also features that we
did not want to give away.
Other open-source packages offering a reduced CPU and memory footprint usually
do so by designing a different API, or require code generation (therefore adding
complexity to the build process). These were not acceptable conditions for us,
as we were not willing to trade off developer productivity for better runtime
performance. To achieve this, we chose to exactly replicate the standard
library interfaces and behavior, which meant the package implementation was the
only area that we were able to work with. The internals of this package make
heavy use of unsafe pointer arithmetics and other performance optimizations,
and therefore are not as approachable as typical Go programs. Basically, we put
a bigger burden on maintainers to achieve better runtime cost without
sacrificing developer productivity.
For these reasons, we also don't believe that this code should be ported upstream
to the standard `encoding/json` package. The standard library has to remain
readable and approachable to maximize stability and maintainability, and make
projects like this one possible because a high quality reference implementation
already exists.
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+970
View File
@@ -0,0 +1,970 @@
package json
import (
"encoding"
"fmt"
"math"
"reflect"
"sort"
"strconv"
"sync"
"time"
"unicode/utf8"
"unsafe"
"github.com/segmentio/asm/base64"
)
const hex = "0123456789abcdef"
func (e encoder) encodeNull(b []byte, p unsafe.Pointer) ([]byte, error) {
return append(b, "null"...), nil
}
func (e encoder) encodeBool(b []byte, p unsafe.Pointer) ([]byte, error) {
if *(*bool)(p) {
return append(b, "true"...), nil
}
return append(b, "false"...), nil
}
func (e encoder) encodeInt(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendInt(b, int64(*(*int)(p))), nil
}
func (e encoder) encodeInt8(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendInt(b, int64(*(*int8)(p))), nil
}
func (e encoder) encodeInt16(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendInt(b, int64(*(*int16)(p))), nil
}
func (e encoder) encodeInt32(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendInt(b, int64(*(*int32)(p))), nil
}
func (e encoder) encodeInt64(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendInt(b, *(*int64)(p)), nil
}
func (e encoder) encodeUint(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendUint(b, uint64(*(*uint)(p))), nil
}
func (e encoder) encodeUintptr(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendUint(b, uint64(*(*uintptr)(p))), nil
}
func (e encoder) encodeUint8(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendUint(b, uint64(*(*uint8)(p))), nil
}
func (e encoder) encodeUint16(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendUint(b, uint64(*(*uint16)(p))), nil
}
func (e encoder) encodeUint32(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendUint(b, uint64(*(*uint32)(p))), nil
}
func (e encoder) encodeUint64(b []byte, p unsafe.Pointer) ([]byte, error) {
return appendUint(b, *(*uint64)(p)), nil
}
func (e encoder) encodeFloat32(b []byte, p unsafe.Pointer) ([]byte, error) {
return e.encodeFloat(b, float64(*(*float32)(p)), 32)
}
func (e encoder) encodeFloat64(b []byte, p unsafe.Pointer) ([]byte, error) {
return e.encodeFloat(b, *(*float64)(p), 64)
}
func (e encoder) encodeFloat(b []byte, f float64, bits int) ([]byte, error) {
switch {
case math.IsNaN(f):
return b, &UnsupportedValueError{Value: reflect.ValueOf(f), Str: "NaN"}
case math.IsInf(f, 0):
return b, &UnsupportedValueError{Value: reflect.ValueOf(f), Str: "inf"}
}
// Convert as if by ES6 number to string conversion.
// This matches most other JSON generators.
// See golang.org/issue/6384 and golang.org/issue/14135.
// Like fmt %g, but the exponent cutoffs are different
// and exponents themselves are not padded to two digits.
abs := math.Abs(f)
fmt := byte('f')
// Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right.
if abs != 0 {
if bits == 64 && (abs < 1e-6 || abs >= 1e21) || bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) {
fmt = 'e'
}
}
b = strconv.AppendFloat(b, f, fmt, -1, int(bits))
if fmt == 'e' {
// clean up e-09 to e-9
n := len(b)
if n >= 4 && b[n-4] == 'e' && b[n-3] == '-' && b[n-2] == '0' {
b[n-2] = b[n-1]
b = b[:n-1]
}
}
return b, nil
}
func (e encoder) encodeNumber(b []byte, p unsafe.Pointer) ([]byte, error) {
n := *(*Number)(p)
if n == "" {
n = "0"
}
d := decoder{}
_, _, _, err := d.parseNumber(stringToBytes(string(n)))
if err != nil {
return b, err
}
return append(b, n...), nil
}
func (e encoder) encodeString(b []byte, p unsafe.Pointer) ([]byte, error) {
s := *(*string)(p)
if len(s) == 0 {
return append(b, `""`...), nil
}
i := 0
j := 0
escapeHTML := (e.flags & EscapeHTML) != 0
b = append(b, '"')
if len(s) >= 8 {
if j = escapeIndex(s, escapeHTML); j < 0 {
return append(append(b, s...), '"'), nil
}
}
for j < len(s) {
c := s[j]
if c >= 0x20 && c <= 0x7f && c != '\\' && c != '"' && (!escapeHTML || (c != '<' && c != '>' && c != '&')) {
// fast path: most of the time, printable ascii characters are used
j++
continue
}
switch c {
case '\\', '"', '\b', '\f', '\n', '\r', '\t':
b = append(b, s[i:j]...)
b = append(b, '\\', escapeByteRepr(c))
i = j + 1
j = j + 1
continue
case '<', '>', '&':
b = append(b, s[i:j]...)
b = append(b, `\u00`...)
b = append(b, hex[c>>4], hex[c&0xF])
i = j + 1
j = j + 1
continue
}
// This encodes bytes < 0x20 except for \t, \n and \r.
if c < 0x20 {
b = append(b, s[i:j]...)
b = append(b, `\u00`...)
b = append(b, hex[c>>4], hex[c&0xF])
i = j + 1
j = j + 1
continue
}
r, size := utf8.DecodeRuneInString(s[j:])
if r == utf8.RuneError && size == 1 {
b = append(b, s[i:j]...)
b = append(b, `\ufffd`...)
i = j + size
j = j + size
continue
}
switch r {
case '\u2028', '\u2029':
// U+2028 is LINE SEPARATOR.
// U+2029 is PARAGRAPH SEPARATOR.
// They are both technically valid characters in JSON strings,
// but don't work in JSONP, which has to be evaluated as JavaScript,
// and can lead to security holes there. It is valid JSON to
// escape them, so we do so unconditionally.
// See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion.
b = append(b, s[i:j]...)
b = append(b, `\u202`...)
b = append(b, hex[r&0xF])
i = j + size
j = j + size
continue
}
j += size
}
b = append(b, s[i:]...)
b = append(b, '"')
return b, nil
}
func (e encoder) encodeToString(b []byte, p unsafe.Pointer, encode encodeFunc) ([]byte, error) {
i := len(b)
b, err := encode(e, b, p)
if err != nil {
return b, err
}
j := len(b)
s := b[i:]
if b, err = e.encodeString(b, unsafe.Pointer(&s)); err != nil {
return b, err
}
n := copy(b[i:], b[j:])
return b[:i+n], nil
}
func (e encoder) encodeBytes(b []byte, p unsafe.Pointer) ([]byte, error) {
v := *(*[]byte)(p)
if v == nil {
return append(b, "null"...), nil
}
n := base64.StdEncoding.EncodedLen(len(v)) + 2
if avail := cap(b) - len(b); avail < n {
newB := make([]byte, cap(b)+(n-avail))
copy(newB, b)
b = newB[:len(b)]
}
i := len(b)
j := len(b) + n
b = b[:j]
b[i] = '"'
base64.StdEncoding.Encode(b[i+1:j-1], v)
b[j-1] = '"'
return b, nil
}
func (e encoder) encodeDuration(b []byte, p unsafe.Pointer) ([]byte, error) {
b = append(b, '"')
b = appendDuration(b, *(*time.Duration)(p))
b = append(b, '"')
return b, nil
}
func (e encoder) encodeTime(b []byte, p unsafe.Pointer) ([]byte, error) {
t := *(*time.Time)(p)
b = append(b, '"')
b = t.AppendFormat(b, time.RFC3339Nano)
b = append(b, '"')
return b, nil
}
func (e encoder) encodeArray(b []byte, p unsafe.Pointer, n int, size uintptr, t reflect.Type, encode encodeFunc) ([]byte, error) {
start := len(b)
var err error
b = append(b, '[')
for i := range n {
if i != 0 {
b = append(b, ',')
}
if b, err = encode(e, b, unsafe.Pointer(uintptr(p)+(uintptr(i)*size))); err != nil {
return b[:start], err
}
}
b = append(b, ']')
return b, nil
}
func (e encoder) encodeSlice(b []byte, p unsafe.Pointer, size uintptr, t reflect.Type, encode encodeFunc) ([]byte, error) {
s := (*slice)(p)
if s.data == nil && s.len == 0 && s.cap == 0 {
return append(b, "null"...), nil
}
return e.encodeArray(b, s.data, s.len, size, t, encode)
}
func (e encoder) encodeMap(b []byte, p unsafe.Pointer, t reflect.Type, encodeKey, encodeValue encodeFunc, sortKeys sortFunc) ([]byte, error) {
m := reflect.NewAt(t, p).Elem()
if m.IsNil() {
return append(b, "null"...), nil
}
keys := m.MapKeys()
if sortKeys != nil && (e.flags&SortMapKeys) != 0 {
sortKeys(keys)
}
start := len(b)
var err error
b = append(b, '{')
for i, k := range keys {
v := m.MapIndex(k)
if i != 0 {
b = append(b, ',')
}
if b, err = encodeKey(e, b, (*iface)(unsafe.Pointer(&k)).ptr); err != nil {
return b[:start], err
}
b = append(b, ':')
if b, err = encodeValue(e, b, (*iface)(unsafe.Pointer(&v)).ptr); err != nil {
return b[:start], err
}
}
b = append(b, '}')
return b, nil
}
type element struct {
key string
val any
raw RawMessage
}
type mapslice struct {
elements []element
}
func (m *mapslice) Len() int { return len(m.elements) }
func (m *mapslice) Less(i, j int) bool { return m.elements[i].key < m.elements[j].key }
func (m *mapslice) Swap(i, j int) { m.elements[i], m.elements[j] = m.elements[j], m.elements[i] }
var mapslicePool = sync.Pool{
New: func() any { return new(mapslice) },
}
func (e encoder) encodeMapStringInterface(b []byte, p unsafe.Pointer) ([]byte, error) {
m := *(*map[string]any)(p)
if m == nil {
return append(b, "null"...), nil
}
if (e.flags & SortMapKeys) == 0 {
// Optimized code path when the program does not need the map keys to be
// sorted.
b = append(b, '{')
if len(m) != 0 {
var err error
i := 0
for k, v := range m {
if i != 0 {
b = append(b, ',')
}
b, _ = e.encodeString(b, unsafe.Pointer(&k))
b = append(b, ':')
b, err = Append(b, v, e.flags)
if err != nil {
return b, err
}
i++
}
}
b = append(b, '}')
return b, nil
}
s := mapslicePool.Get().(*mapslice)
if cap(s.elements) < len(m) {
s.elements = make([]element, 0, align(10, uintptr(len(m))))
}
for key, val := range m {
s.elements = append(s.elements, element{key: key, val: val})
}
sort.Sort(s)
start := len(b)
var err error
b = append(b, '{')
for i, elem := range s.elements {
if i != 0 {
b = append(b, ',')
}
b, _ = e.encodeString(b, unsafe.Pointer(&elem.key))
b = append(b, ':')
b, err = Append(b, elem.val, e.flags)
if err != nil {
break
}
}
for i := range s.elements {
s.elements[i] = element{}
}
s.elements = s.elements[:0]
mapslicePool.Put(s)
if err != nil {
return b[:start], err
}
b = append(b, '}')
return b, nil
}
func (e encoder) encodeMapStringRawMessage(b []byte, p unsafe.Pointer) ([]byte, error) {
m := *(*map[string]RawMessage)(p)
if m == nil {
return append(b, "null"...), nil
}
if (e.flags & SortMapKeys) == 0 {
// Optimized code path when the program does not need the map keys to be
// sorted.
b = append(b, '{')
if len(m) != 0 {
var err error
i := 0
for k, v := range m {
if i != 0 {
b = append(b, ',')
}
// encodeString doesn't return errors so we ignore it here
b, _ = e.encodeString(b, unsafe.Pointer(&k))
b = append(b, ':')
b, err = e.encodeRawMessage(b, unsafe.Pointer(&v))
if err != nil {
break
}
i++
}
}
b = append(b, '}')
return b, nil
}
s := mapslicePool.Get().(*mapslice)
if cap(s.elements) < len(m) {
s.elements = make([]element, 0, align(10, uintptr(len(m))))
}
for key, raw := range m {
s.elements = append(s.elements, element{key: key, raw: raw})
}
sort.Sort(s)
start := len(b)
var err error
b = append(b, '{')
for i, elem := range s.elements {
if i != 0 {
b = append(b, ',')
}
b, _ = e.encodeString(b, unsafe.Pointer(&elem.key))
b = append(b, ':')
b, err = e.encodeRawMessage(b, unsafe.Pointer(&elem.raw))
if err != nil {
break
}
}
for i := range s.elements {
s.elements[i] = element{}
}
s.elements = s.elements[:0]
mapslicePool.Put(s)
if err != nil {
return b[:start], err
}
b = append(b, '}')
return b, nil
}
func (e encoder) encodeMapStringString(b []byte, p unsafe.Pointer) ([]byte, error) {
m := *(*map[string]string)(p)
if m == nil {
return append(b, "null"...), nil
}
if (e.flags & SortMapKeys) == 0 {
// Optimized code path when the program does not need the map keys to be
// sorted.
b = append(b, '{')
if len(m) != 0 {
i := 0
for k, v := range m {
if i != 0 {
b = append(b, ',')
}
// encodeString never returns an error so we ignore it here
b, _ = e.encodeString(b, unsafe.Pointer(&k))
b = append(b, ':')
b, _ = e.encodeString(b, unsafe.Pointer(&v))
i++
}
}
b = append(b, '}')
return b, nil
}
s := mapslicePool.Get().(*mapslice)
if cap(s.elements) < len(m) {
s.elements = make([]element, 0, align(10, uintptr(len(m))))
}
for key, val := range m {
v := val
s.elements = append(s.elements, element{key: key, val: &v})
}
sort.Sort(s)
b = append(b, '{')
for i, elem := range s.elements {
if i != 0 {
b = append(b, ',')
}
// encodeString never returns an error so we ignore it here
b, _ = e.encodeString(b, unsafe.Pointer(&elem.key))
b = append(b, ':')
b, _ = e.encodeString(b, unsafe.Pointer(elem.val.(*string)))
}
for i := range s.elements {
s.elements[i] = element{}
}
s.elements = s.elements[:0]
mapslicePool.Put(s)
b = append(b, '}')
return b, nil
}
func (e encoder) encodeMapStringStringSlice(b []byte, p unsafe.Pointer) ([]byte, error) {
m := *(*map[string][]string)(p)
if m == nil {
return append(b, "null"...), nil
}
stringSize := unsafe.Sizeof("")
if (e.flags & SortMapKeys) == 0 {
// Optimized code path when the program does not need the map keys to be
// sorted.
b = append(b, '{')
if len(m) != 0 {
var err error
i := 0
for k, v := range m {
if i != 0 {
b = append(b, ',')
}
b, _ = e.encodeString(b, unsafe.Pointer(&k))
b = append(b, ':')
b, err = e.encodeSlice(b, unsafe.Pointer(&v), stringSize, sliceStringType, encoder.encodeString)
if err != nil {
return b, err
}
i++
}
}
b = append(b, '}')
return b, nil
}
s := mapslicePool.Get().(*mapslice)
if cap(s.elements) < len(m) {
s.elements = make([]element, 0, align(10, uintptr(len(m))))
}
for key, val := range m {
v := val
s.elements = append(s.elements, element{key: key, val: &v})
}
sort.Sort(s)
start := len(b)
var err error
b = append(b, '{')
for i, elem := range s.elements {
if i != 0 {
b = append(b, ',')
}
b, _ = e.encodeString(b, unsafe.Pointer(&elem.key))
b = append(b, ':')
b, err = e.encodeSlice(b, unsafe.Pointer(elem.val.(*[]string)), stringSize, sliceStringType, encoder.encodeString)
if err != nil {
break
}
}
for i := range s.elements {
s.elements[i] = element{}
}
s.elements = s.elements[:0]
mapslicePool.Put(s)
if err != nil {
return b[:start], err
}
b = append(b, '}')
return b, nil
}
func (e encoder) encodeMapStringBool(b []byte, p unsafe.Pointer) ([]byte, error) {
m := *(*map[string]bool)(p)
if m == nil {
return append(b, "null"...), nil
}
if (e.flags & SortMapKeys) == 0 {
// Optimized code path when the program does not need the map keys to be
// sorted.
b = append(b, '{')
if len(m) != 0 {
i := 0
for k, v := range m {
if i != 0 {
b = append(b, ',')
}
// encodeString never returns an error so we ignore it here
b, _ = e.encodeString(b, unsafe.Pointer(&k))
if v {
b = append(b, ":true"...)
} else {
b = append(b, ":false"...)
}
i++
}
}
b = append(b, '}')
return b, nil
}
s := mapslicePool.Get().(*mapslice)
if cap(s.elements) < len(m) {
s.elements = make([]element, 0, align(10, uintptr(len(m))))
}
for key, val := range m {
s.elements = append(s.elements, element{key: key, val: val})
}
sort.Sort(s)
b = append(b, '{')
for i, elem := range s.elements {
if i != 0 {
b = append(b, ',')
}
// encodeString never returns an error so we ignore it here
b, _ = e.encodeString(b, unsafe.Pointer(&elem.key))
if elem.val.(bool) {
b = append(b, ":true"...)
} else {
b = append(b, ":false"...)
}
}
for i := range s.elements {
s.elements[i] = element{}
}
s.elements = s.elements[:0]
mapslicePool.Put(s)
b = append(b, '}')
return b, nil
}
func (e encoder) encodeStruct(b []byte, p unsafe.Pointer, st *structType) ([]byte, error) {
start := len(b)
var err error
var k string
var n int
b = append(b, '{')
escapeHTML := (e.flags & EscapeHTML) != 0
for i := range st.fields {
f := &st.fields[i]
v := unsafe.Pointer(uintptr(p) + f.offset)
if f.omitempty && f.empty(v) {
continue
}
if escapeHTML {
k = f.html
} else {
k = f.json
}
lengthBeforeKey := len(b)
if n != 0 {
b = append(b, k...)
} else {
b = append(b, k[1:]...)
}
if b, err = f.codec.encode(e, b, v); err != nil {
if err == (rollback{}) {
b = b[:lengthBeforeKey]
continue
}
return b[:start], err
}
n++
}
b = append(b, '}')
return b, nil
}
type rollback struct{}
func (rollback) Error() string { return "rollback" }
func (e encoder) encodeEmbeddedStructPointer(b []byte, p unsafe.Pointer, t reflect.Type, unexported bool, offset uintptr, encode encodeFunc) ([]byte, error) {
p = *(*unsafe.Pointer)(p)
if p == nil {
return b, rollback{}
}
return encode(e, b, unsafe.Pointer(uintptr(p)+offset))
}
func (e encoder) encodePointer(b []byte, p unsafe.Pointer, t reflect.Type, encode encodeFunc) ([]byte, error) {
if p = *(*unsafe.Pointer)(p); p != nil {
if e.ptrDepth++; e.ptrDepth >= startDetectingCyclesAfter {
if _, seen := e.ptrSeen[p]; seen {
// TODO: reconstruct the reflect.Value from p + t so we can set
// the erorr's Value field?
return b, &UnsupportedValueError{Str: fmt.Sprintf("encountered a cycle via %s", t)}
}
if e.ptrSeen == nil {
e.ptrSeen = make(map[unsafe.Pointer]struct{})
}
e.ptrSeen[p] = struct{}{}
defer delete(e.ptrSeen, p)
}
return encode(e, b, p)
}
return e.encodeNull(b, nil)
}
func (e encoder) encodeInterface(b []byte, p unsafe.Pointer) ([]byte, error) {
return Append(b, *(*any)(p), e.flags)
}
func (e encoder) encodeMaybeEmptyInterface(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) {
return Append(b, reflect.NewAt(t, p).Elem().Interface(), e.flags)
}
func (e encoder) encodeUnsupportedTypeError(b []byte, p unsafe.Pointer, t reflect.Type) ([]byte, error) {
return b, &UnsupportedTypeError{Type: t}
}
func (e encoder) encodeRawMessage(b []byte, p unsafe.Pointer) ([]byte, error) {
v := *(*RawMessage)(p)
if v == nil {
return append(b, "null"...), nil
}
var s []byte
if (e.flags & TrustRawMessage) != 0 {
s = v
} else {
var err error
v = skipSpaces(v) // don't assume that a RawMessage starts with a token.
d := decoder{}
s, _, _, err = d.parseValue(v)
if err != nil {
return b, &UnsupportedValueError{Value: reflect.ValueOf(v), Str: err.Error()}
}
}
if (e.flags & EscapeHTML) != 0 {
return appendCompactEscapeHTML(b, s), nil
}
return append(b, s...), nil
}
func (e encoder) encodeJSONMarshaler(b []byte, p unsafe.Pointer, t reflect.Type, pointer bool) ([]byte, error) {
v := reflect.NewAt(t, p)
if !pointer {
v = v.Elem()
}
switch v.Kind() {
case reflect.Ptr, reflect.Interface:
if v.IsNil() {
return append(b, "null"...), nil
}
}
j, err := v.Interface().(Marshaler).MarshalJSON()
if err != nil {
return b, err
}
d := decoder{}
s, _, _, err := d.parseValue(j)
if err != nil {
return b, &MarshalerError{Type: t, Err: err}
}
if (e.flags & EscapeHTML) != 0 {
return appendCompactEscapeHTML(b, s), nil
}
return append(b, s...), nil
}
func (e encoder) encodeTextMarshaler(b []byte, p unsafe.Pointer, t reflect.Type, pointer bool) ([]byte, error) {
v := reflect.NewAt(t, p)
if !pointer {
v = v.Elem()
}
switch v.Kind() {
case reflect.Ptr, reflect.Interface:
if v.IsNil() {
return append(b, `null`...), nil
}
}
s, err := v.Interface().(encoding.TextMarshaler).MarshalText()
if err != nil {
return b, err
}
return e.encodeString(b, unsafe.Pointer(&s))
}
func appendCompactEscapeHTML(dst []byte, src []byte) []byte {
start := 0
escape := false
inString := false
for i, c := range src {
if !inString {
switch c {
case '"': // enter string
inString = true
case ' ', '\n', '\r', '\t': // skip space
if start < i {
dst = append(dst, src[start:i]...)
}
start = i + 1
}
continue
}
if escape {
escape = false
continue
}
if c == '\\' {
escape = true
continue
}
if c == '"' {
inString = false
continue
}
if c == '<' || c == '>' || c == '&' {
if start < i {
dst = append(dst, src[start:i]...)
}
dst = append(dst, `\u00`...)
dst = append(dst, hex[c>>4], hex[c&0xF])
start = i + 1
continue
}
// Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9).
if c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 {
if start < i {
dst = append(dst, src[start:i]...)
}
dst = append(dst, `\u202`...)
dst = append(dst, hex[src[i+2]&0xF])
start = i + 3
continue
}
}
if start < len(src) {
dst = append(dst, src[start:]...)
}
return dst
}
+98
View File
@@ -0,0 +1,98 @@
package json
import (
"unsafe"
)
var endianness int
func init() {
var b [2]byte
*(*uint16)(unsafe.Pointer(&b)) = uint16(0xABCD)
switch b[0] {
case 0xCD:
endianness = 0 // LE
case 0xAB:
endianness = 1 // BE
default:
panic("could not determine endianness")
}
}
// "00010203...96979899" cast to []uint16
var intLELookup = [100]uint16{
0x3030, 0x3130, 0x3230, 0x3330, 0x3430, 0x3530, 0x3630, 0x3730, 0x3830, 0x3930,
0x3031, 0x3131, 0x3231, 0x3331, 0x3431, 0x3531, 0x3631, 0x3731, 0x3831, 0x3931,
0x3032, 0x3132, 0x3232, 0x3332, 0x3432, 0x3532, 0x3632, 0x3732, 0x3832, 0x3932,
0x3033, 0x3133, 0x3233, 0x3333, 0x3433, 0x3533, 0x3633, 0x3733, 0x3833, 0x3933,
0x3034, 0x3134, 0x3234, 0x3334, 0x3434, 0x3534, 0x3634, 0x3734, 0x3834, 0x3934,
0x3035, 0x3135, 0x3235, 0x3335, 0x3435, 0x3535, 0x3635, 0x3735, 0x3835, 0x3935,
0x3036, 0x3136, 0x3236, 0x3336, 0x3436, 0x3536, 0x3636, 0x3736, 0x3836, 0x3936,
0x3037, 0x3137, 0x3237, 0x3337, 0x3437, 0x3537, 0x3637, 0x3737, 0x3837, 0x3937,
0x3038, 0x3138, 0x3238, 0x3338, 0x3438, 0x3538, 0x3638, 0x3738, 0x3838, 0x3938,
0x3039, 0x3139, 0x3239, 0x3339, 0x3439, 0x3539, 0x3639, 0x3739, 0x3839, 0x3939,
}
var intBELookup = [100]uint16{
0x3030, 0x3031, 0x3032, 0x3033, 0x3034, 0x3035, 0x3036, 0x3037, 0x3038, 0x3039,
0x3130, 0x3131, 0x3132, 0x3133, 0x3134, 0x3135, 0x3136, 0x3137, 0x3138, 0x3139,
0x3230, 0x3231, 0x3232, 0x3233, 0x3234, 0x3235, 0x3236, 0x3237, 0x3238, 0x3239,
0x3330, 0x3331, 0x3332, 0x3333, 0x3334, 0x3335, 0x3336, 0x3337, 0x3338, 0x3339,
0x3430, 0x3431, 0x3432, 0x3433, 0x3434, 0x3435, 0x3436, 0x3437, 0x3438, 0x3439,
0x3530, 0x3531, 0x3532, 0x3533, 0x3534, 0x3535, 0x3536, 0x3537, 0x3538, 0x3539,
0x3630, 0x3631, 0x3632, 0x3633, 0x3634, 0x3635, 0x3636, 0x3637, 0x3638, 0x3639,
0x3730, 0x3731, 0x3732, 0x3733, 0x3734, 0x3735, 0x3736, 0x3737, 0x3738, 0x3739,
0x3830, 0x3831, 0x3832, 0x3833, 0x3834, 0x3835, 0x3836, 0x3837, 0x3838, 0x3839,
0x3930, 0x3931, 0x3932, 0x3933, 0x3934, 0x3935, 0x3936, 0x3937, 0x3938, 0x3939,
}
var intLookup = [2]*[100]uint16{&intLELookup, &intBELookup}
func appendInt(b []byte, n int64) []byte {
return formatInteger(b, uint64(n), n < 0)
}
func appendUint(b []byte, n uint64) []byte {
return formatInteger(b, n, false)
}
func formatInteger(out []byte, n uint64, negative bool) []byte {
if !negative {
if n < 10 {
return append(out, byte(n+'0'))
} else if n < 100 {
u := intLELookup[n]
return append(out, byte(u), byte(u>>8))
}
} else {
n = -n
}
lookup := intLookup[endianness]
var b [22]byte
u := (*[11]uint16)(unsafe.Pointer(&b))
i := 11
for n >= 100 {
j := n % 100
n /= 100
i--
u[i] = lookup[j]
}
i--
u[i] = lookup[n]
i *= 2 // convert to byte index
if n < 10 {
i++ // remove leading zero
}
if negative {
i--
b[i] = '-'
}
return append(out, b[i:]...)
}
+594
View File
@@ -0,0 +1,594 @@
package json
import (
"bytes"
"encoding/json"
"io"
"math/bits"
"reflect"
"runtime"
"sync"
"unsafe"
)
// Delim is documented at https://golang.org/pkg/encoding/json/#Delim
type Delim = json.Delim
// InvalidUTF8Error is documented at https://golang.org/pkg/encoding/json/#InvalidUTF8Error
type InvalidUTF8Error = json.InvalidUTF8Error //nolint:staticcheck // compat.
// InvalidUnmarshalError is documented at https://golang.org/pkg/encoding/json/#InvalidUnmarshalError
type InvalidUnmarshalError = json.InvalidUnmarshalError
// Marshaler is documented at https://golang.org/pkg/encoding/json/#Marshaler
type Marshaler = json.Marshaler
// MarshalerError is documented at https://golang.org/pkg/encoding/json/#MarshalerError
type MarshalerError = json.MarshalerError
// Number is documented at https://golang.org/pkg/encoding/json/#Number
type Number = json.Number
// RawMessage is documented at https://golang.org/pkg/encoding/json/#RawMessage
type RawMessage = json.RawMessage
// A SyntaxError is a description of a JSON syntax error.
type SyntaxError = json.SyntaxError
// Token is documented at https://golang.org/pkg/encoding/json/#Token
type Token = json.Token
// UnmarshalFieldError is documented at https://golang.org/pkg/encoding/json/#UnmarshalFieldError
type UnmarshalFieldError = json.UnmarshalFieldError //nolint:staticcheck // compat.
// UnmarshalTypeError is documented at https://golang.org/pkg/encoding/json/#UnmarshalTypeError
type UnmarshalTypeError = json.UnmarshalTypeError
// Unmarshaler is documented at https://golang.org/pkg/encoding/json/#Unmarshaler
type Unmarshaler = json.Unmarshaler
// UnsupportedTypeError is documented at https://golang.org/pkg/encoding/json/#UnsupportedTypeError
type UnsupportedTypeError = json.UnsupportedTypeError
// UnsupportedValueError is documented at https://golang.org/pkg/encoding/json/#UnsupportedValueError
type UnsupportedValueError = json.UnsupportedValueError
// AppendFlags is a type used to represent configuration options that can be
// applied when formatting json output.
type AppendFlags uint32
const (
// EscapeHTML is a formatting flag used to to escape HTML in json strings.
EscapeHTML AppendFlags = 1 << iota
// SortMapKeys is formatting flag used to enable sorting of map keys when
// encoding JSON (this matches the behavior of the standard encoding/json
// package).
SortMapKeys
// TrustRawMessage is a performance optimization flag to skip value
// checking of raw messages. It should only be used if the values are
// known to be valid json (e.g., they were created by json.Unmarshal).
TrustRawMessage
// appendNewline is a formatting flag to enable the addition of a newline
// in Encode (this matches the behavior of the standard encoding/json
// package).
appendNewline
)
// ParseFlags is a type used to represent configuration options that can be
// applied when parsing json input.
type ParseFlags uint32
func (flags ParseFlags) has(f ParseFlags) bool {
return (flags & f) != 0
}
func (f ParseFlags) kind() Kind {
return Kind((f >> kindOffset) & 0xFF)
}
func (f ParseFlags) withKind(kind Kind) ParseFlags {
return (f & ^(ParseFlags(0xFF) << kindOffset)) | (ParseFlags(kind) << kindOffset)
}
const (
// DisallowUnknownFields is a parsing flag used to prevent decoding of
// objects to Go struct values when a field of the input does not match
// with any of the struct fields.
DisallowUnknownFields ParseFlags = 1 << iota
// UseNumber is a parsing flag used to load numeric values as Number
// instead of float64.
UseNumber
// DontCopyString is a parsing flag used to provide zero-copy support when
// loading string values from a json payload. It is not always possible to
// avoid dynamic memory allocations, for example when a string is escaped in
// the json data a new buffer has to be allocated, but when the `wire` value
// can be used as content of a Go value the decoder will simply point into
// the input buffer.
DontCopyString
// DontCopyNumber is a parsing flag used to provide zero-copy support when
// loading Number values (see DontCopyString and DontCopyRawMessage).
DontCopyNumber
// DontCopyRawMessage is a parsing flag used to provide zero-copy support
// when loading RawMessage values from a json payload. When used, the
// RawMessage values will not be allocated into new memory buffers and
// will instead point directly to the area of the input buffer where the
// value was found.
DontCopyRawMessage
// DontMatchCaseInsensitiveStructFields is a parsing flag used to prevent
// matching fields in a case-insensitive way. This can prevent degrading
// performance on case conversions, and can also act as a stricter decoding
// mode.
DontMatchCaseInsensitiveStructFields
// Decode integers into *big.Int.
// Takes precedence over UseNumber for integers.
UseBigInt
// Decode in-range integers to int64.
// Takes precedence over UseNumber and UseBigInt for in-range integers.
UseInt64
// Decode in-range positive integers to uint64.
// Takes precedence over UseNumber, UseBigInt, and UseInt64
// for positive, in-range integers.
UseUint64
// ZeroCopy is a parsing flag that combines all the copy optimizations
// available in the package.
//
// The zero-copy optimizations are better used in request-handler style
// code where none of the values are retained after the handler returns.
ZeroCopy = DontCopyString | DontCopyNumber | DontCopyRawMessage
// validAsciiPrint is an internal flag indicating that the input contains
// only valid ASCII print chars (0x20 <= c <= 0x7E). If the flag is unset,
// it's unknown whether the input is valid ASCII print.
validAsciiPrint ParseFlags = 1 << 28
// noBackslach is an internal flag indicating that the input does not
// contain a backslash. If the flag is unset, it's unknown whether the
// input contains a backslash.
noBackslash ParseFlags = 1 << 29
// Bit offset where the kind of the json value is stored.
//
// See Kind in token.go for the enum.
kindOffset ParseFlags = 16
)
// Kind represents the different kinds of value that exist in JSON.
type Kind uint
const (
Undefined Kind = 0
Null Kind = 1 // Null is not zero, so we keep zero for "undefined".
Bool Kind = 2 // Bit two is set to 1, means it's a boolean.
False Kind = 2 // Bool + 0
True Kind = 3 // Bool + 1
Num Kind = 4 // Bit three is set to 1, means it's a number.
Uint Kind = 5 // Num + 1
Int Kind = 6 // Num + 2
Float Kind = 7 // Num + 3
String Kind = 8 // Bit four is set to 1, means it's a string.
Unescaped Kind = 9 // String + 1
Array Kind = 16 // Equivalent to Delim == '['
Object Kind = 32 // Equivalent to Delim == '{'
)
// Class returns the class of k.
func (k Kind) Class() Kind { return Kind(1 << uint(bits.Len(uint(k))-1)) }
// Append acts like Marshal but appends the json representation to b instead of
// always reallocating a new slice.
func Append(b []byte, x any, flags AppendFlags) ([]byte, error) {
if x == nil {
// Special case for nil values because it makes the rest of the code
// simpler to assume that it won't be seeing nil pointers.
return append(b, "null"...), nil
}
t := reflect.TypeOf(x)
p := (*iface)(unsafe.Pointer(&x)).ptr
cache := cacheLoad()
c, found := cache[typeid(t)]
if !found {
c = constructCachedCodec(t, cache)
}
b, err := c.encode(encoder{flags: flags}, b, p)
runtime.KeepAlive(x)
return b, err
}
// Escape is a convenience helper to construct an escaped JSON string from s.
// The function escales HTML characters, for more control over the escape
// behavior and to write to a pre-allocated buffer, use AppendEscape.
func Escape(s string) []byte {
// +10 for extra escape characters, maybe not enough and the buffer will
// be reallocated.
b := make([]byte, 0, len(s)+10)
return AppendEscape(b, s, EscapeHTML)
}
// AppendEscape appends s to b with the string escaped as a JSON value.
// This will include the starting and ending quote characters, and the
// appropriate characters will be escaped correctly for JSON encoding.
func AppendEscape(b []byte, s string, flags AppendFlags) []byte {
e := encoder{flags: flags}
b, _ = e.encodeString(b, unsafe.Pointer(&s))
return b
}
// Unescape is a convenience helper to unescape a JSON value.
// For more control over the unescape behavior and
// to write to a pre-allocated buffer, use AppendUnescape.
func Unescape(s []byte) []byte {
b := make([]byte, 0, len(s))
return AppendUnescape(b, s, ParseFlags(0))
}
// AppendUnescape appends s to b with the string unescaped as a JSON value.
// This will remove starting and ending quote characters, and the
// appropriate characters will be escaped correctly as if JSON decoded.
// New space will be reallocated if more space is needed.
func AppendUnescape(b []byte, s []byte, flags ParseFlags) []byte {
d := decoder{flags: flags}
buf := new(string)
d.decodeString(s, unsafe.Pointer(buf))
return append(b, *buf...)
}
// Compact is documented at https://golang.org/pkg/encoding/json/#Compact
func Compact(dst *bytes.Buffer, src []byte) error {
return json.Compact(dst, src)
}
// HTMLEscape is documented at https://golang.org/pkg/encoding/json/#HTMLEscape
func HTMLEscape(dst *bytes.Buffer, src []byte) {
json.HTMLEscape(dst, src)
}
// Indent is documented at https://golang.org/pkg/encoding/json/#Indent
func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
return json.Indent(dst, src, prefix, indent)
}
// Marshal is documented at https://golang.org/pkg/encoding/json/#Marshal
func Marshal(x any) ([]byte, error) {
var err error
buf := encoderBufferPool.Get().(*encoderBuffer)
if buf.data, err = Append(buf.data[:0], x, EscapeHTML|SortMapKeys); err != nil {
return nil, err
}
b := make([]byte, len(buf.data))
copy(b, buf.data)
encoderBufferPool.Put(buf)
return b, nil
}
// MarshalIndent is documented at https://golang.org/pkg/encoding/json/#MarshalIndent
func MarshalIndent(x any, prefix, indent string) ([]byte, error) {
b, err := Marshal(x)
if err == nil {
tmp := &bytes.Buffer{}
tmp.Grow(2 * len(b))
Indent(tmp, b, prefix, indent)
b = tmp.Bytes()
}
return b, err
}
// Unmarshal is documented at https://golang.org/pkg/encoding/json/#Unmarshal
func Unmarshal(b []byte, x any) error {
r, err := Parse(b, x, 0)
if len(r) != 0 {
if _, ok := err.(*SyntaxError); !ok {
// The encoding/json package prioritizes reporting errors caused by
// unexpected trailing bytes over other issues; here we emulate this
// behavior by overriding the error.
err = syntaxError(r, "invalid character '%c' after top-level value", r[0])
}
}
return err
}
// Parse behaves like Unmarshal but the caller can pass a set of flags to
// configure the parsing behavior.
func Parse(b []byte, x any, flags ParseFlags) ([]byte, error) {
t := reflect.TypeOf(x)
p := (*iface)(unsafe.Pointer(&x)).ptr
d := decoder{flags: flags | internalParseFlags(b)}
b = skipSpaces(b)
if t == nil || p == nil || t.Kind() != reflect.Ptr {
_, r, _, err := d.parseValue(b)
r = skipSpaces(r)
if err != nil {
return r, err
}
return r, &InvalidUnmarshalError{Type: t}
}
t = t.Elem()
cache := cacheLoad()
c, found := cache[typeid(t)]
if !found {
c = constructCachedCodec(t, cache)
}
r, err := c.decode(d, b, p)
return skipSpaces(r), err
}
// Valid is documented at https://golang.org/pkg/encoding/json/#Valid
func Valid(data []byte) bool {
data = skipSpaces(data)
d := decoder{flags: internalParseFlags(data)}
_, data, _, err := d.parseValue(data)
if err != nil {
return false
}
return len(skipSpaces(data)) == 0
}
// Decoder is documented at https://golang.org/pkg/encoding/json/#Decoder
type Decoder struct {
reader io.Reader
buffer []byte
remain []byte
inputOffset int64
err error
flags ParseFlags
}
// NewDecoder is documented at https://golang.org/pkg/encoding/json/#NewDecoder
func NewDecoder(r io.Reader) *Decoder { return &Decoder{reader: r} }
// Buffered is documented at https://golang.org/pkg/encoding/json/#Decoder.Buffered
func (dec *Decoder) Buffered() io.Reader {
return bytes.NewReader(dec.remain)
}
// Decode is documented at https://golang.org/pkg/encoding/json/#Decoder.Decode
func (dec *Decoder) Decode(v any) error {
raw, err := dec.readValue()
if err != nil {
return err
}
_, err = Parse(raw, v, dec.flags)
return err
}
const (
minBufferSize = 32768
minReadSize = 4096
)
// readValue reads one JSON value from the buffer and returns its raw bytes. It
// is optimized for the "one JSON value per line" case.
func (dec *Decoder) readValue() (v []byte, err error) {
var n int
var r []byte
d := decoder{flags: dec.flags}
for {
if len(dec.remain) != 0 {
v, r, _, err = d.parseValue(dec.remain)
if err == nil {
dec.remain, n = skipSpacesN(r)
dec.inputOffset += int64(len(v) + n)
return
}
if len(r) != 0 {
// Parsing of the next JSON value stopped at a position other
// than the end of the input buffer, which indicaates that a
// syntax error was encountered.
return
}
}
if err = dec.err; err != nil {
if len(dec.remain) != 0 && err == io.EOF {
err = io.ErrUnexpectedEOF
}
return
}
if dec.buffer == nil {
dec.buffer = make([]byte, 0, minBufferSize)
} else {
dec.buffer = dec.buffer[:copy(dec.buffer[:cap(dec.buffer)], dec.remain)]
dec.remain = nil
}
if (cap(dec.buffer) - len(dec.buffer)) < minReadSize {
buf := make([]byte, len(dec.buffer), 2*cap(dec.buffer))
copy(buf, dec.buffer)
dec.buffer = buf
}
n, err = io.ReadFull(dec.reader, dec.buffer[len(dec.buffer):cap(dec.buffer)])
if n > 0 {
dec.buffer = dec.buffer[:len(dec.buffer)+n]
if err != nil {
err = nil
}
} else if err == io.ErrUnexpectedEOF {
err = io.EOF
}
dec.remain, n = skipSpacesN(dec.buffer)
d.flags = dec.flags | internalParseFlags(dec.remain)
dec.inputOffset += int64(n)
dec.err = err
}
}
// DisallowUnknownFields is documented at https://golang.org/pkg/encoding/json/#Decoder.DisallowUnknownFields
func (dec *Decoder) DisallowUnknownFields() { dec.flags |= DisallowUnknownFields }
// UseNumber is documented at https://golang.org/pkg/encoding/json/#Decoder.UseNumber
func (dec *Decoder) UseNumber() { dec.flags |= UseNumber }
// DontCopyString is an extension to the standard encoding/json package
// which instructs the decoder to not copy strings loaded from the json
// payloads when possible.
func (dec *Decoder) DontCopyString() { dec.flags |= DontCopyString }
// DontCopyNumber is an extension to the standard encoding/json package
// which instructs the decoder to not copy numbers loaded from the json
// payloads.
func (dec *Decoder) DontCopyNumber() { dec.flags |= DontCopyNumber }
// DontCopyRawMessage is an extension to the standard encoding/json package
// which instructs the decoder to not allocate RawMessage values in separate
// memory buffers (see the documentation of the DontcopyRawMessage flag for
// more detais).
func (dec *Decoder) DontCopyRawMessage() { dec.flags |= DontCopyRawMessage }
// DontMatchCaseInsensitiveStructFields is an extension to the standard
// encoding/json package which instructs the decoder to not match object fields
// against struct fields in a case-insensitive way, the field names have to
// match exactly to be decoded into the struct field values.
func (dec *Decoder) DontMatchCaseInsensitiveStructFields() {
dec.flags |= DontMatchCaseInsensitiveStructFields
}
// ZeroCopy is an extension to the standard encoding/json package which enables
// all the copy optimizations of the decoder.
func (dec *Decoder) ZeroCopy() { dec.flags |= ZeroCopy }
// InputOffset returns the input stream byte offset of the current decoder position.
// The offset gives the location of the end of the most recently returned token
// and the beginning of the next token.
func (dec *Decoder) InputOffset() int64 {
return dec.inputOffset
}
// Encoder is documented at https://golang.org/pkg/encoding/json/#Encoder
type Encoder struct {
writer io.Writer
prefix string
indent string
buffer *bytes.Buffer
err error
flags AppendFlags
}
// NewEncoder is documented at https://golang.org/pkg/encoding/json/#NewEncoder
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{writer: w, flags: EscapeHTML | SortMapKeys | appendNewline}
}
// Encode is documented at https://golang.org/pkg/encoding/json/#Encoder.Encode
func (enc *Encoder) Encode(v any) error {
if enc.err != nil {
return enc.err
}
var err error
buf := encoderBufferPool.Get().(*encoderBuffer)
buf.data, err = Append(buf.data[:0], v, enc.flags)
if err != nil {
encoderBufferPool.Put(buf)
return err
}
if (enc.flags & appendNewline) != 0 {
buf.data = append(buf.data, '\n')
}
b := buf.data
if enc.prefix != "" || enc.indent != "" {
if enc.buffer == nil {
enc.buffer = new(bytes.Buffer)
enc.buffer.Grow(2 * len(buf.data))
} else {
enc.buffer.Reset()
}
Indent(enc.buffer, buf.data, enc.prefix, enc.indent)
b = enc.buffer.Bytes()
}
if _, err := enc.writer.Write(b); err != nil {
enc.err = err
}
encoderBufferPool.Put(buf)
return err
}
// SetEscapeHTML is documented at https://golang.org/pkg/encoding/json/#Encoder.SetEscapeHTML
func (enc *Encoder) SetEscapeHTML(on bool) {
if on {
enc.flags |= EscapeHTML
} else {
enc.flags &= ^EscapeHTML
}
}
// SetIndent is documented at https://golang.org/pkg/encoding/json/#Encoder.SetIndent
func (enc *Encoder) SetIndent(prefix, indent string) {
enc.prefix = prefix
enc.indent = indent
}
// SetSortMapKeys is an extension to the standard encoding/json package which
// allows the program to toggle sorting of map keys on and off.
func (enc *Encoder) SetSortMapKeys(on bool) {
if on {
enc.flags |= SortMapKeys
} else {
enc.flags &= ^SortMapKeys
}
}
// SetTrustRawMessage skips value checking when encoding a raw json message. It should only
// be used if the values are known to be valid json, e.g. because they were originally created
// by json.Unmarshal.
func (enc *Encoder) SetTrustRawMessage(on bool) {
if on {
enc.flags |= TrustRawMessage
} else {
enc.flags &= ^TrustRawMessage
}
}
// SetAppendNewline is an extension to the standard encoding/json package which
// allows the program to toggle the addition of a newline in Encode on or off.
func (enc *Encoder) SetAppendNewline(on bool) {
if on {
enc.flags |= appendNewline
} else {
enc.flags &= ^appendNewline
}
}
var encoderBufferPool = sync.Pool{
New: func() any { return &encoderBuffer{data: make([]byte, 0, 4096)} },
}
type encoderBuffer struct{ data []byte }
+781
View File
@@ -0,0 +1,781 @@
package json
import (
"bytes"
"encoding/binary"
"math"
"math/bits"
"reflect"
"unicode"
"unicode/utf16"
"unicode/utf8"
"github.com/segmentio/encoding/ascii"
)
// All spaces characters defined in the json specification.
const (
sp = ' '
ht = '\t'
nl = '\n'
cr = '\r'
)
func internalParseFlags(b []byte) (flags ParseFlags) {
// Don't consider surrounding whitespace
b = skipSpaces(b)
b = trimTrailingSpaces(b)
if ascii.ValidPrint(b) {
flags |= validAsciiPrint
}
if bytes.IndexByte(b, '\\') == -1 {
flags |= noBackslash
}
return
}
func skipSpaces(b []byte) []byte {
if len(b) > 0 && b[0] <= 0x20 {
b, _ = skipSpacesN(b)
}
return b
}
func skipSpacesN(b []byte) ([]byte, int) {
for i := range b {
switch b[i] {
case sp, ht, nl, cr:
default:
return b[i:], i
}
}
return nil, 0
}
func trimTrailingSpaces(b []byte) []byte {
if len(b) > 0 && b[len(b)-1] <= 0x20 {
b = trimTrailingSpacesN(b)
}
return b
}
func trimTrailingSpacesN(b []byte) []byte {
i := len(b) - 1
loop:
for ; i >= 0; i-- {
switch b[i] {
case sp, ht, nl, cr:
default:
break loop
}
}
return b[:i+1]
}
// parseInt parses a decimal representation of an int64 from b.
//
// The function is equivalent to calling strconv.ParseInt(string(b), 10, 64) but
// it prevents Go from making a memory allocation for converting a byte slice to
// a string (escape analysis fails due to the error returned by strconv.ParseInt).
//
// Because it only works with base 10 the function is also significantly faster
// than strconv.ParseInt.
func (d decoder) parseInt(b []byte, t reflect.Type) (int64, []byte, error) {
var value int64
var count int
if len(b) == 0 {
return 0, b, syntaxError(b, "cannot decode integer from an empty input")
}
if b[0] == '-' {
const max = math.MinInt64
const lim = max / 10
if len(b) == 1 {
return 0, b, syntaxError(b, "cannot decode integer from '-'")
}
if len(b) > 2 && b[1] == '0' && '0' <= b[2] && b[2] <= '9' {
return 0, b, syntaxError(b, "invalid leading character '0' in integer")
}
for _, c := range b[1:] {
if c < '0' || c > '9' {
if count == 0 {
b, err := d.inputError(b, t)
return 0, b, err
}
break
}
if value < lim {
return 0, b, unmarshalOverflow(b, t)
}
value *= 10
x := int64(c - '0')
if value < (max + x) {
return 0, b, unmarshalOverflow(b, t)
}
value -= x
count++
}
count++
} else {
if len(b) > 1 && b[0] == '0' && '0' <= b[1] && b[1] <= '9' {
return 0, b, syntaxError(b, "invalid leading character '0' in integer")
}
for ; count < len(b) && b[count] >= '0' && b[count] <= '9'; count++ {
x := int64(b[count] - '0')
next := value*10 + x
if next < value {
return 0, b, unmarshalOverflow(b, t)
}
value = next
}
if count == 0 {
b, err := d.inputError(b, t)
return 0, b, err
}
}
if count < len(b) {
switch b[count] {
case '.', 'e', 'E': // was this actually a float?
v, r, _, err := d.parseNumber(b)
if err != nil {
v, r = b[:count+1], b[count+1:]
}
return 0, r, unmarshalTypeError(v, t)
}
}
return value, b[count:], nil
}
// parseUint is like parseInt but for unsigned integers.
func (d decoder) parseUint(b []byte, t reflect.Type) (uint64, []byte, error) {
var value uint64
var count int
if len(b) == 0 {
return 0, b, syntaxError(b, "cannot decode integer value from an empty input")
}
if len(b) > 1 && b[0] == '0' && '0' <= b[1] && b[1] <= '9' {
return 0, b, syntaxError(b, "invalid leading character '0' in integer")
}
for ; count < len(b) && b[count] >= '0' && b[count] <= '9'; count++ {
x := uint64(b[count] - '0')
next := value*10 + x
if next < value {
return 0, b, unmarshalOverflow(b, t)
}
value = next
}
if count == 0 {
b, err := d.inputError(b, t)
return 0, b, err
}
if count < len(b) {
switch b[count] {
case '.', 'e', 'E': // was this actually a float?
v, r, _, err := d.parseNumber(b)
if err != nil {
v, r = b[:count+1], b[count+1:]
}
return 0, r, unmarshalTypeError(v, t)
}
}
return value, b[count:], nil
}
// parseUintHex parses a hexadecimanl representation of a uint64 from b.
//
// The function is equivalent to calling strconv.ParseUint(string(b), 16, 64) but
// it prevents Go from making a memory allocation for converting a byte slice to
// a string (escape analysis fails due to the error returned by strconv.ParseUint).
//
// Because it only works with base 16 the function is also significantly faster
// than strconv.ParseUint.
func (d decoder) parseUintHex(b []byte) (uint64, []byte, error) {
const max = math.MaxUint64
const lim = max / 0x10
var value uint64
var count int
if len(b) == 0 {
return 0, b, syntaxError(b, "cannot decode hexadecimal value from an empty input")
}
parseLoop:
for i, c := range b {
var x uint64
switch {
case c >= '0' && c <= '9':
x = uint64(c - '0')
case c >= 'A' && c <= 'F':
x = uint64(c-'A') + 0xA
case c >= 'a' && c <= 'f':
x = uint64(c-'a') + 0xA
default:
if i == 0 {
return 0, b, syntaxError(b, "expected hexadecimal digit but found '%c'", c)
}
break parseLoop
}
if value > lim {
return 0, b, syntaxError(b, "hexadecimal value out of range")
}
if value *= 0x10; value > (max - x) {
return 0, b, syntaxError(b, "hexadecimal value out of range")
}
value += x
count++
}
return value, b[count:], nil
}
func (d decoder) parseNull(b []byte) ([]byte, []byte, Kind, error) {
if hasNullPrefix(b) {
return b[:4], b[4:], Null, nil
}
if len(b) < 4 {
return nil, b[len(b):], Undefined, unexpectedEOF(b)
}
return nil, b, Undefined, syntaxError(b, "expected 'null' but found invalid token")
}
func (d decoder) parseTrue(b []byte) ([]byte, []byte, Kind, error) {
if hasTruePrefix(b) {
return b[:4], b[4:], True, nil
}
if len(b) < 4 {
return nil, b[len(b):], Undefined, unexpectedEOF(b)
}
return nil, b, Undefined, syntaxError(b, "expected 'true' but found invalid token")
}
func (d decoder) parseFalse(b []byte) ([]byte, []byte, Kind, error) {
if hasFalsePrefix(b) {
return b[:5], b[5:], False, nil
}
if len(b) < 5 {
return nil, b[len(b):], Undefined, unexpectedEOF(b)
}
return nil, b, Undefined, syntaxError(b, "expected 'false' but found invalid token")
}
func (d decoder) parseNumber(b []byte) (v, r []byte, kind Kind, err error) {
if len(b) == 0 {
r, err = b, unexpectedEOF(b)
return
}
// Assume it's an unsigned integer at first.
kind = Uint
i := 0
// sign
if b[i] == '-' {
kind = Int
i++
}
if i == len(b) {
r, err = b[i:], syntaxError(b, "missing number value after sign")
return
}
if b[i] < '0' || b[i] > '9' {
r, err = b[i:], syntaxError(b, "expected digit but got '%c'", b[i])
return
}
// integer part
if b[i] == '0' {
i++
if i == len(b) || (b[i] != '.' && b[i] != 'e' && b[i] != 'E') {
v, r = b[:i], b[i:]
return
}
if '0' <= b[i] && b[i] <= '9' {
r, err = b[i:], syntaxError(b, "cannot decode number with leading '0' character")
return
}
}
for i < len(b) && '0' <= b[i] && b[i] <= '9' {
i++
}
// decimal part
if i < len(b) && b[i] == '.' {
kind = Float
i++
decimalStart := i
for i < len(b) {
if c := b[i]; '0' > c || c > '9' {
if i == decimalStart {
r, err = b[i:], syntaxError(b, "expected digit but found '%c'", c)
return
}
break
}
i++
}
if i == decimalStart {
r, err = b[i:], syntaxError(b, "expected decimal part after '.'")
return
}
}
// exponent part
if i < len(b) && (b[i] == 'e' || b[i] == 'E') {
kind = Float
i++
if i < len(b) {
if c := b[i]; c == '+' || c == '-' {
i++
}
}
if i == len(b) {
r, err = b[i:], syntaxError(b, "missing exponent in number")
return
}
exponentStart := i
for i < len(b) {
if c := b[i]; '0' > c || c > '9' {
if i == exponentStart {
err = syntaxError(b, "expected digit but found '%c'", c)
return
}
break
}
i++
}
}
v, r = b[:i], b[i:]
return
}
func (d decoder) parseUnicode(b []byte) (rune, int, error) {
if len(b) < 4 {
return 0, len(b), syntaxError(b, "unicode code point must have at least 4 characters")
}
u, r, err := d.parseUintHex(b[:4])
if err != nil {
return 0, 4, syntaxError(b, "parsing unicode code point: %s", err)
}
if len(r) != 0 {
return 0, 4, syntaxError(b, "invalid unicode code point")
}
return rune(u), 4, nil
}
func (d decoder) parseString(b []byte) ([]byte, []byte, Kind, error) {
if len(b) < 2 {
return nil, b[len(b):], Undefined, unexpectedEOF(b)
}
if b[0] != '"' {
return nil, b, Undefined, syntaxError(b, "expected '\"' at the beginning of a string value")
}
var n int
if len(b) >= 9 {
// This is an optimization for short strings. We read 8/16 bytes,
// and XOR each with 0x22 (") so that these bytes (and only
// these bytes) are now zero. We use the hasless(u,1) trick
// from https://graphics.stanford.edu/~seander/bithacks.html#ZeroInWord
// to determine whether any bytes are zero. Finally, we CTZ
// to find the index of that byte.
const mask1 = 0x2222222222222222
const mask2 = 0x0101010101010101
const mask3 = 0x8080808080808080
u := binary.LittleEndian.Uint64(b[1:]) ^ mask1
if mask := (u - mask2) & ^u & mask3; mask != 0 {
n = bits.TrailingZeros64(mask)/8 + 2
goto found
}
if len(b) >= 17 {
u = binary.LittleEndian.Uint64(b[9:]) ^ mask1
if mask := (u - mask2) & ^u & mask3; mask != 0 {
n = bits.TrailingZeros64(mask)/8 + 10
goto found
}
}
}
n = bytes.IndexByte(b[1:], '"') + 2
if n <= 1 {
return nil, b[len(b):], Undefined, syntaxError(b, "missing '\"' at the end of a string value")
}
found:
if (d.flags.has(noBackslash) || bytes.IndexByte(b[1:n], '\\') < 0) &&
(d.flags.has(validAsciiPrint) || ascii.ValidPrint(b[1:n])) {
return b[:n], b[n:], Unescaped, nil
}
for i := 1; i < len(b); i++ {
switch b[i] {
case '\\':
if i++; i < len(b) {
switch b[i] {
case '"', '\\', '/', 'n', 'r', 't', 'f', 'b':
case 'u':
_, n, err := d.parseUnicode(b[i+1:])
if err != nil {
return nil, b[i+1+n:], Undefined, err
}
i += n
default:
return nil, b, Undefined, syntaxError(b, "invalid character '%c' in string escape code", b[i])
}
}
case '"':
return b[:i+1], b[i+1:], String, nil
default:
if b[i] < 0x20 {
return nil, b, Undefined, syntaxError(b, "invalid character '%c' in string escape code", b[i])
}
}
}
return nil, b[len(b):], Undefined, syntaxError(b, "missing '\"' at the end of a string value")
}
func (d decoder) parseStringUnquote(b []byte, r []byte) ([]byte, []byte, bool, error) {
s, b, k, err := d.parseString(b)
if err != nil {
return s, b, false, err
}
s = s[1 : len(s)-1] // trim the quotes
if k == Unescaped {
return s, b, false, nil
}
if r == nil {
r = make([]byte, 0, len(s))
}
for len(s) != 0 {
i := bytes.IndexByte(s, '\\')
if i < 0 {
r = appendCoerceInvalidUTF8(r, s)
break
}
r = appendCoerceInvalidUTF8(r, s[:i])
s = s[i+1:]
c := s[0]
switch c {
case '"', '\\', '/':
// simple escaped character
case 'n':
c = '\n'
case 'r':
c = '\r'
case 't':
c = '\t'
case 'b':
c = '\b'
case 'f':
c = '\f'
case 'u':
s = s[1:]
r1, n1, err := d.parseUnicode(s)
if err != nil {
return r, b, true, err
}
s = s[n1:]
if utf16.IsSurrogate(r1) {
if !hasPrefix(s, `\u`) {
r1 = unicode.ReplacementChar
} else {
r2, n2, err := d.parseUnicode(s[2:])
if err != nil {
return r, b, true, err
}
if r1 = utf16.DecodeRune(r1, r2); r1 != unicode.ReplacementChar {
s = s[2+n2:]
}
}
}
r = appendRune(r, r1)
continue
default: // not sure what this escape sequence is
return r, b, false, syntaxError(s, "invalid character '%c' in string escape code", c)
}
r = append(r, c)
s = s[1:]
}
return r, b, true, nil
}
func appendRune(b []byte, r rune) []byte {
n := len(b)
b = append(b, 0, 0, 0, 0)
return b[:n+utf8.EncodeRune(b[n:], r)]
}
func appendCoerceInvalidUTF8(b []byte, s []byte) []byte {
c := [4]byte{}
for _, r := range string(s) {
b = append(b, c[:utf8.EncodeRune(c[:], r)]...)
}
return b
}
func (d decoder) parseObject(b []byte) ([]byte, []byte, Kind, error) {
if len(b) < 2 {
return nil, b[len(b):], Undefined, unexpectedEOF(b)
}
if b[0] != '{' {
return nil, b, Undefined, syntaxError(b, "expected '{' at the beginning of an object value")
}
var err error
a := b
n := len(b)
i := 0
b = b[1:]
for {
b = skipSpaces(b)
if len(b) == 0 {
return nil, b, Undefined, syntaxError(b, "cannot decode object from empty input")
}
if b[0] == '}' {
j := (n - len(b)) + 1
return a[:j], a[j:], Object, nil
}
if i != 0 {
if len(b) == 0 {
return nil, b, Undefined, syntaxError(b, "unexpected EOF after object field value")
}
if b[0] != ',' {
return nil, b, Undefined, syntaxError(b, "expected ',' after object field value but found '%c'", b[0])
}
b = skipSpaces(b[1:])
if len(b) == 0 {
return nil, b, Undefined, unexpectedEOF(b)
}
if b[0] == '}' {
return nil, b, Undefined, syntaxError(b, "unexpected trailing comma after object field")
}
}
_, b, _, err = d.parseString(b)
if err != nil {
return nil, b, Undefined, err
}
b = skipSpaces(b)
if len(b) == 0 {
return nil, b, Undefined, syntaxError(b, "unexpected EOF after object field key")
}
if b[0] != ':' {
return nil, b, Undefined, syntaxError(b, "expected ':' after object field key but found '%c'", b[0])
}
b = skipSpaces(b[1:])
_, b, _, err = d.parseValue(b)
if err != nil {
return nil, b, Undefined, err
}
i++
}
}
func (d decoder) parseArray(b []byte) ([]byte, []byte, Kind, error) {
if len(b) < 2 {
return nil, b[len(b):], Undefined, unexpectedEOF(b)
}
if b[0] != '[' {
return nil, b, Undefined, syntaxError(b, "expected '[' at the beginning of array value")
}
var err error
a := b
n := len(b)
i := 0
b = b[1:]
for {
b = skipSpaces(b)
if len(b) == 0 {
return nil, b, Undefined, syntaxError(b, "missing closing ']' after array value")
}
if b[0] == ']' {
j := (n - len(b)) + 1
return a[:j], a[j:], Array, nil
}
if i != 0 {
if len(b) == 0 {
return nil, b, Undefined, syntaxError(b, "unexpected EOF after array element")
}
if b[0] != ',' {
return nil, b, Undefined, syntaxError(b, "expected ',' after array element but found '%c'", b[0])
}
b = skipSpaces(b[1:])
if len(b) == 0 {
return nil, b, Undefined, unexpectedEOF(b)
}
if b[0] == ']' {
return nil, b, Undefined, syntaxError(b, "unexpected trailing comma after object field")
}
}
_, b, _, err = d.parseValue(b)
if err != nil {
return nil, b, Undefined, err
}
i++
}
}
func (d decoder) parseValue(b []byte) ([]byte, []byte, Kind, error) {
if len(b) == 0 {
return nil, b, Undefined, syntaxError(b, "unexpected end of JSON input")
}
var v []byte
var k Kind
var err error
switch b[0] {
case '{':
v, b, k, err = d.parseObject(b)
case '[':
v, b, k, err = d.parseArray(b)
case '"':
v, b, k, err = d.parseString(b)
case 'n':
v, b, k, err = d.parseNull(b)
case 't':
v, b, k, err = d.parseTrue(b)
case 'f':
v, b, k, err = d.parseFalse(b)
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
v, b, k, err = d.parseNumber(b)
default:
err = syntaxError(b, "invalid character '%c' looking for beginning of value", b[0])
}
return v, b, k, err
}
func hasNullPrefix(b []byte) bool {
return len(b) >= 4 && string(b[:4]) == "null"
}
func hasTruePrefix(b []byte) bool {
return len(b) >= 4 && string(b[:4]) == "true"
}
func hasFalsePrefix(b []byte) bool {
return len(b) >= 5 && string(b[:5]) == "false"
}
func hasPrefix(b []byte, s string) bool {
return len(b) >= len(s) && s == string(b[:len(s)])
}
func hasLeadingSign(b []byte) bool {
return len(b) > 0 && (b[0] == '+' || b[0] == '-')
}
func hasLeadingZeroes(b []byte) bool {
if hasLeadingSign(b) {
b = b[1:]
}
return len(b) > 1 && b[0] == '0' && '0' <= b[1] && b[1] <= '9'
}
func appendToLower(b, s []byte) []byte {
if ascii.Valid(s) { // fast path for ascii strings
i := 0
for j := range s {
c := s[j]
if 'A' <= c && c <= 'Z' {
b = append(b, s[i:j]...)
b = append(b, c+('a'-'A'))
i = j + 1
}
}
return append(b, s[i:]...)
}
for _, r := range string(s) {
b = appendRune(b, foldRune(r))
}
return b
}
func foldRune(r rune) rune {
if r = unicode.SimpleFold(r); 'A' <= r && r <= 'Z' {
r = r + ('a' - 'A')
}
return r
}
+20
View File
@@ -0,0 +1,20 @@
//go:build go1.20
// +build go1.20
package json
import (
"reflect"
"unsafe"
)
func extendSlice(t reflect.Type, s *slice, n int) slice {
arrayType := reflect.ArrayOf(n, t.Elem())
arrayData := reflect.New(arrayType)
reflect.Copy(arrayData.Elem(), reflect.NewAt(t, unsafe.Pointer(s)).Elem())
return slice{
data: unsafe.Pointer(arrayData.Pointer()),
len: s.len,
cap: n,
}
}
+30
View File
@@ -0,0 +1,30 @@
//go:build !go1.20
// +build !go1.20
package json
import (
"reflect"
"unsafe"
)
//go:linkname unsafe_NewArray reflect.unsafe_NewArray
func unsafe_NewArray(rtype unsafe.Pointer, length int) unsafe.Pointer
//go:linkname typedslicecopy reflect.typedslicecopy
//go:noescape
func typedslicecopy(elemType unsafe.Pointer, dst, src slice) int
func extendSlice(t reflect.Type, s *slice, n int) slice {
elemTypeRef := t.Elem()
elemTypePtr := ((*iface)(unsafe.Pointer(&elemTypeRef))).ptr
d := slice{
data: unsafe_NewArray(elemTypePtr, n),
len: s.len,
cap: n,
}
typedslicecopy(elemTypePtr, d, *s)
return d
}
+89
View File
@@ -0,0 +1,89 @@
package json
import (
"math/bits"
"unsafe"
)
const (
lsb = 0x0101010101010101
msb = 0x8080808080808080
)
// escapeIndex finds the index of the first char in `s` that requires escaping.
// A char requires escaping if it's outside of the range of [0x20, 0x7F] or if
// it includes a double quote or backslash. If the escapeHTML mode is enabled,
// the chars <, > and & also require escaping. If no chars in `s` require
// escaping, the return value is -1.
func escapeIndex(s string, escapeHTML bool) int {
chunks := stringToUint64(s)
for _, n := range chunks {
// combine masks before checking for the MSB of each byte. We include
// `n` in the mask to check whether any of the *input* byte MSBs were
// set (i.e. the byte was outside the ASCII range).
mask := n | below(n, 0x20) | contains(n, '"') | contains(n, '\\')
if escapeHTML {
mask |= contains(n, '<') | contains(n, '>') | contains(n, '&')
}
if (mask & msb) != 0 {
return bits.TrailingZeros64(mask&msb) / 8
}
}
for i := len(chunks) * 8; i < len(s); i++ {
c := s[i]
if c < 0x20 || c > 0x7f || c == '"' || c == '\\' || (escapeHTML && (c == '<' || c == '>' || c == '&')) {
return i
}
}
return -1
}
func escapeByteRepr(b byte) byte {
switch b {
case '\\', '"':
return b
case '\b':
return 'b'
case '\f':
return 'f'
case '\n':
return 'n'
case '\r':
return 'r'
case '\t':
return 't'
}
return 0
}
// below return a mask that can be used to determine if any of the bytes
// in `n` are below `b`. If a byte's MSB is set in the mask then that byte was
// below `b`. The result is only valid if `b`, and each byte in `n`, is below
// 0x80.
func below(n uint64, b byte) uint64 {
return n - expand(b)
}
// contains returns a mask that can be used to determine if any of the
// bytes in `n` are equal to `b`. If a byte's MSB is set in the mask then
// that byte is equal to `b`. The result is only valid if `b`, and each
// byte in `n`, is below 0x80.
func contains(n uint64, b byte) uint64 {
return (n ^ expand(b)) - lsb
}
// expand puts the specified byte into each of the 8 bytes of a uint64.
func expand(b byte) uint64 {
return lsb * uint64(b)
}
func stringToUint64(s string) []uint64 {
return *(*[]uint64)(unsafe.Pointer(&sliceHeader{
Data: *(*unsafe.Pointer)(unsafe.Pointer(&s)),
Len: len(s) / 8,
Cap: len(s) / 8,
}))
}
+426
View File
@@ -0,0 +1,426 @@
package json
import (
"strconv"
"sync"
"unsafe"
)
// Tokenizer is an iterator-style type which can be used to progressively parse
// through a json input.
//
// Tokenizing json is useful to build highly efficient parsing operations, for
// example when doing tranformations on-the-fly where as the program reads the
// input and produces the transformed json to an output buffer.
//
// Here is a common pattern to use a tokenizer:
//
// for t := json.NewTokenizer(b); t.Next(); {
// switch k := t.Kind(); k.Class() {
// case json.Null:
// ...
// case json.Bool:
// ...
// case json.Num:
// ...
// case json.String:
// ...
// case json.Array:
// ...
// case json.Object:
// ...
// }
// }
type Tokenizer struct {
// When the tokenizer is positioned on a json delimiter this field is not
// zero. In this case the possible values are '{', '}', '[', ']', ':', and
// ','.
Delim Delim
// This field contains the raw json token that the tokenizer is pointing at.
// When Delim is not zero, this field is a single-element byte slice
// continaing the delimiter value. Otherwise, this field holds values like
// null, true, false, numbers, or quoted strings.
Value RawValue
// When the tokenizer has encountered invalid content this field is not nil.
Err error
// When the value is in an array or an object, this field contains the depth
// at which it was found.
Depth int
// When the value is in an array or an object, this field contains the
// position at which it was found.
Index int
// This field is true when the value is the key of an object.
IsKey bool
// Tells whether the next value read from the tokenizer is a key.
isKey bool
// json input for the tokenizer, pointing at data right after the last token
// that was parsed.
json []byte
// Stack used to track entering and leaving arrays, objects, and keys.
stack *stack
// Decoder used for parsing.
decoder
}
// NewTokenizer constructs a new Tokenizer which reads its json input from b.
func NewTokenizer(b []byte) *Tokenizer {
return &Tokenizer{
json: b,
decoder: decoder{flags: internalParseFlags(b)},
}
}
// Reset erases the state of t and re-initializes it with the json input from b.
func (t *Tokenizer) Reset(b []byte) {
if t.stack != nil {
releaseStack(t.stack)
}
// This code is similar to:
//
// *t = Tokenizer{json: b}
//
// However, it does not compile down to an invocation of duff-copy.
t.Delim = 0
t.Value = nil
t.Err = nil
t.Depth = 0
t.Index = 0
t.IsKey = false
t.isKey = false
t.json = b
t.stack = nil
t.decoder = decoder{flags: internalParseFlags(b)}
}
// Next returns a new tokenizer pointing at the next token, or the zero-value of
// Tokenizer if the end of the json input has been reached.
//
// If the tokenizer encounters malformed json while reading the input the method
// sets t.Err to an error describing the issue, and returns false. Once an error
// has been encountered, the tokenizer will always fail until its input is
// cleared by a call to its Reset method.
func (t *Tokenizer) Next() bool {
if t.Err != nil {
return false
}
// Inlined code of the skipSpaces function, this give a ~15% speed boost.
i := 0
skipLoop:
for _, c := range t.json {
switch c {
case sp, ht, nl, cr:
i++
default:
break skipLoop
}
}
if i > 0 {
t.json = t.json[i:]
}
if len(t.json) == 0 {
t.Reset(nil)
return false
}
var kind Kind
switch t.json[0] {
case '"':
t.Delim = 0
t.Value, t.json, kind, t.Err = t.parseString(t.json)
case 'n':
t.Delim = 0
t.Value, t.json, kind, t.Err = t.parseNull(t.json)
case 't':
t.Delim = 0
t.Value, t.json, kind, t.Err = t.parseTrue(t.json)
case 'f':
t.Delim = 0
t.Value, t.json, kind, t.Err = t.parseFalse(t.json)
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
t.Delim = 0
t.Value, t.json, kind, t.Err = t.parseNumber(t.json)
case '{', '}', '[', ']', ':', ',':
t.Delim, t.Value, t.json = Delim(t.json[0]), t.json[:1], t.json[1:]
switch t.Delim {
case '{':
kind = Object
case '[':
kind = Array
}
default:
t.Delim = 0
t.Value, t.json, t.Err = t.json[:1], t.json[1:], syntaxError(t.json, "expected token but found '%c'", t.json[0])
}
t.Depth = t.depth()
t.Index = t.index()
t.flags = t.flags.withKind(kind)
if t.Delim == 0 {
t.IsKey = t.isKey
} else {
t.IsKey = false
switch t.Delim {
case '{':
t.isKey = true
t.push(inObject)
case '[':
t.push(inArray)
case '}':
t.Err = t.pop(inObject)
t.Depth--
t.Index = t.index()
case ']':
t.Err = t.pop(inArray)
t.Depth--
t.Index = t.index()
case ':':
t.isKey = false
case ',':
if t.stack == nil || len(t.stack.state) == 0 {
t.Err = syntaxError(t.json, "found unexpected comma")
return false
}
if t.stack.is(inObject) {
t.isKey = true
}
t.stack.state[len(t.stack.state)-1].len++
}
}
return (t.Delim != 0 || len(t.Value) != 0) && t.Err == nil
}
func (t *Tokenizer) depth() int {
if t.stack == nil {
return 0
}
return t.stack.depth()
}
func (t *Tokenizer) index() int {
if t.stack == nil {
return 0
}
return t.stack.index()
}
func (t *Tokenizer) push(typ scope) {
if t.stack == nil {
t.stack = acquireStack()
}
t.stack.push(typ)
}
func (t *Tokenizer) pop(expect scope) error {
if t.stack == nil || !t.stack.pop(expect) {
return syntaxError(t.json, "found unexpected character while tokenizing json input")
}
return nil
}
// Kind returns the kind of the value that the tokenizer is currently positioned
// on.
func (t *Tokenizer) Kind() Kind { return t.flags.kind() }
// Bool returns a bool containing the value of the json boolean that the
// tokenizer is currently pointing at.
//
// This method must only be called after checking the kind of the token via a
// call to Kind.
//
// If the tokenizer is not positioned on a boolean, the behavior is undefined.
func (t *Tokenizer) Bool() bool { return t.flags.kind() == True }
// Int returns a byte slice containing the value of the json number that the
// tokenizer is currently pointing at.
//
// This method must only be called after checking the kind of the token via a
// call to Kind.
//
// If the tokenizer is not positioned on an integer, the behavior is undefined.
func (t *Tokenizer) Int() int64 {
i, _, _ := t.parseInt(t.Value, int64Type)
return i
}
// Uint returns a byte slice containing the value of the json number that the
// tokenizer is currently pointing at.
//
// This method must only be called after checking the kind of the token via a
// call to Kind.
//
// If the tokenizer is not positioned on a positive integer, the behavior is
// undefined.
func (t *Tokenizer) Uint() uint64 {
u, _, _ := t.parseUint(t.Value, uint64Type)
return u
}
// Float returns a byte slice containing the value of the json number that the
// tokenizer is currently pointing at.
//
// This method must only be called after checking the kind of the token via a
// call to Kind.
//
// If the tokenizer is not positioned on a number, the behavior is undefined.
func (t *Tokenizer) Float() float64 {
f, _ := strconv.ParseFloat(*(*string)(unsafe.Pointer(&t.Value)), 64)
return f
}
// String returns a byte slice containing the value of the json string that the
// tokenizer is currently pointing at.
//
// This method must only be called after checking the kind of the token via a
// call to Kind.
//
// When possible, the returned byte slice references the backing array of the
// tokenizer. A new slice is only allocated if the tokenizer needed to unescape
// the json string.
//
// If the tokenizer is not positioned on a string, the behavior is undefined.
func (t *Tokenizer) String() []byte {
if t.flags.kind() == Unescaped && len(t.Value) > 1 {
return t.Value[1 : len(t.Value)-1] // unquote
}
s, _, _, _ := t.parseStringUnquote(t.Value, nil)
return s
}
// Remaining returns the number of bytes left to parse.
//
// The position of the tokenizer's current Value within the original byte slice
// can be calculated like so:
//
// end := len(b) - tok.Remaining()
// start := end - len(tok.Value)
//
// And slicing b[start:end] will yield the tokenizer's current Value.
func (t *Tokenizer) Remaining() int {
return len(t.json)
}
// RawValue represents a raw json value, it is intended to carry null, true,
// false, number, and string values only.
type RawValue []byte
// String returns true if v contains a string value.
func (v RawValue) String() bool { return len(v) != 0 && v[0] == '"' }
// Null returns true if v contains a null value.
func (v RawValue) Null() bool { return len(v) != 0 && v[0] == 'n' }
// True returns true if v contains a true value.
func (v RawValue) True() bool { return len(v) != 0 && v[0] == 't' }
// False returns true if v contains a false value.
func (v RawValue) False() bool { return len(v) != 0 && v[0] == 'f' }
// Number returns true if v contains a number value.
func (v RawValue) Number() bool {
if len(v) != 0 {
switch v[0] {
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
return true
}
}
return false
}
// AppendUnquote writes the unquoted version of the string value in v into b.
func (v RawValue) AppendUnquote(b []byte) []byte {
d := decoder{}
s, r, _, err := d.parseStringUnquote(v, b)
if err != nil {
panic(err)
}
if len(r) != 0 {
panic(syntaxError(r, "unexpected trailing tokens after json value"))
}
return append(b, s...)
}
// Unquote returns the unquoted version of the string value in v.
func (v RawValue) Unquote() []byte {
return v.AppendUnquote(nil)
}
type scope int
const (
inArray scope = iota
inObject
)
type state struct {
typ scope
len int
}
type stack struct {
state []state
}
func (s *stack) push(typ scope) {
s.state = append(s.state, state{typ: typ, len: 1})
}
func (s *stack) pop(expect scope) bool {
i := len(s.state) - 1
if i < 0 {
return false
}
if found := s.state[i]; expect != found.typ {
return false
}
s.state = s.state[:i]
return true
}
func (s *stack) is(typ scope) bool {
return len(s.state) != 0 && s.state[len(s.state)-1].typ == typ
}
func (s *stack) depth() int {
return len(s.state)
}
func (s *stack) index() int {
if len(s.state) == 0 {
return 0
}
return s.state[len(s.state)-1].len - 1
}
func acquireStack() *stack {
s, _ := stackPool.Get().(*stack)
if s == nil {
s = &stack{state: make([]state, 0, 4)}
} else {
s.state = s.state[:0]
}
return s
}
func releaseStack(s *stack) {
stackPool.Put(s)
}
var stackPool sync.Pool // *stack