8
"github.com/pelletier/go-toml/v2/internal/characters"
9
"github.com/pelletier/go-toml/v2/internal/danger"
12
// ParserError describes an error relative to the content of the document.
14
// It cannot outlive the instance of Parser it refers to, and may cause panics
15
// if the parser is reset.
16
type ParserError struct {
19
Key []string // optional
22
// Error is the implementation of the error interface.
23
func (e *ParserError) Error() string {
27
// NewParserError is a convenience function to create a ParserError
29
// Warning: Highlight needs to be a subslice of Parser.data, so only slices
30
// returned by Parser.Raw are valid candidates.
31
func NewParserError(highlight []byte, format string, args ...interface{}) error {
34
Message: fmt.Errorf(format, args...).Error(),
38
// Parser scans over a TOML-encoded document and generates an iterative AST.
40
// To prime the Parser, first reset it with the contents of a TOML document.
41
// Then, process all top-level expressions sequentially. See Example.
43
// Don't forget to check Error() after you're done parsing.
45
// Each top-level expression needs to be fully processed before calling
46
// NextExpression() again. Otherwise, calls to various Node methods may panic if
47
// the parser has moved on the next expression.
49
// For performance reasons, go-toml doesn't make a copy of the input bytes to
50
// the parser. Make sure to copy all the bytes you need to outlive the slice
51
// given to the parser.
63
// Data returns the slice provided to the last call to Reset.
64
func (p *Parser) Data() []byte {
68
// Range returns a range description that corresponds to a given slice of the
69
// input. If the argument is not a subslice of the parser input, this function
71
func (p *Parser) Range(b []byte) Range {
73
Offset: uint32(danger.SubsliceOffset(p.data, b)),
74
Length: uint32(len(b)),
78
// Raw returns the slice corresponding to the bytes in the given range.
79
func (p *Parser) Raw(raw Range) []byte {
80
return p.data[raw.Offset : raw.Offset+raw.Length]
83
// Reset brings the parser to its initial state for a given input. It wipes an
84
// reuses internal storage to reduce allocation.
85
func (p *Parser) Reset(b []byte) {
87
p.ref = invalidReference
94
// NextExpression parses the next top-level expression. If an expression was
95
// successfully parsed, it returns true. If the parser is at the end of the
96
// document or an error occurred, it returns false.
98
// Retrieve the parsed expression with Expression().
99
func (p *Parser) NextExpression() bool {
100
if len(p.left) == 0 || p.err != nil {
105
p.ref = invalidReference
108
if len(p.left) == 0 || p.err != nil {
113
p.left, p.err = p.parseNewline(p.left)
116
if len(p.left) == 0 || p.err != nil {
120
p.ref, p.left, p.err = p.parseExpression(p.left)
134
// Expression returns a pointer to the node representing the last successfully
136
func (p *Parser) Expression() *Node {
137
return p.builder.NodeAt(p.ref)
140
// Error returns any error that has occurred during parsing.
141
func (p *Parser) Error() error {
145
// Position describes a position in the input.
146
type Position struct {
147
// Number of bytes from the beginning of the input.
149
// Line number, starting at 1.
151
// Column number, starting at 1.
155
// Shape describes the position of a range in the input.
161
func (p *Parser) position(b []byte) Position {
162
offset := danger.SubsliceOffset(p.data, b)
164
lead := p.data[:offset]
168
Line: bytes.Count(lead, []byte{'\n'}) + 1,
169
Column: len(lead) - bytes.LastIndex(lead, []byte{'\n'}),
173
// Shape returns the shape of the given range in the input. Will
174
// panic if the range is not a subslice of the input.
175
func (p *Parser) Shape(r Range) Shape {
178
Start: p.position(raw),
179
End: p.position(raw[r.Length:]),
183
func (p *Parser) parseNewline(b []byte) ([]byte, error) {
189
_, rest, err := scanWindowsNewline(b)
193
return nil, NewParserError(b[0:1], "expected newline but got %#U", b[0])
196
func (p *Parser) parseComment(b []byte) (reference, []byte, error) {
197
ref := invalidReference
198
data, rest, err := scanComment(b)
199
if p.KeepComments && err == nil {
200
ref = p.builder.Push(Node{
206
return ref, rest, err
209
func (p *Parser) parseExpression(b []byte) (reference, []byte, error) {
210
// expression = ws [ comment ]
211
// expression =/ ws keyval ws [ comment ]
212
// expression =/ ws table ws [ comment ]
213
ref := invalidReference
215
b = p.parseWhitespace(b)
222
ref, rest, err := p.parseComment(b)
223
return ref, rest, err
226
if b[0] == '\n' || b[0] == '\r' {
232
ref, b, err = p.parseTable(b)
234
ref, b, err = p.parseKeyval(b)
241
b = p.parseWhitespace(b)
243
if len(b) > 0 && b[0] == '#' {
244
cref, rest, err := p.parseComment(b)
245
if cref != invalidReference {
246
p.builder.Chain(ref, cref)
248
return ref, rest, err
254
func (p *Parser) parseTable(b []byte) (reference, []byte, error) {
255
// table = std-table / array-table
256
if len(b) > 1 && b[1] == '[' {
257
return p.parseArrayTable(b)
260
return p.parseStdTable(b)
263
func (p *Parser) parseArrayTable(b []byte) (reference, []byte, error) {
264
// array-table = array-table-open key array-table-close
265
// array-table-open = %x5B.5B ws ; [[ Double left square bracket
266
// array-table-close = ws %x5D.5D ; ]] Double right square bracket
267
ref := p.builder.Push(Node{
272
b = p.parseWhitespace(b)
274
k, b, err := p.parseKey(b)
279
p.builder.AttachChild(ref, k)
280
b = p.parseWhitespace(b)
282
b, err = expect(']', b)
287
b, err = expect(']', b)
292
func (p *Parser) parseStdTable(b []byte) (reference, []byte, error) {
293
// std-table = std-table-open key std-table-close
294
// std-table-open = %x5B ws ; [ Left square bracket
295
// std-table-close = ws %x5D ; ] Right square bracket
296
ref := p.builder.Push(Node{
301
b = p.parseWhitespace(b)
303
key, b, err := p.parseKey(b)
308
p.builder.AttachChild(ref, key)
310
b = p.parseWhitespace(b)
312
b, err = expect(']', b)
317
func (p *Parser) parseKeyval(b []byte) (reference, []byte, error) {
318
// keyval = key keyval-sep val
319
ref := p.builder.Push(Node{
323
key, b, err := p.parseKey(b)
325
return invalidReference, nil, err
328
// keyval-sep = ws %x3D ws ; =
330
b = p.parseWhitespace(b)
333
return invalidReference, nil, NewParserError(b, "expected = after a key, but the document ends there")
336
b, err = expect('=', b)
338
return invalidReference, nil, err
341
b = p.parseWhitespace(b)
343
valRef, b, err := p.parseVal(b)
348
p.builder.Chain(valRef, key)
349
p.builder.AttachChild(ref, valRef)
354
//nolint:cyclop,funlen
355
func (p *Parser) parseVal(b []byte) (reference, []byte, error) {
356
// val = string / boolean / array / inline-table / date-time / float / integer
357
ref := invalidReference
360
return ref, nil, NewParserError(b, "expected value, not eof")
370
if scanFollowsMultilineBasicStringDelimiter(b) {
371
raw, v, b, err = p.parseMultilineBasicString(b)
373
raw, v, b, err = p.parseBasicString(b)
377
ref = p.builder.Push(Node{
388
if scanFollowsMultilineLiteralStringDelimiter(b) {
389
raw, v, b, err = p.parseMultilineLiteralString(b)
391
raw, v, b, err = p.parseLiteralString(b)
395
ref = p.builder.Push(Node{
404
if !scanFollowsTrue(b) {
405
return ref, nil, NewParserError(atmost(b, 4), "expected 'true'")
408
ref = p.builder.Push(Node{
413
return ref, b[4:], nil
415
if !scanFollowsFalse(b) {
416
return ref, nil, NewParserError(atmost(b, 5), "expected 'false'")
419
ref = p.builder.Push(Node{
424
return ref, b[5:], nil
426
return p.parseValArray(b)
428
return p.parseInlineTable(b)
430
return p.parseIntOrFloatOrDateTime(b)
434
func atmost(b []byte, n int) []byte {
442
func (p *Parser) parseLiteralString(b []byte) ([]byte, []byte, []byte, error) {
443
v, rest, err := scanLiteralString(b)
445
return nil, nil, nil, err
448
return v, v[1 : len(v)-1], rest, nil
451
func (p *Parser) parseInlineTable(b []byte) (reference, []byte, error) {
452
// inline-table = inline-table-open [ inline-table-keyvals ] inline-table-close
453
// inline-table-open = %x7B ws ; {
454
// inline-table-close = ws %x7D ; }
455
// inline-table-sep = ws %x2C ws ; , Comma
456
// inline-table-keyvals = keyval [ inline-table-sep inline-table-keyvals ]
457
parent := p.builder.Push(Node{
472
b = p.parseWhitespace(b)
475
return parent, nil, NewParserError(previousB[:1], "inline table is incomplete")
483
b, err = expect(',', b)
485
return parent, nil, err
487
b = p.parseWhitespace(b)
492
kv, b, err = p.parseKeyval(b)
494
return parent, nil, err
498
p.builder.AttachChild(parent, kv)
500
p.builder.Chain(child, kv)
507
rest, err := expect('}', b)
509
return parent, rest, err
512
//nolint:funlen,cyclop
513
func (p *Parser) parseValArray(b []byte) (reference, []byte, error) {
514
// array = array-open [ array-values ] ws-comment-newline array-close
515
// array-open = %x5B ; [
516
// array-close = %x5D ; ]
517
// array-values = ws-comment-newline val ws-comment-newline array-sep array-values
518
// array-values =/ ws-comment-newline val ws-comment-newline [ array-sep ]
519
// array-sep = %x2C ; , Comma
520
// ws-comment-newline = *( wschar / [ comment ] newline )
524
parent := p.builder.Push(Node{
528
// First indicates whether the parser is looking for the first element
529
// (non-comment) of the array.
532
lastChild := invalidReference
534
addChild := func(valueRef reference) {
535
if lastChild == invalidReference {
536
p.builder.AttachChild(parent, valueRef)
538
p.builder.Chain(lastChild, valueRef)
545
cref := invalidReference
546
cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
548
return parent, nil, err
551
if cref != invalidReference {
556
return parent, nil, NewParserError(arrayStart[:1], "array is incomplete")
565
return parent, nil, NewParserError(b[0:1], "array cannot start with comma")
569
cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
571
return parent, nil, err
573
if cref != invalidReference {
577
return parent, nil, NewParserError(b[0:1], "array elements must be separated by commas")
580
// TOML allows trailing commas in arrays.
581
if len(b) > 0 && b[0] == ']' {
585
var valueRef reference
586
valueRef, b, err = p.parseVal(b)
588
return parent, nil, err
593
cref, b, err = p.parseOptionalWhitespaceCommentNewline(b)
595
return parent, nil, err
597
if cref != invalidReference {
604
rest, err := expect(']', b)
606
return parent, rest, err
609
func (p *Parser) parseOptionalWhitespaceCommentNewline(b []byte) (reference, []byte, error) {
610
rootCommentRef := invalidReference
611
latestCommentRef := invalidReference
613
addComment := func(ref reference) {
614
if rootCommentRef == invalidReference {
616
} else if latestCommentRef == invalidReference {
617
p.builder.AttachChild(rootCommentRef, ref)
618
latestCommentRef = ref
620
p.builder.Chain(latestCommentRef, ref)
621
latestCommentRef = ref
627
b = p.parseWhitespace(b)
629
if len(b) > 0 && b[0] == '#' {
631
ref, b, err = p.parseComment(b)
633
return invalidReference, nil, err
635
if ref != invalidReference {
644
if b[0] == '\n' || b[0] == '\r' {
645
b, err = p.parseNewline(b)
647
return invalidReference, nil, err
654
return rootCommentRef, b, nil
657
func (p *Parser) parseMultilineLiteralString(b []byte) ([]byte, []byte, []byte, error) {
658
token, rest, err := scanMultilineLiteralString(b)
660
return nil, nil, nil, err
665
// skip the immediate new line
666
if token[i] == '\n' {
668
} else if token[i] == '\r' && token[i+1] == '\n' {
672
return token, token[i : len(token)-3], rest, err
675
//nolint:funlen,gocognit,cyclop
676
func (p *Parser) parseMultilineBasicString(b []byte) ([]byte, []byte, []byte, error) {
677
// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
678
// ml-basic-string-delim
679
// ml-basic-string-delim = 3quotation-mark
680
// ml-basic-body = *mlb-content *( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
682
// mlb-content = mlb-char / newline / mlb-escaped-nl
683
// mlb-char = mlb-unescaped / escaped
684
// mlb-quotes = 1*2quotation-mark
685
// mlb-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
686
// mlb-escaped-nl = escape ws newline *( wschar / newline )
687
token, escaped, rest, err := scanMultilineBasicString(b)
689
return nil, nil, nil, err
694
// skip the immediate new line
695
if token[i] == '\n' {
697
} else if token[i] == '\r' && token[i+1] == '\n' {
703
endIdx := len(token) - len(`"""`)
706
str := token[startIdx:endIdx]
707
verr := characters.Utf8TomlValidAlreadyEscaped(str)
709
return token, str, rest, nil
711
return nil, nil, nil, NewParserError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8")
714
var builder bytes.Buffer
716
// The scanner ensures that the token starts and ends with quotes and that
717
// escapes are balanced.
718
for i < len(token)-3 {
723
// When the last non-whitespace character on a line is an unescaped \,
724
// it will be trimmed along with all whitespace (including newlines) up
725
// to the next non-whitespace character or closing delimiter.
727
isLastNonWhitespaceOnLine := false
730
for ; j < len(token)-3-i; j++ {
735
if token[i+j+1] == '\n' {
739
isLastNonWhitespaceOnLine = true
743
if isLastNonWhitespaceOnLine {
745
for ; i < len(token)-3; i++ {
747
if !(c == '\n' || c == '\r' || c == ' ' || c == '\t') {
764
builder.WriteByte('\b')
766
builder.WriteByte('\f')
768
builder.WriteByte('\n')
770
builder.WriteByte('\r')
772
builder.WriteByte('\t')
774
builder.WriteByte(0x1B)
776
x, err := hexToRune(atmost(token[i+1:], 4), 4)
778
return nil, nil, nil, err
783
x, err := hexToRune(atmost(token[i+1:], 8), 8)
785
return nil, nil, nil, err
791
return nil, nil, nil, NewParserError(token[i:i+1], "invalid escaped character %#U", c)
795
size := characters.Utf8ValidNext(token[i:])
797
return nil, nil, nil, NewParserError(token[i:i+1], "invalid character %#U", c)
799
builder.Write(token[i : i+size])
804
return token, builder.Bytes(), rest, nil
807
func (p *Parser) parseKey(b []byte) (reference, []byte, error) {
808
// key = simple-key / dotted-key
809
// simple-key = quoted-key / unquoted-key
811
// unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
812
// quoted-key = basic-string / literal-string
813
// dotted-key = simple-key 1*( dot-sep simple-key )
815
// dot-sep = ws %x2E ws ; . Period
816
raw, key, b, err := p.parseSimpleKey(b)
818
return invalidReference, nil, err
821
ref := p.builder.Push(Node{
828
b = p.parseWhitespace(b)
829
if len(b) > 0 && b[0] == '.' {
830
b = p.parseWhitespace(b[1:])
832
raw, key, b, err = p.parseSimpleKey(b)
837
p.builder.PushAndChain(Node{
850
func (p *Parser) parseSimpleKey(b []byte) (raw, key, rest []byte, err error) {
852
return nil, nil, nil, NewParserError(b, "expected key but found none")
855
// simple-key = quoted-key / unquoted-key
856
// unquoted-key = 1*( ALPHA / DIGIT / %x2D / %x5F ) ; A-Z / a-z / 0-9 / - / _
857
// quoted-key = basic-string / literal-string
860
return p.parseLiteralString(b)
862
return p.parseBasicString(b)
863
case isUnquotedKeyChar(b[0]):
864
key, rest = scanUnquotedKey(b)
865
return key, key, rest, nil
867
return nil, nil, nil, NewParserError(b[0:1], "invalid character at start of key: %c", b[0])
871
//nolint:funlen,cyclop
872
func (p *Parser) parseBasicString(b []byte) ([]byte, []byte, []byte, error) {
873
// basic-string = quotation-mark *basic-char quotation-mark
874
// quotation-mark = %x22 ; "
875
// basic-char = basic-unescaped / escaped
876
// basic-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
877
// escaped = escape escape-seq-char
878
// escape-seq-char = %x22 ; " quotation mark U+0022
879
// escape-seq-char =/ %x5C ; \ reverse solidus U+005C
880
// escape-seq-char =/ %x62 ; b backspace U+0008
881
// escape-seq-char =/ %x66 ; f form feed U+000C
882
// escape-seq-char =/ %x6E ; n line feed U+000A
883
// escape-seq-char =/ %x72 ; r carriage return U+000D
884
// escape-seq-char =/ %x74 ; t tab U+0009
885
// escape-seq-char =/ %x75 4HEXDIG ; uXXXX U+XXXX
886
// escape-seq-char =/ %x55 8HEXDIG ; UXXXXXXXX U+XXXXXXXX
887
token, escaped, rest, err := scanBasicString(b)
889
return nil, nil, nil, err
893
endIdx := len(token) - len(`"`)
895
// Fast path. If there is no escape sequence, the string should just be
896
// an UTF-8 encoded string, which is the same as Go. In that case,
897
// validate the string and return a direct reference to the buffer.
899
str := token[startIdx:endIdx]
900
verr := characters.Utf8TomlValidAlreadyEscaped(str)
902
return token, str, rest, nil
904
return nil, nil, nil, NewParserError(str[verr.Index:verr.Index+verr.Size], "invalid UTF-8")
909
var builder bytes.Buffer
911
// The scanner ensures that the token starts and ends with quotes and that
912
// escapes are balanced.
913
for i < len(token)-1 {
923
builder.WriteByte('\b')
925
builder.WriteByte('\f')
927
builder.WriteByte('\n')
929
builder.WriteByte('\r')
931
builder.WriteByte('\t')
933
builder.WriteByte(0x1B)
935
x, err := hexToRune(token[i+1:len(token)-1], 4)
937
return nil, nil, nil, err
943
x, err := hexToRune(token[i+1:len(token)-1], 8)
945
return nil, nil, nil, err
951
return nil, nil, nil, NewParserError(token[i:i+1], "invalid escaped character %#U", c)
955
size := characters.Utf8ValidNext(token[i:])
957
return nil, nil, nil, NewParserError(token[i:i+1], "invalid character %#U", c)
959
builder.Write(token[i : i+size])
964
return token, builder.Bytes(), rest, nil
967
func hexToRune(b []byte, length int) (rune, error) {
969
return -1, NewParserError(b, "unicode point needs %d character, not %d", length, len(b))
974
for i, c := range b {
977
case '0' <= c && c <= '9':
979
case 'a' <= c && c <= 'f':
980
d = uint32(c - 'a' + 10)
981
case 'A' <= c && c <= 'F':
982
d = uint32(c - 'A' + 10)
984
return -1, NewParserError(b[i:i+1], "non-hex character")
989
if r > unicode.MaxRune || 0xD800 <= r && r < 0xE000 {
990
return -1, NewParserError(b, "escape sequence is invalid Unicode code point")
996
func (p *Parser) parseWhitespace(b []byte) []byte {
998
// wschar = %x20 ; Space
999
// wschar =/ %x09 ; Horizontal tab
1000
_, rest := scanWhitespace(b)
1006
func (p *Parser) parseIntOrFloatOrDateTime(b []byte) (reference, []byte, error) {
1009
if !scanFollowsInf(b) {
1010
return invalidReference, nil, NewParserError(atmost(b, 3), "expected 'inf'")
1013
return p.builder.Push(Node{
1018
if !scanFollowsNan(b) {
1019
return invalidReference, nil, NewParserError(atmost(b, 3), "expected 'nan'")
1022
return p.builder.Push(Node{
1027
return p.scanIntOrFloat(b)
1031
return p.scanIntOrFloat(b)
1039
for idx, c := range b[:s] {
1044
if idx == 2 && c == ':' || (idx == 4 && c == '-') {
1045
return p.scanDateTime(b)
1051
return p.scanIntOrFloat(b)
1054
func (p *Parser) scanDateTime(b []byte) (reference, []byte, error) {
1055
// scans for contiguous characters in [0-9T:Z.+-], and up to one space if
1056
// followed by a digit.
1064
for ; i < len(b); i++ {
1071
const minOffsetOfTz = 8
1072
if i >= minOffsetOfTz {
1075
case c == 'T' || c == 't' || c == ':' || c == '.':
1077
case c == '+' || c == '-' || c == 'Z' || c == 'z':
1080
if !seenSpace && i+1 < len(b) && isDigit(b[i+1]) {
1082
// Avoid reaching past the end of the document in case the time
1083
// is malformed. See TestIssue585.
1104
kind = LocalDateTime
1113
return p.builder.Push(Node{
1119
//nolint:funlen,gocognit,cyclop
1120
func (p *Parser) scanIntOrFloat(b []byte) (reference, []byte, error) {
1123
if len(b) > 2 && b[0] == '0' && b[1] != '.' && b[1] != 'e' && b[1] != 'E' {
1124
var isValidRune validRuneFn
1128
isValidRune = isValidHexRune
1130
isValidRune = isValidOctalRune
1132
isValidRune = isValidBinaryRune
1137
if isValidRune != nil {
1139
for ; i < len(b); i++ {
1140
if !isValidRune(b[i]) {
1146
return p.builder.Push(Node{
1154
for ; i < len(b); i++ {
1157
if c >= '0' && c <= '9' || c == '+' || c == '-' || c == '_' {
1161
if c == '.' || c == 'e' || c == 'E' {
1168
if scanFollowsInf(b[i:]) {
1169
return p.builder.Push(Node{
1175
return invalidReference, nil, NewParserError(b[i:i+1], "unexpected character 'i' while scanning for a number")
1179
if scanFollowsNan(b[i:]) {
1180
return p.builder.Push(Node{
1186
return invalidReference, nil, NewParserError(b[i:i+1], "unexpected character 'n' while scanning for a number")
1193
return invalidReference, b, NewParserError(b, "incomplete number")
1202
return p.builder.Push(Node{
1208
func isDigit(r byte) bool {
1209
return r >= '0' && r <= '9'
1212
type validRuneFn func(r byte) bool
1214
func isValidHexRune(r byte) bool {
1215
return r >= 'a' && r <= 'f' ||
1216
r >= 'A' && r <= 'F' ||
1217
r >= '0' && r <= '9' ||
1221
func isValidOctalRune(r byte) bool {
1222
return r >= '0' && r <= '7' || r == '_'
1225
func isValidBinaryRune(r byte) bool {
1226
return r == '0' || r == '1' || r == '_'
1229
func expect(x byte, b []byte) ([]byte, error) {
1231
return nil, NewParserError(b, "expected character %c but the document ended here", x)
1235
return nil, NewParserError(b[0:1], "expected character %c", x)