gitech

1
// Copyright 2021 The Gitea Authors. All rights reserved.
2
// SPDX-License-Identifier: MIT
3

4
package csv
5

6
import (
7
	"bytes"
8
	stdcsv "encoding/csv"
9
	"io"
10
	"path/filepath"
11
	"regexp"
12
	"strings"
13

14
	"code.gitea.io/gitea/modules/markup"
15
	"code.gitea.io/gitea/modules/translation"
16
	"code.gitea.io/gitea/modules/util"
17
)
18

19
const (
20
	maxLines        = 10
21
	guessSampleSize = 1e4 // 10k
22
)
23

24
// CreateReader creates a csv.Reader with the given delimiter.
25
func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
26
	rd := stdcsv.NewReader(input)
27
	rd.Comma = delimiter
28
	if delimiter != '\t' && delimiter != ' ' {
29
		// TrimLeadingSpace can't be true when delimiter is a tab or a space as the value for a column might be empty,
30
		// thus would change `\t\t` to just `\t` or `  ` (two spaces) to just ` ` (single space)
31
		rd.TrimLeadingSpace = true
32
	}
33
	return rd
34
}
35

36
// CreateReaderAndDetermineDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
37
// Reads at most guessSampleSize bytes.
38
func CreateReaderAndDetermineDelimiter(ctx *markup.RenderContext, rd io.Reader) (*stdcsv.Reader, error) {
39
	data := make([]byte, guessSampleSize)
40
	size, err := util.ReadAtMost(rd, data)
41
	if err != nil {
42
		return nil, err
43
	}
44

45
	return CreateReader(
46
		io.MultiReader(bytes.NewReader(data[:size]), rd),
47
		determineDelimiter(ctx, data[:size]),
48
	), nil
49
}
50

51
// determineDelimiter takes a RenderContext and if it isn't nil and the Filename has an extension that specifies the delimiter,
52
// it is used as the delimiter. Otherwise we call guessDelimiter with the data passed
53
func determineDelimiter(ctx *markup.RenderContext, data []byte) rune {
54
	extension := ".csv"
55
	if ctx != nil {
56
		extension = strings.ToLower(filepath.Ext(ctx.RelativePath))
57
	}
58

59
	var delimiter rune
60
	switch extension {
61
	case ".tsv":
62
		delimiter = '\t'
63
	case ".psv":
64
		delimiter = '|'
65
	default:
66
		delimiter = guessDelimiter(data)
67
	}
68

69
	return delimiter
70
}
71

72
// quoteRegexp follows the RFC-4180 CSV standard for when double-quotes are used to enclose fields, then a double-quote appearing inside a
73
// field must be escaped by preceding it with another double quote. https://www.ietf.org/rfc/rfc4180.txt
74
// This finds all quoted strings that have escaped quotes.
75
var quoteRegexp = regexp.MustCompile(`"[^"]*"`)
76

77
// removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can reliably have each row on one line
78
// (quoted strings often have new lines within the string)
79
func removeQuotedString(text string) string {
80
	return quoteRegexp.ReplaceAllLiteralString(text, "")
81
}
82

83
// guessDelimiter takes up to maxLines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors.
84
// If more than one delimiter passes, the delimiter that results in the most columns is returned.
85
func guessDelimiter(data []byte) rune {
86
	delimiter := guessFromBeforeAfterQuotes(data)
87
	if delimiter != 0 {
88
		return delimiter
89
	}
90

91
	// Removes quoted values so we don't have columns with new lines in them
92
	text := removeQuotedString(string(data))
93

94
	// Make the text just be maxLines or less, ignoring truncated lines
95
	lines := strings.SplitN(text, "\n", maxLines+1) // Will contain at least one line, and if there are more than MaxLines, the last item holds the rest of the lines
96
	if len(lines) > maxLines {
97
		// If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines
98
		lines = lines[:maxLines]
99
	} else if len(lines) > 1 && len(data) >= guessSampleSize {
100
		// Even with data >= guessSampleSize, we don't have maxLines + 1 (no extra lines, must have really long lines)
101
		// thus the last line is probably have a truncated line. Drop the last line if len(lines) > 1
102
		lines = lines[:len(lines)-1]
103
	}
104

105
	// Put lines back together as a string
106
	text = strings.Join(lines, "\n")
107

108
	delimiters := []rune{',', '\t', ';', '|', '@'}
109
	validDelim := delimiters[0]
110
	validDelimColCount := 0
111
	for _, delim := range delimiters {
112
		csvReader := stdcsv.NewReader(strings.NewReader(text))
113
		csvReader.Comma = delim
114
		if rows, err := csvReader.ReadAll(); err == nil && len(rows) > 0 && len(rows[0]) > validDelimColCount {
115
			validDelim = delim
116
			validDelimColCount = len(rows[0])
117
		}
118
	}
119
	return validDelim
120
}
121

122
// FormatError converts csv errors into readable messages.
123
func FormatError(err error, locale translation.Locale) (string, error) {
124
	if perr, ok := err.(*stdcsv.ParseError); ok {
125
		if perr.Err == stdcsv.ErrFieldCount {
126
			return locale.TrString("repo.error.csv.invalid_field_count", perr.Line), nil
127
		}
128
		return locale.TrString("repo.error.csv.unexpected", perr.Line, perr.Column), nil
129
	}
130

131
	return "", err
132
}
133

134
// Looks for possible delimiters right before or after (with spaces after the former) double quotes with closing quotes
135
var beforeAfterQuotes = regexp.MustCompile(`([,@\t;|]{0,1}) *(?:"[^"]*")+([,@\t;|]{0,1})`)
136

137
// guessFromBeforeAfterQuotes guesses the limiter by finding a double quote that has a valid delimiter before it and a closing quote,
138
// or a double quote with a closing quote and a valid delimiter after it
139
func guessFromBeforeAfterQuotes(data []byte) rune {
140
	rs := beforeAfterQuotes.FindStringSubmatch(string(data)) // returns first match, or nil if none
141
	if rs != nil {
142
		if rs[1] != "" {
143
			return rune(rs[1][0]) // delimiter found left of quoted string
144
		} else if rs[2] != "" {
145
			return rune(rs[2][0]) // delimiter found right of quoted string
146
		}
147
	}
148
	return 0 // no match found
149
}
150