v

1
/*
2
csv reader 1.0 alpha
3

4
Copyright (c) 2023 Dario Deledda. All rights reserved.
5
Use of this source code is governed by an MIT license
6
that can be found in the LICENSE file.
7

8
This file contains tests
9

10
Known limitations:
11
*/
12
import encoding.csv
13
import strings
14
import os
15

16
/******************************************************************************
17
*
18
* Test Data
19
*
20
******************************************************************************/
21
// dataset 1
22
const txt1 = '
23

24
#
25
# pippo
26
#
27
a,b,c,d,e,f,g
28
0,dario,.2,3.2e-2,4,"pero5",6
29
# first comment, test @# again
30
1,2,3,4,5,6,7
31
2,3,4,5,6,7,8
32
3,4,5,6,7,8,9
33

34
a,"b,c,d",0,#,3,"pippo"
35

36
# last comment
37
'
38

39
const target_header_list = [
40
	csv.HeaderItem{
41
		label:  'a'
42
		column: 0
43
		htype:  .int
44
	},
45
	csv.HeaderItem{
46
		label:  'b'
47
		column: 1
48
		htype:  .string
49
	},
50
	csv.HeaderItem{
51
		label:  'c'
52
		column: 2
53
		htype:  .f32
54
	},
55
	csv.HeaderItem{
56
		label:  'd'
57
		column: 3
58
		htype:  .f32
59
	},
60
	csv.HeaderItem{
61
		label:  'e'
62
		column: 4
63
		htype:  .int
64
	},
65
	csv.HeaderItem{
66
		label:  'f'
67
		column: 5
68
		htype:  .string
69
	},
70
	csv.HeaderItem{
71
		label:  'g'
72
		column: 6
73
		htype:  .int
74
	},
75
]
76

77
const target_data = [
78
	['a', 'b', 'c', 'd', 'e', 'f', 'g'],
79
	['0', 'dario', '.2', '3.2e-2', '4', '"pero5"', '6'],
80
	['1', '2', '3', '4', '5', '6', '7'],
81
	['2', '3', '4', '5', '6', '7', '8'],
82
	['3', '4', '5', '6', '7', '8', '9'],
83
	['a', '"b,c,d"', '0', '#', '3', '"pippo"'], // 6 columns for test purpose
84
]
85

86
// dataset 2 crlf string from windows
87
const txt2 = '
88

89
#
90
# pippo
91
#
92
a,b,c,d,e,f,g
93
0,dario,.2,3.2e-2,4,"pero5",6
94
# first comment, test @# again
95
1,2,3,4,5,6,7
96
2,3,4,5,6,7,8
97
3,4,5,6,7,8,9
98

99
a,"b,c,d",,#,3,"pippo"
100

101
# last comment
102
'
103

104
// dataset 3/4
105
const txt3 = 'a,b,c,d\r\n0,1,2,3\r\n4,5,6,7\r\n'
106
const txt4 = 'a,b,c,d\n0,1,2,3\n4,5,6,7\n'
107
/******************************************************************************
108
*
109
* Test Sequential Functions
110
*
111
******************************************************************************/
112
fn test_csv_sequential() {
113
	mut csvr := csv.csv_sequential_reader(scr_buf: txt1.str, scr_buf_len: txt1.len)!
114
	mut data := [][]string{}
115
	for csvr.has_data() > 1 {
116
		data << csvr.get_next_row()!
117
	}
118
	csvr.dispose_csv_reader()
119
	assert data[0][0] == 'a', 'test_csv_sequential1 reading failed!'
120
	// there is a final empty row in txt1
121
	assert data[data.len - 2][0] == 'a', 'test_csv_sequential2 reading failed!'
122
	assert data[data.len - 2][1] == 'b,c,d', 'test_csv_sequential3 reading failed!'
123

124
	csvr = csv.csv_sequential_reader(scr_buf: txt2.str, scr_buf_len: txt2.len)!
125
	csvr.empty_cell = '####'
126
	data = [][]string{}
127
	for csvr.has_data() > 1 {
128
		data << csvr.get_next_row()!
129
	}
130
	csvr.dispose_csv_reader()
131
	assert data[data.len - 2][2] == '####', 'test_csv_sequential4 reading failed!'
132
	assert data[data.len - 2][5] == 'pippo', 'test_csv_sequential5 reading failed!'
133

134
	// create a temp file to test csv parsing from file
135
	file_path_str := os.join_path(os.temp_dir(), 'test_csv.csv')
136
	// println("file_path_str: ${file_path_str}")
137

138
	// test Windows confguration
139
	mut tmp_txt1 := txt1.replace('\n', '\r\n')
140

141
	mut f := os.open_file(file_path_str, 'wb')!
142
	unsafe {
143
		f.write_ptr(tmp_txt1.str, tmp_txt1.len)
144
	}
145
	// f.write_string(tmp_txt1)!
146
	f.close()
147

148
	csvr = csv.csv_sequential_reader(
149
		file_path:    file_path_str
150
		mem_buf_size: 64
151
		end_line_len: csv.endline_crlf_len
152
	)!
153
	data = [][]string{}
154
	for csvr.has_data() > 1 {
155
		data << csvr.get_next_row()!
156
	}
157
	csvr.dispose_csv_reader()
158

159
	assert data[0][0] == 'a', 'test_csv_sequential1 reading failed!'
160
	// there is a final empty row in txt1
161
	assert data[data.len - 2][0] == 'a', 'test_csv_sequential2 reading failed!'
162
	assert data[data.len - 2][1] == 'b,c,d', 'test_csv_sequential3 reading failed!'
163

164
	// remove the temp file
165
	os.rm(file_path_str)!
166
}
167

168
/******************************************************************************
169
*
170
* Test Random Access Functions
171
*
172
******************************************************************************/
173
fn perform_test(mut csvr csv.RandomAccessReader) ! {
174
	csvr.build_header_dict(csv.GetHeaderConf{})!
175

176
	// test the Header reader
177
	// println("csvr.header_list: ${csvr.header_list}")
178
	assert csvr.header_list == target_header_list, 'header_list not matched!'
179

180
	/*	
181
	println("--------------------------------")
182
	for x in csvr.csv_map#[..5] {
183
		println(x.len)
184
		println(x)
185
	}
186
	println("--------------------------------")
187
	*/
188

189
	// test the data reading
190
	mut data := [][]string{len: csvr.csv_map.len}
191
	for x in 0 .. csvr.csv_map.len {
192
		data[x] = csvr.get_row(x)!
193
		// if x % 10000 == 0 {
194
		//	println("#${x:-6d}")
195
		//}
196
	}
197

198
	/*	
199
	// debug print
200
	println("---------------")
201
	for x in 0..csvr.csv_map.len {
202
		println(csvr.get_row(x)!)
203
	}
204
	*/
205

206
	// test if we have the same amount of data rows
207
	assert data.len == csvr.csv_map.len, 'data len not equal'
208

209
	// test the data retriever
210
	for row_count, row in target_data {
211
		// println("${data[row_count]} ${row}")
212
		assert data[row_count] == row, ''
213
	}
214

215
	// test lfcr cr
216
	assert csvr.get_cell(x: 6, y: 4)! == '9'
217

218
	// test the get cell behaviour
219
	assert csvr.get_cell(x: csvr.header_map['b'], y: 1)! == 'dario', 'get_cell failed 1'
220
	assert csvr.get_cell(x: csvr.header_map['g'], y: 5)! == csvr.default_cell, 'get_cell out of data failed 2'
221
	assert csvr.get_cellt(x: 0, y: 1)! == csv.CellValue(0), 'get_cellt [int] failed'
222
	assert csvr.get_cellt(x: 1, y: 1)! == csv.CellValue('dario'), 'get_cellt [string] failed'
223
	assert csvr.get_cellt(x: 2, y: 1)! == csv.CellValue(f32(.2)), 'get_cell [f32] failed'
224

225
	// test the filter quote flag
226
	csvr.quote_remove = true
227
	assert csvr.get_cell(x: 1, y: 5)! == 'b,c,d', 'get_cell filer quote flag failed'
228
}
229

230
fn perform_test2(mut csvr csv.RandomAccessReader) ! {
231
	csvr.build_header_dict(csv.GetHeaderConf{})!
232
	// test the empty cells
233
	assert csvr.get_cell(x: csvr.header_map['c'], y: 5)! == csvr.empty_cell, 'get_cell empty_cell failed 2'
234
}
235

236
fn perform_test3(mut csvr csv.RandomAccessReader) ! {
237
	csvr.build_header_dict(csv.GetHeaderConf{})!
238
	/*
239
	// debug print
240
	println("---------------")
241
	for x in 0..csvr.csv_map.len {
242
		println(csvr.get_row(x)!)
243
	}
244
	*/
245
	assert csvr.get_cell(x: csvr.header_map['d'], y: 2)! == '7', 'test \n \r\n failed'
246
}
247

248
fn test_csv_string() {
249
	// test the csv parsing from RAM
250
	mut csvr := csv.csv_reader_from_string(txt1)!
251
	perform_test(mut csvr)!
252
	csvr.dispose_csv_reader()
253

254
	// create a temp file to test csv parsing from file
255
	file_path_str := os.join_path(os.temp_dir(), 'test_csv.csv')
256
	// println("file_path_str: ${file_path_str}")
257

258
	// test Windows confguration
259
	mut tmp_txt1 := txt1.replace('\n', '\r\n')
260

261
	mut f := os.open_file(file_path_str, 'wb')!
262
	unsafe {
263
		f.write_ptr(tmp_txt1.str, tmp_txt1.len)
264
	}
265
	// f.write_string(tmp_txt1)!
266
	f.close()
267

268
	// parse the temp file
269
	csvr = csv.csv_reader(
270
		file_path:    file_path_str
271
		mem_buf_size: 32
272
		end_line_len: csv.endline_crlf_len
273
	)!
274
	perform_test(mut csvr)!
275
	csvr.dispose_csv_reader()
276

277
	// remove the temp file
278
	os.rm(file_path_str)!
279

280
	csvr = csv.csv_reader_from_string(txt2)!
281
	perform_test2(mut csvr)!
282
	csvr.dispose_csv_reader()
283

284
	// test crlf endline
285
	csvr = csv.csv_reader(
286
		scr_buf:      txt3.str
287
		scr_buf_len:  txt3.len
288
		end_line_len: csv.endline_crlf_len
289
	)!
290
	perform_test3(mut csvr)!
291
	csvr.dispose_csv_reader()
292

293
	// test cr endline
294
	csvr = csv.csv_reader(scr_buf: txt4.str, scr_buf_len: txt4.len, end_line_len: csv.endline_cr_len)!
295
	perform_test3(mut csvr)!
296
	csvr.dispose_csv_reader()
297
}
298

299
fn test_coherence() {
300
	file_path_str := os.join_path(os.temp_dir(), 'test_csv.csv')
301
	mut f := os.open_file(file_path_str, 'w')!
302
	mut b := strings.new_builder(64536)
303
	mut i := u64(0)
304
	mut sum := u64(0)
305
	for rows in 0 .. 1000 {
306
		for col in 0 .. 1000 {
307
			if col > 0 {
308
				b.write_u8(`,`)
309
			}
310
			b.write_string(i.str())
311
			i++
312
			sum += i
313
		}
314
		b.write_string('\n')
315
	}
316
	f.write_string(b.str())!
317
	f.close()
318

319
	sum -= i
320
	// println('sum: ${sum}')
321

322
	// parse the temp file
323
	mut csvr := csv.csv_reader(
324
		file_path:    file_path_str
325
		mem_buf_size: 32
326
		end_line_len: csv.endline_cr_len
327
	)!
328

329
	mut sum1 := u64(0)
330
	for row_index in 0 .. csvr.csv_map.len {
331
		row := csvr.get_row(row_index)!
332
		for x in row {
333
			sum1 += u64(x.int())
334
		}
335
	}
336
	// println('sum: ${sum1}')
337

338
	csvr.dispose_csv_reader()
339

340
	// remove the temp file
341
	os.rm(file_path_str)!
342

343
	assert sum == sum1, 'csv coherence test failed'
344
}
345

346
// Debug code
347
fn main() {
348
	test_csv_string()
349
}
350