src/parser/csvStream.ts - master - grigoriygrisha/excel-loader

src/parser/csvStream.ts
182 строки5 KB
grigoriy-grisha
first_commit
26 июл 2026, 21:46
26 июл 2026, 21:466cc93a5
import type { CellValue, ColumnMeta, Row } from '../core/types.js'
import { inferColumnType, type ParseOptions, type StreamingParser } from './types.js'

export class CsvParser implements StreamingParser {
  private columnsValue: ColumnMeta[] = []
  private readonly file: File
  private readonly options: ParseOptions
  private readonly sampleForTypes: CellValue[][] = []

  constructor(file: File, options: ParseOptions) {
    this.file = file
    this.options = options
  }

  getColumns(): ColumnMeta[] {
    return this.columnsValue
  }

  async *parse(): AsyncIterable<Row> {
    const stream = this.file
      .stream()
      .pipeThrough(new TextDecoderStream('utf-8'))

    const reader = stream.getReader()
    let buffer = ''
    let rowIndex = 0
    let headerParsed = false
    let columnCount = 0
    const delimiter = await this.resolveDelimiter()

    try {
      while (true) {
        const { done, value } = await reader.read()
        if (done) break

        buffer += value
        const lines = buffer.split(/\r\n|\n|\r/)
        buffer = lines.pop() ?? ''

        for (const line of lines) {
          if (line === '') continue
          const cells = parseCsvLine(line, delimiter)

          if (!headerParsed) {
            if (this.options.hasHeader) {
              columnCount = cells.length
              this.columnsValue = cells.map((name, i) => ({
                index: i,
                name: String(name ?? `Column ${i + 1}`),
                type: 'string',
              }))
              headerParsed = true
              continue
            }
            columnCount = cells.length
            headerParsed = true
          }

          const normalized = normalizeRow(cells, columnCount)
          if (this.sampleForTypes.length < 200) {
            this.sampleForTypes.push(normalized)
          }
          yield { rowIndex: rowIndex++, values: normalized }
        }
      }

      if (buffer !== '') {
        const cells = parseCsvLine(buffer, delimiter)
        if (!headerParsed && !this.options.hasHeader) {
          columnCount = cells.length
        }
        if (headerParsed || this.options.hasHeader) {
          const normalized = normalizeRow(cells, columnCount)
          if (this.sampleForTypes.length < 200) this.sampleForTypes.push(normalized)
          yield { rowIndex: rowIndex++, values: normalized }
        }
      }

      this.finalizeColumns(columnCount)
    } finally {
      reader.releaseLock()
    }
  }

  private async resolveDelimiter(): Promise<string> {
    if (this.options.csvDelimiter) return this.options.csvDelimiter

    const peek = await this.file.slice(0, 64 * 1024).text()
    const firstLine = peek.split(/\r\n|\n|\r/, 1)[0] ?? ''
    if (!firstLine) return ','

    const candidates = [',', ';', '\t', '|']
    let best = ','
    let bestCount = 1
    for (const c of candidates) {
      const count = countUnquotedOccurrences(firstLine, c)
      if (count > bestCount) {
        best = c
        bestCount = count
      }
    }
    return best
  }

  private finalizeColumns(columnCount: number): void {
    if (this.columnsValue.length > 0) return

    const byColumn: CellValue[][] = Array.from({ length: columnCount }, () => [])
    for (const row of this.sampleForTypes) {
      for (let i = 0; i < columnCount; i++) byColumn[i].push(row[i] ?? null)
    }

    this.columnsValue = Array.from({ length: columnCount }, (_, i) =>
      inferColumnType(i, byColumn[i], `Column ${i + 1}`),
    )
  }
}

export function parseCsvLine(line: string, delimiter: string): CellValue[] {
  const cells: CellValue[] = []
  let current = ''
  let inQuotes = false

  for (let i = 0; i < line.length; i++) {
    const ch = line[i]

    if (inQuotes) {
      if (ch === '"') {
        if (line[i + 1] === '"') {
          current += '"'
          i++
        } else {
          inQuotes = false
        }
      } else {
        current += ch
      }
    } else if (ch === '"') {
      inQuotes = true
    } else if (ch === delimiter) {
      cells.push(coerceCell(current))
      current = ''
    } else {
      current += ch
    }
  }

  cells.push(coerceCell(current))
  return cells
}

function normalizeRow(cells: CellValue[], columnCount: number): CellValue[] {
  if (cells.length === columnCount) return cells
  if (cells.length < columnCount) {
    return [...cells, ...Array<CellValue>(columnCount - cells.length).fill(null)]
  }
  return cells.slice(0, columnCount)
}

function coerceCell(raw: string): CellValue {
  const trimmed = raw.trim()
  if (trimmed === '') return null
  if (trimmed === 'true') return true
  if (trimmed === 'false') return false

  if (/^-?\d+(\.\d+)?$/.test(trimmed)) {
    const n = Number(trimmed)
    if (Number.isFinite(n)) return n
  }
  return raw
}

function countUnquotedOccurrences(line: string, delimiter: string): number {
  let count = 0
  let inQuotes = false
  for (let i = 0; i < line.length; i++) {
    const ch = line[i]
    if (ch === '"') inQuotes = !inQuotes
    else if (ch === delimiter && !inQuotes) count++
  }
  return count
}