codegen/parser.nim - master - Team8/kittycad

codegen/parser.nim
983 строки23 KB
levovix
add: codegen: AUTO() for childCount, childAt, childIndex
07 янв 2026, 21:39
07 янв 2026, 21:394f739c9
import std/[importutils, typetraits, sequtils, strformat, strutils]
import ../thirdparty/print/print {.all.}
import ./[lexer]

export print.print, print.newNameNode, print.newTopPairNode, print.printNodes


type
  ParseError* = object of ValueError

  ClexId* = enum
    Eof = cint(256)
    Lexer_error = cint(257)
    IntlitT = cint(258)
    FloatlitT = cint(259)
    Id = cint(260)
    Dqstring = cint(261)
    Sqstring = cint(262)
    CharlitT = cint(263)
    Eq = cint(264)
    Noteq = cint(265)
    Lesseq = cint(266)
    Greatereq = cint(267)
    Andand = cint(268)
    Oror = cint(269)
    Shl = cint(270)
    Shr = cint(271)
    Plusplus = cint(272)
    Minusminus = cint(273)
    Pluseq = cint(274)
    Minuseq = cint(275)
    Muleq = cint(276)
    Diveq = cint(277)
    Modeq = cint(278)
    Andeq = cint(279)
    Oreq = cint(280)
    Xoreq = cint(281)
    Arrow = cint(282)
    Eqarrow = cint(283)
    Shleq = cint(284)
    Shreq = cint(285)
    First_unused_token = cint(286)
  

  Node* = ref object of RootObj


  AccessKind* = enum
    Public
    Private
    Protected

  Inherit* = object
    typ*: Node
    access*: AccessKind
  

  Unknown* = ref object of Node
    token*: string
  

  Stmts* = ref object of Node
    childs*: seq[Node]
  
  
  Namespace* = ref object of Stmts
    name*: string
  

  TypeImpl* = ref object of Namespace
  
  Class* = ref object of TypeImpl
    defaultAccess*: AccessKind
    inherits*: seq[Inherit]

  
  Ident* = ref object of Node
    name*: string
    namespace*: Ident
  
  
  VarDecl* = ref object of Node
    name*: string
    typ*: Node
    init*: Node
    access*: AccessKind
    pragma*: seq[Node]  # any additional (meta) info
  
  VarSection* = ref object of Stmts
    # todo: always flatten it into var decls and remove, it's pretty hard to handle for no reason

  
  Enum* = ref object of TypeImpl

  EnumFld* = ref object of Node
    name*: string
    init*: Node
  

  IntLit* = ref object of Node
    val*: int64
    
  FloatLit* = ref object of Node
    val*: float64
  
  StrLit* = ref object of Node
    val*: string
  
  CharLit* = ref object of Node
    val*: string
  

  ExprList* = ref object of Stmts


  Call* = ref object of Stmts
    callable*: Node

  ProcDecl* = ref object of Stmts
    name*: string
    rettype*: Node
    pragma*: seq[Node]  # any additional (meta) info


  GenericsExpr* = ref object of Ident
    types*: seq[Node]
  
  PtrTy* = ref object of GenericsExpr
  RefTy* = ref object of GenericsExpr


  ContextWhere* = enum
    InToplevel
    InClass
    InEnum
    InExpr
    InArgs

  ParseContext* = object
    where*: ContextWhere
    classname*: string

  ParseGlobals* = object
    access*: AccessKind



var anyplace_pragmas = @[
  "DONT_SERIALIZE",
  "TRACE_AS_TEMPORARY",
  "DONT_DUMP",
  "CHILD",
]

var anyplace_pragmas_with_args = @[
  "AUTO",
]

var postfix_pragmas = @[
  "override",
  "const",
] & anyplace_pragmas

var prefix_pragmas = @[
  "inline",
  "static",
  "virtual",
] & anyplace_pragmas

var prefix_pragmas_with_args = anyplace_pragmas_with_args

var postfix_pragmas_with_args = anyplace_pragmas_with_args



proc toString*(s: cstring, len: int): string =
  result.setLen len
  if len > 0:
    copyMem(result[0].addr, s, len)


proc formatToken*(t: clong): string =
  if t < 256: result.add cast[char](t)
  else:
    result = $t.ClexId


proc str*(lex: ptr stb_lexer): string =
  lex.string.toString(lex.string_len)


proc formatToken*(t: clong, lex: ptr stb_lexer): string =
  result.add t.formatToken
  if t == Id.clong:
    result.add " " & lex.str


proc lineInfo*(lex: ptr stb_lexer): string =
  var loc: stb_lex_location
  lex.get_location(lex.where_firstchar, loc.addr)
  let filename = lex.filename.toString(lex.filename_len)
  &"{filename}:{loc.line_number}:{loc.line_offset + 1}"


proc error*(lex: ptr stb_lexer, s: string) {.noreturn.} =
  raise ParseError.newException(&"{lex.lineInfo} {s}")


proc sure_get_token(lex: ptr stb_lexer) =
  if lex.get_token() == 0: lex.error "unexpected EOF"


converter toClong(b: char): clong = b.clong
converter toClong(b: ClexId): clong = b.clong


proc `{}`(lex: ptr stb_lexer): (stb_lexer, string) =
  (system.`[]`(lex), lex.str)
  
proc `{}=`(lex: ptr stb_lexer, v: (stb_lexer, string)) =
  system.`[]`(lex) = v[0]
  if v[1].len != 0:
    copyMem lex.string, v[1][0].addr, v[1].len
  lex.string_len = v[1].len.cint




method newNodeM*(x: Node): print.Node {.base.} =
  privateAccess print.Node

  var nodes: seq[print.Node]
  for n, e in x.fieldPairs2:
    nodes.add(newFieldPairNode(newNameNode(n), newNodeFromBaseType(e)))

  result = print.Node(kind: nkObject, value: $type(x), nodes: nodes)

method typename*(x: Node): string {.base.} =
  "Node"

method equals(a: Node, b: Node): bool {.base, noSideEffect.} =
  not system.`==`(b, nil)


proc newNodeFromBaseType*(x: Node): print.Node =
  privateAccess print.Node
  
  if x == nil: print.Node(kind: nkNil, value: "nil")
  else: newNodeM(x)
  
proc newNode*(x: Node): print.Node =
  privateAccess print.Node
  
  if x == nil: print.Node(kind: nkNil, value: "nil")
  else: newNodeM(x)


proc newNode*[T](x: seq[T]): print.Node =
  privateAccess print.Node
  
  mixin newNodeFromBaseType
  var nodes: seq[print.Node]
  for e in x:
    nodes.add(newNodeFromBaseType(e))
  print.Node(kind: nkSeq, nodes: nodes)


proc newNodeFromBaseType*[T](x: T): print.Node =
  when T is Node: newNode(x.Node)
  else:
    newNode(x.distinctBase(recursive = true))
    


func `==`*(a, b: Node): bool =
  result = (system.`==`(a, nil) and system.`==`(b, nil)) or
  ((not system.`==`(a, nil)) and (not system.`==`(b, nil)) and equals(a, b) and equals(b, a))

func `==`*(a: Node, b: typeof(nil)): bool = system.`==`(a, b)
func `!=`*(a: Node, b: typeof(nil)): bool = system.`!=`(a, b)

func `!=`*(a, b: Node): bool =
  not parser.`==`(a, b)

func `==`*(a, b: seq[Node]): bool =
  if a.len != b.len: return false
  result = true
  for i in 0..<a.len:
    if parser.`!=`(a[i], b[i]):
      return false


template genPrint(t: typedesc) =
  method newNodeM*(x: t): print.Node =
    privateAccess print.Node

    var nodes: seq[print.Node]
    for n, e in x.fieldPairs2:
      nodes.add(newFieldPairNode(newNameNode(n), newNodeFromBaseType(e)))

    result = print.Node(kind: nkObject, value: $type(x), nodes: nodes)
  
  method typename*(x: t): string =
    $t
  
  method equals(a: t, b: Node): bool {.noSideEffect.} =
    if system.`==`(b, nil): return false
    result = true
    for k, av, bv in fieldPairs(a[], b.t[]):
      when av is Node:
        if parser.`!=`(av, bv): return false
      else:
        if av != bv: return false


genPrint Unknown

genPrint Stmts

genPrint Namespace
genPrint TypeImpl
genPrint Class

genPrint Ident

genPrint VarDecl
genPrint VarSection

genPrint Enum
genPrint EnumFld

genPrint IntLit
genPrint FloatLit
genPrint StrLit
genPrint CharLit

genPrint ExprList

genPrint Call
genPrint ProcDecl

genPrint GenericsExpr
genPrint PtrTy
genPrint RefTy



proc fullName*(n: Ident): string =
  if n.namespace != nil:
    fullName(n.namespace) & "::" & n.name
  else:
    n.name
    
proc fullName*(n: Node): string =
  if n of Ident:
    fullName(n.Ident)
  else:
    raise ParseError.newException("expected Ident")



using lex: ptr stb_lexer
using context: ParseContext
using globals: var ParseGlobals


proc parseType(lex, context, globals; allowProcs = false): Node
proc parseAux(lex, context, globals): Node
proc tryParseProcDecl(typ: Ident, name: string, inner: var Node, lex, context, globals): bool


proc render*(n: Node): string =
  if n of Ident: n.fullName
  elif n of ProcDecl: n.ProcDecl.rettype.render & " " & n.ProcDecl.name & "(" & n.ProcDecl.childs.map(render).join(", ") & ")"
  else: raise ValueError.newException("unimplemented render for node of type " & n.typename)



proc parseNameNamespace(lex; inner: var Ident, context, globals): bool =
  let old_lex = lex{}

  if lex.token == ':':
    lex.sure_get_token()
    if lex.token == ':':
      lex.sure_get_token()
      if lex.token == Id:
        inner = Ident(name: lex.str, namespace: inner)
        discard lex.get_token()
        return true
  
  lex{} = old_lex



proc parseGenericsExpr(lex; inner: var Ident, context, globals): bool =
  let old_lex = lex{}

  if lex.token == '<':
    let res = GenericsExpr(name: inner.name, namespace: inner.namespace)
    lex.sure_get_token()

    while true:
      if lex.token == IntlitT:
        res.types.add IntLit(val: lex.int_number)
        lex.sure_get_token()
      
      elif lex.token == FloatlitT:
        res.types.add FloatLit(val: lex.real_number)
        lex.sure_get_token()
      
      else:
        if lex.token != Id:
          lex{} = old_lex
          return false  # not actually a GenericsExpr
        
        let typ = parseType(lex, context, globals, allowProcs = true)
        
        res.types.add typ
      
      if lex.token == '>':
        discard lex.get_token()
        break
      elif lex.token == Shr:  # that's why using <> as brackets is bad design
        lex.token = '>'
        break

      if lex.token != ',':
        lex{} = old_lex
        return false  # not actually a GenericsExpr
      else:
        lex.sure_get_token()
    
    inner = res
    return true



proc parseType(lex, context, globals; allowProcs = false): Node =
  if lex.token != Id: lex.error "expected Id"

  result = Ident(name: lex.str)
  discard lex.get_token()
  
  while parseNameNamespace(lex, result.Ident, context, globals): discard
  
  discard parseGenericsExpr(lex, result.Ident, context, globals)

  var isConst = false
  while true:
    if lex.token == Id and lex.str == "const":
      isConst = true
      lex.sure_get_token()
  
    elif lex.token == '*':
      result = PtrTy(name: (if isConst: "const*" else: "*"), types: @[result])
      isConst = false
      lex.sure_get_token()
    
    else:
      break
    
  if lex.token == '&':
    result = RefTy(name: (if isConst: "const&" else: "&"), types: @[result])
    isConst = false
    lex.sure_get_token()
    
  if lex.token == AndAnd:
    result = RefTy(name: "&&", types: @[result])
    lex.sure_get_token()
  
  if allowProcs and lex.token == '(':
    discard tryParseProcDecl(result.Ident, "", result, lex, context, globals)



proc parseNamespace(lex, context, globals): Namespace =
  assert lex.token == Id and lex.str == "namespace"
  result = Namespace()
  lex.sure_get_token()

  if lex.token != Id: lex.error "expected Id"
  result.name = lex.str
  lex.sure_get_token()

  if lex.token != '{': lex.error "expected {"
  lex.sure_get_token()
  
  while true:
    if lex.token == '}':
      discard lex.get_token()
      break
    result.childs.add parseAux(lex, context, globals)



proc parseClass(lex, context, globals): Class =
  assert lex.token == Id and lex.str in ["class", "struct"]
  result = Class()
  var context = context
  
  result.defaultAccess =
    if lex.str == "struct": Public
    else: Private
  lex.sure_get_token()
  
  if lex.token != Id: lex.error "expected Id"
  result.name = lex.str
  context.classname = result.name
  lex.sure_get_token()
  
  if lex.token == ':':
    lex.sure_get_token()
      
    while true:
      var access = result.defaultAccess
      
      if lex.token == Id and lex.str in ["public", "private", "protected"]:
        access = case lex.str
          of "public": Public
          of "private": Private
          else: Protected

        lex.sure_get_token()

      result.inherits.add Inherit(typ: parseType(lex, context, globals), access: access)

      if lex.token == '{':
        break
      
      elif lex.token == ',':
        lex.sure_get_token()
        continue
      
      elif lex.token == ';':
        break
      
      else: lex.error "expected {"

  if lex.token == '{':
    lex.sure_get_token()
    context.where = InClass
    globals.access = result.defaultAccess
    
    while true:
      if lex.token == '}':
        lex.sure_get_token()
        break
      result.Stmts.childs.add parseAux(lex, context, globals)

  if lex.token != ';': lex.error "expected ;"
  discard lex.get_token()



proc propogateInit(self: VarSection, init: Node) =
  for i in countdown(self.childs.high, 0):
    let child = self.childs[i]
    
    if (not (child of VarDecl)) or child.VarDecl.init != nil:
      break
    
    child.VarDecl.init = init


proc propogatePragma(self: VarSection, pragma: Node) =
  for x in self.childs:
    if x == nil: continue
    x.VarDecl.pragma.add pragma


proc addPragma(self: Node, pragma: Node) =
  if self of VarSection:
    propogatePragma(self.VarSection, pragma)
  elif self of VarDecl:
    self.VarDecl.pragma.add pragma
  elif self of ProcDecl:
    self.ProcDecl.pragma.add pragma
  else:
    raise ParseError.newException("a pragma cannot be attached to a node of type " & self.typename)



proc tryParsePostfixPragma(lex, context, globals; res: Node): bool =
  if lex.token == Id and lex.str in postfix_pragmas:
    res.addPragma Ident(name: lex.str)
    discard lex.get_token()
    true
  
  elif lex.token == Id and lex.str in postfix_pragmas_with_args:
    var pragma = GenericsExpr(name: lex.str)
    lex.sure_get_token()
    
    if lex.token != '(': lex.error "expected an '('"
    lex.sure_get_token()
    
    var context = context
    context.where = InArgs
    while lex.token != ')':
      pragma.types.add parseAux(lex, context, globals)
    discard lex.get_token()
    
    res.addPragma pragma
    
    true
  
  elif lex.token == '=':
    let old_lex = lex{}
    discard lex.get_token()
    
    if lex.token == Id and lex.str == "default":
      res.addPragma Ident(name: "=default")
      discard lex.get_token()
      true
    
    elif lex.token == IntlitT and lex.int_number == 0:
      res.addPragma Ident(name: "=0")
      discard lex.get_token()
      true
    
    else:
      lex{} = old_lex
      false
  
  else:
    false

proc parsePostfixPragmas(lex, context, globals; res: Node) =
  while tryParsePostfixPragma(lex, context, globals, res): discard



proc skipBody(lex, context, globals): bool =
  if lex.token == '{':
    result = true

    lex.sure_get_token()
    var level = 1
    while level > 0 and lex.token != Eof:
      if   lex.token == '{': inc level
      elif lex.token == '}': dec level
      else:                  discard

      lex.sure_get_token()
  
  else:
    result = false



proc tryParseProcDecl(typ: Ident, name: string, inner: var Node, lex, context, globals): bool =
  ## try parse proc decl as an expression (eg. in typedesc)
  let old_lex = lex{}
  var context = context
  let res = ProcDecl(name: name, rettype: typ)
  
  if lex.token != '(':
    lex{} = old_lex
    return false
  lex.sure_get_token()
  
  context.where = InArgs
  while true:
    if lex.token == ')':
      lex.sure_get_token()
      break

    let a = parseAux(lex, context, globals)
    res.childs.add a

    if lex.token == ',':
      discard lex.get_token()
    elif lex.token != ')':
      lex.error "expected )"
  
  parsePostfixPragmas(lex, context, globals, res)

  inner = res
  return true

proc tryParseProcDecl(typ: Ident, inner: var Node, lex, context, globals): bool =
  ## try parse proc decl as a statement
  assert lex.token == Id
  let old_lex = lex{}
  var name = lex.str
  lex.sure_get_token()

  if name == "operator":
    case lex.token
    of '=', '+', '-', '/', '*':
      name = name & lex.token.char
      lex.sure_get_token()
    else: lex.error "unexpected operator name"
  
  result = tryParseProcDecl(typ, name, inner, lex, context, globals)
  
  if result:
    if skipBody(lex, context, globals):
      discard
    else:
      if lex.token != ';':
        lex.error "expected ;"
      discard lex.get_token()
  else:
    lex{} = old_lex



proc parseVarSection(typ: Node, lex, context, globals): Node =
  ## or a single VarDecl, if context.where == InArgs
  assert lex.token == Id
  var res = VarSection()

  var context = context
  let initWhere = context.where

  while true:
    if lex.token == Id:
      res.childs.add VarDecl(
        typ: typ,
        name: lex.str,
        access: (if initWhere == InArgs: Public else: globals.access)
      )
      lex.sure_get_token()
    
    if lex.token == '=':
      context.where = InExpr
      lex.sure_get_token()
      propogateInit res, parseAux(lex, context, globals)
    
    elif lex.token == '{':
      context.where = InExpr
      propogateInit res, parseAux(lex, context, globals)
    
    if initWhere != InArgs and lex.token == ',':
      lex.sure_get_token()
    else:
      break
  
  if lex.token == '(':
    lex.error "unexpeted function decl"
  
  if initWhere != InArgs:
    parsePostfixPragmas(lex, context, globals, res)

    if lex.token != ';': lex.error "expected ;"
    discard lex.get_token()

  result = res
  
  if initWhere == InArgs:
    result = result.VarSection.childs[0]



var last_lex: stb_lexer



proc parseAux(lex, context, globals): Node =
  if last_lex == lex[]:
    lex.error "infinite loop detected"
  last_lex = lex[]

  var context = context
  
  case lex.token
  of '{':
    result =
      if context.where == InExpr: ExprList()
      else: Stmts()
    lex.sure_get_token()

    while true:
      if lex.token == '}':
        discard lex.get_token()
        break

      result.Stmts.childs.add parseAux(lex, context, globals)

      if context.where == InExpr and lex.token == ',':
        discard lex.get_token()
      elif context.where != InExpr and lex.token == ';':
        discard lex.get_token()
      else:
        discard
  

  of '-':
    lex.sure_get_token()
    if lex.token == IntLitT:
      result = IntLit(val: -lex.int_number)
      discard lex.get_token()

    elif lex.token == FloatLitT:
      result = FloatLit(val: -lex.real_number)
      discard lex.get_token()
    
    else:
      lex.error("expected a number")
    

  of IntLitT:
    result = IntLit(val: lex.int_number)
    discard lex.get_token()
    
  of FloatLitT:
    result = FloatLit(val: lex.real_number)
    discard lex.get_token()
    
  of Dqstring:
    result = StrLit(val: lex.str)
    discard lex.get_token()
    
  of Sqstring, CharLitT:
    result = CharLit(val: lex.str)
    discard lex.get_token()
  

  of ';':
    discard lex.get_token()


  of Id:
    case lex.str
    of "namespace":
      result = parseNamespace(lex, context, globals)
      

    of "struct", "class":
      result = parseClass(lex, context, globals)
    

    of "enum":
      result = Enum()
      lex.sure_get_token()

      if lex.token != Id: lex.error "expected Id"
      result.Enum.name = lex.str
      lex.sure_get_token()
      
      if lex.token != '{': lex.error "expected {"
      lex.sure_get_token()

      context.where = InEnum
      while true:
        if lex.token == '}':
          lex.sure_get_token()
          break
        
        result.Stmts.childs.add parseAux(lex, context, globals)

      if lex.token != ';': lex.error "expected ;"
      discard lex.get_token()

    
    of "public", "private", "protected":
      globals.access = case lex.str
        of "public": Public
        of "private": Private
        else: Protected
      lex.sure_get_token()

      if lex.token != ':': lex.error "expected :"
      discard lex.get_token()
    

    of "EXAMPLES":
      while lex.str != "END_EXAMPLES":
        lex.sure_get_token()
    

    of "SERGEN_IGNORE":
      while lex.str != "SERGEN_IGNORE_END":
        lex.sure_get_token()
    

    elif context.where == InEnum:
      result = EnumFld(name: lex.str)
      lex.sure_get_token()

      case lex.token
      of ',':
        lex.sure_get_token()

      of '=':
        context.where = InExpr
        lex.sure_get_token()

        result.EnumFld.init = parseAux(lex, context, globals)
        
        if lex.token == ',':
          discard lex.get_token()
      
      else: discard
    
    else:
      var pragmas: seq[Ident]
      while lex.token == Id and (lex.str in prefix_pragmas or lex.str in prefix_pragmas_with_args):
        var context = context
        if lex.str in prefix_pragmas_with_args:
          var pragma = GenericsExpr(name: lex.str)
          lex.sure_get_token()
          
          if lex.token != '(': lex.error "expected an '('"
          lex.sure_get_token()
          
          var context = context
          context.where = InArgs
          while lex.token != ')':
            pragma.types.add parseAux(lex, context, globals)
          discard lex.get_token()
          
          pragmas.add pragma
        
        else:
          pragmas.add Ident(name: lex.str)
          lex.sure_get_token()
      
      result = parseType(lex, context, globals)

      if (
        context.where == InClass and
        result of Ident and
        result.Ident.name == context.classname and
        tryParseProcDecl(result.Ident, "=new", result, lex, context, globals)
      ):
        for pragma in pragmas:
          result.addPragma pragma


      elif lex.token == '(':
        result = Call(callable: result)
        lex.sure_get_token()
        
        context.where = InExpr
        while true:
          if lex.token == ')':
            discard lex.get_token()
            break

          result.Stmts.childs.add parseAux(lex, context, globals)

          if lex.token == ',':
            discard lex.get_token()

      elif lex.token == Id and context.where != InArgs and tryParseProcDecl(result.Ident, result, lex, context, globals):
        for pragma in pragmas:
          result.addPragma pragma
      
      elif lex.token == Id:
        result = parseVarSection(result, lex, context, globals)
        for pragma in pragmas:
          result.addPragma pragma

  of '#':
    lex.sure_get_token()
    if lex.token == Id and lex.str == "if":
      lex.sure_get_token()  #  пока что этого хватит
    
    elif lex.token == Id and lex.str == "endif":
      lex.sure_get_token()


  else:
    result = Unknown(token: lex.token.formatToken(lex))
    discard lex.get_token()
  

  if result of Stmts:
    result.Stmts.childs = result.Stmts.childs.filterIt(it != nil)



proc parse*(lex): Node =
  result = Stmts()

  discard lex.get_token()

  var glob: ParseGlobals
  while lex.token != Eof:
    result.Stmts.childs.add parseAux(lex, ParseContext(), glob)
  
  result.Stmts.childs = result.Stmts.childs.filterIt(it != nil)