githubmirror/cpython

Parser/tokenizer/tokenizer.h

14 строк476 B

Lysandros Nikolaou

gh-104169: Refactor tokenizer into lexer and wrappers (#110684)

11 окт 2023, 18:14

11 окт 2023, 18:1401481f2

#ifndef Py_TOKENIZER_H
#define Py_TOKENIZER_H

#include "Python.h"

struct tok_state *_PyTokenizer_FromString(const char *, int, int);
struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int);
struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int);
struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
                                              const char *, const char *);

#define tok_dump _Py_tok_dump

#endif /* !Py_TOKENIZER_H */