cython

1
# Grammar for Cython, based on the Grammar for Python 3
2

3
# Note: This grammar is not yet used by the Cython parser and is subject to change.
4

5
# Start symbols for the grammar:
6
#       single_input is a single interactive statement;
7
#       file_input is a module or sequence of commands read from an input file;
8
#       eval_input is the input for the eval() functions.
9
# NB: compound_stmt in single_input is followed by extra NEWLINE!
10
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
11
file_input: (NEWLINE | stmt)* ENDMARKER
12
eval_input: testlist NEWLINE* ENDMARKER
13

14
decorator: '@' dotted_PY_NAME [ '(' [arglist] ')' ] NEWLINE
15
decorators: decorator+
16
decorated: decorators (classdef | funcdef | async_funcdef | cdef_stmt)
17
async_funcdef: 'async' funcdef
18
funcdef: 'def' PY_NAME parameters ['->' test] ':' suite
19
parameters: '(' [typedargslist] ')'
20
typedargslist: (tfpdef ['=' (test | '*')] (',' tfpdef ['=' (test | '*')])* [','
21
       ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
22
     |  '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) [',' ellipsis]
23
tfpdef: maybe_typed_name [('not' | 'or') 'None'] [':' test]
24
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
25
       ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
26
     |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
27
vfpdef: maybe_typed_name ['not' 'None']
28

29
stmt: simple_stmt | compound_stmt | cdef_stmt | ctypedef_stmt | DEF_stmt | IF_stmt
30
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
31
small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
32
             import_stmt | global_stmt | nonlocal_stmt | assert_stmt | print_stmt)
33
expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
34
                     ('=' (yield_expr|testlist_star_expr))*)
35
testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
36
augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
37
            '<<=' | '>>=' | '**=' | '//=')
38
print_stmt: 'print' ( [ test (',' test)* [','] ] |
39
                      '>>' test [ (',' test)+ [','] ] )
40
# For normal assignments, additional restrictions enforced by the interpreter
41
del_stmt: 'del' exprlist
42
pass_stmt: 'pass'
43
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
44
break_stmt: 'break'
45
continue_stmt: 'continue'
46
return_stmt: 'return' [testlist]
47
yield_stmt: yield_expr
48
raise_stmt: 'raise' [test ['from' test]]
49
# raise_stmt: 'raise' [test [',' test [',' test]]]
50
import_stmt: import_PY_NAME | import_from
51
import_PY_NAME: ('import' | 'cimport') dotted_as_PY_NAMEs
52
# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
53
import_from: ('from' (('.' | '...')* dotted_PY_NAME | ('.' | '...')+)
54
              ('import' | 'cimport') ('*' | '(' import_as_PY_NAMEs ')' | import_as_PY_NAMEs))
55
import_as_PY_NAME: PY_NAME ['as' PY_NAME]
56
dotted_as_PY_NAME: dotted_PY_NAME ['as' PY_NAME]
57
import_as_PY_NAMEs: import_as_PY_NAME (',' import_as_PY_NAME)* [',']
58
dotted_as_PY_NAMEs: dotted_as_PY_NAME (',' dotted_as_PY_NAME)*
59
dotted_PY_NAME: PY_NAME ('.' PY_NAME)*
60
global_stmt: 'global' PY_NAME (',' PY_NAME)*
61
nonlocal_stmt: 'nonlocal' PY_NAME (',' PY_NAME)*
62
exec_stmt: 'exec' expr ['in' test [',' test]]
63
assert_stmt: 'assert' test [',' test]
64

65
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
66
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
67
while_stmt: 'while' test ':' suite ['else' ':' suite]
68
for_stmt: 'for' exprlist ('in' testlist | for_from_clause)':' suite ['else' ':' suite]
69
for_from_clause: 'from' expr comp_op PY_NAME comp_op expr ['by' expr]
70
try_stmt: ('try' ':' suite
71
           ((except_clause ':' suite)+
72
            ['else' ':' suite]
73
            ['finally' ':' suite] |
74
           'finally' ':' suite))
75
with_stmt: 'with' with_item (',' with_item)*  ':' suite
76
with_item: test ['as' expr]
77
# NB compile.c makes sure that the default except clause is last
78
except_clause: 'except' [test [('as' | ',') test]]
79
suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
80

81
test: or_test ['if' or_test 'else' test] | lambdef
82
test_nocond: or_test | lambdef_nocond
83
lambdef: 'lambda' [varargslist] ':' test
84
lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
85
or_test: and_test ('or' and_test)*
86
and_test: not_test ('and' not_test)*
87
not_test: 'not' not_test | comparison
88
comparison: expr (comp_op expr)*
89
# <> isn't actually a valid comparison operator in Python. It's here for the
90
# sake of a __future__ import described in PEP 401
91
comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
92
star_expr: '*' expr
93
expr: xor_expr ('|' xor_expr)*
94
xor_expr: and_expr ('^' and_expr)*
95
and_expr: shift_expr ('&' shift_expr)*
96
shift_expr: arith_expr (('<<'|'>>') arith_expr)*
97
arith_expr: term (('+'|'-') term)*
98
term: factor (('*'|'/'|'%'|'//') factor)*
99
factor: ('+'|'-'|'~') factor | power | address | size_of | cast
100
power: atom_expr ['**' factor]
101
atom_expr: ['await'] atom trailer*
102
atom: ('(' [yield_expr|testlist_comp] ')' |
103
       '[' [testlist_comp] ']' |
104
       '{' [dictorsetmaker] '}' |
105
       new_expr |
106
       PY_NAME | NUMBER | STRING+ | ellipsis | 'None' | 'True' | 'False')
107
testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
108
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' (PY_NAME | 'sizeof')
109
subscriptlist: subscript (',' subscript)* [',']
110
subscript: test | [test] ':' [test] [sliceop]
111
sliceop: ':' [test]
112
exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
113
testlist: test (',' test)* [',']
114
dictorsetmaker: ( ((test ':' test | '**' expr)
115
                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
116
                  ((test | star_expr)
117
                   (comp_for | (',' (test | star_expr))* [','])) )
118

119
classdef: 'class' PY_NAME ['(' [arglist] ')'] ':' suite
120

121
arglist: argument (',' argument)*  [',']
122

123
# The reason that keywords are test nodes instead of NAME is that using NAME
124
# results in an ambiguity. ast.c makes sure it's a NAME.
125
# "test '=' test" is really "keyword '=' test", but we have no such token.
126
# These need to be in a single rule to avoid grammar that is ambiguous
127
# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
128
# we explicitly match '*' here, too, to give it proper precedence.
129
# Illegal combinations and orderings are blocked in ast.c:
130
# multiple (test comp_for) arguments are blocked; keyword unpackings
131
# that precede iterable unpackings are blocked; etc.
132
argument: ( test [comp_for] |
133
            test '=' test |
134
            '**' expr |
135
            star_expr )
136

137
comp_iter: comp_for | comp_if
138
comp_for: 'for' exprlist ('in' or_test | for_from_clause) [comp_iter]
139
comp_if: 'if' test_nocond [comp_iter]
140

141
# not used in grammar, but may appear in "node" passed from Parser to Compiler
142
encoding_decl: NAME
143

144
yield_expr: 'yield' [yield_arg]
145
yield_arg: 'from' test | testlist
146

147

148
# Cython extensions
149

150
# Accommodate to Py2 tokenizer.
151
ellipsis: '...' | '.' '.' '.'
152

153
signedness: 'unsigned' | 'signed'
154
longness: 'char' | 'short' | 'long' | 'long' 'long'
155
# TODO: [unsigned] double doesn't make sens, but we need long double
156
int_type: signedness [longness] | longness | [signedness] [longness] ('int' | 'double')  | 'complex'
157

158
type: ['const'] (NAME ('.' PY_NAME)* | int_type | '(' type ')') ['complex'] [type_qualifiers]
159
maybe_typed_name: ['const'] (NAME [('.' PY_NAME)* ['complex'] [type_qualifiers] NAME] | (int_type | '(' type ')') ['complex'] [type_qualifiers] NAME)
160
teplate_params: '[' NAME (',' NAME)* ']'
161
type_qualifiers: type_qualifier+
162
type_qualifier: '*' | '**' | '&' | type_index ('.' NAME [type_index])*
163
# TODO: old buffer syntax
164
type_index: '[' [(NUMBER | type (',' type)* | (memory_view_index (',' memory_view_index)*))] ']'
165
memory_view_index: ':' [':'] [NUMBER]
166

167
address: '&' factor
168
cast: '<' type ['?'] '>' factor
169
size_of: 'sizeof' '(' (type) ')'
170
type_id: 'typeid' '(' (type) ')'
171
new_expr: 'new' type '(' [arglist] ')'
172

173
# TODO: Restrict cdef_stmt to "top-level" statements.
174
cdef_stmt: ('cdef' | 'cpdef') (cvar_def | cdef_type_decl | extern_block)
175
cdef_type_decl: ctype_decl | fused | cclass
176
ctype_decl: struct | enum | cppclass
177
# TODO: Does the cdef/ctypedef distinction even make sense for fused?
178
ctypedef_stmt: 'ctypedef' (cvar_decl | struct | enum | fused)
179

180
# Note: these two are similar but can't be used in an or clause
181
# as it would cause ambiguity in the LL(1) parser.
182
# Requires a type
183
cvar_decl: [visibility] type cname (NEWLINE | cfunc)
184
# Allows an assignment
185
cvar_def: [visibility] maybe_typed_name (['=' test] (',' PY_NAME ['=' test])* NEWLINE | cfunc)
186

187
visibility: 'public' | 'api' | 'readonly'
188
# TODO: Standardize gil_spec first or last.
189
cfunc: [teplate_params] parameters [gil_spec] [exception_value] [gil_spec] (':' suite | NEWLINE)
190
exception_value: 'except' (['?'] expr | '*' | '+' [PY_NAME])
191
gil_spec: 'with' ('gil' | 'nogil') | 'nogil'
192

193
cname: NAME [STRING]
194
cclass: classdef
195
fused: 'fused' PY_NAME ':' NEWLINE INDENT ( type NEWLINE)+ DEDENT
196
enum: 'enum' [cname] (NEWLINE | ':' enum_suite)
197
enum_suite: NEWLINE INDENT (cname ['=' NUMBER] NEWLINE | pass_stmt NEWLINE)+ DEDENT
198
struct: ('struct' | 'union') cname (NEWLINE | (':' struct_suite))
199
struct_suite: NEWLINE INDENT (cvar_decl | pass_stmt NEWLINE)+ DEDENT
200
cppclass: 'cppclass' cname [teplate_params] [cppclass_bases] (NEWLINE | ':' cppclass_suite)
201
cppclass_bases: '(' dotted_PY_NAME (',' dotted_PY_NAME [teplate_params])*')'
202
cppclass_suite: NEWLINE INDENT (cvar_decl | ctype_decl | pass_stmt NEWLINE)+ DEDENT
203
# TODO: C++ constructors, operators
204

205
extern_block: 'extern' (cvar_decl | 'from' ('*' | STRING) ['namespace' STRING] [gil_spec] ':' (pass_stmt | extern_suite))
206
extern_suite: NEWLINE INDENT (['cdef' | 'cpdef'] (cvar_decl | cdef_type_decl) | ctypedef_stmt)+ DEDENT
207

208
cy_type_kwd: 'struct' | 'union' | 'fused' | 'cppclass' | 'int' | 'double' | 'complex'
209
cy_kwd: cy_type_kwd | signedness | longness | visibility | 'gil' | 'nogil' | 'namespace' | 'const' | 'by' | 'extern'
210
PY_NAME: NAME | cy_kwd
211

212
# TODO: Do we really want these? Don't play well with include...
213
DEF_stmt: 'DEF' NAME '=' testlist
214
IF_stmt: 'IF' test ':' suite ('ELIF' test ':' suite)* ['ELSE' ':' suite]
215