7
cython.declare(os=object, re=object, operator=object, textwrap=object,
8
Template=object, Naming=object, Options=object, StringEncoding=object,
9
Utils=object, SourceDescriptor=object, StringIOTree=object,
10
DebugFlags=object, defaultdict=object,
11
closing=object, partial=object, wraps=object)
19
from string import Template
20
from functools import partial, wraps
21
from contextlib import closing, contextmanager
22
from collections import defaultdict
26
from . import DebugFlags
27
from . import StringEncoding
29
from .Scanning import SourceDescriptor
30
from ..StringIOTree import StringIOTree
33
renamed_py2_builtins_map = {34
# builtins that had different names in Py2 code
38
'raw_input' : 'input',
41
ctypedef_builtins_map = {42
# types of builtins in "ctypedef class" statements which we don't
43
# import either because the names conflict with C types or because
44
# the type simply is not exposed.
45
'py_int' : '&PyLong_Type',
46
'py_long' : '&PyLong_Type',
47
'py_float' : '&PyFloat_Type',
48
'wrapper_descriptor' : '&PyWrapperDescr_Type',
51
basicsize_builtins_map = {52
# builtins whose type has a different tp_basicsize than sizeof(...)
53
'PyTypeObject': 'PyHeapTypeObject',
56
# Builtins as of Python version ...
57
KNOWN_PYTHON_BUILTINS_VERSION = (3, 13, 0, 'alpha', 5)
58
KNOWN_PYTHON_BUILTINS = frozenset([
69
'ConnectionAbortedError',
71
'ConnectionRefusedError',
72
'ConnectionResetError',
89
'_IncompleteInputError',
98
'ModuleNotFoundError',
101
'NotADirectoryError',
103
'NotImplementedError',
106
'PendingDeprecationWarning',
108
'ProcessLookupError',
109
'PythonFinalizationError',
115
'StopAsyncIteration',
126
'UnicodeDecodeError',
127
'UnicodeEncodeError',
129
'UnicodeTranslateError',
216
uncachable_builtins = [
217
# Global/builtin names that cannot be cached because they may or may not
218
# be available at import time, for various reasons:
220
'_IncompleteInputError',
221
'PythonFinalizationError',
223
'BaseExceptionGroup',
230
'breakpoint', # might deserve an implementation in Cython
231
## - platform specific
234
'_', # e.g. used by gettext
237
special_py_methods = cython.declare(frozenset, frozenset((
238
'__cinit__', '__dealloc__', '__richcmp__', '__next__',
239
'__await__', '__aiter__', '__anext__',
240
'__getbuffer__', '__releasebuffer__',
243
modifier_output_mapper = {244
'inline': 'CYTHON_INLINE'
247
cleanup_level_for_type_prefix = cython.declare(object, {256
An include file and/or verbatim C code to be included in the
261
# pieces {order: unicode}: pieces of C code to be generated.262
# For the included file, the key "order" is zero.
263
# For verbatim include code, the "order" is the "order"
264
# attribute of the original IncludeCode where this piece
265
# of C code was first added. This is needed to prevent
266
# duplication if the same include code is found through
268
# location int: where to put this include in the C sources, one
269
# of the constants INITIAL, EARLY, LATE
270
# order int: sorting order (automatically set by increasing counter)
272
# Constants for location. If the same include occurs with different
273
# locations, the earliest one takes precedense.
278
counter = 1 # Counter for "order"
280
def __init__(self, include=None, verbatim=None, late=True, initial=False):
281
self.order = self.counter
282
type(self).counter += 1
286
if include[0] == '<' and include[-1] == '>':
287
self.pieces[0] = '#include {}'.format(include)288
late = False # system include is never late
290
self.pieces[0] = '#include "{}"'.format(include)293
self.pieces[self.order] = verbatim
296
self.location = self.INITIAL
298
self.location = self.LATE
300
self.location = self.EARLY
302
def dict_update(self, d, key):
304
Insert `self` in dict `d` with key `key`. If that key already
305
exists, update the attributes of the existing value with `self`.
309
other.location = min(self.location, other.location)
310
other.pieces.update(self.pieces)
319
Return the main piece of C code, corresponding to the include
320
file. If there was no include file, return None.
322
return self.pieces.get(0)
324
def write(self, code):
325
# Write values of self.pieces dict, sorted by the keys
326
for k in sorted(self.pieces):
327
code.putln(self.pieces[k])
330
def get_utility_dir():
331
# make this a function and not global variables:
332
# http://trac.cython.org/cython_trac/ticket/475
333
Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
334
return os.path.join(Cython_dir, "Utility")
336
read_utilities_hook = None
338
Override the hook for reading a utilities file that contains code fragments used
341
The hook functions takes the path of the utilities file, and returns a list
342
of strings, one per line.
344
The default behavior is to open a file relative to get_utility_dir().
347
def read_utilities_from_utility_dir(path):
349
Read all lines of the file at the provided path from a path relative
350
to get_utility_dir().
352
filename = os.path.join(get_utility_dir(), path)
353
with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f:
356
# by default, read utilities from the utility directory.
357
read_utilities_hook = read_utilities_from_utility_dir
359
class UtilityCodeBase:
361
Support for loading utility code from a file.
363
Code sections in the file can be specified as follows:
365
##### MyUtility.proto #####
369
##### MyUtility.init #####
371
[code run at module initialization]
373
##### MyUtility #####
374
#@requires: MyOtherUtility
379
##### MyUtility #####
380
#@substitute: tempita
382
[requires tempita substitution
383
- context can't be specified here though so only
384
tempita utility that requires no external context
385
will benefit from this tag
386
- only necessary when @required from non-tempita code]
388
for prototypes and implementation respectively. For non-python or
389
-cython files backslashes should be used instead. 5 to 30 comment
390
characters may be used on either side.
392
If the @cname decorator is not used and this is a CythonUtilityCode,
393
one should pass in the 'name' keyword argument to be used for name
394
mangling of such entries.
397
is_cython_utility = False
401
def _add_utility(cls, utility, name, type, lines, begin_lineno, tags=None):
405
code = '\n'.join(lines)
406
if tags and 'substitute' in tags and 'naming' in tags['substitute']:
408
new_code = Template(code).substitute(vars(Naming))
409
except (KeyError, ValueError) as e:
411
f"Error parsing templated utility code '{name}.{type}' at line {begin_lineno:d}: {e}")414
f"Found useless 'substitute: naming' declaration without replacements. ({name}.{type}:{begin_lineno:d})")417
# remember correct line numbers at least until after templating
418
code = '\n' * begin_lineno + code
425
all_tags = utility[2]
426
all_tags[type] = code
429
all_tags = utility[2]
430
for name, values in tags.items():
431
all_tags.setdefault(name, set()).update(values)
434
def load_utilities_from_file(cls, path):
435
utilities = cls._utility_cache.get(path)
439
_, ext = os.path.splitext(path)
440
if ext in ('.pyx', '.py', '.pxd', '.pxi'):442
strip_comments = partial(re.compile(r'^\s*#(?!\s*cython\s*:).*').sub, '')
446
strip_comments = partial(re.compile(r'^\s*//.*|/\*[^*]*\*/').sub, '')
447
rstrip = partial(re.compile(r'\s+(\\?)$').sub, r'\1')
448
match_special = re.compile(
449
(r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'450
r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)') %
451
{'C': comment}).match452
match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match
454
all_lines = read_utilities_hook(path)
456
utilities = defaultdict(lambda: [None, None, {}])458
tags = defaultdict(set)
459
utility = name = type = None
462
for lineno, line in enumerate(all_lines):
463
m = match_special(line)
466
cls._add_utility(utility, name, type, lines, begin_lineno, tags)
468
begin_lineno = lineno + 1
472
name = m.group('name')473
mtype = match_type(name)
475
name, type = mtype.groups()
478
utility = utilities[name]
480
tags[m.group('tag')].add(m.group('value'))481
lines.append('') # keep line number correct483
lines.append(rstrip(strip_comments(line)))
486
raise ValueError("Empty utility code file")488
# Don't forget to add the last utility code
489
cls._add_utility(utility, name, type, lines, begin_lineno, tags)
491
utilities = dict(utilities) # un-defaultdict-ify
492
cls._utility_cache[path] = utilities
496
def load(cls, util_code_name, from_file, **kwargs):
498
Load utility code from a file specified by from_file (relative to
499
Cython/Utility) and name util_code_name.
502
if '::' in util_code_name:
503
from_file, util_code_name = util_code_name.rsplit('::', 1)505
utilities = cls.load_utilities_from_file(from_file)
506
proto, impl, tags = utilities[util_code_name]
509
if "substitute" in tags and "tempita" in tags["substitute"]:
510
if not issubclass(cls, TempitaUtilityCode):
511
return TempitaUtilityCode.load(util_code_name, from_file, **kwargs)
512
orig_kwargs = kwargs.copy()
513
for name, values in tags.items():
516
# only pass lists when we have to: most argument expect one value or None
517
if name == 'requires':
519
values = [cls.load(dep, from_file, **orig_kwargs)
520
for dep in sorted(values)]
522
# dependencies are rarely unique, so use load_cached() when we can
523
values = [cls.load_cached(dep, from_file)
524
for dep in sorted(values)]
525
elif name == 'substitute':
526
# don't want to pass "naming" or "tempita" to the constructor
527
# since these will have been handled
528
values = values - {'naming', 'tempita'}533
elif len(values) == 1:
534
values = list(values)[0]
535
kwargs[name] = values
537
if proto is not None:
538
kwargs['proto'] = proto
540
kwargs['impl'] = impl
542
if 'name' not in kwargs:
543
kwargs['name'] = util_code_name
545
if 'file' not in kwargs and from_file:
546
kwargs['file'] = from_file
550
def load_cached(cls, utility_code_name, from_file, __cache={}):552
Calls .load(), but using a per-type cache based on utility name and file name.
554
key = (utility_code_name, from_file, cls)
559
code = __cache[key] = cls.load(utility_code_name, from_file)
563
def load_as_string(cls, util_code_name, from_file, include_requires=False, **kwargs):
565
Load a utility code as a string. Returns (proto, implementation).
567
If 'include_requires=True', concatenates all requirements before the actually
568
requested utility code, separately for proto and impl part.
570
util = cls.load(util_code_name, from_file, **kwargs)
572
if not include_requires:
573
return util.format_code(util.proto), util.format_code(util.impl)
575
protos, impls = [], []
576
def prepend(util_code):
577
if util_code.requires:
578
for dep in util_code.requires:
581
protos.append(util_code.format_code(util_code.proto))
583
impls.append(util_code.format_code(util_code.impl))
586
return "".join(protos), "".join(impls)
588
def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub):
590
Format a code section for output.
593
code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n'597
return "<%s(%s)>" % (type(self).__name__, self.name)
599
def get_tree(self, **kwargs):
602
def __deepcopy__(self, memodict=None):
603
# No need to deep-copy utility code since it's essentially immutable.
607
class UtilityCode(UtilityCodeBase):
609
Stores utility code to add during code generation.
611
See GlobalState.put_utility_code.
613
hashes/equals by instance
616
impl implementation code
617
init code to call on module initialization
618
requires utility code dependencies
619
proto_block the place in the resulting file where the prototype should
621
name name of the utility code (or None)
622
file filename of the utility code file this utility was loaded
626
def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
627
proto_block='utility_code_proto', name=None, file=None):
628
# proto_block: Which code block to dump prototype in. See GlobalState.
632
self.cleanup = cleanup
633
self.requires = requires
635
self.specialize_list = []
636
self.proto_block = proto_block
641
return hash((self.proto, self.impl))
643
def __eq__(self, other):
646
self_type, other_type = type(self), type(other)
647
if self_type is not other_type and not (isinstance(other, self_type) or isinstance(self, other_type)):
650
self_init = getattr(self, 'init', None)
651
other_init = getattr(other, 'init', None)
652
self_proto = getattr(self, 'proto', None)
653
other_proto = getattr(other, 'proto', None)
654
return (self_init, self_proto, self.impl) == (other_init, other_proto, other.impl)
656
def none_or_sub(self, s, context):
658
Format a string in this utility code with context. If None, do nothing.
664
def specialize(self, pyrex_type=None, **data):
666
if pyrex_type is not None:
667
data['type'] = pyrex_type.empty_declaration_code()
668
data['type_name'] = pyrex_type.specialization_name()
669
name = "%s[%s]" % (name, data['type_name'])
670
# Dicts aren't hashable...
671
key = tuple(sorted(data.items()))
673
return self._cache[key]
675
if self.requires is None:
678
requires = [r.specialize(data) for r in self.requires]
680
s = self._cache[key] = UtilityCode(
681
self.none_or_sub(self.proto, data),
682
self.none_or_sub(self.impl, data),
683
self.none_or_sub(self.init, data),
684
self.none_or_sub(self.cleanup, data),
690
self.specialize_list.append(s)
693
def put_code(self, output):
695
for dependency in self.requires:
696
output.use_utility_code(dependency)
698
writer = output[self.proto_block]
699
writer.putln(f"/* {self.name}.proto */")700
proto, result_is_module_specific = process_utility_ccode(self, output, self.proto)
701
if result_is_module_specific:
704
# can be reused across modules
705
writer.put_or_include(proto, f'{self.name}_proto')707
impl, result_is_module_specific = process_utility_ccode(self, output, self.impl)
708
writer = output['utility_code_def']
709
writer.putln(f"/* {self.name} */")710
if result_is_module_specific:
713
# can be reused across modules
714
writer.put_or_include(impl, f'{self.name}_impl')716
writer = output['init_globals']
717
writer.putln(f"/* {self.name}.init */")718
if isinstance(self.init, str):
719
writer.put(self.format_code(self.init))
721
self.init(writer, output.module_pos)
722
# 'init' code can end with an 'if' statement for an error condition like:
723
# if (check_ok()) ; else
724
writer.putln(writer.error_goto_if_PyErr(output.module_pos))
726
if self.cleanup and Options.generate_cleanup_code:
727
writer = output['cleanup_globals']
728
writer.putln(f"/* {self.name}.cleanup */")729
if isinstance(self.cleanup, str):
730
writer.put_or_include(
731
self.format_code(self.cleanup),
732
f'{self.name}_cleanup')734
self.cleanup(writer, output.module_pos)
737
def add_macro_processor(*macro_names, regex=None, is_module_specific=False, _last_macro_processor = [None]):
738
"""Decorator to chain the code macro processors below.
740
last_processor = _last_macro_processor[0]
742
def build_processor(func):
744
def process(utility_code: UtilityCode, output, code_string: str):
745
# First, call the processing chain in FIFO function definition order.
746
result_is_module_specific = False
747
if last_processor is not None:
748
code_string, result_is_module_specific = last_processor(utility_code, output, code_string)
750
# Detect if we need to do something.
752
for macro in macro_names:
753
if macro in code_string:
756
return code_string, result_is_module_specific
760
code_string = func(utility_code, output, code_string)
762
code_string = re.sub(regex, partial(func, output), code_string)
764
# Make sure we found and replaced all macro occurrences.
765
for macro in macro_names:
766
if macro in code_string:
767
raise RuntimeError(f"Left-over utility code macro '{macro}()' found in '{utility_code.name}'")769
result_is_module_specific |= is_module_specific
770
return code_string, result_is_module_specific
772
_last_macro_processor[0] = process
775
return build_processor
780
regex=r'CSTRING\(\s*"""([^"]*(?:"[^"]+)*)"""\s*\)',
782
def _wrap_c_string(_, matchobj):
783
"""Replace CSTRING('''xyz''') by a C compatible string, taking care of line breaks.785
content = matchobj.group(1).replace('"', '\042')787
f'"{line}\\n"\n' if not line.endswith('\\') or line.endswith('\\\\') else f'"{line[:-1]}"\n'788
for line in content.splitlines())
791
@add_macro_processor()
792
def _format_impl_code(utility_code: UtilityCode, _, impl):
793
return utility_code.format_code(impl)
797
'CALL_UNBOUND_METHOD',
798
is_module_specific=True,
800
r'CALL_UNBOUND_METHOD\('801
r'([a-zA-Z_]+),' # type cname
802
r'\s*"([^"]+)",' # method name
803
r'\s*([^),]+)' # object cname
804
r'((?:,[^),]+)*)' # args*
808
def _inject_unbound_method(output, matchobj):
809
"""Replace 'UNBOUND_METHOD(type, "name")' by a constant Python identifier cname.
811
type_cname, method_name, obj_cname, args = matchobj.groups()
812
type_cname = '&%s' % type_cname
813
args = [arg.strip() for arg in args[1:].split(',')] if args else []814
assert len(args) < 3, f"CALL_UNBOUND_METHOD() does not support {len(args):d} call arguments"815
return output.cached_unbound_method_call_code(obj_cname, type_cname, method_name, args)
819
'PYIDENT', 'PYUNICODE',
820
is_module_specific=True,
821
regex=r'PY(IDENT|UNICODE)\("([^"]+)"\)',823
def _inject_string_constant(output, matchobj):
824
"""Replace 'PYIDENT("xyz")' by a constant Python identifier cname.826
str_type, name = matchobj.groups()
827
return output.get_py_string_const(
828
StringEncoding.EncodedString(name), identifier=str_type == 'IDENT').cname
833
# As long as we use the same C access macros for these names, they are not module specific.
834
# is_module_specific=True,
835
regex=r'EMPTY\((bytes|unicode|tuple)\)',
837
def _inject_empty_collection_constant(output, matchobj):
838
"""Replace 'EMPTY(bytes|tuple|...)' by a constant Python identifier cname.
840
type_name = matchobj.group(1)
841
return getattr(Naming, f'empty_{type_name}')844
@add_macro_processor()
845
def process_utility_ccode(utility_code, _, code_string):
846
"""Entry point for code processors, must be defined last.
851
def sub_tempita(s, context, file=None, name=None, __cache={}):852
"Run tempita on string s with given context."
857
name = f"{file}:{name}"859
context['__name'] = name
862
template = __cache[s]
864
from ..Tempita import Template
865
template = __cache[s] = Template(s, name=name)
867
return template.substitute(context)
870
class TempitaUtilityCode(UtilityCode):
871
def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs):
874
proto = sub_tempita(proto, context, file, name)
875
impl = sub_tempita(impl, context, file, name)
876
init = sub_tempita(init, context, file, name)
878
proto, impl, init=init, name=name, file=file, **kwargs)
881
def load_cached(cls, utility_code_name, from_file=None, context=None, __cache={}):882
context_key = tuple(sorted(context.items())) if context else None
883
assert hash(context_key) is not None # raise TypeError if not hashable
884
key = (cls, from_file, utility_code_name, context_key)
889
code = __cache[key] = cls.load(utility_code_name, from_file, context=context)
892
def none_or_sub(self, s, context):
894
Format a string in this utility code with context. If None, do nothing.
898
return sub_tempita(s, context, self.file, self.name)
901
class LazyUtilityCode(UtilityCodeBase):
903
Utility code that calls a callback with the root code writer when
904
available. Useful when you only have 'env' but not 'code'.
909
def __init__(self, callback):
910
self.callback = callback
912
def put_code(self, globalstate):
913
utility = self.callback(globalstate.rootwriter)
914
globalstate.use_utility_code(utility)
918
# return_label string function return point label
919
# error_label string error catch point label
920
# error_without_exception boolean Can go to the error label without an exception (e.g. __next__ can return NULL)
921
# continue_label string loop continue point label
922
# break_label string loop break point label
923
# return_from_error_cleanup_label string
924
# label_counter integer counter for naming labels
925
# in_try_finally boolean inside try of try...finally
926
# exc_vars (string * 3) exception variables for reraise, or None
927
# can_trace boolean line tracing is supported in the current context
928
# scope Scope the scope object of the current function
930
# Not used for now, perhaps later
931
def __init__(self, owner, names_taken=set(), scope=None):
932
self.names_taken = names_taken
936
self.error_label = None
937
self.label_counter = 0
938
self.labels_used = set()
939
self.return_label = self.new_label()
940
self.new_error_label()
941
self.continue_label = None
942
self.break_label = None
943
self.yield_labels = []
945
self.in_try_finally = 0
947
self.current_except = None
948
self.can_trace = False
949
self.gil_owned = True
951
self.temps_allocated = [] # of (name, type, manage_ref, static)
952
self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status953
self.temps_used_type = {} # name -> (type, manage_ref)954
self.zombie_temps = set() # temps that must not be reused after release
955
self.temp_counter = 0
956
self.closure_temps = None
958
# This is used to collect temporaries, useful to find out which temps
959
# need to be privatized in parallel sections
960
self.collect_temps_stack = []
962
# This is used for the error indicator, which needs to be local to the
963
# function. It used to be global, which relies on the GIL being held.
964
# However, exceptions may need to be propagated through 'nogil'
965
# sections, in which case we introduce a race condition.
966
self.should_declare_error_indicator = False
967
self.uses_error_indicator = False
969
self.error_without_exception = False
971
self.needs_refnanny = False
975
def validate_exit(self):
976
# validate that all allocated temps have been freed
977
if self.temps_allocated:
978
leftovers = self.temps_in_use()
980
msg = "TEMPGUARD: Temps left over at end of '%s': %s" % (self.scope.name, ', '.join([
981
'%s [%s]' % (name, ctype)
982
for name, ctype, is_pytemp in sorted(leftovers)]),
985
raise RuntimeError(msg)
989
def new_label(self, name=None):
990
n: cython.size_t = self.label_counter
991
self.label_counter = n + 1
992
label = "%s%d" % (Naming.label_prefix, n)
997
def new_yield_label(self, expr_type='yield'):
998
label = self.new_label('resume_from_%s' % expr_type)999
num_and_label = (len(self.yield_labels) + 1, label)
1000
self.yield_labels.append(num_and_label)
1001
return num_and_label
1003
def new_error_label(self, prefix=""):
1004
old_err_lbl = self.error_label
1005
self.error_label = self.new_label(prefix + 'error')
1008
def get_loop_labels(self):
1010
self.continue_label,
1013
def set_loop_labels(self, labels):
1014
(self.continue_label,
1015
self.break_label) = labels
1017
def new_loop_labels(self, prefix=""):
1018
old_labels = self.get_loop_labels()
1019
self.set_loop_labels(
1020
(self.new_label(prefix + "continue"),
1021
self.new_label(prefix + "break")))
1024
def get_all_labels(self):
1026
self.continue_label,
1031
def set_all_labels(self, labels):
1032
(self.continue_label,
1035
self.error_label) = labels
1037
def all_new_labels(self):
1038
old_labels = self.get_all_labels()
1040
for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']):
1042
new_labels.append(self.new_label(name))
1044
new_labels.append(old_label)
1045
self.set_all_labels(new_labels)
1048
def use_label(self, lbl):
1049
self.labels_used.add(lbl)
1051
def label_used(self, lbl):
1052
return lbl in self.labels_used
1056
def allocate_temp(self, type, manage_ref, static=False, reusable=True):
1058
Allocates a temporary (which may create a new one or get a previously
1059
allocated and released one of the same type). Type is simply registered
1060
and handed back, but will usually be a PyrexType.
1062
If type.needs_refcounting, manage_ref comes into play. If manage_ref is set to
1063
True, the temp will be decref-ed on return statements and in exception
1064
handling clauses. Otherwise the caller has to deal with any reference
1065
counting of the variable.
1067
If not type.needs_refcounting, then manage_ref will be ignored, but it
1068
still has to be passed. It is recommended to pass False by convention
1069
if it is known that type will never be a reference counted type.
1071
static=True marks the temporary declaration with "static".
1072
This is only used when allocating backing store for a module-level
1075
if reusable=False, the temp will not be reused after release.
1077
A C string referring to the variable is returned.
1079
if type.is_cv_qualified and not type.is_reference:
1080
type = type.cv_base_type
1081
elif type.is_reference and not type.is_fake_reference:
1082
type = type.ref_base_type
1083
elif type.is_cfunction:
1084
from . import PyrexTypes
1085
type = PyrexTypes.c_ptr_type(type) # A function itself isn't an l-value
1086
elif type.is_cpp_class and not type.is_fake_reference and self.scope.directives['cpp_locals']:
1087
self.scope.use_utility_code(UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp"))1088
if not type.needs_refcounting:
1089
# Make manage_ref canonical, so that manage_ref will always mean
1090
# a decref is needed.
1093
freelist = self.temps_free.get((type, manage_ref))
1094
if reusable and freelist is not None and freelist[0]:
1095
result = freelist[0].pop()
1096
freelist[1].remove(result)
1099
self.temp_counter += 1
1100
result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)
1101
if result not in self.names_taken: break
1102
self.temps_allocated.append((result, type, manage_ref, static))
1104
self.zombie_temps.add(result)
1105
self.temps_used_type[result] = (type, manage_ref)
1106
if DebugFlags.debug_temp_code_comments:
1107
self.owner.putln("/* %s allocated (%s)%s */" % (result, type, "" if reusable else " - zombie"))1109
if self.collect_temps_stack:
1110
self.collect_temps_stack[-1].add((result, type))
1114
def release_temp(self, name):
1116
Releases a temporary so that it can be reused by other code needing
1117
a temp of the same type.
1119
type, manage_ref = self.temps_used_type[name]
1120
freelist = self.temps_free.get((type, manage_ref))
1121
if freelist is None:
1122
freelist = ([], set()) # keep order in list and make lookups in set fast
1123
self.temps_free[(type, manage_ref)] = freelist
1124
if name in freelist[1]:
1125
raise RuntimeError("Temp %s freed twice!" % name)1126
if name not in self.zombie_temps:
1127
freelist[0].append(name)
1128
freelist[1].add(name)
1129
if DebugFlags.debug_temp_code_comments:
1130
self.owner.putln("/* %s released %s*/" % (1131
name, " - zombie" if name in self.zombie_temps else ""))
1133
def temps_in_use(self):
1134
"""Return a list of (cname,type,manage_ref) tuples of temp names and their type
1135
that are currently in use.
1138
for name, type, manage_ref, static in self.temps_allocated:
1139
freelist = self.temps_free.get((type, manage_ref))
1140
if freelist is None or name not in freelist[1]:
1141
used.append((name, type, manage_ref and type.needs_refcounting))
1144
def temps_holding_reference(self):
1145
"""Return a list of (cname,type) tuples of temp names and their type
1146
that are currently in use. This includes only temps
1147
with a reference counted type which owns its reference.
1149
return [(name, type)
1150
for name, type, manage_ref in self.temps_in_use()
1151
if manage_ref and type.needs_refcounting]
1153
def all_managed_temps(self):
1154
"""Return a list of (cname, type) tuples of refcount-managed Python objects.
1156
return [(cname, type)
1157
for cname, type, manage_ref, static in self.temps_allocated
1160
def all_free_managed_temps(self):
1161
"""Return a list of (cname, type) tuples of refcount-managed Python
1162
objects that are not currently in use. This is used by
1163
try-except and try-finally blocks to clean up temps in the
1166
return sorted([ # Enforce deterministic order.
1168
for (type, manage_ref), freelist in self.temps_free.items() if manage_ref
1169
for cname in freelist[0]
1172
def start_collecting_temps(self):
1174
Useful to find out which temps were used in a code block
1176
self.collect_temps_stack.append(set())
1178
def stop_collecting_temps(self):
1179
return self.collect_temps_stack.pop()
1181
def init_closure_temps(self, scope):
1182
self.closure_temps = ClosureTempAllocator(scope)
1186
"""Global info about a Python number constant held by GlobalState.
1190
py_type string int, long, float
1191
value_code string evaluation code if different from value
1194
def __init__(self, cname, value, py_type, value_code=None):
1197
self.py_type = py_type
1198
self.value_code = value_code or value
1202
"""Global info about a generic constant held by GlobalState.
1207
def __init__(self, cname, type):
1212
cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object,
1213
replace_identifier=object, find_alphanums=object)
1214
possible_unicode_identifier = re.compile(r"(?![0-9])\w+$", re.U).match
1215
possible_bytes_identifier = re.compile(br"(?![0-9])\w+$").match
1216
replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub
1217
find_alphanums = re.compile('([a-zA-Z0-9]+)').findall1220
"""Global info about a C string constant held by GlobalState.
1223
# text EncodedString or BytesLiteral
1224
# py_strings {(identifier, encoding) : PyStringConst}1226
def __init__(self, cname, text, byte_string):
1229
self.escaped_value = StringEncoding.escape_byte_string(byte_string)
1230
self.py_strings = None
1232
def get_py_string_const(self, encoding, identifier=None):
1235
is_unicode: cython.bint
1237
if identifier or encoding is None:
1239
encoding = encoding_key = None
1244
encoding = encoding.lower()
1245
if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):1249
encoding_key = ''.join(find_alphanums(encoding))
1253
elif identifier is None:
1254
if isinstance(text, bytes):
1255
intern = bool(possible_bytes_identifier(text))
1257
intern = bool(possible_unicode_identifier(text))
1261
key = (intern, is_unicode, encoding_key)
1262
if self.py_strings is None:
1263
self.py_strings = {}1266
return self.py_strings[key]
1271
f"{Naming.interned_prefixes['str'] if intern else Naming.py_const_prefix}"1272
f"{'u' if is_unicode else 'b'}"1273
f"{'_' + encoding_key if encoding_key else ''}"1274
f"_{self.cname[len(Naming.const_prefix):]}"1277
py_string = PyStringConst(pystring_cname, encoding, intern, is_unicode)
1278
self.py_strings[key] = py_string
1283
"""Global info about a Python string constant held by GlobalState.
1288
# is_unicode boolean
1290
def __init__(self, cname, encoding, intern=False, is_unicode=False):
1292
self.encoding = encoding
1293
self.is_unicode = is_unicode
1294
self.intern = intern
1296
def __lt__(self, other):
1297
return self.cname < other.cname
1301
# filename_table {string : int} for finding filename table indexes1302
# filename_list [string] filenames in filename table order
1303
# input_file_contents dict contents (=list of lines) of any file that was used as input
1304
# to create this output C code. This is
1305
# used to annotate the comments.
1307
# utility_codes set IDs of used utility code (to avoid reinsertion)
1309
# declared_cnames {string:Entry} used in a transition phase to merge pxd-declared1310
# constants etc. into the pyx-declared ones (i.e,
1311
# check if constants are already added).
1312
# In time, hopefully the literals etc. will be
1313
# supplied directly instead.
1315
# const_cnames_used dict global counter for unique constant identifiers
1318
# parts {string:CCodeWriter}1325
# directives set Temporary variable used to track
1326
# the current set of directives in the code generation
1334
'utility_code_proto_before_types',
1335
'numeric_typedefs', # Let these detailed individual parts stay!,
1336
'complex_type_declarations', # as the proper solution is to make a full DAG...
1337
'type_declarations', # More coarse-grained blocks would simply hide
1338
'utility_code_proto', # the ugliness, not fix it
1339
'module_declarations',
1341
'before_global_var',
1347
'module_state_clear',
1348
'module_state_traverse',
1349
'module_state_defines', # redefines names used in module_state/_clear/_traverse
1350
'module_code', # user code goes here
1352
'initfunc_declarations',
1359
'init_globals', # (utility code called at init-time)
1363
'utility_code_pragmas', # silence some irrelevant warnings in utility code
1365
'utility_code_pragmas_end', # clean-up the utility_code_pragmas
1369
# h files can only have a much smaller list of sections
1372
'utility_code_proto_before_types',
1373
'type_declarations',
1374
'utility_code_proto',
1378
def __init__(self, writer, module_node, code_config, common_utility_include_dir=None):
1379
self.filename_table = {}1380
self.filename_list = []
1381
self.input_file_contents = {}1382
self.utility_codes = set()
1383
self.declared_cnames = {}1384
self.in_utility_code_generation = False
1385
self.code_config = code_config
1386
self.common_utility_include_dir = common_utility_include_dir
1388
self.module_node = module_node # because some utility code generation needs it
1389
# (generating backwards-compatible Get/ReleaseBuffer
1391
self.const_cnames_used = {}1392
self.string_const_index = {}1393
self.dedup_const_index = {}1394
self.pyunicode_ptr_const_index = {}1395
self.codeobject_constants = []
1396
self.num_const_index = {}1397
self.arg_default_constants = []
1398
self.const_array_counters = {} # counts of differently prefixed arrays of constants1399
self.cached_cmethods = {}1400
self.initialised_constants = set()
1402
writer.set_global_state(self)
1403
self.rootwriter = writer
1405
def initialize_main_c_code(self):
1406
rootwriter = self.rootwriter
1407
for i, part in enumerate(self.code_layout):
1408
w = self.parts[part] = rootwriter.insertion_point()
1410
w.putln("/* #### Code section: %s ### */" % part)1412
if not Options.cache_builtins:
1413
del self.parts['cached_builtins']
1415
w = self.parts['cached_builtins']
1416
w.start_initcfunc("int __Pyx_InitCachedBuiltins(void)")1418
w = self.parts['cached_constants']
1419
w.start_initcfunc("int __Pyx_InitCachedConstants(void)", refnanny=True)1420
w.put_setup_refcount_context(StringEncoding.EncodedString("__Pyx_InitCachedConstants"))1422
w = self.parts['init_globals']
1423
w.start_initcfunc("int __Pyx_InitGlobals(void)")1425
w = self.parts['init_constants']
1426
w.start_initcfunc("int __Pyx_InitConstants(void)")1428
if not Options.generate_cleanup_code:
1429
del self.parts['cleanup_globals']
1431
w = self.parts['cleanup_globals']
1432
w.start_initcfunc("void __Pyx_CleanupGlobals(void)")1434
code = self.parts['utility_code_proto']
1436
code.putln("/* --- Runtime support code (head) --- */")1438
code = self.parts['utility_code_def']
1439
if self.code_config.emit_linenums:
1440
code.write('\n#line 1 "cython_utility"\n')1442
code.putln("/* --- Runtime support code --- */")1444
def initialize_main_h_code(self):
1445
rootwriter = self.rootwriter
1446
for part in self.h_code_layout:
1447
self.parts[part] = rootwriter.insertion_point()
1449
def finalize_main_c_code(self):
1450
self.close_global_decls()
1455
code = self.parts['utility_code_def']
1456
util = TempitaUtilityCode.load_cached("TypeConversions", "TypeConversion.c")1457
code.put(util.format_code(util.impl))
1461
# utility code pragmas
1463
code = self.parts['utility_code_pragmas']
1464
util = UtilityCode.load_cached("UtilityCodePragmas", "ModuleSetupCode.c")1465
code.putln(util.format_code(util.impl))
1467
code = self.parts['utility_code_pragmas_end']
1468
util = UtilityCode.load_cached("UtilityCodePragmasEnd", "ModuleSetupCode.c")1469
code.putln(util.format_code(util.impl))
1472
def __getitem__(self, key):
1473
return self.parts[key]
1476
# Global constants, interned objects, etc.
1478
def close_global_decls(self):
1479
# This is called when it is known that no more global declarations will
1481
self.generate_const_declarations()
1482
if Options.cache_builtins:
1483
w = self.parts['cached_builtins']
1484
w.putln("return 0;")1485
if w.label_used(w.error_label):
1486
w.put_label(w.error_label)
1487
w.putln("return -1;")1489
w.exit_cfunc_scope()
1491
w = self.parts['cached_constants']
1492
w.put_finish_refcount_context()
1493
w.putln("return 0;")1494
if w.label_used(w.error_label):
1495
w.put_label(w.error_label)
1496
w.put_finish_refcount_context()
1497
w.putln("return -1;")1499
w.exit_cfunc_scope()
1501
for part in ['init_globals', 'init_constants']:
1502
w = self.parts[part]
1503
w.putln("return 0;")1504
if w.label_used(w.error_label):
1505
w.put_label(w.error_label)
1506
w.putln("return -1;")1508
w.exit_cfunc_scope()
1510
if Options.generate_cleanup_code:
1511
w = self.parts['cleanup_globals']
1513
w.exit_cfunc_scope()
1515
if Options.generate_cleanup_code:
1516
w = self.parts['cleanup_module']
1518
w.exit_cfunc_scope()
1520
def put_pyobject_decl(self, entry):
1521
self['global_var'].putln("static PyObject *%s;" % entry.cname)1523
# constant handling at code generation time
1525
def get_cached_constants_writer(self, target=None):
1526
if target is not None:
1527
if target in self.initialised_constants:
1528
# Return None on second/later calls to prevent duplicate creation code.
1530
self.initialised_constants.add(target)
1531
return self.parts['cached_constants']
1533
def get_int_const(self, str_value, longness=False):
1534
py_type = longness and 'long' or 'int'
1536
c = self.num_const_index[(str_value, py_type)]
1538
c = self.new_num_const(str_value, py_type)
1541
def get_float_const(self, str_value, value_code):
1543
c = self.num_const_index[(str_value, 'float')]
1545
c = self.new_num_const(str_value, 'float', value_code)
1548
def get_py_const(self, prefix, dedup_key=None):
1549
if dedup_key is not None:
1550
const = self.dedup_const_index.get(dedup_key)
1551
if const is not None:
1553
const = self.new_array_const_cname(prefix)
1554
if dedup_key is not None:
1555
self.dedup_const_index[dedup_key] = const
1558
def get_argument_default_const(self, type):
1559
cname = self.new_const_cname('')1560
c = PyObjectConst(cname, type)
1561
self.arg_default_constants.append(c)
1562
# Argument default constants aren't currently cleaned up.
1563
# If that changes, it needs to account for the fact that they
1564
# aren't just Python objects
1567
def get_string_const(self, text):
1568
# return a C string constant, creating a new one if necessary
1570
byte_string = text.utf8encode()
1572
byte_string = text.byteencode()
1574
c = self.string_const_index[byte_string]
1576
c = self.new_string_const(text, byte_string)
1579
def get_pyunicode_ptr_const(self, text):
1580
# return a Py_UNICODE[] constant, creating a new one if necessary
1581
assert text.is_unicode
1583
c = self.pyunicode_ptr_const_index[text]
1585
c = self.pyunicode_ptr_const_index[text] = self.new_const_cname()
1588
def get_py_string_const(self, text, identifier=None):
1589
# return a Python string constant, creating a new one if necessary
1590
c_string = self.get_string_const(text)
1591
py_string = c_string.get_py_string_const(text.encoding, identifier)
1594
def get_py_codeobj_const(self, node):
1595
idx = len(self.codeobject_constants)
1596
name = f"{Naming.codeobjtab_cname}[{idx}]"1597
self.codeobject_constants.append(node)
1600
def get_interned_identifier(self, text):
1601
return self.get_py_string_const(text, identifier=True)
1603
def new_string_const(self, text, byte_string):
1604
cname = self.new_string_const_cname(byte_string)
1605
c = StringConst(cname, text, byte_string)
1606
self.string_const_index[byte_string] = c
1609
def new_num_const(self, value, py_type, value_code=None):
1610
cname = self.new_num_const_cname(value, py_type)
1611
c = NumConst(cname, value, py_type, value_code)
1612
self.num_const_index[(value, py_type)] = c
1615
def new_string_const_cname(self, bytes_value):
1616
# Create a new globally-unique nice name for a C string constant.
1617
value = bytes_value.decode('ASCII', 'ignore')1618
return self.new_const_cname(value=value)
1620
def unique_const_cname(self, format_str): # type: (str) -> str
1621
used = self.const_cnames_used
1622
cname = value = format_str.format(sep='', counter='')
1623
while cname in used:
1624
counter = used[value] = used[value] + 1
1625
cname = format_str.format(sep='_', counter=counter)
1629
def new_num_const_cname(self, value, py_type): # type: (str, str) -> str
1630
if py_type == 'long':
1633
prefix = Naming.interned_prefixes[py_type]
1635
value = value.replace('.', '_').replace('+', '_').replace('-', 'neg_')1637
# update tests/run/large_integer_T5290.py in case the amount is changed
1638
cname = self.unique_const_cname(
1639
prefix + "large{counter}_" + value[:18] + "_xxx_" + value[-18:])1641
cname = "%s%s" % (prefix, value)
1644
def new_const_cname(self, prefix='', value=''):
1645
value = replace_identifier('_', value)[:32].strip('_')1646
name_suffix = self.unique_const_cname(value + "{sep}{counter}")1648
prefix = Naming.interned_prefixes[prefix]
1650
prefix = Naming.const_prefix
1651
return "%s%s" % (prefix, name_suffix)
1653
def new_array_const_cname(self, prefix: str):
1654
count = self.const_array_counters.get(prefix, 0)
1655
self.const_array_counters[prefix] = count+1
1656
return f"{Naming.pyrex_prefix}{prefix}[{count}]"1658
def get_cached_unbound_method(self, type_cname, method_name):
1659
key = (type_cname, method_name)
1661
cname = self.cached_cmethods[key]
1663
cname = self.cached_cmethods[key] = self.new_const_cname(
1664
'umethod', '%s_%s' % (type_cname, method_name))
1667
def cached_unbound_method_call_code(self, obj_cname, type_cname, method_name, arg_cnames):
1668
# admittedly, not the best place to put this method, but it is reused by UtilityCode and ExprNodes ...
1669
utility_code_name = "CallUnboundCMethod%d" % len(arg_cnames)
1670
self.use_utility_code(UtilityCode.load_cached(utility_code_name, "ObjectHandling.c"))
1671
cache_cname = self.get_cached_unbound_method(type_cname, method_name)
1672
args = [obj_cname] + arg_cnames
1673
return "__Pyx_%s(&%s, %s)" % (
1679
def add_cached_builtin_decl(self, entry):
1680
if entry.is_builtin and entry.is_const:
1681
if self.should_declare(entry.cname, entry):
1682
self.put_pyobject_decl(entry)
1684
if name in renamed_py2_builtins_map:
1685
name = renamed_py2_builtins_map[name]
1686
self.put_cached_builtin_init(
1687
entry.pos, StringEncoding.EncodedString(name),
1690
def put_cached_builtin_init(self, pos, name, cname):
1691
w = self.parts['cached_builtins']
1692
interned_cname = self.get_interned_identifier(name).cname
1693
self.use_utility_code(
1694
UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c"))1695
w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % (1701
def generate_const_declarations(self):
1702
self.generate_cached_methods_decls()
1703
self.generate_object_constant_decls()
1704
self.generate_codeobject_constants()
1705
# generate code for string and numeric constants as late as possible
1706
# to allow new constants be to created by the earlier stages.
1707
# (although the constants themselves are written early)
1708
self.generate_string_constants()
1709
self.generate_num_constants()
1711
def _generate_module_array_traverse_and_clear(self, struct_attr_cname, count, may_have_refcycles=True):
1712
counter_type = 'int' if count < 2**15 else 'Py_ssize_t'
1713
visit_call = "Py_VISIT" if may_have_refcycles else "__Pyx_VISIT_CONST"
1715
writer = self.parts['module_state_traverse']
1716
writer.putln(f"for ({counter_type} i=0; i<{count}; ++i) {{ {visit_call}(traverse_module_state->{struct_attr_cname}[i]); }}")1718
writer = self.parts['module_state_clear']
1719
writer.putln(f"for ({counter_type} i=0; i<{count}; ++i) {{ Py_CLEAR(clear_module_state->{struct_attr_cname}[i]); }}")1721
def generate_object_constant_decls(self):
1722
consts = [(len(c.cname), c.cname, c)
1723
for c in self.arg_default_constants]
1725
for _, cname, c in consts:
1726
self.parts['module_state'].putln("%s;" % c.type.declaration_code(cname))1727
self.parts['module_state_defines'].putln(
1728
"#define %s %s->%s" % (cname, Naming.modulestateglobal_cname, cname))
1729
if not c.type.needs_refcounting:
1730
# Note that py_constants is used for all argument defaults
1731
# which aren't necessarily PyObjects, so aren't appropriate
1735
self.parts['module_state_clear'].put_xdecref_clear(
1736
f"clear_module_state->{cname}",1738
clear_before_decref=True,
1742
if c.type.is_memoryviewslice:
1743
# TODO: Implement specific to type like CodeWriter.put_xdecref_clear()
1744
cname += "->memview"
1746
self.parts['module_state_traverse'].putln(
1747
f"Py_VISIT(traverse_module_state->{cname});")1749
for prefix, count in sorted(self.const_array_counters.items()):
1750
# name the struct attribute and the global "define" slightly differently
1751
# to avoid the global define getting in the way
1752
struct_attr_cname = f"{Naming.pyrex_prefix}_{prefix}"1753
global_cname = f"{Naming.pyrex_prefix}{prefix}"1754
self.parts['module_state'].putln(f"PyObject *{struct_attr_cname}[{count}];")1755
self.parts['module_state_defines'].putln(
1756
f"#define {global_cname} {Naming.modulestateglobal_cname}->{struct_attr_cname}")1758
# The constant tuples/slices that we create can never participate in reference cycles.
1759
self._generate_module_array_traverse_and_clear(struct_attr_cname, count, may_have_refcycles=False)
1761
cleanup_level = cleanup_level_for_type_prefix(prefix)
1762
if cleanup_level is not None and cleanup_level <= Options.generate_cleanup_code:
1763
part_writer = self.parts['cleanup_globals']
1764
part_writer.putln(f"for (Py_ssize_t i=0; i<{count}; ++i) {{ Py_CLEAR({global_cname}[i]); }}")1766
def generate_cached_methods_decls(self):
1767
if not self.cached_cmethods:
1770
decl = self.parts['decls']
1771
init = self.parts['init_constants']
1773
for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()):
1774
cnames.append(cname)
1775
method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname
1776
decl.putln('static __Pyx_CachedCFunction %s = {0, 0, 0, 0, 0};' % (1778
# split type reference storage as it might not be static
1779
init.putln('%s.type = (PyObject*)%s;' % (1781
# method name string isn't static in limited api
1782
init.putln('%s.method_name = &%s;' % (1783
cname, method_name_cname))
1785
if Options.generate_cleanup_code:
1786
cleanup = self.parts['cleanup_globals']
1787
for cname in cnames:
1788
cleanup.putln("Py_CLEAR(%s.method);" % cname)1790
def generate_string_constants(self):
1791
c_consts = [(len(c.cname), c.cname, c) for c in self.string_const_index.values()]
1794
longest_pystring = 0
1797
def normalise_encoding_name(py_string):
1798
if py_string.encoding and py_string.encoding not in (
1799
'ASCII', 'USASCII', 'US-ASCII', 'UTF8', 'UTF-8'):
1800
return f'"{py_string.encoding.lower()}"'1804
decls_writer = self.parts['string_decls']
1805
for _, cname, c in c_consts:
1806
cliteral = StringEncoding.split_string_literal(c.escaped_value)
1808
f'static const char {cname}[] = "{cliteral}";',1809
safe=True) # Braces in user strings are not for indentation.
1810
if c.py_strings is not None:
1811
if len(c.escaped_value) > longest_pystring:
1812
# This is not an accurate count since it adds up C escape characters,
1813
# but it's probably good enough for an upper bound.
1814
longest_pystring = len(c.escaped_value)
1815
for py_string in c.py_strings.values():
1816
encodings.add(normalise_encoding_name(py_string))
1817
py_strings.append((c.cname, len(py_string.cname), py_string))
1819
for c, cname in sorted(self.pyunicode_ptr_const_index.items()):
1820
utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c)
1822
# Narrow and wide representations differ
1823
decls_writer.putln("#ifdef Py_UNICODE_WIDE")1824
decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf32_array))1826
decls_writer.putln("#else")1827
decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array))1828
decls_writer.putln("#endif")1835
w = self.parts['pystring_table']
1838
# We use only type size macros from "pyport.h" here.
1839
w.put(textwrap.dedent("""\1842
#if %(max_length)d <= 65535
1843
const unsigned short n;
1844
#elif %(max_length)d / 2 < INT_MAX
1845
const unsigned int n;
1846
#elif %(max_length)d / 2 < LONG_MAX
1847
const unsigned long n;
1851
#if %(num_encodings)d <= 31
1852
const unsigned int encoding : 5;
1853
#elif %(num_encodings)d <= 255
1854
const unsigned char encoding;
1855
#elif %(num_encodings)d <= 65535
1856
const unsigned short encoding;
1858
const Py_ssize_t encoding;
1860
const unsigned int is_unicode : 1;
1861
const unsigned int intern : 1;
1862
} __Pyx_StringTabEntry;
1864
max_length=longest_pystring,
1865
num_encodings=len(encodings),
1868
py_string_count = len(py_strings)
1869
self.parts['module_state'].putln(f"PyObject *{Naming.stringtab_cname}[{py_string_count}];")1870
self._generate_module_array_traverse_and_clear(Naming.stringtab_cname, py_string_count, may_have_refcycles=False)
1872
encodings = sorted(encodings)
1873
encodings.sort(key=len) # stable sort to make sure '0' comes first, index 0
1874
assert not encodings or '0' not in encodings or encodings[0] == '0', encodings
1875
encodings_map = {encoding: i for i, encoding in enumerate(encodings)}1876
w.putln("static const char * const %s[] = { %s };" % (1877
Naming.stringtab_encodings_cname,
1878
', '.join(encodings),
1881
w.putln("static const __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname)1882
for n, (c_cname, _, py_string) in enumerate(py_strings):
1883
encodings_index = encodings_map[normalise_encoding_name(py_string)]
1884
is_unicode = py_string.is_unicode
1886
self.parts['module_state_defines'].putln("#define %s %s->%s[%s]" % (1888
Naming.modulestateglobal_cname,
1889
Naming.stringtab_cname,
1892
w.putln("{%s, sizeof(%s), %d, %d, %d}, /* PyObject cname: %s */" % (1900
w.putln("{0, 0, 0, 0, 0}")1903
self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c"))1905
init_constants = self.parts['init_constants']
1906
init_constants.putln(
1907
"if (__Pyx_InitStrings(%s, %s->%s, %s) < 0) %s;" % (
1908
Naming.stringtab_cname,
1909
Naming.modulestateglobal_cname,
1910
Naming.stringtab_cname,
1911
Naming.stringtab_encodings_cname,
1912
init_constants.error_goto(self.module_pos)))
1914
def generate_codeobject_constants(self):
1915
w = self.parts['init_codeobjects']
1916
init_function = "int __Pyx_CreateCodeObjects(void)"
1918
if not self.codeobject_constants:
1919
w.start_initcfunc(init_function)
1920
w.putln("return 0;")1921
w.exit_cfunc_scope()
1925
# Create a downsized config struct and build code objects from it.
1926
max_flags = 0x3ff # to be adapted when we start using new flags
1929
max_posonly_args = 1
1933
for node in self.codeobject_constants:
1934
def_node = node.def_node
1935
if not def_node.is_generator_expression:
1936
max_func_args = max(max_func_args, len(def_node.args) - def_node.num_kwonly_args)
1937
max_kwonly_args = max(max_kwonly_args, def_node.num_kwonly_args)
1938
max_posonly_args = max(max_posonly_args, def_node.num_posonly_args)
1939
max_vars = max(max_vars, len(node.varnames))
1940
max_line = max(max_line, def_node.pos[1])
1941
max_positions = max(max_positions, len(def_node.node_positions))
1943
# Even for full 64-bit line/column values, one entry in the line table can never be larger than 45 bytes.
1944
max_linetable_len = max_positions * 47
1946
w.put(textwrap.dedent(f"""\
1948
unsigned int argcount : {max_func_args.bit_length()};1949
unsigned int num_posonly_args : {max_posonly_args.bit_length()};1950
unsigned int num_kwonly_args : {max_kwonly_args.bit_length()};1951
unsigned int nlocals : {max_vars.bit_length()};1952
unsigned int flags : {max_flags.bit_length()};1953
unsigned int first_line : {max_line.bit_length()};1954
unsigned int line_table_length : {max_linetable_len.bit_length()};1955
}} __Pyx_PyCode_New_function_description;
1958
self.use_utility_code(UtilityCode.load_cached("NewCodeObj", "ModuleSetupCode.c"))1960
w.start_initcfunc(init_function)
1962
w.putln("PyObject* tuple_dedup_map = PyDict_New();")1963
w.putln("if (unlikely(!tuple_dedup_map)) return -1;")1965
for node in self.codeobject_constants:
1966
node.generate_codeobj(w, "bad")
1968
w.putln("Py_DECREF(tuple_dedup_map);")1969
w.putln("return 0;")1972
w.putln("Py_DECREF(tuple_dedup_map);")1973
w.putln("return -1;")1974
w.exit_cfunc_scope()
1977
code_object_count = len(self.codeobject_constants)
1978
self.parts['module_state'].putln(f"PyObject *{Naming.codeobjtab_cname}[{code_object_count}];")1979
# The code objects that we generate only contain plain constants and can never participate in reference cycles.
1980
self._generate_module_array_traverse_and_clear(Naming.codeobjtab_cname, code_object_count, may_have_refcycles=False)
1982
self.parts['module_state_defines'].putln("#define %s %s->%s" % (1983
Naming.codeobjtab_cname, Naming.modulestateglobal_cname, Naming.codeobjtab_cname
1986
def generate_num_constants(self):
1987
consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c)
1988
for c in self.num_const_index.values()]
1990
init_constants = self.parts['init_constants']
1991
for py_type, _, _, value, value_code, c in consts:
1993
self.parts['module_state'].putln("PyObject *%s;" % cname)1994
self.parts['module_state_defines'].putln("#define %s %s->%s" % (1995
cname, Naming.modulestateglobal_cname, cname))
1996
self.parts['module_state_clear'].putln(
1997
"Py_CLEAR(clear_module_state->%s);" % cname)
1998
self.parts['module_state_traverse'].putln(
1999
"__Pyx_VISIT_CONST(traverse_module_state->%s);" % cname)
2000
if py_type == 'float':
2001
function = 'PyFloat_FromDouble(%s)'
2002
elif py_type == 'long':
2003
function = 'PyLong_FromString("%s", 0, 0)'2004
elif Utils.long_literal(value):
2005
function = 'PyLong_FromString("%s", 0, 0)'2006
elif len(value.lstrip('-')) > 4:2007
function = "PyLong_FromLong(%sL)"
2009
function = "PyLong_FromLong(%s)"
2010
init_constants.putln('%s = %s; %s' % (2011
cname, function % value_code,
2012
init_constants.error_goto_if_null(cname, self.module_pos)))
2014
# The functions below are there in a transition phase only
2015
# and will be deprecated. They are called from Nodes.BlockNode.
2016
# The copy&paste duplication is intentional in order to be able
2017
# to see quickly how BlockNode worked, until this is replaced.
2019
def should_declare(self, cname, entry):
2020
if cname in self.declared_cnames:
2021
other = self.declared_cnames[cname]
2022
assert str(entry.type) == str(other.type)
2023
assert entry.init == other.init
2026
self.declared_cnames[cname] = entry
2033
def lookup_filename(self, source_desc):
2034
entry = source_desc.get_filenametable_entry()
2036
index = self.filename_table[entry]
2038
index = len(self.filename_list)
2039
self.filename_list.append(source_desc)
2040
self.filename_table[entry] = index
2043
def commented_file_contents(self, source_desc):
2045
return self.input_file_contents[source_desc]
2048
source_file = source_desc.get_lines(encoding='ASCII',
2049
error_handling='ignore')
2051
F = [' * ' + line.rstrip().replace(
2052
'*/', '*[inserted by cython to avoid comment closer]/'
2054
'/*', '/[inserted by cython to avoid comment start]*'
2056
for line in source_file]
2058
if hasattr(source_file, 'close'):
2060
if not F: F.append('')2061
self.input_file_contents[source_desc] = F
2065
# Utility code state
2068
def use_utility_code(self, utility_code):
2070
Adds code to the C file. utility_code should
2071
a) implement __eq__/__hash__ for the purpose of knowing whether the same
2072
code has already been included
2073
b) implement put_code, which takes a globalstate instance
2077
if utility_code and utility_code not in self.utility_codes:
2078
self.utility_codes.add(utility_code)
2079
utility_code.put_code(self)
2081
def use_entry_utility_code(self, entry):
2084
if entry.utility_code:
2085
self.use_utility_code(entry.utility_code)
2086
if entry.utility_code_definition:
2087
self.use_utility_code(entry.utility_code_definition)
2090
def funccontext_property(func):
2091
name = func.__name__
2092
attribute_of = operator.attrgetter(name)
2094
return attribute_of(self.funcstate)
2095
def set(self, value):
2096
setattr(self.funcstate, name, value)
2097
return property(get, set)
2101
# emit_linenums boolean write #line pragmas?
2102
# emit_code_comments boolean copy the original code into C comments?
2103
# c_line_in_traceback boolean append the c file and line number to the traceback for exceptions?
2105
def __init__(self, emit_linenums=True, emit_code_comments=True, c_line_in_traceback=True):
2106
self.emit_code_comments = emit_code_comments
2107
self.emit_linenums = emit_linenums
2108
self.c_line_in_traceback = c_line_in_traceback
2113
Utility class to output C code.
2115
When creating an insertion point one must care about the state that is
2117
- formatting state (level, bol) is cloned and used in insertion points
2119
- labels, temps, exc_vars: One must construct a scope in which these can
2120
exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for
2121
sanity checking and forward compatibility). Created insertion points
2122
looses this scope and cannot access it.
2123
- marker: Not copied to insertion point
2124
- filename_table, filename_list, input_file_contents: All codewriters
2125
coming from the same root share the same instances simultaneously.
2128
# f file output file
2129
# buffer StringIOTree
2131
# level int indentation level
2132
# bol bool beginning of line?
2133
# marker string comment to emit before next line
2134
# funcstate FunctionState contains state local to a C function used for code
2135
# generation (labels and temps state etc.)
2136
# globalstate GlobalState contains state global for a C file (input file info,
2137
# utility code, declared constants etc.)
2138
# pyclass_stack list used during recursive code generation to pass information
2139
# about the current class one is in
2140
# code_config CCodeConfig configuration options for the C code writer
2142
@cython.locals(create_from='CCodeWriter')
2143
def __init__(self, create_from=None, buffer=None, copy_formatting=False):
2144
if buffer is None: buffer = StringIOTree()
2145
self.buffer = buffer
2146
self.last_pos = None
2147
self.last_marked_pos = None
2148
self.pyclass_stack = []
2150
self.funcstate = None
2151
self.globalstate = None
2152
self.code_config = None
2157
if create_from is not None:
2158
# Use same global state
2159
self.set_global_state(create_from.globalstate)
2160
self.funcstate = create_from.funcstate
2161
# Clone formatting state
2163
self.level = create_from.level
2164
self.bol = create_from.bol
2165
self.call_level = create_from.call_level
2166
self.last_pos = create_from.last_pos
2167
self.last_marked_pos = create_from.last_marked_pos
2169
def create_new(self, create_from, buffer, copy_formatting):
2170
# polymorphic constructor -- very slightly more versatile
2171
# than using __class__
2172
result = CCodeWriter(create_from, buffer, copy_formatting)
2175
def set_global_state(self, global_state):
2176
assert self.globalstate is None # prevent overwriting once it's set
2177
self.globalstate = global_state
2178
self.code_config = global_state.code_config
2180
def copyto(self, f):
2181
self.buffer.copyto(f)
2184
return self.buffer.getvalue()
2188
self._write_lines(s)
2190
self._write_to_buffer(s)
2192
def _write_lines(self, s):
2193
# Cygdb needs to know which Cython source line corresponds to which C line.
2194
# Therefore, we write this information into "self.buffer.markers" and then write it from there
2195
# into cython_debug/cython_debug_info_* (see ModuleNode._serialize_lineno_map).
2196
filename_line = self.last_marked_pos[:2] if self.last_marked_pos else (None, 0)
2197
self.buffer.markers.extend([filename_line] * s.count('\n'))2199
self._write_to_buffer(s)
2201
def _write_to_buffer(self, s):
2202
self.buffer.write(s)
2204
def insertion_point(self):
2205
other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
2208
def new_writer(self):
2210
Creates a new CCodeWriter connected to the same global state, which
2211
can later be inserted using insert.
2213
return CCodeWriter(create_from=self)
2215
def insert(self, writer):
2217
Inserts the contents of another code writer (created with
2218
the same global state) in the current location.
2220
It is ok to write to the inserted writer also after insertion.
2222
assert writer.globalstate is self.globalstate
2223
self.buffer.insert(writer.buffer)
2225
# Properties delegated to function scope
2226
@funccontext_property
2227
def label_counter(self): pass
2228
@funccontext_property
2229
def return_label(self): pass
2230
@funccontext_property
2231
def error_label(self): pass
2232
@funccontext_property
2233
def labels_used(self): pass
2234
@funccontext_property
2235
def continue_label(self): pass
2236
@funccontext_property
2237
def break_label(self): pass
2238
@funccontext_property
2239
def return_from_error_cleanup_label(self): pass
2240
@funccontext_property
2241
def yield_labels(self): pass
2243
def label_interceptor(self, new_labels, orig_labels, skip_to_label=None, pos=None, trace=True):
2245
Helper for generating multiple label interceptor code blocks.
2247
@param new_labels: the new labels that should be intercepted
2248
@param orig_labels: the original labels that we should dispatch to after the interception
2249
@param skip_to_label: a label to skip to before starting the code blocks
2250
@param pos: the node position to mark for each interceptor block
2251
@param trace: add a trace line for the pos marker or not
2253
for label, orig_label in zip(new_labels, orig_labels):
2254
if not self.label_used(label):
2257
# jump over the whole interception block
2258
self.put_goto(skip_to_label)
2259
skip_to_label = None
2262
self.mark_pos(pos, trace=trace)
2263
self.put_label(label)
2264
yield (label, orig_label)
2265
self.put_goto(orig_label)
2267
# Functions delegated to function scope
2268
def new_label(self, name=None): return self.funcstate.new_label(name)
2269
def new_error_label(self, *args): return self.funcstate.new_error_label(*args)
2270
def new_yield_label(self, *args): return self.funcstate.new_yield_label(*args)
2271
def get_loop_labels(self): return self.funcstate.get_loop_labels()
2272
def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)
2273
def new_loop_labels(self, *args): return self.funcstate.new_loop_labels(*args)
2274
def get_all_labels(self): return self.funcstate.get_all_labels()
2275
def set_all_labels(self, labels): return self.funcstate.set_all_labels(labels)
2276
def all_new_labels(self): return self.funcstate.all_new_labels()
2277
def use_label(self, lbl): return self.funcstate.use_label(lbl)
2278
def label_used(self, lbl): return self.funcstate.label_used(lbl)
2281
def enter_cfunc_scope(self, scope=None):
2282
self.funcstate = FunctionState(self, scope=scope)
2284
def exit_cfunc_scope(self):
2285
self.funcstate.validate_exit()
2286
self.funcstate = None
2288
def start_initcfunc(self, signature, scope=None, refnanny=False):
2290
Init code helper function to start a cfunc scope and generate
2291
the prototype and function header ("static SIG {") of the function.2293
proto = self.globalstate.parts['initfunc_declarations']
2294
proto.putln(f"static CYTHON_SMALL_CODE {signature}; /*proto*/")2295
self.enter_cfunc_scope(scope)
2297
self.putln(f"static {signature} {{")2299
self.put_declare_refcount_context()
2303
def get_py_int(self, str_value, longness):
2304
return self.globalstate.get_int_const(str_value, longness).cname
2306
def get_py_float(self, str_value, value_code):
2307
return self.globalstate.get_float_const(str_value, value_code).cname
2309
def get_py_const(self, prefix, dedup_key=None):
2310
return self.globalstate.get_py_const(prefix, dedup_key)
2312
def get_string_const(self, text):
2313
return self.globalstate.get_string_const(text).cname
2315
def get_pyunicode_ptr_const(self, text):
2316
return self.globalstate.get_pyunicode_ptr_const(text)
2318
def get_py_string_const(self, text, identifier=None):
2319
return self.globalstate.get_py_string_const(text, identifier).cname
2321
def get_py_codeobj_const(self, node):
2322
return self.globalstate.get_py_codeobj_const(node)
2324
def get_argument_default_const(self, type):
2325
return self.globalstate.get_argument_default_const(type).cname
2327
def intern(self, text):
2328
return self.get_py_string_const(text)
2330
def intern_identifier(self, text):
2331
return self.get_py_string_const(text, identifier=True)
2333
def get_cached_constants_writer(self, target=None):
2334
return self.globalstate.get_cached_constants_writer(target)
2338
def putln(self, code="", safe=False):
2339
if self.last_pos and self.bol:
2341
if self.code_config.emit_linenums and self.last_marked_pos:
2342
source_desc, line, _ = self.last_marked_pos
2343
self._write_lines(f'\n#line {line} "{source_desc.get_escaped_description()}"\n')2349
self._write_lines("\n")2352
def mark_pos(self, pos, trace=True):
2355
if self.last_marked_pos and self.last_marked_pos[:2] == pos[:2]:
2357
self.last_pos = (pos, trace)
2359
def emit_marker(self):
2360
pos, trace = self.last_pos
2361
self.last_marked_pos = pos
2362
self.last_pos = None
2363
self._write_lines("\n")2364
if self.code_config.emit_code_comments:
2366
self._write_lines(self._build_marker(pos))
2368
self.write_trace_line(pos)
2370
def write_trace_line(self, pos):
2371
if self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']:
2374
f'__Pyx_TraceLine({pos[1]:d},{self.pos_to_offset(pos):d},{not self.funcstate.gil_owned:d},{self.error_goto(pos)})\n')2376
def _build_marker(self, pos):
2377
source_desc, line, col = pos
2378
assert isinstance(source_desc, SourceDescriptor)
2379
contents = self.globalstate.commented_file_contents(source_desc)
2380
lines = contents[max(0, line-3):line] # line numbers start at 1
2381
lines[-1] += ' # <<<<<<<<<<<<<<'
2382
lines += contents[line:line+2]
2383
code = "\n".join(lines)
2384
return f'/* "{source_desc.get_escaped_description()}":{line:d}\n{code}\n*/\n'2386
def put_safe(self, code):
2387
# put code, but ignore {}2391
def put_or_include(self, code, name):
2392
include_dir = self.globalstate.common_utility_include_dir
2393
if include_dir and len(code) > 1024:
2394
hash = hashlib.sha256(code.encode('utf8')).hexdigest()2395
include_file = f"{name}_{hash}.h"2396
path = os.path.join(include_dir, include_file)
2397
if not os.path.exists(path):
2398
tmp_path = f'{path}.tmp{os.getpid()}'2399
with Utils.open_new_file(tmp_path) as f:
2401
shutil.move(tmp_path, path)
2402
# We use forward slashes in the include path to assure identical code generation
2403
# under Windows and Posix. C/C++ compilers should still understand it.
2404
c_path = path.replace('\\', '/')2405
code = f'#include "{c_path}"\n'2408
def put(self, code):
2411
dl = code.count("{")2415
dl -= code.count("}")2418
elif dl == 0 and code[0] == "}":
2419
# special cases like "} else {" need a temporary dedent2431
def increase_indent(self):
2434
def decrease_indent(self):
2437
def begin_block(self):
2439
self.increase_indent()
2441
def end_block(self):
2442
self.decrease_indent()
2446
self._write_to_buffer(" " * self.level)2448
def get_py_version_hex(self, pyversion):
2449
return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
2451
def put_label(self, lbl):
2452
if lbl in self.funcstate.labels_used:
2453
self.putln("%s:;" % lbl)2455
def put_goto(self, lbl):
2456
self.funcstate.use_label(lbl)
2457
self.putln("goto %s;" % lbl)2459
def put_var_declaration(self, entry, storage_class="",
2460
dll_linkage=None, definition=True):
2461
#print "Code.put_var_declaration:", entry.name, "definition =", definition ###
2462
if entry.visibility == 'private' and not (definition or entry.defined_in_pxd):
2463
#print "...private and not definition, skipping", entry.cname ###
2465
if entry.visibility == "private" and not entry.used:
2466
#print "...private and not used, skipping", entry.cname ###
2468
if not entry.cf_used:
2469
self.put('CYTHON_UNUSED ')2471
self.put("%s " % storage_class)2472
if entry.is_cpp_optional:
2473
self.put(entry.type.cpp_optional_declaration_code(
2474
entry.cname, dll_linkage=dll_linkage))
2476
self.put(entry.type.declaration_code(
2477
entry.cname, dll_linkage=dll_linkage))
2478
if entry.init is not None:
2479
self.put_safe(" = %s" % entry.type.literal_code(entry.init))2480
elif entry.type.is_pyobject:
2483
self.funcstate.scope.use_entry_utility_code(entry)
2485
def put_temp_declarations(self, func_context):
2486
for name, type, manage_ref, static in func_context.temps_allocated:
2487
if type.is_cpp_class and not type.is_fake_reference and func_context.scope.directives['cpp_locals']:
2488
decl = type.cpp_optional_declaration_code(name)
2490
decl = type.declaration_code(name)
2491
if type.is_pyobject:
2492
self.putln("%s = NULL;" % decl)2493
elif type.is_memoryviewslice:
2494
self.putln("%s = %s;" % (decl, type.literal_code(type.default_value)))2496
self.putln("%s%s;" % (static and "static " or "", decl))2498
if func_context.should_declare_error_indicator:
2499
if self.funcstate.uses_error_indicator:
2502
unused = 'CYTHON_UNUSED '
2503
# Initialize these variables to silence compiler warnings
2504
self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname))2505
self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cname))2506
self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname))2508
def put_generated_by(self):
2509
self.putln(Utils.GENERATED_BY_MARKER)
2512
def put_h_guard(self, guard):
2513
self.putln("#ifndef %s" % guard)2514
self.putln("#define %s" % guard)2516
def unlikely(self, cond):
2517
if Options.gcc_branch_hints:
2518
return 'unlikely(%s)' % cond
2522
def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper):
2525
return '%s ' % ' '.join([mapper(m,m) for m in modifiers])
2527
# Python objects and reference counting
2529
def entry_as_pyobject(self, entry):
2531
if (not entry.is_self_arg and not entry.type.is_complete()
2532
or entry.type.is_extension_type):
2533
return "(PyObject *)" + entry.cname
2537
def as_pyobject(self, cname, type):
2538
from .PyrexTypes import py_object_type, typecast
2539
return typecast(py_object_type, type, cname)
2541
def put_gotref(self, cname, type):
2542
type.generate_gotref(self, cname)
2544
def put_giveref(self, cname, type):
2545
type.generate_giveref(self, cname)
2547
def put_xgiveref(self, cname, type):
2548
type.generate_xgiveref(self, cname)
2550
def put_xgotref(self, cname, type):
2551
type.generate_xgotref(self, cname)
2553
def put_incref(self, cname, type, nanny=True):
2554
# Note: original put_Memslice_Incref/Decref also added in some utility code
2555
# this is unnecessary since the relevant utility code is loaded anyway if a memoryview is used
2556
# and so has been removed. However, it's potentially a feature that might be useful here
2557
type.generate_incref(self, cname, nanny=nanny)
2559
def put_xincref(self, cname, type, nanny=True):
2560
type.generate_xincref(self, cname, nanny=nanny)
2562
def put_decref(self, cname, type, nanny=True, have_gil=True):
2563
type.generate_decref(self, cname, nanny=nanny, have_gil=have_gil)
2565
def put_xdecref(self, cname, type, nanny=True, have_gil=True):
2566
type.generate_xdecref(self, cname, nanny=nanny, have_gil=have_gil)
2568
def put_decref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True):
2569
type.generate_decref_clear(self, cname, clear_before_decref=clear_before_decref,
2570
nanny=nanny, have_gil=have_gil)
2572
def put_xdecref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True):
2573
type.generate_xdecref_clear(self, cname, clear_before_decref=clear_before_decref,
2574
nanny=nanny, have_gil=have_gil)
2576
def put_decref_set(self, cname, type, rhs_cname):
2577
type.generate_decref_set(self, cname, rhs_cname)
2579
def put_xdecref_set(self, cname, type, rhs_cname):
2580
type.generate_xdecref_set(self, cname, rhs_cname)
2582
def put_incref_memoryviewslice(self, slice_cname, type, have_gil):
2583
# TODO ideally this would just be merged into "put_incref"
2584
type.generate_incref_memoryviewslice(self, slice_cname, have_gil=have_gil)
2586
def put_var_incref_memoryviewslice(self, entry, have_gil):
2587
self.put_incref_memoryviewslice(entry.cname, entry.type, have_gil=have_gil)
2589
def put_var_gotref(self, entry):
2590
self.put_gotref(entry.cname, entry.type)
2592
def put_var_giveref(self, entry):
2593
self.put_giveref(entry.cname, entry.type)
2595
def put_var_xgotref(self, entry):
2596
self.put_xgotref(entry.cname, entry.type)
2598
def put_var_xgiveref(self, entry):
2599
self.put_xgiveref(entry.cname, entry.type)
2601
def put_var_incref(self, entry, **kwds):
2602
self.put_incref(entry.cname, entry.type, **kwds)
2604
def put_var_xincref(self, entry, **kwds):
2605
self.put_xincref(entry.cname, entry.type, **kwds)
2607
def put_var_decref(self, entry, **kwds):
2608
self.put_decref(entry.cname, entry.type, **kwds)
2610
def put_var_xdecref(self, entry, **kwds):
2611
self.put_xdecref(entry.cname, entry.type, **kwds)
2613
def put_var_decref_clear(self, entry, **kwds):
2614
self.put_decref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds)
2616
def put_var_decref_set(self, entry, rhs_cname, **kwds):
2617
self.put_decref_set(entry.cname, entry.type, rhs_cname, **kwds)
2619
def put_var_xdecref_set(self, entry, rhs_cname, **kwds):
2620
self.put_xdecref_set(entry.cname, entry.type, rhs_cname, **kwds)
2622
def put_var_xdecref_clear(self, entry, **kwds):
2623
self.put_xdecref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds)
2625
def put_var_decrefs(self, entries, used_only = 0):
2626
for entry in entries:
2627
if not used_only or entry.used:
2628
if entry.xdecref_cleanup:
2629
self.put_var_xdecref(entry)
2631
self.put_var_decref(entry)
2633
def put_var_xdecrefs(self, entries):
2634
for entry in entries:
2635
self.put_var_xdecref(entry)
2637
def put_var_xdecrefs_clear(self, entries):
2638
for entry in entries:
2639
self.put_var_xdecref_clear(entry)
2641
def put_init_to_py_none(self, cname, type, nanny=True):
2642
from .PyrexTypes import py_object_type, typecast
2643
py_none = typecast(type, py_object_type, "Py_None")
2645
self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))2647
self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))2649
def put_init_var_to_py_none(self, entry, template = "%s", nanny=True):
2650
code = template % entry.cname
2651
#if entry.type.is_extension_type:
2652
# code = "((PyObject*)%s)" % code
2653
self.put_init_to_py_none(code, entry.type, nanny)
2654
if entry.in_closure:
2655
self.put_giveref('Py_None')2657
def put_pymethoddef(self, entry, term, allow_skip=True, wrapper_code_writer=None):
2658
is_number_slot = False
2659
if entry.is_special or entry.name == '__getattribute__':
2660
from . import TypeSlots
2661
if entry.name not in special_py_methods:
2662
if TypeSlots.is_binop_number_slot(entry.name):
2663
# It's useful if numeric binops are created with meth coexist
2664
# so they can be called directly by looking up the name, skipping the
2665
# dispatch wrapper that enables the reverse slots. This is most useful
2666
# when c_api_binop_methods is False, but there's no reason not to do it
2668
is_number_slot = True
2669
elif entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']:
2671
# Python's typeobject.c will automatically fill in our slot
2672
# in add_operators() (called by PyType_Ready) with a value
2673
# that's better than ours.
2677
method_flags = entry.signature.method_flags()
2678
if not method_flags:
2680
if entry.is_special:
2681
method_flags += [TypeSlots.method_coexist]
2682
func_ptr = wrapper_code_writer.put_pymethoddef_wrapper(entry) if wrapper_code_writer else entry.func_cname
2683
# Add required casts, but try not to shadow real warnings.
2684
cast = entry.signature.method_function_type()
2685
if cast != 'PyCFunction':
2686
func_ptr = '(void*)(%s)%s' % (cast, func_ptr)
2687
entry_name = entry.name.as_c_string_literal()
2689
# Unlike most special functions, binop numeric operator slots are actually generated here
2690
# (to ensure that they can be looked up). However, they're sometimes guarded by the preprocessor
2691
# so a bit of extra logic is needed
2692
slot = TypeSlots.get_slot_table(self.globalstate.directives).get_slot_by_method_name(entry.name)
2693
preproc_guard = slot.preprocessor_guard_code()
2695
self.putln(preproc_guard)
2697
'{%s, (PyCFunction)%s, %s, %s}%s' % (2700
"|".join(method_flags),
2701
entry.doc_cname if entry.doc else '0',
2703
if is_number_slot and preproc_guard:
2704
self.putln("#endif")2706
def put_pymethoddef_wrapper(self, entry):
2707
func_cname = entry.func_cname
2708
if entry.is_special:
2709
method_flags = entry.signature.method_flags() or []
2710
from .TypeSlots import method_noargs
2711
if method_noargs in method_flags:
2712
# Special NOARGS methods really take no arguments besides 'self', but PyCFunction expects one.
2713
func_cname = Naming.method_wrapper_prefix + func_cname
2714
self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {" % func_cname)2715
func_call = "%s(self)" % entry.func_cname
2716
if entry.name == "__next__":
2717
self.putln("PyObject *res = %s;" % func_call)2718
# tp_iternext can return NULL without an exception
2719
self.putln("if (!res && !PyErr_Occurred()) { PyErr_SetNone(PyExc_StopIteration); }")2720
self.putln("return res;")2722
self.putln("return %s;" % func_call)2728
def use_fast_gil_utility_code(self):
2729
if self.globalstate.directives['fast_gil']:
2730
self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c"))2732
self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c"))2734
def put_ensure_gil(self, declare_gilstate=True, variable=None):
2736
Acquire the GIL. The generated code is safe even when no PyThreadState
2737
has been allocated for this thread (for threads not initialized by
2738
using the Python API). Additionally, the code generated by this method
2739
may be called recursively.
2741
self.globalstate.use_utility_code(
2742
UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))2743
self.use_fast_gil_utility_code()
2745
variable = '__pyx_gilstate_save'
2746
if declare_gilstate:
2747
self.put("PyGILState_STATE ")2748
self.putln("%s = __Pyx_PyGILState_Ensure();" % variable)2750
def put_release_ensured_gil(self, variable=None):
2752
Releases the GIL, corresponds to `put_ensure_gil`.
2754
self.use_fast_gil_utility_code()
2756
variable = '__pyx_gilstate_save'
2757
self.putln("__Pyx_PyGILState_Release(%s);" % variable)2759
def put_acquire_freethreading_lock(self):
2760
self.globalstate.use_utility_code(
2761
UtilityCode.load_cached("AccessPyMutexForFreeThreading", "ModuleSetupCode.c"))2762
self.putln("#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING")2763
self.putln(f"PyMutex_Lock(&{Naming.parallel_freethreading_mutex});")2764
self.putln("#endif")2766
def put_release_freethreading_lock(self):
2767
self.putln("#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING")2768
self.putln(f"PyMutex_Unlock(&{Naming.parallel_freethreading_mutex});")2769
self.putln("#endif")2771
def put_acquire_gil(self, variable=None, unknown_gil_state=True):
2773
Acquire the GIL. The thread's thread state must have been initialized
2774
by a previous `put_release_gil`
2776
self.use_fast_gil_utility_code()
2777
self.putln("__Pyx_FastGIL_Forget();")2779
self.putln('_save = %s;' % variable)2780
if unknown_gil_state:
2781
self.putln("if (_save) {")2782
self.putln("Py_BLOCK_THREADS")2783
if unknown_gil_state:
2786
def put_release_gil(self, variable=None, unknown_gil_state=True):
2787
"Release the GIL, corresponds to `put_acquire_gil`."
2788
self.use_fast_gil_utility_code()
2789
self.putln("PyThreadState *_save;")2790
self.putln("_save = NULL;")2791
if unknown_gil_state:
2792
# we don't *know* that we don't have the GIL (since we may be inside a nogil function,
2793
# and Py_UNBLOCK_THREADS is unsafe without the GIL)
2794
self.putln("if (PyGILState_Check()) {")2795
self.putln("Py_UNBLOCK_THREADS")2796
if unknown_gil_state:
2799
self.putln('%s = _save;' % variable)2800
self.putln("__Pyx_FastGIL_Remember();")2802
def declare_gilstate(self):
2803
self.putln("PyGILState_STATE __pyx_gilstate_save;")2807
def put_error_if_neg(self, pos, value):
2808
# TODO this path is almost _never_ taken, yet this macro makes is slower!
2809
# return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))2810
return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))2812
def put_error_if_unbound(self, pos, entry, in_nogil_context=False, unbound_check_code=None):
2813
if entry.from_closure:
2814
func = '__Pyx_RaiseClosureNameError'
2815
self.globalstate.use_utility_code(
2816
UtilityCode.load_cached("RaiseClosureNameError", "ObjectHandling.c"))2817
elif entry.type.is_memoryviewslice and in_nogil_context:
2818
func = '__Pyx_RaiseUnboundMemoryviewSliceNogil'
2819
self.globalstate.use_utility_code(
2820
UtilityCode.load_cached("RaiseUnboundMemoryviewSliceNogil", "ObjectHandling.c"))2821
elif entry.type.is_cpp_class and entry.is_cglobal:
2822
func = '__Pyx_RaiseCppGlobalNameError'
2823
self.globalstate.use_utility_code(
2824
UtilityCode.load_cached("RaiseCppGlobalNameError", "ObjectHandling.c"))2825
elif entry.type.is_cpp_class and entry.is_variable and not entry.is_member and entry.scope.is_c_class_scope:
2826
# there doesn't seem to be a good way to detecting an instance-attribute of a C class
2827
# (is_member is only set for class attributes)
2828
func = '__Pyx_RaiseCppAttributeError'
2829
self.globalstate.use_utility_code(
2830
UtilityCode.load_cached("RaiseCppAttributeError", "ObjectHandling.c"))2832
func = '__Pyx_RaiseUnboundLocalError'
2833
self.globalstate.use_utility_code(
2834
UtilityCode.load_cached("RaiseUnboundLocalError", "ObjectHandling.c"))2836
if not unbound_check_code:
2837
unbound_check_code = entry.type.check_for_null_code(entry.cname)
2838
self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % (2842
self.error_goto(pos)))
2844
def set_error_info(self, pos, used=False):
2845
self.funcstate.should_declare_error_indicator = True
2847
self.funcstate.uses_error_indicator = True
2848
return "__PYX_MARK_ERR_POS(%s, %s)" % (
2849
self.lookup_filename(pos[0]),
2852
def error_goto(self, pos, used=True):
2853
lbl = self.funcstate.error_label
2854
self.funcstate.use_label(lbl)
2856
return 'goto %s;' % lbl
2857
self.funcstate.should_declare_error_indicator = True
2859
self.funcstate.uses_error_indicator = True
2860
return "__PYX_ERR(%s, %s, %s)" % (
2861
self.lookup_filename(pos[0]),
2865
def error_goto_if(self, cond, pos):
2866
return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos))
2868
def error_goto_if_null(self, cname, pos):
2869
return self.error_goto_if("!%s" % cname, pos)2871
def error_goto_if_neg(self, cname, pos):
2872
# Add extra parentheses to silence clang warnings about constant conditions.
2873
return self.error_goto_if("(%s < 0)" % cname, pos)2875
def error_goto_if_PyErr(self, pos):
2876
return self.error_goto_if("PyErr_Occurred()", pos)2878
def lookup_filename(self, filename):
2879
return self.globalstate.lookup_filename(filename)
2881
def put_declare_refcount_context(self):
2882
self.putln('__Pyx_RefNannyDeclarations')2884
def put_setup_refcount_context(self, name, acquire_gil=False):
2885
name = name.as_c_string_literal() # handle unicode names
2887
self.globalstate.use_utility_code(
2888
UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))2889
self.putln('__Pyx_RefNannySetupContext(%s, %d);' % (name, acquire_gil and 1 or 0))2891
def put_finish_refcount_context(self, nogil=False):
2892
self.putln("__Pyx_RefNannyFinishContextNogil()" if nogil else "__Pyx_RefNannyFinishContext();")2894
def put_add_traceback(self, qualified_name, include_cline=True):
2896
Build a Python traceback for propagating exceptions.
2898
qualified_name should be the qualified name of the function.
2900
qualified_name = qualified_name.as_c_string_literal() # handle unicode names
2903
Naming.clineno_cname if include_cline else 0,
2904
Naming.lineno_cname,
2905
Naming.filename_cname,
2908
self.funcstate.uses_error_indicator = True
2909
self.putln('__Pyx_AddTraceback(%s, %s, %s, %s);' % format_tuple)2911
def put_unraisable(self, qualified_name, nogil=False):
2913
Generate code to print a Python warning for an unraisable exception.
2915
qualified_name should be the qualified name of the function.
2919
Naming.clineno_cname,
2920
Naming.lineno_cname,
2921
Naming.filename_cname,
2922
self.globalstate.directives['unraisable_tracebacks'],
2925
self.funcstate.uses_error_indicator = True
2926
self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %d, %d);' % format_tuple)2927
self.globalstate.use_utility_code(
2928
UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))2930
def is_tracing(self):
2931
return self.globalstate.directives['profile'] or self.globalstate.directives['linetrace']
2933
def pos_to_offset(self, pos):
2935
Calculate a fake 'instruction offset' from a node position as 31 bit int (32 bit signed).
2937
scope = self.funcstate.scope
2938
while scope and pos not in scope.node_positions_to_offset:
2939
scope = scope.parent_scope
2940
return scope.node_positions_to_offset[pos] if scope else 0
2942
def put_trace_declarations(self, is_generator=False):
2943
self.putln('__Pyx_TraceDeclarationsGen' if is_generator else '__Pyx_TraceDeclarationsFunc')2945
def put_trace_frame_init(self, codeobj=None):
2947
self.putln('__Pyx_TraceFrameInit(%s)' % codeobj)2949
def put_trace_start(self, name, pos, nogil=False, is_generator=False, is_cpdef_func=False):
2950
trace_func = "__Pyx_TraceStartGen" if is_generator else "__Pyx_TraceStartFunc"
2953
f'{name.as_c_string_literal()}, '2954
f'{Naming.filetable_cname}[{self.lookup_filename(pos[0])}], '2956
f'{self.pos_to_offset(pos):d}, '2958
f'{Naming.skip_dispatch_cname if is_cpdef_func else "0"}, '2959
f'{self.error_goto(pos)}'2963
def put_trace_exit(self):
2964
self.putln("__Pyx_PyMonitoring_ExitScope();")2966
def put_trace_yield(self, retvalue_cname, pos):
2967
error_goto = self.error_goto(pos)
2968
self.putln(f"__Pyx_TraceYield({retvalue_cname}, {self.pos_to_offset(pos)}, {error_goto});")2970
def put_trace_resume(self, pos):
2971
scope = self.funcstate.scope
2972
# pos[1] is probably not the first line, so try to find the first line of the generator function.
2973
first_line = scope.scope_class.pos[1] if scope.scope_class else pos[1]
2974
name = scope.name.as_c_string_literal()
2975
filename_index = self.lookup_filename(pos[0])
2976
error_goto = self.error_goto(pos)
2978
'__Pyx_TraceResumeGen('2980
f'{Naming.filetable_cname}[{filename_index}], '2982
f'{self.pos_to_offset(pos)}, '2987
def put_trace_exception(self, pos, reraise=False, fresh=False):
2988
self.putln(f"__Pyx_TraceException({self.pos_to_offset(pos)}, {bool(reraise):d}, {bool(fresh):d});")2990
def put_trace_exception_propagating(self):
2991
self.putln(f"__Pyx_TraceException({Naming.lineno_cname}, 0, 0);")2993
def put_trace_exception_handled(self, pos):
2994
self.putln(f"__Pyx_TraceExceptionHandled({self.pos_to_offset(pos)});")2996
def put_trace_unwind(self, pos, nogil=False):
2997
self.putln(f"__Pyx_TraceExceptionUnwind({self.pos_to_offset(pos)}, {bool(nogil):d});")2999
def put_trace_stopiteration(self, pos, value):
3000
error_goto = self.error_goto(pos)
3001
self.putln(f"__Pyx_TraceStopIteration({value}, {self.pos_to_offset(pos)}, {error_goto});")3003
def put_trace_return(self, retvalue_cname, pos, return_type=None, nogil=False):
3005
trace_func = "__Pyx_TraceReturnValue"
3007
if return_type is None or return_type.is_pyobject:
3009
elif return_type.is_void:
3010
retvalue_cname = 'Py_None'
3011
elif return_type.to_py_function:
3012
trace_func = "__Pyx_TraceReturnCValue"
3013
extra_arg = f", {return_type.to_py_function}"3015
# We don't have a Python visible return value but we still need to report that we returned.
3016
# 'None' may not be a misleading (it's false, for one), but it's hopefully better than nothing.
3017
retvalue_cname = 'Py_None'
3019
error_handling = self.error_goto(pos)
3020
self.putln(f"{trace_func}({retvalue_cname}{extra_arg}, {self.pos_to_offset(pos)}, {bool(nogil):d}, {error_handling});")3022
def putln_openmp(self, string):
3023
self.putln("#ifdef _OPENMP")3025
self.putln("#endif /* _OPENMP */")3027
def undef_builtin_expect(self, cond):
3029
Redefine the macros likely() and unlikely to no-ops, depending on
3032
self.putln("#if %s" % cond)3033
self.putln(" #undef likely")3034
self.putln(" #undef unlikely")3035
self.putln(" #define likely(x) (x)")3036
self.putln(" #define unlikely(x) (x)")3037
self.putln("#endif")3039
def redef_builtin_expect(self, cond):
3040
self.putln("#if %s" % cond)3041
self.putln(" #undef likely")3042
self.putln(" #undef unlikely")3043
self.putln(" #define likely(x) __builtin_expect(!!(x), 1)")3044
self.putln(" #define unlikely(x) __builtin_expect(!!(x), 0)")3045
self.putln("#endif")3048
class PyrexCodeWriter:
3049
# f file output file
3050
# level int indentation level
3052
def __init__(self, outfile_name):
3053
self.f = Utils.open_new_file(outfile_name)
3056
def putln(self, code):
3057
self.f.write("%s%s\n" % (" " * self.level, code))3068
Can be used for writing out some Cython code.
3071
def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'):
3072
self.buffer = buffer or StringIOTree()
3073
self.level = indent_level
3074
self.original_level = indent_level
3075
self.context = context
3076
self.encoding = encoding
3077
self._insertion_points = {}3079
def indent(self, levels=1):
3080
self.level += levels
3083
def dedent(self, levels=1):
3084
self.level -= levels
3087
def indenter(self, line):
3089
with pyx_code.indenter("for i in range(10):"):3090
pyx_code.putln("print i")3098
return self.buffer.empty()
3101
result = self.buffer.getvalue()
3102
if isinstance(result, bytes):
3103
result = result.decode(self.encoding)
3106
def putln(self, line, context=None):
3108
if self.context is not None:
3109
context = self.context
3110
if context is not None:
3111
line = sub_tempita(line, context)
3112
# Avoid indenting empty lines.
3113
self.buffer.write(f"{self.level * ' '}{line}\n" if line else "\n")3115
def put_chunk(self, chunk, context=None):
3117
if self.context is not None:
3118
context = self.context
3119
if context is not None:
3120
chunk = sub_tempita(chunk, context)
3122
chunk = _indent_chunk(chunk, self.level * 4)
3123
self.buffer.write(chunk)
3125
def insertion_point(self):
3126
return type(self)(self.buffer.insertion_point(), self.level, self.context)
3129
# resets the buffer so that nothing gets written. Most useful
3130
# for abandoning all work in a specific insertion point
3132
self.level = self.original_level
3134
def named_insertion_point(self, name):
3135
self._insertion_points[name] = self.insertion_point()
3137
def __getitem__(self, name):
3138
return self._insertion_points[name]
3143
def _indent_chunk(chunk: str, indentation_length: cython.int) -> str:
3144
"""Normalise leading space to the intended indentation and strip empty lines.
3146
assert '\t' not in chunk
3147
lines = chunk.splitlines(keepends=True)
3150
last_line = lines[-1].rstrip(' ')3152
lines[-1] = last_line
3158
# Count minimal (non-empty) indentation and strip empty lines.
3159
min_indentation: cython.int = len(chunk) + 1
3160
line_indentation: cython.int
3163
for i, line in enumerate(lines):
3164
line_indentation = _count_indentation(line)
3165
if line_indentation + 1 == len(line):
3167
elif line_indentation < min_indentation:
3168
min_indentation = line_indentation
3170
if min_indentation > len(chunk):
3174
if min_indentation < indentation_length:
3175
add_indent = ' ' * (indentation_length - min_indentation)
3177
add_indent + line if line != '\n' else '\n'
3180
elif min_indentation > indentation_length:
3181
start: cython.int = min_indentation - indentation_length
3183
line[start:] if line != '\n' else '\n'
3187
return ''.join(lines)
3190
@cython.exceptval(-1)
3192
def _count_indentation(s: str) -> cython.int:
3195
for i, ch in enumerate(s):
3201
class ClosureTempAllocator:
3202
def __init__(self, klass):
3204
self.temps_allocated = {}3205
self.temps_free = {}3206
self.temps_count = 0
3209
for type, cnames in self.temps_allocated.items():
3210
self.temps_free[type] = list(cnames)
3212
def allocate_temp(self, type):
3213
if type not in self.temps_allocated:
3214
self.temps_allocated[type] = []
3215
self.temps_free[type] = []
3216
elif self.temps_free[type]:
3217
return self.temps_free[type].pop(0)
3218
cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count)
3219
self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True)
3220
self.temps_allocated[type].append(cname)
3221
self.temps_count += 1