Python314/Doc/html/library/token.html - master - shapovalovav/AI-Agent

Python314/Doc/html/library/token.html
1 013 строк64 KB
Anatoly1147
first_commit
11 июл 2026, 17:02
11 июл 2026, 17:02f6c78bd
<!DOCTYPE html>

<html lang="en" data-content_root="../">
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<meta property="og:title" content="token — Constants used with Python parse trees" />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://docs.python.org/3/library/token.html" />
<meta property="og:site_name" content="Python documentation" />
<meta property="og:description" content="Source code: Lib/token.py This module provides constants which represent the numeric values of leaf nodes of the parse tree (terminal tokens). Refer to the file Grammar/Tokens in the Python distrib..." />
<meta property="og:image" content="_static/og-image.png" />
<meta property="og:image:alt" content="Python documentation" />
<meta name="description" content="Source code: Lib/token.py This module provides constants which represent the numeric values of leaf nodes of the parse tree (terminal tokens). Refer to the file Grammar/Tokens in the Python distrib..." />
<meta name="theme-color" content="#3776ab">
<meta property="og:image:width" content="200">
<meta property="og:image:height" content="200">

    <title>token — Constants used with Python parse trees &#8212; Python 3.14.4 documentation</title><meta name="viewport" content="width=device-width, initial-scale=1.0">
    
    <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
    <link rel="stylesheet" type="text/css" href="../_static/classic.css?v=234b1a7c" />
    <link rel="stylesheet" type="text/css" href="../_static/pydoctheme.css?v=82640b3f" />
    <link id="pygments_dark_css" media="(prefers-color-scheme: dark)" rel="stylesheet" type="text/css" href="../_static/pygments_dark.css?v=5349f25f" />
    
    <script src="../_static/documentation_options.js?v=1885ab2e"></script>
    <script src="../_static/doctools.js?v=9bcbadda"></script>
    <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
    
    <script src="../_static/sidebar.js"></script>
    
    <link rel="search" type="application/opensearchdescription+xml"
          title="Search within Python 3.14.4 documentation"
          href="../_static/opensearch.xml"/>
    <link rel="author" title="About these documents" href="../about.html" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="copyright" title="Copyright" href="../copyright.html" />
    <link rel="next" title="keyword — Testing for Python keywords" href="keyword.html" />
    <link rel="prev" title="symtable — Access to the compiler’s symbol tables" href="symtable.html" />
    
      
      <link rel="canonical" href="https://docs.python.org/3/library/token.html">
      
    

    
    <style>
      @media only screen {
        table.full-width-table {
            width: 100%;
        }
      }
    </style>
<link rel="stylesheet" href="../_static/pydoctheme_dark.css" media="(prefers-color-scheme: dark)" id="pydoctheme_dark_css">
    <link rel="shortcut icon" type="image/png" href="../_static/py.svg">
            <script type="text/javascript" src="../_static/copybutton.js"></script>
            <script type="text/javascript" src="../_static/menu.js"></script>
            <script type="text/javascript" src="../_static/search-focus.js"></script>
            <script type="text/javascript" src="../_static/themetoggle.js"></script> 
            <script type="text/javascript" src="../_static/rtd_switcher.js"></script>
            <meta name="readthedocs-addons-api-version" content="1">

  </head>
<body>
<div class="mobile-nav">
    <input type="checkbox" id="menuToggler" class="toggler__input" aria-controls="navigation"
           aria-pressed="false" aria-expanded="false" role="button" aria-label="Menu">
    <nav class="nav-content" role="navigation">
        <label for="menuToggler" class="toggler__label">
            <span></span>
        </label>
        <span class="nav-items-wrapper">
            <a href="https://www.python.org/" class="nav-logo">
                <img src="../_static/py.svg" alt="Python logo">
            </a>
            <span class="version_switcher_placeholder"></span>
            <form role="search" class="search" action="../search.html" method="get">
                <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" class="search-icon">
                    <path fill-rule="nonzero" fill="currentColor" d="M15.5 14h-.79l-.28-.27a6.5 6.5 0 001.48-5.34c-.47-2.78-2.79-5-5.59-5.34a6.505 6.505 0 00-7.27 7.27c.34 2.8 2.56 5.12 5.34 5.59a6.5 6.5 0 005.34-1.48l.27.28v.79l4.25 4.25c.41.41 1.08.41 1.49 0 .41-.41.41-1.08 0-1.49L15.5 14zm-6 0C7.01 14 5 11.99 5 9.5S7.01 5 9.5 5 14 7.01 14 9.5 11.99 14 9.5 14z"></path>
                </svg>
                <input placeholder="Quick search" aria-label="Quick search" type="search" name="q">
                <input type="submit" value="Go">
            </form>
        </span>
    </nav>
    <div class="menu-wrapper">
        <nav class="menu" role="navigation" aria-label="main navigation">
            <div class="language_switcher_placeholder"></div>
            
<label class="theme-selector-label">
    Theme
    <select class="theme-selector" oninput="activateTheme(this.value)">
        <option value="auto" selected>Auto</option>
        <option value="light">Light</option>
        <option value="dark">Dark</option>
    </select>
</label>
  <div>
    <h4>Previous topic</h4>
    <p class="topless"><a href="symtable.html"
                          title="previous chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">symtable</span></code> — Access to the compiler’s symbol tables</a></p>
  </div>
  <div>
    <h4>Next topic</h4>
    <p class="topless"><a href="keyword.html"
                          title="next chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">keyword</span></code> — Testing for Python keywords</a></p>
  </div>
  <script>
    document.addEventListener('DOMContentLoaded', () => {
        const title = document.querySelector('meta[property="og:title"]').content;
        const elements = document.querySelectorAll('.improvepage');
        const pageurl = window.location.href.split('?')[0];
        elements.forEach(element => {
            const url = new URL(element.href.split('?')[0].replace("-nojs", ""));
            url.searchParams.set('pagetitle', title);
            url.searchParams.set('pageurl', pageurl);
            url.searchParams.set('pagesource', "library/token.rst");
            element.href = url.toString();
        });
    });
  </script>
  <div role="note" aria-label="source link">
    <h3>This page</h3>
    <ul class="this-page-menu">
      <li><a href="../bugs.html">Report a bug</a></li>
      <li><a class="improvepage" href="../improve-page-nojs.html">Improve this page</a></li>
      <li>
        <a href="https://github.com/python/cpython/blob/main/Doc/library/token.rst?plain=1"
            rel="nofollow">Show source
        </a>
      </li>
      
    </ul>
  </div>
        </nav>
    </div>
</div>

  
    <div class="related" role="navigation" aria-label="Related">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../genindex.html" title="General Index"
             accesskey="I">index</a></li>
        <li class="right" >
          <a href="../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="right" >
          <a href="keyword.html" title="keyword — Testing for Python keywords"
             accesskey="N">next</a> |</li>
        <li class="right" >
          <a href="symtable.html" title="symtable — Access to the compiler’s symbol tables"
             accesskey="P">previous</a> |</li>

          <li><img src="../_static/py.svg" alt="Python logo" style="vertical-align: middle; margin-top: -1px"></li>
          <li><a href="https://www.python.org/">Python</a> &#187;</li>
          <li class="switchers">
            <div class="language_switcher_placeholder"></div>
            <div class="version_switcher_placeholder"></div>
          </li>
          <li>
              
          </li>
    <li id="cpython-language-and-version">
      <a href="../index.html">3.14.4 Documentation</a> &#187;
    </li>

          <li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> &#187;</li>
          <li class="nav-item nav-item-2"><a href="language.html" accesskey="U">Python Language Services</a> &#187;</li>
        <li class="nav-item nav-item-this"><a href=""><code class="xref py py-mod docutils literal notranslate"><span class="pre">token</span></code> — Constants used with Python parse trees</a></li>
                <li class="right">
                    

    <div class="inline-search" role="search">
        <form class="inline-search" action="../search.html" method="get">
          <input placeholder="Quick search" aria-label="Quick search" type="search" name="q" id="search-box">
          <input type="submit" value="Go">
        </form>
    </div>
                     |
                </li>
            <li class="right">
<label class="theme-selector-label">
    Theme
    <select class="theme-selector" oninput="activateTheme(this.value)">
        <option value="auto" selected>Auto</option>
        <option value="light">Light</option>
        <option value="dark">Dark</option>
    </select>
</label> |</li>
            
      </ul>
    </div>    

    <div class="document">
      <div class="documentwrapper">
        <div class="bodywrapper">
          <div class="body" role="main">
            
  <section id="module-token">
<span id="token-constants-used-with-python-parse-trees"></span><h1><code class="xref py py-mod docutils literal notranslate"><span class="pre">token</span></code> — Constants used with Python parse trees<a class="headerlink" href="#module-token" title="Link to this heading">¶</a></h1>
<p><strong>Source code:</strong> <a class="extlink-source reference external" href="https://github.com/python/cpython/tree/3.14/Lib/token.py">Lib/token.py</a></p>
<hr class="docutils" />
<p>This module provides constants which represent the numeric values of leaf nodes
of the parse tree (terminal tokens).  Refer to the file <code class="file docutils literal notranslate"><span class="pre">Grammar/Tokens</span></code>
in the Python distribution for the definitions of the names in the context of
the language grammar.  The specific numeric values which the names map to may
change between Python versions.</p>
<p>The module also provides a mapping from numeric codes to names and some
functions.  The functions mirror definitions in the Python C header files.</p>
<p>Note that a token’s value may depend on tokenizer options. For example, a
<code class="docutils literal notranslate"><span class="pre">&quot;+&quot;</span></code> token may be reported as either <a class="reference internal" href="#token.PLUS" title="token.PLUS"><code class="xref py py-data docutils literal notranslate"><span class="pre">PLUS</span></code></a> or <a class="reference internal" href="#token.OP" title="token.OP"><code class="xref py py-data docutils literal notranslate"><span class="pre">OP</span></code></a>, or
a <code class="docutils literal notranslate"><span class="pre">&quot;match&quot;</span></code> token may be either <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate"><span class="pre">NAME</span></code></a> or <a class="reference internal" href="#token.SOFT_KEYWORD" title="token.SOFT_KEYWORD"><code class="xref py py-data docutils literal notranslate"><span class="pre">SOFT_KEYWORD</span></code></a>.</p>
<dl class="py data">
<dt class="sig sig-object py" id="token.tok_name">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">tok_name</span></span><a class="headerlink" href="#token.tok_name" title="Link to this definition">¶</a></dt>
<dd><p>Dictionary mapping the numeric values of the constants defined in this module
back to name strings, allowing more human-readable representation of parse trees
to be generated.</p>
</dd></dl>

<dl class="py function">
<dt class="sig sig-object py" id="token.ISTERMINAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ISTERMINAL</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#token.ISTERMINAL" title="Link to this definition">¶</a></dt>
<dd><p>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> for terminal token values.</p>
</dd></dl>

<dl class="py function">
<dt class="sig sig-object py" id="token.ISNONTERMINAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ISNONTERMINAL</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#token.ISNONTERMINAL" title="Link to this definition">¶</a></dt>
<dd><p>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> for non-terminal token values.</p>
</dd></dl>

<dl class="py function">
<dt class="sig sig-object py" id="token.ISEOF">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ISEOF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#token.ISEOF" title="Link to this definition">¶</a></dt>
<dd><p>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> if <em>x</em> is the marker indicating the end of input.</p>
</dd></dl>

<p>The token constants are:</p>
<dl class="py data">
<dt class="sig sig-object py" id="token.NAME">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NAME</span></span><a class="headerlink" href="#token.NAME" title="Link to this definition">¶</a></dt>
<dd><p>Token value that indicates an <a class="reference internal" href="../reference/lexical_analysis.html#identifiers"><span class="std std-ref">identifier or keyword</span></a>.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NUMBER">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NUMBER</span></span><a class="headerlink" href="#token.NUMBER" title="Link to this definition">¶</a></dt>
<dd><p>Token value that indicates a <a class="reference internal" href="../reference/lexical_analysis.html#numbers"><span class="std std-ref">numeric literal</span></a></p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.STRING">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">STRING</span></span><a class="headerlink" href="#token.STRING" title="Link to this definition">¶</a></dt>
<dd><p>Token value that indicates a <a class="reference internal" href="../reference/lexical_analysis.html#strings"><span class="std std-ref">string or byte literal</span></a>,
excluding <a class="reference internal" href="../reference/lexical_analysis.html#f-strings"><span class="std std-ref">formatted string literals</span></a>.
The token string is not interpreted:
it includes the surrounding quotation marks and the prefix (if given);
backslashes are included literally, without processing escape sequences.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.OP">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">OP</span></span><a class="headerlink" href="#token.OP" title="Link to this definition">¶</a></dt>
<dd><p>A generic token value that indicates an
<a class="reference internal" href="../reference/lexical_analysis.html#operators"><span class="std std-ref">operator</span></a> or <a class="reference internal" href="../reference/lexical_analysis.html#delimiters"><span class="std std-ref">delimiter</span></a>.</p>
<div class="impl-detail compound">
<p><strong>CPython implementation detail:</strong> This value is only reported by the <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module.
Internally, the tokenizer uses
<a class="reference internal" href="#token-operators-delimiters"><span class="std std-ref">exact token types</span></a> instead.</p>
</div>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.COMMENT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">COMMENT</span></span><a class="headerlink" href="#token.COMMENT" title="Link to this definition">¶</a></dt>
<dd><p>Token value used to indicate a comment.
The parser ignores <code class="xref py py-data docutils literal notranslate"><span class="pre">COMMENT</span></code> tokens.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NEWLINE">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NEWLINE</span></span><a class="headerlink" href="#token.NEWLINE" title="Link to this definition">¶</a></dt>
<dd><p>Token value that indicates the end of a <a class="reference internal" href="../reference/lexical_analysis.html#logical-lines"><span class="std std-ref">logical line</span></a>.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NL</span></span><a class="headerlink" href="#token.NL" title="Link to this definition">¶</a></dt>
<dd><p>Token value used to indicate a non-terminating newline.
<code class="xref py py-data docutils literal notranslate"><span class="pre">NL</span></code> tokens are generated when a logical line of code is continued
over multiple physical lines. The parser ignores <code class="xref py py-data docutils literal notranslate"><span class="pre">NL</span></code> tokens.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.INDENT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">INDENT</span></span><a class="headerlink" href="#token.INDENT" title="Link to this definition">¶</a></dt>
<dd><p>Token value used at the beginning of a <a class="reference internal" href="../reference/lexical_analysis.html#logical-lines"><span class="std std-ref">logical line</span></a>
to indicate the start of an <a class="reference internal" href="../reference/lexical_analysis.html#indentation"><span class="std std-ref">indented block</span></a>.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.DEDENT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DEDENT</span></span><a class="headerlink" href="#token.DEDENT" title="Link to this definition">¶</a></dt>
<dd><p>Token value used at the beginning of a <a class="reference internal" href="../reference/lexical_analysis.html#logical-lines"><span class="std std-ref">logical line</span></a>
to indicate the end of an <a class="reference internal" href="../reference/lexical_analysis.html#indentation"><span class="std std-ref">indented block</span></a>.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.FSTRING_START">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">FSTRING_START</span></span><a class="headerlink" href="#token.FSTRING_START" title="Link to this definition">¶</a></dt>
<dd><p>Token value used to indicate the beginning of an
<a class="reference internal" href="../reference/lexical_analysis.html#f-strings"><span class="std std-ref">f-string literal</span></a>.</p>
<div class="impl-detail compound">
<p><strong>CPython implementation detail:</strong> The token string includes the prefix and the opening quote(s), but none
of the contents of the literal.</p>
</div>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.FSTRING_MIDDLE">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">FSTRING_MIDDLE</span></span><a class="headerlink" href="#token.FSTRING_MIDDLE" title="Link to this definition">¶</a></dt>
<dd><p>Token value used for literal text inside an <a class="reference internal" href="../reference/lexical_analysis.html#f-strings"><span class="std std-ref">f-string literal</span></a>,
including format specifications.</p>
<div class="impl-detail compound">
<p><strong>CPython implementation detail:</strong> Replacement fields (that is, the non-literal parts of f-strings) use
the same tokens as other expressions, and are delimited by
<a class="reference internal" href="#token.LBRACE" title="token.LBRACE"><code class="xref py py-data docutils literal notranslate"><span class="pre">LBRACE</span></code></a>, <a class="reference internal" href="#token.RBRACE" title="token.RBRACE"><code class="xref py py-data docutils literal notranslate"><span class="pre">RBRACE</span></code></a>, <a class="reference internal" href="#token.EXCLAMATION" title="token.EXCLAMATION"><code class="xref py py-data docutils literal notranslate"><span class="pre">EXCLAMATION</span></code></a> and <a class="reference internal" href="#token.COLON" title="token.COLON"><code class="xref py py-data docutils literal notranslate"><span class="pre">COLON</span></code></a>
tokens.</p>
</div>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.FSTRING_END">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">FSTRING_END</span></span><a class="headerlink" href="#token.FSTRING_END" title="Link to this definition">¶</a></dt>
<dd><p>Token value used to indicate the end of a <a class="reference internal" href="../reference/lexical_analysis.html#f-strings"><span class="std std-ref">f-string</span></a>.</p>
<div class="impl-detail compound">
<p><strong>CPython implementation detail:</strong> The token string contains the closing quote(s).</p>
</div>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.TSTRING_START">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TSTRING_START</span></span><a class="headerlink" href="#token.TSTRING_START" title="Link to this definition">¶</a></dt>
<dd><p>Token value used to indicate the beginning of a template string literal.</p>
<div class="impl-detail compound">
<p><strong>CPython implementation detail:</strong> The token string includes the prefix and the opening quote(s), but none
of the contents of the literal.</p>
</div>
<div class="versionadded">
<p><span class="versionmodified added">Added in version 3.14.</span></p>
</div>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.TSTRING_MIDDLE">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TSTRING_MIDDLE</span></span><a class="headerlink" href="#token.TSTRING_MIDDLE" title="Link to this definition">¶</a></dt>
<dd><p>Token value used for literal text inside a template string literal
including format specifications.</p>
<div class="impl-detail compound">
<p><strong>CPython implementation detail:</strong> Replacement fields (that is, the non-literal parts of t-strings) use
the same tokens as other expressions, and are delimited by
<a class="reference internal" href="#token.LBRACE" title="token.LBRACE"><code class="xref py py-data docutils literal notranslate"><span class="pre">LBRACE</span></code></a>, <a class="reference internal" href="#token.RBRACE" title="token.RBRACE"><code class="xref py py-data docutils literal notranslate"><span class="pre">RBRACE</span></code></a>, <a class="reference internal" href="#token.EXCLAMATION" title="token.EXCLAMATION"><code class="xref py py-data docutils literal notranslate"><span class="pre">EXCLAMATION</span></code></a> and <a class="reference internal" href="#token.COLON" title="token.COLON"><code class="xref py py-data docutils literal notranslate"><span class="pre">COLON</span></code></a>
tokens.</p>
</div>
<div class="versionadded">
<p><span class="versionmodified added">Added in version 3.14.</span></p>
</div>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.TSTRING_END">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TSTRING_END</span></span><a class="headerlink" href="#token.TSTRING_END" title="Link to this definition">¶</a></dt>
<dd><p>Token value used to indicate the end of a template string literal.</p>
<div class="impl-detail compound">
<p><strong>CPython implementation detail:</strong> The token string contains the closing quote(s).</p>
</div>
<div class="versionadded">
<p><span class="versionmodified added">Added in version 3.14.</span></p>
</div>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.ENDMARKER">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ENDMARKER</span></span><a class="headerlink" href="#token.ENDMARKER" title="Link to this definition">¶</a></dt>
<dd><p>Token value that indicates the end of input.
Used in <a class="reference internal" href="../reference/toplevel_components.html#top-level"><span class="std std-ref">top-level grammar rules</span></a>.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.ENCODING">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ENCODING</span></span><a class="headerlink" href="#token.ENCODING" title="Link to this definition">¶</a></dt>
<dd><p>Token value that indicates the encoding used to decode the source bytes
into text. The first token returned by <a class="reference internal" href="tokenize.html#tokenize.tokenize" title="tokenize.tokenize"><code class="xref py py-func docutils literal notranslate"><span class="pre">tokenize.tokenize()</span></code></a> will
always be an <code class="docutils literal notranslate"><span class="pre">ENCODING</span></code> token.</p>
<div class="impl-detail compound">
<p><strong>CPython implementation detail:</strong> This token type isn’t used by the C tokenizer but is needed for
the <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module.</p>
</div>
</dd></dl>

<p>The following token types are not produced by the <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module,
and are defined for special uses in the tokenizer or parser:</p>
<dl class="py data">
<dt class="sig sig-object py" id="token.TYPE_IGNORE">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TYPE_IGNORE</span></span><a class="headerlink" href="#token.TYPE_IGNORE" title="Link to this definition">¶</a></dt>
<dd><p>Token value indicating that a <code class="docutils literal notranslate"><span class="pre">type:</span> <span class="pre">ignore</span></code> comment was recognized.
Such tokens are produced instead of regular <a class="reference internal" href="#token.COMMENT" title="token.COMMENT"><code class="xref py py-data docutils literal notranslate"><span class="pre">COMMENT</span></code></a> tokens only
with the <a class="reference internal" href="ast.html#ast.PyCF_TYPE_COMMENTS" title="ast.PyCF_TYPE_COMMENTS"><code class="xref py py-data docutils literal notranslate"><span class="pre">PyCF_TYPE_COMMENTS</span></code></a> flag.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.TYPE_COMMENT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TYPE_COMMENT</span></span><a class="headerlink" href="#token.TYPE_COMMENT" title="Link to this definition">¶</a></dt>
<dd><p>Token value indicating that a type comment was recognized.
Such tokens are produced instead of regular <a class="reference internal" href="#token.COMMENT" title="token.COMMENT"><code class="xref py py-data docutils literal notranslate"><span class="pre">COMMENT</span></code></a> tokens only
with the <a class="reference internal" href="ast.html#ast.PyCF_TYPE_COMMENTS" title="ast.PyCF_TYPE_COMMENTS"><code class="xref py py-data docutils literal notranslate"><span class="pre">PyCF_TYPE_COMMENTS</span></code></a> flag.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.SOFT_KEYWORD">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">SOFT_KEYWORD</span></span><a class="headerlink" href="#token.SOFT_KEYWORD" title="Link to this definition">¶</a></dt>
<dd><p>Token value indicating a <a class="reference internal" href="../reference/lexical_analysis.html#soft-keywords"><span class="std std-ref">soft keyword</span></a>.</p>
<p>The tokenizer never produces this value.
To check for a soft keyword, pass a <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate"><span class="pre">NAME</span></code></a> token’s string to
<a class="reference internal" href="keyword.html#keyword.issoftkeyword" title="keyword.issoftkeyword"><code class="xref py py-func docutils literal notranslate"><span class="pre">keyword.issoftkeyword()</span></code></a>.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.ERRORTOKEN">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ERRORTOKEN</span></span><a class="headerlink" href="#token.ERRORTOKEN" title="Link to this definition">¶</a></dt>
<dd><p>Token value used to indicate wrong input.</p>
<p>The <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module generally indicates errors by
raising exceptions instead of emitting this token.
It can also emit tokens such as <a class="reference internal" href="#token.OP" title="token.OP"><code class="xref py py-data docutils literal notranslate"><span class="pre">OP</span></code></a> or <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate"><span class="pre">NAME</span></code></a> with strings that
are later rejected by the parser.</p>
</dd></dl>

<p id="token-operators-delimiters">The remaining tokens represent specific <a class="reference internal" href="../reference/lexical_analysis.html#operators"><span class="std std-ref">operators</span></a> and
<a class="reference internal" href="../reference/lexical_analysis.html#delimiters"><span class="std std-ref">delimiters</span></a>.
(The <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module reports these as <a class="reference internal" href="#token.OP" title="token.OP"><code class="xref py py-data docutils literal notranslate"><span class="pre">OP</span></code></a>; see <code class="docutils literal notranslate"><span class="pre">exact_type</span></code>
in the <code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code> documentation for details.)</p>
<table class="docutils align-left">
<thead>
<tr class="row-odd"><th class="head"><p>Token</p></th>
<th class="head"><p>Value</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.LPAR">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LPAR</span></span><a class="headerlink" href="#token.LPAR" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;(&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.RPAR">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RPAR</span></span><a class="headerlink" href="#token.RPAR" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;)&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.LSQB">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LSQB</span></span><a class="headerlink" href="#token.LSQB" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;[&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.RSQB">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RSQB</span></span><a class="headerlink" href="#token.RSQB" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;]&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.COLON">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">COLON</span></span><a class="headerlink" href="#token.COLON" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;:&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.COMMA">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">COMMA</span></span><a class="headerlink" href="#token.COMMA" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;,&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.SEMI">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">SEMI</span></span><a class="headerlink" href="#token.SEMI" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;;&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.PLUS">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">PLUS</span></span><a class="headerlink" href="#token.PLUS" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;+&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.MINUS">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">MINUS</span></span><a class="headerlink" href="#token.MINUS" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;-&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.STAR">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">STAR</span></span><a class="headerlink" href="#token.STAR" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;*&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.SLASH">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">SLASH</span></span><a class="headerlink" href="#token.SLASH" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;/&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.VBAR">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">VBAR</span></span><a class="headerlink" href="#token.VBAR" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;|&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.AMPER">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">AMPER</span></span><a class="headerlink" href="#token.AMPER" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&amp;&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.LESS">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LESS</span></span><a class="headerlink" href="#token.LESS" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&lt;&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.GREATER">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">GREATER</span></span><a class="headerlink" href="#token.GREATER" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&gt;&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.EQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">EQUAL</span></span><a class="headerlink" href="#token.EQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.DOT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOT</span></span><a class="headerlink" href="#token.DOT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;.&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.PERCENT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">PERCENT</span></span><a class="headerlink" href="#token.PERCENT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;%&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.LBRACE">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LBRACE</span></span><a class="headerlink" href="#token.LBRACE" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;{&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.RBRACE">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RBRACE</span></span><a class="headerlink" href="#token.RBRACE" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;}&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.EQEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">EQEQUAL</span></span><a class="headerlink" href="#token.EQEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;==&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.NOTEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NOTEQUAL</span></span><a class="headerlink" href="#token.NOTEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;!=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.LESSEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LESSEQUAL</span></span><a class="headerlink" href="#token.LESSEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&lt;=&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.GREATEREQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">GREATEREQUAL</span></span><a class="headerlink" href="#token.GREATEREQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&gt;=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.TILDE">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TILDE</span></span><a class="headerlink" href="#token.TILDE" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;~&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.CIRCUMFLEX">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">CIRCUMFLEX</span></span><a class="headerlink" href="#token.CIRCUMFLEX" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;^&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.LEFTSHIFT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LEFTSHIFT</span></span><a class="headerlink" href="#token.LEFTSHIFT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&lt;&lt;&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.RIGHTSHIFT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RIGHTSHIFT</span></span><a class="headerlink" href="#token.RIGHTSHIFT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&gt;&gt;&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.DOUBLESTAR">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOUBLESTAR</span></span><a class="headerlink" href="#token.DOUBLESTAR" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;**&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.PLUSEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">PLUSEQUAL</span></span><a class="headerlink" href="#token.PLUSEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;+=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.MINEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">MINEQUAL</span></span><a class="headerlink" href="#token.MINEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;-=&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.STAREQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">STAREQUAL</span></span><a class="headerlink" href="#token.STAREQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;*=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.SLASHEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">SLASHEQUAL</span></span><a class="headerlink" href="#token.SLASHEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;/=&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.PERCENTEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">PERCENTEQUAL</span></span><a class="headerlink" href="#token.PERCENTEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;%=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.AMPEREQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">AMPEREQUAL</span></span><a class="headerlink" href="#token.AMPEREQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&amp;=&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.VBAREQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">VBAREQUAL</span></span><a class="headerlink" href="#token.VBAREQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;|=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.CIRCUMFLEXEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">CIRCUMFLEXEQUAL</span></span><a class="headerlink" href="#token.CIRCUMFLEXEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;^=&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.LEFTSHIFTEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LEFTSHIFTEQUAL</span></span><a class="headerlink" href="#token.LEFTSHIFTEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&lt;&lt;=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.RIGHTSHIFTEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RIGHTSHIFTEQUAL</span></span><a class="headerlink" href="#token.RIGHTSHIFTEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&gt;&gt;=&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.DOUBLESTAREQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOUBLESTAREQUAL</span></span><a class="headerlink" href="#token.DOUBLESTAREQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;**=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.DOUBLESLASH">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOUBLESLASH</span></span><a class="headerlink" href="#token.DOUBLESLASH" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;//&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.DOUBLESLASHEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOUBLESLASHEQUAL</span></span><a class="headerlink" href="#token.DOUBLESLASHEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;//=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.AT">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">AT</span></span><a class="headerlink" href="#token.AT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&#64;&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.ATEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ATEQUAL</span></span><a class="headerlink" href="#token.ATEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;&#64;=&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.RARROW">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RARROW</span></span><a class="headerlink" href="#token.RARROW" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;-&gt;&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.ELLIPSIS">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ELLIPSIS</span></span><a class="headerlink" href="#token.ELLIPSIS" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;...&quot;</span></code></p></td>
</tr>
<tr class="row-even"><td><dl class="py data">
<dt class="sig sig-object py" id="token.COLONEQUAL">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">COLONEQUAL</span></span><a class="headerlink" href="#token.COLONEQUAL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;:=&quot;</span></code></p></td>
</tr>
<tr class="row-odd"><td><dl class="py data">
<dt class="sig sig-object py" id="token.EXCLAMATION">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">EXCLAMATION</span></span><a class="headerlink" href="#token.EXCLAMATION" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

</td>
<td><p><code class="docutils literal notranslate"><span class="pre">&quot;!&quot;</span></code></p></td>
</tr>
</tbody>
</table>
<p>The following non-token constants are provided:</p>
<dl class="py data">
<dt class="sig sig-object py" id="token.N_TOKENS">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">N_TOKENS</span></span><a class="headerlink" href="#token.N_TOKENS" title="Link to this definition">¶</a></dt>
<dd><p>The number of token types defined in this module.</p>
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.EXACT_TOKEN_TYPES">
<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">EXACT_TOKEN_TYPES</span></span><a class="headerlink" href="#token.EXACT_TOKEN_TYPES" title="Link to this definition">¶</a></dt>
<dd><p>A dictionary mapping the string representation of a token to its numeric code.</p>
<div class="versionadded">
<p><span class="versionmodified added">Added in version 3.8.</span></p>
</div>
</dd></dl>

<div class="versionchanged">
<p><span class="versionmodified changed">Changed in version 3.5: </span>Added <code class="xref py py-data docutils literal notranslate"><span class="pre">AWAIT</span></code> and <code class="xref py py-data docutils literal notranslate"><span class="pre">ASYNC</span></code> tokens.</p>
</div>
<div class="versionchanged">
<p><span class="versionmodified changed">Changed in version 3.7: </span>Added <a class="reference internal" href="#token.COMMENT" title="token.COMMENT"><code class="xref py py-data docutils literal notranslate"><span class="pre">COMMENT</span></code></a>, <a class="reference internal" href="#token.NL" title="token.NL"><code class="xref py py-data docutils literal notranslate"><span class="pre">NL</span></code></a> and <a class="reference internal" href="#token.ENCODING" title="token.ENCODING"><code class="xref py py-data docutils literal notranslate"><span class="pre">ENCODING</span></code></a> tokens.</p>
</div>
<div class="versionchanged">
<p><span class="versionmodified changed">Changed in version 3.7: </span>Removed <code class="xref py py-data docutils literal notranslate"><span class="pre">AWAIT</span></code> and <code class="xref py py-data docutils literal notranslate"><span class="pre">ASYNC</span></code> tokens. “async” and “await” are
now tokenized as <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate"><span class="pre">NAME</span></code></a> tokens.</p>
</div>
<div class="versionchanged">
<p><span class="versionmodified changed">Changed in version 3.8: </span>Added <a class="reference internal" href="#token.TYPE_COMMENT" title="token.TYPE_COMMENT"><code class="xref py py-data docutils literal notranslate"><span class="pre">TYPE_COMMENT</span></code></a>, <a class="reference internal" href="#token.TYPE_IGNORE" title="token.TYPE_IGNORE"><code class="xref py py-data docutils literal notranslate"><span class="pre">TYPE_IGNORE</span></code></a>, <a class="reference internal" href="#token.COLONEQUAL" title="token.COLONEQUAL"><code class="xref py py-data docutils literal notranslate"><span class="pre">COLONEQUAL</span></code></a>.
Added <code class="xref py py-data docutils literal notranslate"><span class="pre">AWAIT</span></code> and <code class="xref py py-data docutils literal notranslate"><span class="pre">ASYNC</span></code> tokens back (they’re needed
to support parsing older Python versions for <a class="reference internal" href="ast.html#ast.parse" title="ast.parse"><code class="xref py py-func docutils literal notranslate"><span class="pre">ast.parse()</span></code></a> with
<code class="docutils literal notranslate"><span class="pre">feature_version</span></code> set to 6 or lower).</p>
</div>
<div class="versionchanged">
<p><span class="versionmodified changed">Changed in version 3.12: </span>Added <a class="reference internal" href="#token.EXCLAMATION" title="token.EXCLAMATION"><code class="xref py py-data docutils literal notranslate"><span class="pre">EXCLAMATION</span></code></a>.</p>
</div>
<div class="versionchanged">
<p><span class="versionmodified changed">Changed in version 3.13: </span>Removed <code class="xref py py-data docutils literal notranslate"><span class="pre">AWAIT</span></code> and <code class="xref py py-data docutils literal notranslate"><span class="pre">ASYNC</span></code> tokens again.</p>
</div>
</section>


            <div class="clearer"></div>
          </div>
        </div>
      </div>
      <div class="sphinxsidebar" role="navigation" aria-label="Main">
        <div class="sphinxsidebarwrapper">
  <div>
    <h4>Previous topic</h4>
    <p class="topless"><a href="symtable.html"
                          title="previous chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">symtable</span></code> — Access to the compiler’s symbol tables</a></p>
  </div>
  <div>
    <h4>Next topic</h4>
    <p class="topless"><a href="keyword.html"
                          title="next chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">keyword</span></code> — Testing for Python keywords</a></p>
  </div>
  <script>
    document.addEventListener('DOMContentLoaded', () => {
        const title = document.querySelector('meta[property="og:title"]').content;
        const elements = document.querySelectorAll('.improvepage');
        const pageurl = window.location.href.split('?')[0];
        elements.forEach(element => {
            const url = new URL(element.href.split('?')[0].replace("-nojs", ""));
            url.searchParams.set('pagetitle', title);
            url.searchParams.set('pageurl', pageurl);
            url.searchParams.set('pagesource', "library/token.rst");
            element.href = url.toString();
        });
    });
  </script>
  <div role="note" aria-label="source link">
    <h3>This page</h3>
    <ul class="this-page-menu">
      <li><a href="../bugs.html">Report a bug</a></li>
      <li><a class="improvepage" href="../improve-page-nojs.html">Improve this page</a></li>
      <li>
        <a href="https://github.com/python/cpython/blob/main/Doc/library/token.rst?plain=1"
            rel="nofollow">Show source
        </a>
      </li>
      
    </ul>
  </div>
        </div>
<div id="sidebarbutton" title="Collapse sidebar">
<span>«</span>
</div>

      </div>
      <div class="clearer"></div>
    </div>  
    <div class="related" role="navigation" aria-label="Related">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../genindex.html" title="General Index"
             >index</a></li>
        <li class="right" >
          <a href="../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="right" >
          <a href="keyword.html" title="keyword — Testing for Python keywords"
             >next</a> |</li>
        <li class="right" >
          <a href="symtable.html" title="symtable — Access to the compiler’s symbol tables"
             >previous</a> |</li>

          <li><img src="../_static/py.svg" alt="Python logo" style="vertical-align: middle; margin-top: -1px"></li>
          <li><a href="https://www.python.org/">Python</a> &#187;</li>
          <li class="switchers">
            <div class="language_switcher_placeholder"></div>
            <div class="version_switcher_placeholder"></div>
          </li>
          <li>
              
          </li>
    <li id="cpython-language-and-version">
      <a href="../index.html">3.14.4 Documentation</a> &#187;
    </li>

          <li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> &#187;</li>
          <li class="nav-item nav-item-2"><a href="language.html" >Python Language Services</a> &#187;</li>
        <li class="nav-item nav-item-this"><a href=""><code class="xref py py-mod docutils literal notranslate"><span class="pre">token</span></code> — Constants used with Python parse trees</a></li>
                <li class="right">
                    

    <div class="inline-search" role="search">
        <form class="inline-search" action="../search.html" method="get">
          <input placeholder="Quick search" aria-label="Quick search" type="search" name="q" id="search-box">
          <input type="submit" value="Go">
        </form>
    </div>
                     |
                </li>
            <li class="right">
<label class="theme-selector-label">
    Theme
    <select class="theme-selector" oninput="activateTheme(this.value)">
        <option value="auto" selected>Auto</option>
        <option value="light">Light</option>
        <option value="dark">Dark</option>
    </select>
</label> |</li>
            
      </ul>
    </div>  
    <div class="footer">
    &copy; <a href="../copyright.html">Copyright</a> 2001 Python Software Foundation.
    <br>
    This page is licensed under the Python Software Foundation License Version 2.
    <br>
    Examples, recipes, and other code in the documentation are additionally licensed under the Zero Clause BSD License.
    <br>
    
      See <a href="/license.html">History and License</a> for more information.<br>
    
    
    <br>

    The Python Software Foundation is a non-profit corporation.
<a href="https://www.python.org/psf/donations/">Please donate.</a>
<br>
    <br>
      Last updated on Apr 07, 2026 (13:52 UTC).
    
      <a href="/bugs.html">Found a bug</a>?
    
    <br>

    Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3.
    </div>

  </body>
</html>