gettext-tools/src/x-python.c - master - niceSOFT/gettext

gettext-tools/src/x-python.c
1 938 строк59 KB
Bruno Haible
Use countof, part 3.
12 май 2026, 03:16
12 май 2026, 03:16474d099
/* xgettext Python backend.
   Copyright (C) 2002-2026 Free Software Foundation, Inc.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.  */

/* Written by Bruno Haible.  */

#include <config.h>

/* Specification.  */
#include "x-python.h"

#include <assert.h>
#include <errno.h>
#include <stdbool.h>
#include <stdcountof.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define SB_NO_APPENDF
#include <error.h>
#include "attribute.h"
#include "message.h"
#include "rc-str-list.h"
#include "xgettext.h"
#include "xg-pos.h"
#include "xg-encoding.h"
#include "xg-mixed-string.h"
#include "xg-arglist-context.h"
#include "xg-arglist-callshape.h"
#include "xg-arglist-parser.h"
#include "xg-message.h"
#include "if-error.h"
#include "xstrerror.h"
#include "progname.h"
#include "basename-lgpl.h"
#include "xerror.h"
#include "xvasprintf.h"
#include "xalloc.h"
#include "string-buffer.h"
#include "c-strstr.h"
#include "c-ctype.h"
#include "po-charset.h"
#include "uniname.h"
#include "unistr.h"
#include "gettext.h"

#define _(s) gettext(s)

#undef max /* clean up after MSVC's <stdlib.h> */
#define max(a,b) ((a) > (b) ? (a) : (b))


/* The Python syntax is defined in the Python Reference Manual
   /usr/share/doc/packages/python/html/ref/index.html.
   See also Python-3.7.17/Parser/tokenizer.c, Python-3.7.17/Python/compile.c,
   Python-3.7.17/Objects/bytesobject.c, Python-3.7.17/Objects/unicodeobject.c.
   For the f-strings, refer to https://peps.python.org/pep-0498/
   and https://docs.python.org/3/reference/lexical_analysis.html#literals .  */


/* ====================== Keyword set customization.  ====================== */

/* If true extract all strings.  */
static bool extract_all = false;

static hash_table keywords;
static bool default_keywords = true;


void
x_python_extract_all ()
{
  extract_all = true;
}


void
x_python_keyword (const char *name)
{
  if (name == NULL)
    default_keywords = false;
  else
    {
      if (keywords.table == NULL)
        hash_init (&keywords, 100);

      const char *end;
      struct callshape shape;
      split_keywordspec (name, &end, &shape);
      if (split_keywordspec_ok (name, end - name))
        /* The characters between name and end should form a valid
           C identifier.  */
        insert_keyword_callshape (&keywords, name, end - name, &shape);
    }
}

/* Finish initializing the keywords hash table.
   Called after argument processing, before each file is processed.  */
static void
init_keywords ()
{
  if (default_keywords)
    {
      /* When adding new keywords here, also update the documentation in
         xgettext.texi!  */
      x_python_keyword ("gettext");
      x_python_keyword ("ugettext");
      x_python_keyword ("dgettext:2");
      x_python_keyword ("ngettext:1,2");
      x_python_keyword ("ungettext:1,2");
      x_python_keyword ("dngettext:2,3");
      x_python_keyword ("_");
      default_keywords = false;
    }
}

void
init_flag_table_python ()
{
  xgettext_record_flag ("gettext:1:pass-python-format");
  xgettext_record_flag ("ugettext:1:pass-python-format");
  xgettext_record_flag ("dgettext:2:pass-python-format");
  xgettext_record_flag ("ngettext:1:pass-python-format");
  xgettext_record_flag ("ngettext:2:pass-python-format");
  xgettext_record_flag ("ungettext:1:pass-python-format");
  xgettext_record_flag ("ungettext:2:pass-python-format");
  xgettext_record_flag ("dngettext:2:pass-python-format");
  xgettext_record_flag ("dngettext:3:pass-python-format");
  xgettext_record_flag ("_:1:pass-python-format");
  /* xgettext_record_flag ("%:1:python-format"); // % is an infix operator! */

  xgettext_record_flag ("gettext:1:pass-python-brace-format");
  xgettext_record_flag ("ugettext:1:pass-python-brace-format");
  xgettext_record_flag ("dgettext:2:pass-python-brace-format");
  xgettext_record_flag ("ngettext:1:pass-python-brace-format");
  xgettext_record_flag ("ngettext:2:pass-python-brace-format");
  xgettext_record_flag ("ungettext:1:pass-python-brace-format");
  xgettext_record_flag ("ungettext:2:pass-python-brace-format");
  xgettext_record_flag ("dngettext:2:pass-python-brace-format");
  xgettext_record_flag ("dngettext:3:pass-python-brace-format");
  xgettext_record_flag ("_:1:pass-python-brace-format");
  /* xgettext_record_flag ("format:1:python-brace-format"); */
}


/* ======================== Reading of characters.  ======================== */

/* The input file stream.  */
static FILE *fp;


/* 0. Terminate line by \n, regardless whether the external
   representation of a line terminator is CR (Mac), and CR/LF
   (DOS/Windows), as Python treats them equally.  */
static int
phase0_getc ()
{
  int c;

  c = getc (fp);
  if (c == EOF)
    {
      if (ferror (fp))
        error (EXIT_FAILURE, errno, _("error while reading \"%s\""),
               real_file_name);
      return EOF;
    }

  if (c == '\r')
    {
      int c1 = getc (fp);

      if (c1 != EOF && c1 != '\n')
        ungetc (c1, fp);

      /* Seen line terminator CR or CR/LF.  */
      return '\n';
    }

  return c;
}

/* Supports only one pushback character, and not '\n'.  */
MAYBE_UNUSED static inline void
phase0_ungetc (int c)
{
  if (c != EOF)
    ungetc (c, fp);
}


/* 1. line_number handling.  */

/* Maximum used, roughly a safer MB_LEN_MAX.  */
#define MAX_PHASE1_PUSHBACK 16
static unsigned char phase1_pushback[MAX_PHASE1_PUSHBACK];
static int phase1_pushback_length;

/* Read the next single byte from the input file.  */
static int
phase1_getc ()
{
  int c;

  if (phase1_pushback_length)
    c = phase1_pushback[--phase1_pushback_length];
  else
    c = phase0_getc ();

  if (c == '\n')
    ++line_number;

  return c;
}

/* Supports MAX_PHASE1_PUSHBACK characters of pushback.  */
static void
phase1_ungetc (int c)
{
  if (c != EOF)
    {
      if (c == '\n')
        --line_number;

      if (phase1_pushback_length == countof (phase1_pushback))
        abort ();
      phase1_pushback[phase1_pushback_length++] = c;
    }
}


/* Phase 2: Conversion to Unicode.
   This is done early because PEP 0263 specifies that conversion to Unicode
   conceptually occurs before tokenization.  A test case where it matters
   is with encodings like BIG5: when a double-byte character ending in 0x5C
   is followed by '\' or 'u0021', the tokenizer must not treat the second
   half of the double-byte character as a backslash.  */

/* End-of-file indicator for functions returning an UCS-4 character.  */
#define UEOF -1

static lexical_context_ty lexical_context;

static int phase2_pushback[max (9, UNINAME_MAX + 3)];
static int phase2_pushback_length;

/* Read the next Unicode UCS-4 character from the input file.  */
static int
phase2_getc ()
{
  if (phase2_pushback_length)
    return phase2_pushback[--phase2_pushback_length];

  if (xgettext_current_source_encoding == po_charset_ascii)
    {
      int c = phase1_getc ();
      if (c == EOF)
        return UEOF;
      if (!c_isascii (c))
        {
          multiline_error (xstrdup (""),
                           xasprintf ("%s\n%s\n",
                                      non_ascii_error_message (lexical_context,
                                                               real_file_name,
                                                               line_number),
                                      _("\
Please specify the source encoding through --from-code or through a comment\n\
as specified in https://www.python.org/peps/pep-0263.html.\n")));
          exit (EXIT_FAILURE);
        }
      return c;
    }
  else if (xgettext_current_source_encoding != po_charset_utf8)
    {
#if HAVE_ICONV
      /* Use iconv on an increasing number of bytes.  Read only as many bytes
         through phase1_getc as needed.  This is needed to give reasonable
         interactive behaviour when fp is connected to an interactive tty.  */
      unsigned char buf[MAX_PHASE1_PUSHBACK];
      size_t bufcount;

      {
        int c = phase1_getc ();
        if (c == EOF)
          return UEOF;
        buf[0] = (unsigned char) c;
        bufcount = 1;
      }

      for (;;)
        {
          unsigned char scratchbuf[6];
          const char *inptr = (const char *) &buf[0];
          size_t insize = bufcount;
          char *outptr = (char *) &scratchbuf[0];
          size_t outsize = sizeof (scratchbuf);

          size_t res = iconv (xgettext_current_source_iconv,
                              (ICONV_CONST char **) &inptr, &insize,
                              &outptr, &outsize);
          /* We expect that a character has been produced if and only if
             some input bytes have been consumed.  */
          if ((insize < bufcount) != (outsize < sizeof (scratchbuf)))
            abort ();
          if (outsize == sizeof (scratchbuf))
            {
              /* No character has been produced.  Must be an error.  */
              if (res != (size_t)(-1))
                abort ();

              if (errno == EILSEQ)
                {
                  /* An invalid multibyte sequence was encountered.  */
                  goto invalid;
                }
              else if (errno == EINVAL)
                {
                  /* An incomplete multibyte character.  */
                  if (bufcount == MAX_PHASE1_PUSHBACK)
                    {
                      /* An overlong incomplete multibyte sequence was
                         encountered.  */
                      multiline_error (xstrdup (""),
                                       xasprintf (_("\
%s:%d: Long incomplete multibyte sequence.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in https://www.python.org/peps/pep-0263.html.\n"),
                                       real_file_name, line_number));
                      exit (EXIT_FAILURE);
                    }

                  /* Read one more byte and retry iconv.  */
                  int c = phase1_getc ();
                  if (c == EOF)
                    goto incomplete_at_eof;
                  if (c == '\n')
                    goto incomplete_at_eol;
                  buf[bufcount++] = (unsigned char) c;
                }
              else
                if_error (IF_SEVERITY_FATAL_ERROR,
                          real_file_name, line_number, (size_t)(-1), false,
                          "%s", xstrerror (_("iconv failure"), errno));
            }
          else
            {
              size_t outbytes = sizeof (scratchbuf) - outsize;
              size_t bytes = bufcount - insize;

              /* We expect that one character has been produced.  */
              if (bytes == 0)
                abort ();
              if (outbytes == 0)
                abort ();
              /* Push back the unused bytes.  */
              while (insize > 0)
                phase1_ungetc (buf[--insize]);
              /* Convert the character from UTF-8 to UCS-4.  */
              ucs4_t uc;
              if (u8_mbtoucr (&uc, scratchbuf, outbytes) < (int) outbytes)
                {
                  /* scratchbuf contains an out-of-range Unicode character
                     (> 0x10ffff).  */
                  goto invalid;
                }
              return uc;
            }
        }
#else
      /* If we don't have iconv(), the only supported values for
         xgettext_global_source_encoding and thus also for
         xgettext_current_source_encoding are ASCII and UTF-8.  */
      abort ();
#endif
    }
  else
    {
      /* Read an UTF-8 encoded character.
         Reject invalid input, like u8_mbtouc does.  */
      int c;
      ucs4_t uc;

      c = phase1_getc ();
      if (c == EOF)
        return UEOF;
      if (c < 0x80)
        {
          uc = c;
        }
      else if (c < 0xc2)
        goto invalid;
      else if (c < 0xe0)
        {
          int c1 = phase1_getc ();
          if (c1 == EOF)
            goto incomplete_at_eof;
          if (c1 == '\n')
            goto incomplete_at_eol;
          if ((c1 ^ 0x80) < 0x40)
            uc = ((unsigned int) (c & 0x1f) << 6)
                 | (unsigned int) (c1 ^ 0x80);
          else
            goto invalid;
        }
      else if (c < 0xf0)
        {
          int c1 = phase1_getc ();
          if (c1 == EOF)
            goto incomplete_at_eof;
          if (c1 == '\n')
            goto incomplete_at_eol;
          if ((c1 ^ 0x80) < 0x40
              && (c >= 0xe1 || c1 >= 0xa0)
              && (c != 0xed || c1 < 0xa0))
            {
              int c2 = phase1_getc ();
              if (c2 == EOF)
                goto incomplete_at_eof;
              if (c2 == '\n')
                goto incomplete_at_eol;
              if ((c2 ^ 0x80) < 0x40)
                uc = ((unsigned int) (c & 0x0f) << 12)
                     | ((unsigned int) (c1 ^ 0x80) << 6)
                     | (unsigned int) (c2 ^ 0x80);
              else
                goto invalid;
            }
          else
            goto invalid;
        }
      else if (c < 0xf8)
        {
          int c1 = phase1_getc ();
          if (c1 == EOF)
            goto incomplete_at_eof;
          if (c1 == '\n')
            goto incomplete_at_eol;
          if ((c1 ^ 0x80) < 0x40
              && (c >= 0xf1 || c1 >= 0x90)
              && (c < 0xf4 || (c == 0xf4 && c1 < 0x90)))
            {
              int c2 = phase1_getc ();
              if (c2 == EOF)
                goto incomplete_at_eof;
              if (c2 == '\n')
                goto incomplete_at_eol;
              if ((c2 ^ 0x80) < 0x40)
                {
                  int c3 = phase1_getc ();
                  if (c3 == EOF)
                    goto incomplete_at_eof;
                  if (c3 == '\n')
                    goto incomplete_at_eol;
                  if ((c3 ^ 0x80) < 0x40)
                    uc = ((unsigned int) (c & 0x07) << 18)
                         | ((unsigned int) (c1 ^ 0x80) << 12)
                         | ((unsigned int) (c2 ^ 0x80) << 6)
                         | (unsigned int) (c3 ^ 0x80);
                  else
                    goto invalid;
                }
              else
                goto invalid;
            }
          else
            goto invalid;
        }
      else
        goto invalid;

      return uc;
    }

 invalid:
  /* An invalid multibyte sequence was encountered.  */
  multiline_error (xstrdup (""),
                   xasprintf (_("\
%s:%d: Invalid multibyte sequence.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in https://www.python.org/peps/pep-0263.html.\n"),
                   real_file_name, line_number));
  exit (EXIT_FAILURE);

 incomplete_at_eof:
  multiline_error (xstrdup (""),
                   xasprintf (_("\
%s:%d: Incomplete multibyte sequence at end of file.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in https://www.python.org/peps/pep-0263.html.\n"),
                   real_file_name, line_number));
  exit (EXIT_FAILURE);

 incomplete_at_eol:
  multiline_error (xstrdup (""),
                   xasprintf (_("\
%s:%d: Incomplete multibyte sequence at end of line.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in https://www.python.org/peps/pep-0263.html.\n"),
                   real_file_name, line_number - 1));
  exit (EXIT_FAILURE);
}

/* Supports max (9, UNINAME_MAX + 3) pushback characters.  */
static void
phase2_ungetc (int c)
{
  if (c != UEOF)
    {
      if (phase2_pushback_length == countof (phase2_pushback))
        abort ();
      phase2_pushback[phase2_pushback_length++] = c;
    }
}


/* ========================= Accumulating strings.  ======================== */

/* See xg-mixed-string.h for the API.  */


/* ======================== Accumulating comments.  ======================== */


/* Accumulating a single comment line.  */

static struct mixed_string_buffer comment_buffer;

static inline void
comment_start ()
{
  mixed_string_buffer_init (&comment_buffer, lc_comment,
                            logical_file_name, line_number);
}

static inline bool
comment_at_start ()
{
  return mixed_string_buffer_is_empty (&comment_buffer);
}

static inline void
comment_add (int c)
{
  mixed_string_buffer_append_unicode (&comment_buffer, c);
}

static inline const char *
comment_line_end ()
{
  char *buffer =
    mixed_string_contents_free1 (mixed_string_buffer_result (&comment_buffer));
  size_t buflen = strlen (buffer);

  while (buflen >= 1
         && (buffer[buflen - 1] == ' ' || buffer[buflen - 1] == '\t'))
    --buflen;
  buffer[buflen] = '\0';
  savable_comment_add (buffer);
  lexical_context = lc_outside;
  return buffer;
}


/* These are for tracking whether comments count as immediately before
   keyword.  */
static int last_comment_line;
static int last_non_comment_line;


/* ======================== Recognizing comments.  ======================== */


/* Recognizing the "coding" comment.
   As specified in PEP 0263, it takes the form
     "coding" [":"|"="] {alphanumeric or "-" or "_" or "*"}*
   or
     "set" "fileencoding" "=" {alphanumeric or "-" or "_" or "*"}*
   and is located in a comment in a line that
     - is either the first or second line,
     - is not a continuation line,
     - in the first form, contains no other tokens except this comment.  */

/* Canonicalized encoding name for the current input file.  */
static const char *xgettext_current_file_source_encoding;

#if HAVE_ICONV
/* Converter from xgettext_current_file_source_encoding to UTF-8 (except from
   ASCII or UTF-8, when this conversion is a no-op).  */
static iconv_t xgettext_current_file_source_iconv;
#endif

static inline void
set_current_file_source_encoding (const char *canon_encoding)
{
  xgettext_current_file_source_encoding = canon_encoding;

  if (xgettext_current_file_source_encoding != po_charset_ascii
      && xgettext_current_file_source_encoding != po_charset_utf8)
    {
#if HAVE_ICONV
      iconv_t cd =
        iconv_open (po_charset_utf8, xgettext_current_file_source_encoding);

      if (cd == (iconv_t)(-1))
        error_at_line (EXIT_FAILURE, 0, logical_file_name, line_number - 1,
                       _("Cannot convert from \"%s\" to \"%s\". %s relies on iconv(), and iconv() does not support this conversion."),
                       xgettext_current_file_source_encoding, po_charset_utf8,
                       last_component (program_name));
      xgettext_current_file_source_iconv = cd;
#else
      error_at_line (EXIT_FAILURE, 0, logical_file_name, line_number - 1,
                     _("Cannot convert from \"%s\" to \"%s\". %s relies on iconv(). This version was built without iconv()."),
                     xgettext_current_file_source_encoding, po_charset_utf8,
                     last_component (program_name));
#endif
    }

  xgettext_current_source_encoding = xgettext_current_file_source_encoding;
#if HAVE_ICONV
  xgettext_current_source_iconv = xgettext_current_file_source_iconv;
#endif
}

static inline void
try_to_extract_coding (const char *comment)
{
  const char *p = c_strstr (comment, "coding");

  if (p != NULL)
    {
      p += 6;
      if (*p == ':' || *p == '=')
        {
          p++;
          while (*p == ' ' || *p == '\t')
            p++;
          {
            const char *encoding_start = p;

            while (c_isalnum (*p) || *p == '-' || *p == '_' || *p == '.')
              p++;
            {
              const char *encoding_end = p;

              if (encoding_end > encoding_start)
                {
                  /* Extract the encoding string.  */
                  size_t encoding_len = encoding_end - encoding_start;

                  char *encoding = XNMALLOC (encoding_len + 1, char);
                  memcpy (encoding, encoding_start, encoding_len);
                  encoding[encoding_len] = '\0';

                  {
                    /* Canonicalize it.  */
                    const char *canon_encoding = po_charset_canonicalize (encoding);
                    if (canon_encoding == NULL)
                      {
                        error_at_line (0, 0,
                                       logical_file_name, line_number - 1,
                                       _("Unknown encoding \"%s\". Proceeding with ASCII instead."),
                                       encoding);
                        canon_encoding = po_charset_ascii;
                      }

                    /* Activate it.  */
                    set_current_file_source_encoding (canon_encoding);
                  }

                  free (encoding);
                }
            }
          }
        }
    }
}

/* Tracking whether the current line is a continuation line or contains a
   non-blank character.  */
static bool continuation_or_nonblank_line;


/* Phase 3: Outside strings, replace backslash-newline with nothing and a
   comment with nothing.  */

static int
phase3_getc ()
{
  for (;;)
    {
      int c;

      c = phase2_getc ();
      if (c == '\\')
        {
          c = phase2_getc ();
          if (c != '\n')
            {
              phase2_ungetc (c);
              /* This shouldn't happen usually, because "A backslash is
                 illegal elsewhere on a line outside a string literal."  */
              return '\\';
            }
          /* Eat backslash-newline.  */
          continuation_or_nonblank_line = true;
        }
      else if (c == '#')
        {
          /* Eat a comment.  */
          last_comment_line = line_number;
          comment_start ();
          for (;;)
            {
              c = phase2_getc ();
              if (c == UEOF || c == '\n')
                break;
              /* We skip all leading white space, but not EOLs.  */
              if (!(comment_at_start () && (c == ' ' || c == '\t')))
                comment_add (c);
            }
          const char *comment = comment_line_end ();
          if (line_number - 1 <= 2 && !continuation_or_nonblank_line)
            try_to_extract_coding (comment);
          continuation_or_nonblank_line = false;
          return c;
        }
      else
        {
          if (c == '\n')
            continuation_or_nonblank_line = false;
          else if (!(c == ' ' || c == '\t' || c == '\f'))
            continuation_or_nonblank_line = true;
          return c;
        }
    }
}

/* Supports only one pushback character.  */
static void
phase3_ungetc (int c)
{
  phase2_ungetc (c);
}


/* ========================= Accumulating strings.  ======================== */

/* Return value of phase7_getuc when EOF is reached.  */
#define P7_EOF (-1)
#define P7_STRING_END (-2)
#define P7_498_START_OF_EXPRESSION (-3) /* { */

/* Convert an UTF-16 or UTF-32 code point to a return value that can be
   distinguished from a single-byte return value.  */
#define UNICODE(code) (0x100 + (code))

/* Test a return value of phase7_getuc whether it designates an UTF-16 or
   UTF-32 code point.  */
#define IS_UNICODE(p7_result) ((p7_result) >= 0x100)

/* Extract the UTF-16 or UTF-32 code of a return value that satisfies
   IS_UNICODE.  */
#define UNICODE_VALUE(p7_result) ((p7_result) - 0x100)


/* ========================== Reading of tokens.  ========================== */


enum token_type_ty
{
  token_type_eof,
  token_type_lparen,            /* ( */
  token_type_rparen,            /* ) */
  token_type_comma,             /* , */
  token_type_lbracket,          /* [ */
  token_type_rbracket,          /* ] */
  token_type_string,            /* "abc", 'abc', """abc""", '''abc''' */
  token_type_498,               /* f"abc", f'abc', f"""abc""", f'''abc''' */
  token_type_l498,              /* left part of f-string: f"abc{, f'abc{, f"""abc{, f'''abc{ */
  token_type_m498,              /* middle part of f-string: }abc{ */
  token_type_r498,              /* right part of f-string: }abc", }abc', }abc""", }abc''' */
  token_type_symbol,            /* symbol, number */
  token_type_plus,              /* + */
  token_type_other              /* misc. operator */
};
typedef enum token_type_ty token_type_ty;

typedef struct token_ty token_ty;
struct token_ty
{
  token_type_ty type;
  char *string;                       /* for token_type_symbol */
  mixed_string_ty *mixed_string;      /* for token_type_string, token_type_498 */
  refcounted_string_list_ty *comment; /* for token_type_string, token_type_498 */
  int line_number;
};

/* Free the memory pointed to by a 'struct token_ty'.  */
static inline void
free_token (token_ty *tp)
{
  if (tp->type == token_type_symbol)
    free (tp->string);
  if (tp->type == token_type_string || tp->type == token_type_498)
    {
      mixed_string_free (tp->mixed_string);
      drop_reference (tp->comment);
    }
}


/* There are two different input syntaxes for byte strings, b"abc" and br"abc",
   and two different input syntaxes for Unicode strings, u"abc" and ur"abc";
   the 'u' may be omitted.
   Which escape sequences are understood, i.e. what is interpreted specially
   after backslash?
    b"abc"    \<nl> \\ \' \" \a\b\f\n\r\t\v \ooo \xnn
    br"abc"
    u"abc"    \<nl> \\ \' \" \a\b\f\n\r\t\v \ooo \xnn \unnnn \Unnnnnnnn \N{...}
    ur"abc"                                           \unnnn
   The \unnnn values are UTF-16 values; a single \Unnnnnnnn can expand to two
   \unnnn items.  The \ooo and \xnn values are in the current source encoding
   for byte strings, and Unicode code points for Unicode strings.
 */

static int
phase7_getuc (int quote_char,
              bool triple, bool interpret_ansic, bool interpret_unicode,
              bool f_string,
              unsigned int *backslash_counter)
{
  for (;;)
    {
      int c;

      /* Use phase 2, because phase 3 elides comments.  */
      c = phase2_getc ();

      if (c == UEOF)
        return P7_EOF;

      if (c == quote_char && (interpret_ansic || (*backslash_counter & 1) == 0))
        {
          if (triple)
            {
              int c1 = phase2_getc ();
              if (c1 == quote_char)
                {
                  int c2 = phase2_getc ();
                  if (c2 == quote_char)
                    return P7_STRING_END;
                  phase2_ungetc (c2);
                }
              phase2_ungetc (c1);
              return UNICODE (c);
            }
          else
            return P7_STRING_END;
        }

      if (c == '\n')
        {
          if (triple)
            {
              *backslash_counter = 0;
              return UNICODE ('\n');
            }
          /* In r"..." and ur"..." strings, newline is only allowed
             immediately after an odd number of backslashes (although the
             backslashes are not interpreted!).  */
          if (!(interpret_ansic || (*backslash_counter & 1) == 0))
            {
              *backslash_counter = 0;
              return UNICODE ('\n');
            }
          phase2_ungetc (c);
          if_error (IF_SEVERITY_WARNING,
                    logical_file_name, line_number, (size_t)(-1), false,
                    _("unterminated string"));
          return P7_STRING_END;
        }

      if (f_string)
        {
          if (c == '{')
            {
              int c1 = phase2_getc ();
              if (c1 == '{')
                return UNICODE ('{');
              phase2_ungetc (c1);
              return P7_498_START_OF_EXPRESSION;
            }
          if (c == '}')
            {
              int c1 = phase2_getc ();
              if (c1 == '}')
                return UNICODE ('}');
              phase2_ungetc (c1);
            }
        }

      if (c != '\\')
        {
          *backslash_counter = 0;
          return UNICODE (c);
        }

      /* Backslash handling.  */

      if (!interpret_ansic && !interpret_unicode)
        {
          ++*backslash_counter;
          return UNICODE ('\\');
        }

      /* Dispatch according to the character following the backslash.  */
      c = phase2_getc ();
      if (c == UEOF)
        {
          ++*backslash_counter;
          return UNICODE ('\\');
        }

      if (interpret_ansic)
        switch (c)
          {
          case '\n':
            continue;
          case '\\':
            ++*backslash_counter;
            return UNICODE (c);
          case '\'': case '"':
            *backslash_counter = 0;
            return UNICODE (c);
          case 'a':
            *backslash_counter = 0;
            return UNICODE ('\a');
          case 'b':
            *backslash_counter = 0;
            return UNICODE ('\b');
          case 'f':
            *backslash_counter = 0;
            return UNICODE ('\f');
          case 'n':
            *backslash_counter = 0;
            return UNICODE ('\n');
          case 'r':
            *backslash_counter = 0;
            return UNICODE ('\r');
          case 't':
            *backslash_counter = 0;
            return UNICODE ('\t');
          case 'v':
            *backslash_counter = 0;
            return UNICODE ('\v');
          case '0': case '1': case '2': case '3': case '4':
          case '5': case '6': case '7':
            {
              int n = c - '0';

              c = phase2_getc ();
              if (c != UEOF)
                {
                  if (c >= '0' && c <= '7')
                    {
                      n = (n << 3) + (c - '0');
                      c = phase2_getc ();
                      if (c != UEOF)
                        {
                          if (c >= '0' && c <= '7')
                            n = (n << 3) + (c - '0');
                          else
                            phase2_ungetc (c);
                        }
                    }
                  else
                    phase2_ungetc (c);
                }
              *backslash_counter = 0;
              /* <https://docs.python.org/3.12/reference/lexical_analysis.html#escape-sequences>
                 says: "In a bytes literal, hexadecimal and octal escapes denote
                        the byte with the given value.
                        In a string literal, these escapes denote a Unicode
                        character with the given value."  */
              if (interpret_unicode)
                return UNICODE (n);
              else
                return (unsigned char) n;
            }
          case 'x':
            {
              int c1 = phase2_getc ();

              int n1;
              if (c1 >= '0' && c1 <= '9')
                n1 = c1 - '0';
              else if (c1 >= 'A' && c1 <= 'F')
                n1 = c1 - 'A' + 10;
              else if (c1 >= 'a' && c1 <= 'f')
                n1 = c1 - 'a' + 10;
              else
                n1 = -1;

              if (n1 >= 0)
                {
                  int c2 = phase2_getc ();

                  int n2;
                  if (c2 >= '0' && c2 <= '9')
                    n2 = c2 - '0';
                  else if (c2 >= 'A' && c2 <= 'F')
                    n2 = c2 - 'A' + 10;
                  else if (c2 >= 'a' && c2 <= 'f')
                    n2 = c2 - 'a' + 10;
                  else
                    n2 = -1;

                  if (n2 >= 0)
                    {
                      int n = (n1 << 4) + n2;
                      *backslash_counter = 0;
                      /* <https://docs.python.org/3.12/reference/lexical_analysis.html#escape-sequences>
                         says:
                         "In a bytes literal, hexadecimal and octal escapes denote
                          the byte with the given value.
                          In a string literal, these escapes denote a Unicode
                          character with the given value."  */
                      if (interpret_unicode)
                        return UNICODE (n);
                      else
                        return (unsigned char) n;
                    }

                  phase2_ungetc (c2);
                }
              phase2_ungetc (c1);
              phase2_ungetc (c);
              ++*backslash_counter;
              return UNICODE ('\\');
            }
          }

      if (interpret_ansic && interpret_unicode)
        {
          if (c == 'u')
            {
              unsigned char buf[4];
              unsigned int n = 0;

              for (int i = 0; i < 4; i++)
                {
                  int c1 = phase2_getc ();

                  if (c1 >= '0' && c1 <= '9')
                    n = (n << 4) + (c1 - '0');
                  else if (c1 >= 'A' && c1 <= 'F')
                    n = (n << 4) + (c1 - 'A' + 10);
                  else if (c1 >= 'a' && c1 <= 'f')
                    n = (n << 4) + (c1 - 'a' + 10);
                  else
                    {
                      phase2_ungetc (c1);
                      while (--i >= 0)
                        phase2_ungetc (buf[i]);
                      phase2_ungetc (c);
                      ++*backslash_counter;
                      return UNICODE ('\\');
                    }

                  buf[i] = c1;
                }
              *backslash_counter = 0;
              return UNICODE (n);
            }

          if (c == 'U')
            {
              unsigned char buf[8];
              unsigned int n = 0;
              int i;

              for (i = 0; i < 8; i++)
                {
                  int c1 = phase2_getc ();

                  if (c1 >= '0' && c1 <= '9')
                    n = (n << 4) + (c1 - '0');
                  else if (c1 >= 'A' && c1 <= 'F')
                    n = (n << 4) + (c1 - 'A' + 10);
                  else if (c1 >= 'a' && c1 <= 'f')
                    n = (n << 4) + (c1 - 'a' + 10);
                  else
                    {
                      phase2_ungetc (c1);
                      while (--i >= 0)
                        phase2_ungetc (buf[i]);
                      phase2_ungetc (c);
                      ++*backslash_counter;
                      return UNICODE ('\\');
                    }

                  buf[i] = c1;
                }
              if (n < 0x110000)
                {
                  *backslash_counter = 0;
                  return UNICODE (n);
                }

              if_error (IF_SEVERITY_WARNING,
                        logical_file_name, line_number, (size_t)(-1), false,
                        _("invalid Unicode character"));

              while (--i >= 0)
                phase2_ungetc (buf[i]);
              phase2_ungetc (c);
              ++*backslash_counter;
              return UNICODE ('\\');
            }

          if (c == 'N')
            {
              int c1 = phase2_getc ();
              if (c1 == '{')
                {
                  unsigned char buf[UNINAME_MAX + 1];
                  int i;

                  for (i = 0; i < UNINAME_MAX; i++)
                    {
                      int c2 = phase2_getc ();
                      if (!(c2 >= ' ' && c2 <= '~'))
                        {
                          phase2_ungetc (c2);
                          while (--i >= 0)
                            phase2_ungetc (buf[i]);
                          phase2_ungetc (c1);
                          phase2_ungetc (c);
                          ++*backslash_counter;
                          return UNICODE ('\\');
                        }
                      if (c2 == '}')
                        break;
                      buf[i] = c2;
                    }
                  buf[i] = '\0';

                  unsigned int n = unicode_name_character ((char *) buf);
                  if (n != UNINAME_INVALID)
                    {
                      *backslash_counter = 0;
                      return UNICODE (n);
                    }

                  phase2_ungetc ('}');
                  while (--i >= 0)
                    phase2_ungetc (buf[i]);
                }
              phase2_ungetc (c1);
              phase2_ungetc (c);
              ++*backslash_counter;
              return UNICODE ('\\');
            }
        }

      phase2_ungetc (c);
      ++*backslash_counter;
      return UNICODE ('\\');
    }
}


/* Combine characters into tokens.  Discard whitespace except newlines at
   the end of logical lines.  */

/* Number of open f-strings f"...{ or f'...{ or f"""...{ or f'''...{ or
   fr"...{ or fr'...{ or fr"""...{ or fr'''...{ */
static int f_string_depth;

/* Information per f-string nesting level.  */
struct f_string_level
{
  /* Describes the start and end sequence of the f-string.
     Only relevant for levels > 0.  */
  int quote_char;
  bool interpret_ansic;
  bool triple;
  /* Number of open '{' tokens.  */
  int brace_depth;
};

/* Stack of f-string nesting levels.
   The "current" element is f_string_stack[f_string_depth].  */
static struct f_string_level *f_string_stack;
/* Number of allocated elements in f_string_stack.  */
static size_t f_string_stack_alloc;

/* Adds a new f_string_stack level after f_string_depth was incremented.  */
static void
new_f_string_level (int quote_char, bool interpret_ansic, bool triple)
{
  if (f_string_depth == f_string_stack_alloc)
    {
      f_string_stack_alloc = 2 * f_string_stack_alloc + 1;
      /* Now f_string_depth < f_string_stack_alloc.  */
      f_string_stack =
        (struct f_string_level *)
        xrealloc (f_string_stack,
                  f_string_stack_alloc * sizeof (struct f_string_level));
    }
  f_string_stack[f_string_depth].quote_char = quote_char;
  f_string_stack[f_string_depth].interpret_ansic = interpret_ansic;
  f_string_stack[f_string_depth].triple = triple;
  f_string_stack[f_string_depth].brace_depth = 0;
}

/* Number of pending open parentheses/brackets.  */
static int open_pb;

static token_ty phase5_pushback[2];
static int phase5_pushback_length;

static void
phase5_get (token_ty *tp)
{
  if (phase5_pushback_length)
    {
      *tp = phase5_pushback[--phase5_pushback_length];
      return;
    }

  for (;;)
    {
      int c;

      tp->line_number = line_number;
      c = phase3_getc ();

      switch (c)
        {
        case UEOF:
          tp->type = token_type_eof;
          return;

        case ' ':
        case '\t':
        case '\f':
          /* Ignore whitespace and comments.  */
          continue;

        case '\n':
          if (last_non_comment_line > last_comment_line)
            savable_comment_reset ();
          /* Ignore newline if and only if it is used for implicit line
             joining.  */
          if (open_pb > 0 || f_string_stack[f_string_depth].brace_depth > 0)
            continue;
          tp->type = token_type_other;
          return;
        }

      last_non_comment_line = tp->line_number;

      switch (c)
        {
        case '.':
          {
            int c1 = phase3_getc ();
            phase3_ungetc (c1);
            if (!(c1 >= '0' && c1 <= '9'))
              {

                tp->type = token_type_other;
                return;
              }
          }
          FALLTHROUGH;
        case 'A':           case 'C': case 'D': case 'E':
        case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
        case 'M': case 'N': case 'O': case 'P': case 'Q':
        case 'S': case 'T':           case 'V': case 'W': case 'X':
        case 'Y': case 'Z':
        case '_':
        case 'a':           case 'c': case 'd': case 'e':
        case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
        case 'm': case 'n': case 'o': case 'p': case 'q':
        case 's': case 't':           case 'v': case 'w': case 'x':
        case 'y': case 'z':
        case '0': case '1': case '2': case '3': case '4':
        case '5': case '6': case '7': case '8': case '9':
        symbol:
          /* Symbol, or part of a number.  */
          {
            struct string_buffer buffer;
            sb_init (&buffer);
            for (;;)
              {
                sb_xappend1 (&buffer, c);
                c = phase3_getc ();
                switch (c)
                  {
                  case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
                  case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
                  case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
                  case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
                  case 'Y': case 'Z':
                  case '_':
                  case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
                  case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
                  case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
                  case 's': case 't': case 'u': case 'v': case 'w': case 'x':
                  case 'y': case 'z':
                  case '0': case '1': case '2': case '3': case '4':
                  case '5': case '6': case '7': case '8': case '9':
                    continue;
                  default:
                    phase3_ungetc (c);
                    break;
                  }
                break;
              }
            tp->string = sb_xdupfree_c (&buffer);
            tp->type = token_type_symbol;
            return;
          }

        /* Strings.  */
          {
            int quote_char;
            bool interpret_ansic;
            bool interpret_unicode;
            bool f_string;
            bool triple;
            unsigned int backslash_counter;

            case 'B': case 'b':
              {
                int c1 = phase2_getc ();
                if (c1 == '"' || c1 == '\'')
                  {
                    quote_char = c1;
                    interpret_ansic = true;
                    interpret_unicode = false;
                    f_string = false;
                    goto string;
                  }
                if (c1 == 'R' || c1 == 'r')
                  {
                    int c2 = phase2_getc ();
                    if (c2 == '"' || c2 == '\'')
                      {
                        quote_char = c2;
                        interpret_ansic = false;
                        interpret_unicode = false;
                        f_string = false;
                        goto string;
                      }
                    phase2_ungetc (c2);
                  }
                phase2_ungetc (c1);
                goto symbol;
              }

            case 'F': case 'f':
              {
                int c1 = phase2_getc ();
                if (c1 == '"' || c1 == '\'')
                  {
                    quote_char = c1;
                    interpret_ansic = true;
                    interpret_unicode = true;
                    f_string = true;
                    goto string;
                  }
                if (c1 == 'R' || c1 == 'r')
                  {
                    int c2 = phase2_getc ();
                    if (c2 == '"' || c2 == '\'')
                      {
                        quote_char = c2;
                        interpret_ansic = false;
                        interpret_unicode = true;
                        f_string = true;
                        goto string;
                      }
                    phase2_ungetc (c2);
                  }
                phase2_ungetc (c1);
                goto symbol;
              }

            case 'R': case 'r':
              {
                int c1 = phase2_getc ();
                if (c1 == '"' || c1 == '\'')
                  {
                    quote_char = c1;
                    interpret_ansic = false;
                    interpret_unicode = true;
                    f_string = false;
                    goto string;
                  }
                if (c1 == 'F' || c1 == 'f')
                  {
                    int c2 = phase2_getc ();
                    if (c2 == '"' || c2 == '\'')
                      {
                        quote_char = c2;
                        interpret_ansic = false;
                        interpret_unicode = true;
                        f_string = true;
                        goto string;
                      }
                    phase2_ungetc (c2);
                  }
                phase2_ungetc (c1);
                goto symbol;
              }

            case 'U': case 'u':
              {
                int c1 = phase2_getc ();
                if (c1 == '"' || c1 == '\'')
                  {
                    quote_char = c1;
                    interpret_ansic = true;
                    interpret_unicode = true;
                    f_string = false;
                    goto string;
                  }
                phase2_ungetc (c1);
                goto symbol;
              }

            case '"': case '\'':
              quote_char = c;
              interpret_ansic = true;
              interpret_unicode = true;
              f_string = false;
            string:
              triple = false;
              lexical_context = lc_string;
              {
                int c1 = phase2_getc ();
                if (c1 == quote_char)
                  {
                    int c2 = phase2_getc ();
                    if (c2 == quote_char)
                      triple = true;
                    else
                      {
                        phase2_ungetc (c2);
                        phase2_ungetc (c1);
                      }
                  }
                else
                  phase2_ungetc (c1);
              }
              backslash_counter = 0;
              {
                struct mixed_string_buffer msb;
                /* Start accumulating the string.  */
                mixed_string_buffer_init (&msb, lexical_context,
                                          logical_file_name, line_number);

                for (;;)
                  {
                    int uc = phase7_getuc (quote_char, triple, interpret_ansic,
                                           interpret_unicode, f_string,
                                           &backslash_counter);

                    /* Keep line_number in sync.  */
                    msb.line_number = line_number;

                    if (uc == P7_EOF || uc == P7_STRING_END)
                      {
                        tp->mixed_string = mixed_string_buffer_result (&msb);
                        tp->comment = add_reference (savable_comment);
                        tp->type = (f_string ? token_type_498 : token_type_string);
                        break;
                      }

                    if (uc == P7_498_START_OF_EXPRESSION) /* implies f_string */
                      {
                        mixed_string_buffer_destroy (&msb);
                        tp->type = token_type_l498;
                        f_string_depth++;
                        new_f_string_level (quote_char, interpret_ansic, triple);
                        break;
                      }

                    if (IS_UNICODE (uc))
                      {
                        assert (UNICODE_VALUE (uc) >= 0
                                && UNICODE_VALUE (uc) < 0x110000);
                        mixed_string_buffer_append_unicode (&msb,
                                                            UNICODE_VALUE (uc));
                      }
                    else
                      mixed_string_buffer_append_char (&msb, uc);
                  }
                lexical_context = lc_outside;
              }
              return;
          }

        case '{':
          f_string_stack[f_string_depth].brace_depth++;
          tp->type = token_type_other;
          return;

        case '}':
          if (f_string_stack[f_string_depth].brace_depth > 0)
            f_string_stack[f_string_depth].brace_depth--;
          else if (f_string_depth > 0)
            {
              /* Middle or right part of f-string.  */
              int quote_char = f_string_stack[f_string_depth].quote_char;
              bool interpret_ansic = f_string_stack[f_string_depth].interpret_ansic;
              bool triple = f_string_stack[f_string_depth].triple;
              unsigned int backslash_counter = 0;
              for (;;)
                {
                  int uc = phase7_getuc (quote_char, triple, interpret_ansic,
                                         true, true,
                                         &backslash_counter);

                  if (uc == P7_EOF || uc == P7_STRING_END)
                    {
                      tp->type = token_type_r498;
                      f_string_depth--;
                      break;
                    }

                  if (uc == P7_498_START_OF_EXPRESSION)
                    {
                      tp->type = token_type_m498;
                      break;
                    }
                }
              return;
            }
          tp->type = token_type_other;
          return;

        case '(':
          open_pb++;
          tp->type = token_type_lparen;
          return;

        case ')':
          if (open_pb > 0)
            open_pb--;
          tp->type = token_type_rparen;
          return;

        case ',':
          tp->type = token_type_comma;
          return;

        case '[':
          open_pb++;
          tp->type = token_type_lbracket;
          return;

        case ']':
          if (open_pb > 0)
            open_pb--;
          tp->type = token_type_rbracket;
          return;

        case '+':
          tp->type = token_type_plus;
          return;

        default:
          /* We could carefully recognize each of the 2 and 3 character
             operators, but it is not necessary, as we only need to recognize
             gettext invocations.  Don't bother.  */
          tp->type = token_type_other;
          return;
        }
    }
}

/* Supports only one pushback token.  */
static void
phase5_unget (token_ty *tp)
{
  if (tp->type != token_type_eof)
    {
      if (phase5_pushback_length == countof (phase5_pushback))
        abort ();
      phase5_pushback[phase5_pushback_length++] = *tp;
    }
}


/* Combine adjacent strings to form a single string.  Note that the end
   of a logical line appears as a token of its own, therefore strings that
   belong to different logical lines will not be concatenated.  */

static void
x_python_lex (token_ty *tp)
{
  phase5_get (tp);
  if (tp->type == token_type_string || tp->type == token_type_498)
    {
      mixed_string_ty *sum = tp->mixed_string;

      for (;;)
        {
          token_ty token2;
          phase5_get (&token2);

          token_ty token3;
          token_ty *tp2 = NULL;

          switch (token2.type)
            {
            case token_type_plus:
              {
                phase5_get (&token3);

                if (token3.type == token_type_string
                    || token3.type == token_type_498)
                  {
                    free_token (&token2);
                    tp2 = &token3;
                  }
                else
                  phase5_unget (&token3);
              }
              break;
            case token_type_string:
            case token_type_498:
              tp2 = &token2;
              break;
            default:
              break;
            }

          if (tp2)
            {
              sum = mixed_string_concat_free1 (sum, tp2->mixed_string);

              free_token (tp2);
              continue;
            }
          phase5_unget (&token2);
          break;
        }
      tp->mixed_string = sum;
    }
}


/* ========================= Extracting strings.  ========================== */


/* Context lookup table.  */
static flag_context_list_table_ty *flag_context_list_table;


/* Maximum supported nesting depth.  */
#define MAX_NESTING_DEPTH 1000

/* Current nesting depths.  */
static int paren_nesting_depth;
static int bracket_nesting_depth;


/* The file is broken into tokens.  Scan the token stream, looking for
   a keyword, followed by a left paren, followed by a string.  When we
   see this sequence, we have something to remember.  We assume we are
   looking at a valid C or C++ program, and leave the complaints about
   the grammar to the compiler.

     Normal handling: Look for
       keyword ( ... msgid ... )
     Plural handling: Look for
       keyword ( ... msgid ... msgid_plural ... )

   We use recursion because the arguments before msgid or between msgid
   and msgid_plural can contain subexpressions of the same form.  */


/* Extract messages until the next balanced closing parenthesis or bracket.
   Extracted messages are added to MLP.
   DELIM can be either token_type_rparen or token_type_rbracket, or
   token_type_eof to accept both.
   Return true upon eof, false upon closing parenthesis or bracket.  */
static bool
extract_balanced (message_list_ty *mlp,
                  token_type_ty delim,
                  flag_region_ty *outer_region,
                  flag_context_list_iterator_ty context_iter,
                  struct arglist_parser *argparser)
{
  /* Current argument number.  */
  int arg = 1;
  /* 0 when no keyword has been seen.  1 right after a keyword is seen.  */
  int state;
  /* Parameters of the keyword just seen.  Defined only in state 1.  */
  const struct callshapes *next_shapes = NULL;
  /* Context iterator that will be used if the next token is a '('.  */
  flag_context_list_iterator_ty next_context_iter =
    passthrough_context_list_iterator;
  /* Current region.  */
  flag_region_ty *inner_region =
    inheriting_region (outer_region,
                       flag_context_list_iterator_advance (&context_iter));

  /* Start state is 0.  */
  state = 0;

  for (;;)
    {
      token_ty token;
      x_python_lex (&token);

      switch (token.type)
        {
        case token_type_symbol:
          {
            void *keyword_value;
            if (hash_find_entry (&keywords, token.string, strlen (token.string),
                                 &keyword_value)
                == 0)
              {
                next_shapes = (const struct callshapes *) keyword_value;
                state = 1;
              }
            else
              state = 0;
          }
          next_context_iter =
            flag_context_list_iterator (
              flag_context_list_table_lookup (
                flag_context_list_table,
                token.string, strlen (token.string)));
          free (token.string);
          break;

        case token_type_lparen:
          if (++paren_nesting_depth > MAX_NESTING_DEPTH)
            if_error (IF_SEVERITY_FATAL_ERROR,
                      logical_file_name, line_number, (size_t)(-1), false,
                      _("too many open parentheses"));
          if (extract_balanced (mlp, token_type_rparen,
                                inner_region, next_context_iter,
                                arglist_parser_alloc (mlp,
                                                      state ? next_shapes : NULL)))
            {
              arglist_parser_done (argparser, arg);
              unref_region (inner_region);
              return true;
            }
          paren_nesting_depth--;
          next_context_iter = null_context_list_iterator;
          state = 0;
          break;

        case token_type_rparen:
          if (delim == token_type_rparen || delim == token_type_eof)
            {
              arglist_parser_done (argparser, arg);
              unref_region (inner_region);
              return false;
            }
          next_context_iter = null_context_list_iterator;
          state = 0;
          break;

        case token_type_comma:
          arg++;
          unref_region (inner_region);
          inner_region =
            inheriting_region (outer_region,
                               flag_context_list_iterator_advance (
                                 &context_iter));
          next_context_iter = passthrough_context_list_iterator;
          state = 0;
          break;

        case token_type_lbracket:
          if (++bracket_nesting_depth > MAX_NESTING_DEPTH)
            if_error (IF_SEVERITY_FATAL_ERROR,
                      logical_file_name, line_number, (size_t)(-1), false,
                      _("too many open brackets"));
          if (extract_balanced (mlp, token_type_rbracket,
                                null_context_region (),
                                null_context_list_iterator,
                                arglist_parser_alloc (mlp, NULL)))
            {
              arglist_parser_done (argparser, arg);
              unref_region (inner_region);
              return true;
            }
          bracket_nesting_depth--;
          next_context_iter = null_context_list_iterator;
          state = 0;
          break;

        case token_type_rbracket:
          if (delim == token_type_rbracket || delim == token_type_eof)
            {
              arglist_parser_done (argparser, arg);
              unref_region (inner_region);
              return false;
            }
          next_context_iter = null_context_list_iterator;
          state = 0;
          break;

        case token_type_string:
        case token_type_498:
          {
            lex_pos_ty pos;
            pos.file_name = logical_file_name;
            pos.line_number = token.line_number;

            if (extract_all)
              {
                char *string = mixed_string_contents (token.mixed_string);
                mixed_string_free (token.mixed_string);
                remember_a_message (mlp, NULL, string, true, false,
                                    inner_region, &pos,
                                    NULL, token.comment, true);
              }
            else
              arglist_parser_remember (argparser, arg, token.mixed_string,
                                       inner_region,
                                       pos.file_name, pos.line_number,
                                       token.comment, true);
          }
          drop_reference (token.comment);
          next_context_iter = null_context_list_iterator;
          state = 0;
          break;

        case token_type_eof:
          arglist_parser_done (argparser, arg);
          unref_region (inner_region);
          return true;

        case token_type_l498:
        case token_type_m498:
        case token_type_r498:
        case token_type_plus:
        case token_type_other:
          next_context_iter = null_context_list_iterator;
          state = 0;
          break;

        default:
          abort ();
        }
    }
}


void
extract_python (FILE *f,
                const char *real_filename, const char *logical_filename,
                flag_context_list_table_ty *flag_table,
                msgdomain_list_ty *mdlp)
{
  message_list_ty *mlp = mdlp->item[0]->messages;

  fp = f;
  real_file_name = real_filename;
  logical_file_name = xstrdup (logical_filename);
  line_number = 1;

  phase1_pushback_length = 0;

  lexical_context = lc_outside;

  phase2_pushback_length = 0;

  last_comment_line = -1;
  last_non_comment_line = -1;

  /* For Python, the default source file encoding is UTF-8.  This is specified
     in PEP 3120.  */
  xgettext_current_file_source_encoding =
   (xgettext_global_source_encoding != NULL ? xgettext_global_source_encoding :
    po_charset_utf8);
#if HAVE_ICONV
  xgettext_current_file_source_iconv = xgettext_global_source_iconv;
#endif

  xgettext_current_source_encoding = xgettext_current_file_source_encoding;
#if HAVE_ICONV
  xgettext_current_source_iconv = xgettext_current_file_source_iconv;
#endif

  continuation_or_nonblank_line = false;

  open_pb = 0;

  phase5_pushback_length = 0;

  f_string_depth = 0;
  new_f_string_level (0, false, false);

  flag_context_list_table = flag_table;
  paren_nesting_depth = 0;
  bracket_nesting_depth = 0;

  init_keywords ();

  /* Eat tokens until eof is seen.  When extract_balanced returns
     due to an unbalanced closing parenthesis, just restart it.  */
  while (!extract_balanced (mlp, token_type_eof,
                            null_context_region (), null_context_list_iterator,
                            arglist_parser_alloc (mlp, NULL)))
    ;

  fp = NULL;
  real_file_name = NULL;
  logical_file_name = NULL;
  line_number = 0;
}