ru_tts

1
/* transcription.c -- Phonetic transcription related functions
2
 *
3
 * Copyright (C) 1990, 1991 Speech Research Laboratory, Minsk
4
 * Copyright (C) 2005 Igor Poretsky <poretsky@mlbox.ru>
5
 * Copyright (C) 2021 Boris Lobanov <lobbormef@gmail.com>
6
 * Copyright (C) 2021 Alexander Ivanov <ivalex01@gmail.com>
7
 *
8
 * SPDX-License-Identifier: MIT
9
 */
10

11
#include <stdint.h>
12
#include <stdlib.h>
13
#include <string.h>
14

15
#include "transcription.h"
16
#include "numerics.h"
17
#include "sink.h"
18

19

20
/* Internal flags */
21
#define CLAUSE_START 0x10
22
#define WEAK_STRESS 0x20
23

24

25
/* Local macros */
26
#define PAIR(a, b) ((((uint16_t)(a)) << 8) | (((uint16_t)(b)) & 0xFF))
27
#ifdef _MSC_VER
28
#define strdup(p) _strdup(p)
29
#endif
30

31

32
/* Main punctuations */
33
const char *punctuations = ",.;:?!-";
34

35

36
/* Local data */
37

38
/* Special symbols */
39
static const char *symbols = " ,.;:?!()-+=\"$%&*";
40

41
/* Accepted characters list */
42
static const char *char_list = "TNRLMDPZG^JH_WC[FOE\\UQYX]`a'-*()%\"/&$><@+=";
43

44
/* Special symbols treated as blanks */
45
static const char *blanks = "\t#'/<>@";
46

47
/* Internal letters representation table */
48
static const char *letters = "`ABCDEFGHIJKLMNOPQRSTU_VXYZWa[^]+=";
49

50
/* Phonetic letter classes */
51
static const char *consonants = "JMNRL^HC[WSPFTK_ZBVDG";
52
static const char *vocalics = "`EI\\QUaYOA";
53
static const char *ndts = "NDTS";
54
static const char *bgdjz = "BGD_Z";
55

56
/* Phoncodes */
57
static const uint8_t vocal_phoncodes[] = { 0, 3, 4, 1, 2 };
58
static const uint8_t ndts_soft_phs[] = { 19, 24, 30, 38 };
59
static const uint8_t hard_consonant_phs[] =
60
  {
61
    10, 15, 16, 8,  14, 33, 40, 32, 39,
62
    36, 35, 26, 34, 27, 28,
63
    9,  7,  20, 6,  21, 22
64
  };
65
static const uint8_t soft_consonant_phs[] =
66
  {
67
    10, 18, 19, 13, 17, 33, 41, 32, 39,
68
    36, 38, 29, 37, 30, 31,
69
    9,  12, 23, 11, 24, 25
70
  };
71

72
/* Predefined transcription blocks */
73
static const uint8_t transcription_blocks[] =
74
  {
75
    3, 27, 3, 53,
76
    3, 3, 53, 16,
77
    3, 3, 53, 8,
78
    3, 3, 53, 17,
79
    3, 3, 53, 15,
80
    3, 21, 3, 53,
81
    3, 26, 3, 53,
82
    3, 7, 3, 53,
83
    3, 22, 3, 53,
84
    3, 33, 3, 53,
85
    10, 5, 28, 8, 2, 53, 27, 28, 2, 10, 3,
86
    3, 40, 2, 53,
87
    3, 9, 3, 53,
88
    3, 36, 2, 53,
89
    3, 32, 3, 53,
90
    3, 39, 2, 53,
91
    3, 3, 53, 34,
92
    2, 1, 53,
93
    3, 10, 3, 53,
94
    3, 10, 1, 53,
95
    2, 0, 53,
96
    3, 10, 2, 53,
97
    2, 4, 53,
98
    13, 18, 2, 53, 40, 28, 5, 10, 43, 7, 16, 2, 53, 28,
99
    14, 27, 11, 1, 53, 8, 21, 4, 10, 43, 7, 16, 2, 53, 28,
100
    3, 10, 0, 53,
101
    2, 3, 53,
102
    8, 28, 2, 6, 4, 53, 33, 31, 5,
103
    0,
104
    8, 0, 15, 16, 1, 53, 9, 5, 30,
105
    15, 2, 27, 28, 8, 4, 53, 30, 43, 35, 28, 1, 53, 26, 28, 0,
106
    15, 7, 2, 28, 8, 4, 53, 30, 43, 35, 28, 1, 53, 26, 28, 0,
107
    10, 26, 8, 2, 32, 3, 53, 16, 27, 2, 34,
108
    8, 28, 2, 6, 4, 53, 33, 31, 5,
109
    5, 21, 8, 1, 53, 23,
110
    10, 2, 15, 26, 3, 8, 35, 3, 53, 16, 21,
111
    8, 21, 1, 53, 14, 2, 8, 2, 34,
112
    6, 20, 1, 53, 17, 36, 3,
113
    6, 18, 3, 53, 19, 36, 3,
114
    9, 26, 2, 8, 2, 53, 22, 8, 2, 34,
115
    5, 29, 17, 0, 53, 35,
116
    11, 8, 2, 6, 19, 2, 53, 10, 3, 27, 38, 2,
117

118
    4, 1, 53, 6, 2, /* 42: O+GO */
119
    4, 2, 6, 1, 53, /* 43: OGO+ */
120
    3, 2, 6, 2, /* 44: OGO */
121
    4, 3, 53, 6, 2, /* 45: E+GO (1) */
122
    4, 3, 6, 1, 53, /* 46: EGO+ (1) */
123
    3, 3, 6, 2, /* 47: EGO (1) */
124
    5, 10, 3, 53, 6, 2, /* 48: E+GO (2) */
125
    5, 10, 3, 6, 1, 53, /* 49: EGO+ (2) */
126
    4, 10, 3, 6, 2, /* 50: EGO (2) */
127
    3, 27, 35, 2 /* 51: TSA */
128
  };
129

130
/* Clause termination pairs */
131
static const uint16_t clause_terminations[] =
132
  {
133
    PAIR(',', ' '),
134
    PAIR(',', ','),
135
    PAIR(',', ';'),
136
    PAIR(',', ':'),
137
    PAIR(',', '-'),
138
    PAIR('.', ' '),
139
    PAIR('.', '.'),
140
    PAIR(';', ' '),
141
    PAIR(':', ' '),
142
    PAIR('.', ':'),
143
    PAIR('?', ' '),
144
    PAIR('?', '.'),
145
    PAIR('?', ','),
146
    PAIR('!', ' '),
147
    PAIR('!', '.'),
148
    PAIR('!', ',')
149
  };
150

151

152
/* Local subroutines */
153

154
/* Put specified block to the transcription buffer */
155
static void put_transcription_block(sink_t *consumer, uint8_t n)
156
{
157
  transcription_state_t *transcription = consumer->user_data;
158
  const uint8_t *block = list_item(transcription_blocks, n);
159
  uint8_t i;
160

161
  for (i = 1; i <= block[0]; i++)
162
    {
163
      uint8_t c = block[i];
164
      sink_put(consumer, ((c == 53) && (transcription->flags & WEAK_STRESS)) ? 54 : c);
165
    }
166
}
167

168
/*
169
 * Detect specified suffix in the input text
170
 * and make all necessary changes in the referenced control data.
171
 *
172
 * Returns non-zero value when the match succeeded.
173
 */
174
static int detect_suffix(input_t *input, transcription_state_t *transcription, const char *suffix)
175
{
176
  int rc = 0;
177
  uint8_t n = strlen(suffix);
178
  uint8_t i;
179

180
  transcription->flags &= ~WEAK_STRESS;
181
  for (i = 0; i < n; i++)
182
    if (input->start[i] != suffix[i])
183
      {
184
        if ((suffix[i] == '+') && (input->start[i] == '='))
185
          transcription->flags |= WEAK_STRESS;
186
        else break;
187
      }
188

189
  if ((i == n) && (((input->start + n) >= input->end) || !strchr(letters, input->start[n])))
190
    {
191
      input->start += n - 1;
192
      rc = 1;
193
    }
194

195
  return rc;
196
}
197

198
/*
199
 * Check if clause transcription is terminated. If it is the case,
200
 * terminates transcription and flushes it to the consumer.
201
 * Returns non-zero value on success.
202
 */
203
static int check_clause_termination(input_t *input, sink_t *consumer)
204
{
205
  transcription_state_t *transcription = consumer->user_data;
206
  char *s = memchr(symbols + 1, input->start[0], 6);
207
  int result = 0;
208

209
  if (s)
210
    {
211
      char c = *(input->start++);
212
      char nextc = (input->start < input->end) ? input->start[0] : ' ';
213
      uint16_t termination = PAIR(c, strchr(punctuations, nextc) ? nextc : ' ');
214
      uint8_t i;
215
      for (i = 0; (i < (sizeof(clause_terminations) / sizeof(uint16_t))) && (clause_terminations[i] != termination); i++);
216
      transcription->clause_type = i & 0x0F;
217
      transcription->flags |= CLAUSE_DONE;
218
      sink_put(consumer, s - symbols + 43);
219
      sink_flush(consumer);
220
      result = 1;
221
    }
222

223
  return result;
224
}
225

226
/*
227
 * Get consonant phoncode by index from specified set
228
 * voicifying the unvoiced ones.
229
 */
230
static uint8_t voicify(const uint8_t *phs, uint8_t idx)
231
{
232
  return phs[(idx < 15) ? (idx + 6) : idx];
233
}
234

235
/*
236
 * Get consonant phoncode by index from specified set
237
 * unvoicifying the voiced ones.
238
 */
239
static uint8_t unvoicify(const uint8_t *phs, uint8_t idx)
240
{
241
  return phs[(idx < 15) ? idx : (idx - 6)];
242
}
243

244
/*
245
 * Get hard consonant phoncode by its index
246
 * unvoicifying the voiced ones and changing if necessary
247
 * in accordance with the following character.
248
 */
249
static uint8_t unvoicify_hard(uint8_t idx, char following)
250
{
251
  return (((idx != 10) && (idx != 16)) || (following != 'W')) ?
252
    unvoicify(hard_consonant_phs, idx) : 36;
253
}
254

255
/*
256
 * Get correct consonant phoncode by its index
257
 * depending on the following character specified.
258
 */
259
static uint8_t correct_consonant(uint8_t idx, char following)
260
{
261
  if (memchr(consonants + 5, following, 10))
262
    return unvoicify_hard(idx, following);
263
  else if (strchr(bgdjz, following))
264
    return (((idx != 10) && (idx != 16)) || (following != '_')) ?
265
      voicify(hard_consonant_phs, idx) : 9;
266
  return ((idx != 16) || (following != '_')) ?
267
    hard_consonant_phs[idx] : 9;
268
}
269

270
/* Transcription cycle initialization actions */
271
static void transcription_init(sink_t *consumer)
272
{
273
  uint8_t *buffer = consumer->buffer;
274
  memset(buffer, 43, TRANSCRIPTION_BUFFER_SIZE);
275
  consumer->buffer_offset = TRANSCRIPTION_START;
276
}
277

278

279
/* Common functions */
280

281
/* Get an item from the list by its number */
282
const uint8_t *list_item(const uint8_t *lst, uint8_t n)
283
{
284
  const uint8_t *item = lst;
285
  uint8_t i;
286

287
  for (i = 0; i < n; i++)
288
    item += item[0] + 1;
289
  return item;
290
}
291

292
/* Transcribe specified text clause by clause and pass result to the consumer */
293
void process_text(const char *text, sink_t *consumer)
294
{
295
  input_t input;
296
  transcription_state_t *transcription = consumer->user_data;
297
  char *s;
298
  int accented = 0;
299

300
  consumer->custom_reset = transcription_init;
301
  input.text = strdup(text);
302
  if (!input.text)
303
    return;
304

305
  input.start = input.text;
306
  input.end = input.text;
307
  transcription->flags = 0;
308

309
  for (s = input.text; *s; s++)
310
    {
311
      unsigned char c = *s;
312

313
      switch (c)
314
        {
315
        case '\n':
316
        case '\r':
317
          c = '\r';
318
          break;
319
        case 'j':
320
        case 'J':
321
          c = '_';
322
          break;
323
        case 'q':
324
        case 'Q':
325
        case 'x':
326
        case 'X':
327
          c = 'K';
328
          break;
329
        case 'w':
330
        case 'W':
331
          c = 'U';
332
          break;
333
        case 'y':
334
        case 'Y':
335
          c = 'I';
336
          break;
337
        case 163:
338
        case 179:
339
          c = '\\';
340
          break;
341
        default:
342
          if (strchr(blanks, c))
343
            c = ' ';
344
          else if (c > 191)
345
            c = letters[(c - 192) & 31];
346
          else if ((c >= 'a') && (c <= 'z'))
347
            c -= 0x20;
348
          else if (((c < 'A') && !strchr(symbols, c) && !IS_DIGIT(c)) || (c > 'Z'))
349
            c = 0;
350
          break;
351
        }
352

353
      if (c)
354
        {
355
          const char *sptr = strchr(symbols, c);
356
          if (sptr)
357
            {
358
              int sidx = sptr - symbols;
359
              unsigned char nextc = s[1];
360
              if ((sidx > 6) ||
361
                  ((transcription->flags & CLAUSE_START) &&
362
                   ((c != ' ') || (nextc == '\r') ||
363
                    IS_DIGIT(nextc) || (nextc >= 'A'))))
364
                *(input.end)++ = c;
365
            }
366
          else
367
            {
368
              *(input.end)++ = c;
369
              transcription->flags |= CLAUSE_START;
370
            }
371
        }
372
    }
373
  *(input.end) = 0;
374

375
  for (s = input.start; (s < input.end) && (s[0] < 'A') && !IS_DIGIT(s[0]); s++);
376
  if (s >= input.end)
377
    {
378
      free(input.text);
379
      return;
380
    }
381

382
  while ((input.start < input.end) && !consumer->status)
383
    {
384
      unsigned char last_char = 0;
385

386
      while ((input.start < input.end) && memchr(symbols, input.start[0], 7))
387
        input.start++;
388
      sink_reset(consumer);
389
      for (transcription->flags = CLAUSE_START; (input.start < input.end) && (consumer->buffer_offset < TRANSCRIPTION_MAXLEN) && !consumer->status; input.start++)
390
        {
391
          char *s;
392
          unsigned char c = input.start[0];
393

394
          if (transcription->flags & CLAUSE_START)
395
            {
396
              accented = 0;
397
              for (s = input.start; s < input.end; s++)
398
                if ((s[0] == '+') || (s[0] == '='))
399
                  {
400
                    accented = 1;
401
                    break;
402
                  }
403
                else if (s[0] < 'A')
404
                  break;
405
            }
406

407
          s = strchr(char_list, c);
408
          if (s)
409
            {
410
              uint8_t char_index = s - char_list;
411
              if ((char_index < 17) &&
412
                  (last_char != '+') &&
413
                  (last_char != '=') &&
414
                  (last_char < 'A') &&
415
                  (input.start[1] < 'A'))
416
                {
417
                  put_transcription_block(consumer, char_index);
418
                  transcription->flags |= CLAUSE_START;
419
                  last_char = c;
420
                  continue;
421
                }
422
              else if (char_index > 26)
423
                {
424
                  uint8_t prev = sink_last(consumer);
425
                  if (((c != '+') && (c != '=')) || (prev > 5))
426
                    {
427
                      if ((prev < 43) || (prev > 52))
428
                        sink_put(consumer, 43);
429
                      put_transcription_block(consumer, char_index);
430
                      if ((c != '-') && (input.start[1] >= 'A'))
431
                        sink_put(consumer, 43);
432
                      transcription->flags |= CLAUSE_START;
433
                    }
434
                  else
435
                    {
436
                      sink_put(consumer, (c != '+') ? 54 : 53);
437
                      transcription->flags &= ~CLAUSE_START;
438
                    }
439
                  last_char = c;
440
                  continue;
441
                }
442
            }
443

444
          /* Some suffixes are transcribed specially when accented */
445
          if (accented)
446
            {
447
              if (detect_suffix(&input, transcription, "O+GO"))
448
                {
449
                  put_transcription_block(consumer, 42);
450
                  last_char = 'A';
451
                  continue;
452
                }
453
              else if (detect_suffix(&input, transcription, "E+GO"))
454
                {
455
                  s = input.start - 4;
456
                  put_transcription_block(consumer, ((s >= input.text) && strchr(consonants, *s)) ? 45 : 48);
457
                  last_char = 'A';
458
                  continue;
459
                }
460
              else if (detect_suffix(&input, transcription, "EGO+"))
461
                {
462
                  s = input.start - 4;
463
                  put_transcription_block(consumer, ((s >= input.text) && strchr(consonants, *s)) ? 46 : 49);
464
                  last_char = 'O';
465
                  continue;
466
                }
467
              else if (detect_suffix(&input, transcription, "OGO+"))
468
                {
469
                  put_transcription_block(consumer, 43);
470
                  last_char = 'O';
471
                  continue;
472
                }
473
              else if (detect_suffix(&input, transcription, "EGO"))
474
                {
475
                  s = input.start - 3;
476
                  put_transcription_block(consumer, ((s >= input.text) && strchr(consonants, *s)) ? 47 : 50);
477
                  last_char = 'A';
478
                  continue;
479
                }
480
              else if (detect_suffix(&input, transcription, "OGO"))
481
                {
482
                  put_transcription_block(consumer, 44);
483
                  last_char = 'A';
484
                  continue;
485
                }
486
              else if (detect_suffix(&input, transcription, "TSQ"))
487
                {
488
                  put_transcription_block(consumer, 51);
489
                  last_char = 'A';
490
                  continue;
491
                }
492
              else if (detect_suffix(&input, transcription, "TXSQ"))
493
                {
494
                  put_transcription_block(consumer, 51);
495
                  last_char = 'A';
496
                  continue;
497
                }
498
            }
499

500
          if (IS_DIGIT(c))
501
            {
502
              sink_flush(consumer);
503
              process_number(&input, consumer);
504
              if (check_clause_termination(&input, consumer))
505
                transcription->flags |= CLAUSE_START;
506
              else sink_flush(consumer);
507
              last_char = ' ';
508
              if (input.start[0] != ' ')
509
                input.start--;
510
              continue;
511
            }
512

513
          if (check_clause_termination(&input, consumer))
514
            break;
515

516
          s = strchr(vocalics, c);
517
          if (s)
518
            {
519
              uint8_t vc = (c == 'I') ? 5 :
520
                (((c == 'O') &&accented && (input.start[1] != '+') && (input.start[1] != '=')) ?
521
                 2 : vocal_phoncodes[(s - vocalics) % 5]);
522
              transcription->flags &= ~CLAUSE_START;
523
              if (input.start > input.text)
524
                {
525
                  unsigned char prevc = *(input.start - 1);
526
                  if (prevc != 'X')
527
                    {
528
                      if ((strchr(vocalics, prevc) || memchr(symbols, prevc, 13) ||
529
                           (prevc == ']')) &&
530
                          strchr("`QE\\", c))
531
                        sink_put(consumer, 10);
532
                    }
533
                  else if (strchr("`QE\\IO", c))
534
                    sink_put(consumer, 10);
535
                }
536
              else if (strchr("`QE\\", c))
537
                sink_put(consumer, 10);
538
              sink_put(consumer, vc);
539
              last_char = c;
540
              continue;
541
            }
542

543
          if (strchr(ndts, c))
544
            {
545
              unsigned char nextc = input.start[1];
546
              if (memchr(vocalics, nextc, 5) || (nextc == 'X'))
547
                {
548
                  s = strchr(ndts, last_char);
549
                  if (s)
550
                    sink_replace(consumer, ndts_soft_phs[s - ndts]);
551
                }
552
            }
553

554
          s = strchr(consonants, c);
555
          if (s)
556
            {
557
              unsigned char nextc = ((input.end - input.start) > 1) ? input.start[1] : ',';
558
              uint8_t idx = s - consonants;
559
              transcription->flags &= ~CLAUSE_START;
560
              if (idx < 9)
561
                {
562
                  if (nextc == 'X')
563
                    {
564
                      input.start++;
565
                      sink_put(consumer, soft_consonant_phs[idx]);
566
                    }
567
                  else sink_put(consumer, (memchr(vocalics, nextc, 5) ? soft_consonant_phs : hard_consonant_phs)[idx]);
568
                }
569
              else if (nextc == 'X')
570
                {
571
                  input.start++;
572
                  nextc = ((input.end - input.start) > 1) ? input.start[1] : ',';
573
                  if ((memchr(symbols + 1, nextc, 6) && (sink_last(consumer) != 43)) || memchr(consonants + 5, nextc, 10))
574
                    sink_put(consumer, unvoicify(soft_consonant_phs, idx));
575
                  else if (strchr(bgdjz, nextc))
576
                    sink_put(consumer, voicify(soft_consonant_phs, idx));
577
                  else sink_put(consumer, soft_consonant_phs[idx]);
578
                }
579
              else if (memchr(vocalics, nextc, 5))
580
                sink_put(consumer, soft_consonant_phs[idx]);
581
              else if (memchr(symbols + 1, nextc, 6))
582
                sink_put(consumer, (sink_last(consumer) != 43) ? unvoicify_hard(idx, nextc) : hard_consonant_phs[idx]);
583
              else sink_put(consumer, correct_consonant(idx, (nextc != ' ') ? nextc : input.start[2]));
584
            }
585
          else if (c != ']')
586
            {
587
              transcription->flags |= CLAUSE_START;
588
              sink_put(consumer, (c != '#') ? 43 : 42);
589
            }
590
          else transcription->flags &= ~CLAUSE_START;
591
          last_char = c;
592
        }
593
      sink_flush(consumer);
594
    }
595
  free(input.text);
596
}
597