scripts/segmentImport/normalizeCore.ts - master - pkopachevsky/langboard

scripts/segmentImport/normalizeCore.ts
501 строка16 KB
pkopachevsky
Release 0.18.0 maintainability refactor
27 июл 2026, 23:03
27 июл 2026, 23:037cb5dd4
import { basename } from "node:path";
import { slugifyTurkish } from "./common.js";
import type { ManifestAnnotation, SegmentImportManifest } from "./types.js";

type MarkdownHeading = {
  level: number;
  title: string;
  line: number;
};

type TargetCandidate = {
  target: string;
  bodyMarkdown: string;
  line: number;
};

type ResolvedCandidate = TargetCandidate & {
  matchedText: string;
  offsets: number[];
};

export type NormalizeMarkdownInput = {
  markdown: string;
  inputPath: string;
  workSlug: string;
  sectionSlug: string;
  order: number;
  segmentSlug?: string;
  sectionTitle?: string;
  kind?: "line" | "paragraph" | "utterance";
  sourceOverride?: string;
  translationOverride?: string;
};

export type NormalizeMarkdownResult = {
  manifest: SegmentImportManifest;
  warnings: string[];
};

const TURKISH_SPECIFIC = /[çğıöşüÇĞİÖŞÜâîûÂÎÛ]/;
const CYRILLIC = /[А-Яа-яЁё]/;
const COMMON_TURKISH_WORDS = new Set([
  "ama",
  "bir",
  "bu",
  "böyle",
  "çok",
  "çünkü",
  "daha",
  "de",
  "da",
  "gibi",
  "gün",
  "her",
  "için",
  "ile",
  "ne",
  "o",
  "olarak",
  "önce",
  "sonra",
  "şu",
  "var",
  "ve",
  "yok",
]);
const STRUCTURAL_RUSSIAN =
  /разбор|структура|нюанс|смысл|пример|итог|важн|предложен|перевод|глагол|инфинитив|разбира|шаг|прямая речь|грамматическ|особенност/i;

export function normalizeMarkdown(input: NormalizeMarkdownInput): NormalizeMarkdownResult {
  const warnings: string[] = [];
  const sourceText = input.sourceOverride?.trim() || extractSourceText(input.markdown);
  if (!sourceText) {
    throw new Error("could not infer source sentence; pass --source or add it manually in a strict manifest");
  }

  const translation = input.translationOverride?.trim() || extractTranslation(input.markdown);
  if (!translation) warnings.push("translation was not inferred; pass --translation to set it explicitly");

  const segmentSlug = input.segmentSlug ?? segmentSlugFromPath(input.inputPath);
  const annotations = buildAnnotations({
    markdown: input.markdown,
    sourceText,
    translation,
    workSlug: input.workSlug,
    segmentSlug,
    warnings,
  });

  if (!input.sectionTitle) {
    warnings.push(
      `section title defaults to slug ${JSON.stringify(input.sectionSlug)}; pass --section-title if the section may be created`,
    );
  }

  return {
    warnings,
    manifest: {
      schemaVersion: 1,
      source: {
        path: input.inputPath,
        format: "freeform-md",
      },
      work: {
        mode: "ensure",
        id: `work:${input.workSlug}`,
        slug: input.workSlug,
      },
      section: {
        mode: "ensure",
        id: `section:${input.workSlug}:${input.sectionSlug}`,
        slug: input.sectionSlug,
        kind: "section",
        title: input.sectionTitle ?? input.sectionSlug,
        order: 0,
      },
      segment: {
        mode: "ensure",
        id: `segment:${input.workSlug}:${segmentSlug}`,
        slug: segmentSlug,
        kind: input.kind ?? "paragraph",
        order: input.order,
        sourceText,
        ...(translation ? { translation } : {}),
      },
      concepts: [],
      words: [],
      annotations,
    },
  };
}

export function segmentSlugFromPath(path: string) {
  const name = basename(path).replace(/\.md$/i, "");
  const slug = slugifyTurkish(name.replace(/_/g, "-"));
  return slug.replace(/^(sentence|sent|line|verse|paragraph)(\d+)$/i, "$1-$2");
}

export function extractSourceText(markdown: string) {
  const sourceSection = findHeadingSection(
    markdown,
    /^(?!.*разбор).*предложение\s*$|исходн(?:ый|ого)?\s+текст|source\s+text/i,
  );
  if (sourceSection) {
    const contextualCandidates = textCandidates(sourceSection.body);
    const contextual = contextualCandidates.find(isPlausibleSourceText);
    if (contextual) return contextual;
  }

  const codeText = extractFirstTextCodeBlock(markdown);
  if (codeText && isPlausibleSourceText(codeText)) return codeText;

  return boldCandidates(markdown).find(isLikelyTurkish);
}

export function extractTranslation(markdown: string) {
  const section = findHeadingSection(markdown, /перевод/i);
  if (!section) return undefined;

  const lines = section.body.split(/\r?\n/);
  for (let index = 0; index < lines.length; index += 1) {
    const first = displayText(lines[index]);
    if (!isTranslationText(first)) continue;
    if (!first.endsWith(":")) return first;

    for (let next = index + 1; next < lines.length; next += 1) {
      const continuation = displayText(lines[next]);
      if (!continuation || isSeparator(lines[next])) continue;
      if (isTranslationText(continuation) && isQuotedLine(lines[next], continuation)) {
        return `${first} ${continuation}`;
      }
      break;
    }
    return first;
  }

  return undefined;
}

function buildAnnotations(input: {
  markdown: string;
  sourceText: string;
  translation?: string;
  workSlug: string;
  segmentSlug: string;
  warnings: string[];
}) {
  const lineAnnotation: ManifestAnnotation = {
    mode: "ensure",
    id: `annotation:${input.workSlug}:${input.segmentSlug}`,
    kind: "line",
    title: lineAnnotationTitle(input.segmentSlug),
    hint: input.translation ?? firstSubstantiveRussianText(input.markdown) ?? "",
    bodyMarkdown: input.markdown.trim(),
    target: { type: "fullSegment" },
    tags: [],
    links: [],
  };

  const candidates = collectTargetCandidates(input.markdown)
    .map((candidate) => resolveCandidate(input.sourceText, candidate, input.warnings))
    .filter((candidate): candidate is ResolvedCandidate => Boolean(candidate));
  const grouped = groupCandidates(candidates);
  const annotations: ManifestAnnotation[] = [lineAnnotation];
  const usedIds = new Map<string, number>();

  for (const group of grouped.values()) {
    const sourceOffsets = group[0].offsets;
    const assignments = assignOffsets(group, sourceOffsets, input.warnings);
    for (const assignment of assignments) {
      const selectedText = input.sourceText.slice(
        assignment.offset,
        assignment.offset + assignment.candidate.matchedText.length,
      );
      if (wordCount(selectedText) === 1) {
        input.warnings.push(
          `skipped one-word target ${JSON.stringify(selectedText)}: add a canonical Word and constituent.wordId manually`,
        );
        continue;
      }

      const baseId = `annotation:${input.workSlug}:${input.segmentSlug}-${slugifyTurkish(selectedText)}`;
      const id = uniqueId(baseId, usedIds);
      annotations.push({
        mode: "ensure",
        id,
        kind: "phrase",
        title: selectedText,
        hint: firstSubstantiveRussianText(assignment.candidate.bodyMarkdown) ?? selectedText,
        bodyMarkdown: assignment.candidate.bodyMarkdown.trim() || selectedText,
        target: {
          type: "exactText",
          text: selectedText,
          occurrence: occurrenceAtOffset(input.sourceText, selectedText, assignment.offset),
        },
        tags: [],
        links: [],
      });
    }
  }

  if (annotations.length === 1) {
    input.warnings.push(
      "no unambiguous multiword child annotations were inferred; the full analysis remains in the line annotation",
    );
  }

  return annotations;
}

function collectTargetCandidates(markdown: string) {
  const headings = collectHeadings(markdown)
    .filter((heading) => heading.level >= 2)
    .map<TargetCandidate>((heading) => ({
      target: headingTarget(heading.title),
      bodyMarkdown: sectionBody(markdown, heading),
      line: heading.line,
    }));

  const analysisSection = findHeadingSection(markdown, /разбор\s+по|анализ/i);
  const labels = analysisSection ? standaloneBoldCandidates(analysisSection.body, analysisSection.startLine) : [];
  return [...headings, ...labels]
    .filter((candidate) => candidate.target && !CYRILLIC.test(candidate.target))
    .sort((left, right) => left.line - right.line);
}

function groupCandidates(candidates: ResolvedCandidate[]) {
  const groups = new Map<string, ResolvedCandidate[]>();
  for (const candidate of candidates) {
    const key = candidate.matchedText.toLocaleLowerCase("tr-TR");
    groups.set(key, [...(groups.get(key) ?? []), candidate]);
  }
  return groups;
}

function standaloneBoldCandidates(body: string, startLine: number) {
  const lines = body.split(/\r?\n/);
  return lines.flatMap<TargetCandidate>((line, index) => {
    const match = line.match(/^\s*(?:>\s*)?\*\*([^*\n]+)\*\*[.:]?\s*$/);
    if (!match) return [];
    const nextBoundary = lines.findIndex((candidate, candidateIndex) => {
      if (candidateIndex <= index) return false;
      return /^#{1,6}\s+/.test(candidate) || /^\s*(?:>\s*)?\*\*[^*\n]+\*\*[.:]?\s*$/.test(candidate);
    });
    return [
      {
        target: cleanInline(match[1]),
        bodyMarkdown: lines.slice(index + 1, nextBoundary === -1 ? undefined : nextBoundary).join("\n"),
        line: startLine + index + 1,
      },
    ];
  });
}

function resolveCandidate(
  sourceText: string,
  candidate: TargetCandidate,
  warnings: string[],
): ResolvedCandidate | undefined {
  if (!candidate.target || candidate.target === sourceText) return undefined;
  const variants = targetVariants(candidate.target);
  for (const variant of variants) {
    const exactOffsets = findTargetOccurrences(sourceText, variant);
    if (exactOffsets.length > 0) return { ...candidate, matchedText: variant, offsets: exactOffsets };

    const insensitiveOffsets = findCaseInsensitiveOccurrences(sourceText, variant);
    if (insensitiveOffsets.length > 0) {
      const matchedText = sourceText.slice(insensitiveOffsets[0], insensitiveOffsets[0] + variant.length);
      return { ...candidate, matchedText, offsets: insensitiveOffsets };
    }
  }

  if (looksLikeTargetText(candidate.target)) {
    warnings.push(`target not found in source: ${candidate.target}`);
  }
  return undefined;
}

function assignOffsets(group: ResolvedCandidate[], offsets: number[], warnings: string[]) {
  if (offsets.length === 1) return [{ candidate: group[0], offset: offsets[0] }];
  if (group.length === offsets.length) return group.map((candidate, index) => ({ candidate, offset: offsets[index] }));

  for (const candidate of group) {
    warnings.push(
      `skipped ambiguous repeated target ${JSON.stringify(candidate.matchedText)}: ${offsets.length} source occurrences for ${group.length} analysis heading(s)`,
    );
  }
  return [];
}

function uniqueId(baseId: string, usedIds: Map<string, number>) {
  const count = (usedIds.get(baseId) ?? 0) + 1;
  usedIds.set(baseId, count);
  return count === 1 ? baseId : `${baseId}-${count}`;
}

function occurrenceAtOffset(sourceText: string, selectedText: string, offset: number) {
  const offsets = findOccurrences(sourceText, selectedText);
  const index = offsets.indexOf(offset);
  return index === -1 ? 1 : index + 1;
}

function targetVariants(value: string) {
  const cleaned = headingTarget(value);
  const unquoted = cleaned.replace(/^["“”«»]+|["“”«».,!?;:]+$/g, "").trim();
  return [...new Set([cleaned, unquoted].filter(Boolean))];
}

function findOccurrences(text: string, match: string) {
  const offsets: number[] = [];
  let index = text.indexOf(match);
  while (index !== -1) {
    offsets.push(index);
    index = text.indexOf(match, index + Math.max(match.length, 1));
  }
  return offsets;
}

function findCaseInsensitiveOccurrences(text: string, match: string) {
  const loweredText = text.toLocaleLowerCase("tr-TR");
  const loweredMatch = match.toLocaleLowerCase("tr-TR");
  return findOccurrences(loweredText, loweredMatch).filter((offset) => {
    return (
      text.slice(offset, offset + match.length).toLocaleLowerCase("tr-TR") === loweredMatch &&
      isFragmentBoundary(text, match, offset)
    );
  });
}

function findTargetOccurrences(text: string, match: string) {
  return findOccurrences(text, match).filter((offset) => isFragmentBoundary(text, match, offset));
}

function isFragmentBoundary(text: string, match: string, offset: number) {
  const before = offset > 0 ? text[offset - 1] : undefined;
  const afterOffset = offset + match.length;
  const after = afterOffset < text.length ? text[afterOffset] : undefined;
  const startsWithWordCharacter = /^[\p{L}\p{N}]/u.test(match);
  const endsWithWordCharacter = /[\p{L}\p{N}]$/u.test(match);
  if (startsWithWordCharacter && before && /[\p{L}\p{N}'’]/u.test(before)) return false;
  if (endsWithWordCharacter && after && /[\p{L}\p{N}'’]/u.test(after)) return false;
  return true;
}

function textCandidates(markdown: string) {
  const codeText = extractFirstTextCodeBlock(markdown);
  return [...(codeText ? [codeText] : []), ...boldCandidates(markdown)];
}

function boldCandidates(markdown: string) {
  return [...markdown.matchAll(/(?:^|\n)\s*>?\s*\*\*([^*\n]+)\*\*/g)].map((match) => cleanInline(match[1]));
}

function extractFirstTextCodeBlock(markdown: string) {
  const match = markdown.match(/```(?:text)?\s*\n([\s\S]*?)```/i);
  const text = match?.[1].trim();
  return text || undefined;
}

function isPlausibleSourceText(value: string) {
  const text = value.trim();
  return text.length >= 2 && !CYRILLIC.test(text) && (text.match(/\p{L}+/gu)?.length ?? 0) >= 2;
}

function isLikelyTurkish(value: string) {
  if (!isPlausibleSourceText(value)) return false;
  if (TURKISH_SPECIFIC.test(value)) return true;
  const words = value.toLocaleLowerCase("tr-TR").match(/\p{L}+/gu) ?? [];
  return words.some((word) => COMMON_TURKISH_WORDS.has(word));
}

function findHeadingSection(markdown: string, titlePattern: RegExp) {
  const lines = markdown.split(/\r?\n/);
  const heading = collectHeadings(markdown).find((candidate) => titlePattern.test(candidate.title));
  if (!heading) return undefined;
  const end = lines.findIndex((line, index) => {
    if (index <= heading.line) return false;
    const match = line.match(/^(#{1,6})\s+/);
    return Boolean(match && match[1].length <= heading.level);
  });
  return {
    body: lines.slice(heading.line + 1, end === -1 ? undefined : end).join("\n"),
    startLine: heading.line + 1,
  };
}

function collectHeadings(markdown: string) {
  return markdown.split(/\r?\n/).flatMap<MarkdownHeading>((line, index) => {
    const match = line.match(/^(#{1,6})\s+(.+)$/);
    if (!match) return [];
    return [{ level: match[1].length, title: cleanInline(match[2]), line: index }];
  });
}

function headingTarget(title: string) {
  return cleanInline(title)
    .replace(/^\s*\d+\s*[.)-]\s*/, "")
    .replace(/^[^\p{L}\p{N}'’]+/u, "")
    .trim();
}

function sectionBody(markdown: string, heading: MarkdownHeading) {
  const lines = markdown.split(/\r?\n/);
  const end = lines.findIndex((line, index) => {
    if (index <= heading.line) return false;
    const match = line.match(/^(#{1,6})\s+/);
    return Boolean(match && match[1].length <= heading.level);
  });
  return lines.slice(heading.line + 1, end === -1 ? undefined : end).join("\n");
}

function displayText(line: string) {
  const trimmed = line.trim();
  if (!trimmed || isSeparator(trimmed)) return "";
  const withoutQuote = trimmed.replace(/^>\s*/, "");
  const bold = withoutQuote.match(/^\*\*([\s\S]+?)\*\*[.:]?$/)?.[1];
  return cleanInline(bold ?? withoutQuote);
}

function isTranslationText(value: string) {
  return Boolean(value && CYRILLIC.test(value) && !STRUCTURAL_RUSSIAN.test(value));
}

function isQuotedLine(rawLine: string, value: string) {
  return /^\s*>/.test(rawLine) || /^[«“"]/.test(value);
}

function isSeparator(value: string) {
  return /^\s*-{3,}\s*$/.test(value);
}

function cleanInline(value: string) {
  return value
    .replace(/[*_`]/g, "")
    .replace(/\\"/g, '"')
    .replace(/^[-–—:：\s]+|[-–—\s]+$/g, "")
    .trim();
}

function looksLikeTargetText(value: string) {
  if (!value || CYRILLIC.test(value)) return false;
  return TURKISH_SPECIFIC.test(value) || /^[A-Za-z0-9'’\-\s!?.,:;"“”«»]+$/.test(value);
}

function firstSubstantiveRussianText(markdown: string) {
  const text = markdown
    .split(/\r?\n/)
    .map((line) => displayText(line.replace(/^#{1,6}\s+/, "")))
    .find((line) => line && CYRILLIC.test(line) && !STRUCTURAL_RUSSIAN.test(line));
  if (!text) return undefined;
  return text.length > 120 ? `${text.slice(0, 117)}...` : text;
}

function lineAnnotationTitle(segmentSlug: string) {
  const sentence = segmentSlug.match(/^sentence-(\d+)$/i);
  return sentence ? `Предложение ${sentence[1]}` : `Разбор ${segmentSlug}`;
}

function wordCount(value: string) {
  return value.trim().split(/\s+/).filter(Boolean).length;
}