src/EditorFeatures/CSharpTest2/EmbeddedLanguages/RegularExpressions/CSharpRegexParserTests.cs - main - githubmirror/roslyn

src/EditorFeatures/CSharpTest2/EmbeddedLanguages/RegularExpressions/CSharpRegexParserTests.cs
392 строки14 KB
Jason Malinowski
Switch code that was using Assert.True/False as a replacement for .Fail
21 апр 2026, 23:28
21 апр 2026, 23:283bf59ba
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

#nullable disable

using System;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Xml.Linq;
using Microsoft.CodeAnalysis.CSharp.EmbeddedLanguages.VirtualChars;
using Microsoft.CodeAnalysis.EmbeddedLanguages.Common;
using Microsoft.CodeAnalysis.EmbeddedLanguages.RegularExpressions;
using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars;
using Microsoft.CodeAnalysis.Text;
using Roslyn.Test.Utilities;
using Roslyn.Utilities;
using Xunit;

namespace Microsoft.CodeAnalysis.CSharp.UnitTests.EmbeddedLanguages.RegularExpressions;

using RegexToken = EmbeddedSyntaxToken<RegexKind>;
using RegexTrivia = EmbeddedSyntaxTrivia<RegexKind>;

public sealed partial class CSharpRegexParserTests
{
    private readonly IVirtualCharService _service = CSharpVirtualCharService.Instance;
    private const string _statementPrefix = "var v = ";

    private static SyntaxToken GetStringToken(string text)
    {
        var statement = _statementPrefix + text;
        var parsedStatement = SyntaxFactory.ParseStatement(statement);
        var token = parsedStatement.DescendantTokens().ToArray()[3];
        Assert.Equal(SyntaxKind.StringLiteralToken, token.Kind());

        return token;
    }

    private void Test(
        string stringText, string expected, RegexOptions options,
        bool runtimeHasBug = false)
    {
        var (tree, sourceText) = TryParseTree(stringText, options, conversionFailureOk: false, runtimeHasBug);

        TryParseSubTrees(stringText, options, runtimeHasBug);

        var actual = TreeToText(sourceText, tree)
            .Replace("&quot;", """
            "
            """);
        AssertEx.Equal(expected, actual);
    }

    private void TryParseSubTrees(string stringText, RegexOptions options, bool runtimeHasBug = false)
    {
        // Trim the input from the right and make sure tree invariants hold
        var current = stringText;
        while (current is not """
            @""
            """ and not """
            ""
            """)
        {
            current = current[..^2] + """
                "
                """;
            TryParseTree(current, options, conversionFailureOk: true, runtimeHasBug);
        }

        // Trim the input from the left and make sure tree invariants hold
        current = stringText;
        while (current is not """
            @""
            """ and not """
            ""
            """)
        {
            if (current[0] == '@')
            {
                current = """
                    @"
                    """ + current[3..];
            }
            else
            {
                current = """
                    "
                    """ + current[2..];
            }

            TryParseTree(current, options, conversionFailureOk: true, runtimeHasBug);
        }

        for (var start = stringText[0] == '@' ? 2 : 1; start < stringText.Length - 1; start++)
        {
            TryParseTree(
                stringText[..start] +
                stringText[(start + 1)..],
                options,
                conversionFailureOk: true,
                runtimeHasBug);
        }
    }

    private (SyntaxToken, RegexTree, VirtualCharSequence) JustParseTree(
        string stringText, RegexOptions options, bool conversionFailureOk)
    {
        var token = GetStringToken(stringText);
        var allChars = _service.TryConvertToVirtualChars(token);
        if (allChars.IsDefault)
        {
            Assert.True(conversionFailureOk, "Failed to convert text to token.");
            return (token, null, allChars);
        }

        var tree = RegexParser.TryParse(allChars, options);
        return (token, tree, allChars);
    }

    private (RegexTree, SourceText) TryParseTree(
        string stringText, RegexOptions options, bool conversionFailureOk,
        bool runtimeHasBug = false)
    {
        var (token, tree, allChars) = JustParseTree(stringText, options, conversionFailureOk);
        if (tree == null)
        {
            Assert.True(allChars.IsDefault);
            return default;
        }

        CheckInvariants(tree, allChars);
        var sourceText = token.SyntaxTree.GetText();
        var treeAndText = (tree, sourceText);

        Regex regex = null;
        try
        {
            regex = new Regex(token.ValueText, options);
        }
        catch (ArgumentException ex)
        {
            if (runtimeHasBug)
            {
                // The .NET runtime has a bug where it rejects this pattern. Our parser correctly
                // accepts it. Skip runtime validation. See https://github.com/dotnet/runtime/issues/111633
                return treeAndText;
            }

            Assert.NotEmpty(tree.Diagnostics);

            // Ensure the diagnostic we emit is the same as the .NET one. Note: we can only
            // do this in en-US as that's the only culture where we control the text exactly
            // and can ensure it exactly matches Regex.  We depend on localization to do a
            // good enough job here for other languages.
            if (Thread.CurrentThread.CurrentCulture.Name == "en-US")
            {
                var result = tree.Diagnostics.Any(d => ex.Message.Contains(d.Message));
                Assert.True(result);
            }

            return treeAndText;
        }

        if (!tree.Diagnostics.IsEmpty)
        {
            var expectedDiagnostics = CreateDiagnosticsElement(sourceText, tree);
            Assert.Fail("Expected diagnostics: \r\n" + expectedDiagnostics.ToString().Replace("""
                "
                """, """
                ""
                """));
        }

        Assert.True(regex.GetGroupNumbers().OrderBy(v => v).SequenceEqual(
            tree.CaptureNumbersToSpan.Keys.OrderBy(v => v)));

        Assert.True(regex.GetGroupNames().Where(v => !int.TryParse(v, out _)).OrderBy(v => v).SequenceEqual(
            tree.CaptureNamesToSpan.Keys.OrderBy(v => v)));

        return treeAndText;
    }

    private static string TreeToText(SourceText text, RegexTree tree)
    {
        var element = new XElement("Tree",
            NodeToElement(tree.Root));

        if (tree.Diagnostics.Length > 0)
        {
            element.Add(CreateDiagnosticsElement(text, tree));
        }

        element.Add(new XElement("Captures",
            tree.CaptureNumbersToSpan.OrderBy(kvp => kvp.Key).Select(kvp =>
                new XElement("Capture", new XAttribute("Name", kvp.Key), new XAttribute("Span", kvp.Value), GetTextAttribute(text, kvp.Value))),
            tree.CaptureNamesToSpan.OrderBy(kvp => kvp.Key).Select(kvp =>
                new XElement("Capture", new XAttribute("Name", kvp.Key), new XAttribute("Span", kvp.Value), GetTextAttribute(text, kvp.Value)))));

        return element.ToString();
    }

    private static XElement CreateDiagnosticsElement(SourceText text, RegexTree tree)
        => new("Diagnostics",
            tree.Diagnostics.Select(d =>
                new XElement("Diagnostic",
                    new XAttribute("Message", d.Message),
                    new XAttribute("Span", d.Span),
                    GetTextAttribute(text, d.Span))));

    private static XAttribute GetTextAttribute(SourceText text, TextSpan span)
        => new("Text", text.ToString(span));

    private static XElement NodeToElement(RegexNode node)
    {
        if (node is RegexAlternationNode alternationNode)
            return AlternationToElement(alternationNode, alternationNode.SequenceList.NodesAndTokens.Length);

        var element = new XElement(node.Kind.ToString());
        foreach (var child in node)
            element.Add(child.IsNode ? NodeToElement(child.Node) : TokenToElement(child.Token));

        return element;
    }

    private static XElement AlternationToElement(RegexAlternationNode alternationNode, int end)
    {
        // to keep tests in sync with how we used to structure alternations, we specially handle this node.
        // First, if the node only has a single element, then just print that element as that's what would
        // normally be inlined into the parent.
        if (end == 1)
            return NodeToElement(alternationNode.SequenceList.NodesAndTokens[0].Node);

        var element = new XElement(alternationNode.Kind.ToString());
        element.Add(AlternationToElement(alternationNode, end - 2));
        element.Add(TokenToElement(alternationNode.SequenceList.NodesAndTokens[end - 2].Token));
        element.Add(NodeToElement(alternationNode.SequenceList.NodesAndTokens[end - 1].Node));
        return element;
    }

    private static XElement TokenToElement(RegexToken token)
    {
        var element = new XElement(token.Kind.ToString());

        if (token.Value != null)
        {
            element.Add(new XAttribute("value", token.Value));
        }

        if (token.LeadingTrivia.Length > 0)
        {
            element.Add(new XElement("Trivia", token.LeadingTrivia.Select(t => TriviaToElement(t))));
        }

        if (token.VirtualChars.Length > 0)
        {
            element.Add(token.VirtualChars.CreateString());
        }

        return element;
    }

    private static XElement TriviaToElement(RegexTrivia trivia)
        => new(
            trivia.Kind.ToString(),
            trivia.VirtualChars.CreateString());

    private static void CheckInvariants(RegexTree tree, VirtualCharSequence allChars)
    {
        var root = tree.Root;
        var position = 0;
        CheckInvariants(root, ref position, allChars);
        Assert.Equal(allChars.Length, position);
    }

    private static void CheckInvariants(RegexNode node, ref int position, VirtualCharSequence allChars)
    {
        foreach (var child in node)
        {
            if (child.IsNode)
            {
                CheckInvariants(child.Node, ref position, allChars);
            }
            else
            {
                CheckInvariants(child.Token, ref position, allChars);
            }
        }
    }

    private static void CheckInvariants(RegexToken token, ref int position, VirtualCharSequence allChars)
    {
        CheckInvariants(token.LeadingTrivia, ref position, allChars);
        CheckCharacters(token.VirtualChars, ref position, allChars);
    }

    private static void CheckInvariants(ImmutableArray<RegexTrivia> leadingTrivia, ref int position, VirtualCharSequence allChars)
    {
        foreach (var trivia in leadingTrivia)
        {
            CheckInvariants(trivia, ref position, allChars);
        }
    }

    private static void CheckInvariants(RegexTrivia trivia, ref int position, VirtualCharSequence allChars)
    {
        switch (trivia.Kind)
        {
            case RegexKind.CommentTrivia:
            case RegexKind.WhitespaceTrivia:
                break;
            default:
                Assert.Fail("Incorrect trivia kind");
                return;
        }

        CheckCharacters(trivia.VirtualChars, ref position, allChars);
    }

    private static void CheckCharacters(VirtualCharSequence virtualChars, ref int position, VirtualCharSequence allChars)
    {
        for (var i = 0; i < virtualChars.Length; i++)
        {
            Assert.Equal(allChars[position + i], virtualChars[i]);
        }

        position += virtualChars.Length;
    }

    private static string And(params ReadOnlySpan<string> regexes)
    {
        var conj = $"({regexes[^1]})";
        for (var i = regexes.Length - 2; i >= 0; i--)
            conj = $"(?({regexes[i]}){conj}|[0-[0]])";

        return conj;
    }

    private static string Not(string regex)
        => $"(?({regex})[0-[0]]|.*)";

    [ConditionalFact(typeof(WindowsOnly), Reason = "Deep recursion test relies on Windows stack size (~1MB) to trigger stack overflow; Linux has 8MB stack")]
    public void TestDeepRecursion()
    {
        var (token, tree, chars) =
            JustParseTree(
                """
                @"((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                (((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
                ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((("
                """, RegexOptions.None, conversionFailureOk: false);
        Assert.False(token.IsMissing);
        Assert.False(chars.IsDefaultOrEmpty());
        Assert.Null(tree);
    }

    [Fact]
    public void TestNoStackOverflow()
    {
        for (var i = 1; i < 1200; i++)
        {
            var text = new string('(', i);
            var (token, _, chars) = JustParseTree($"""
                @"{text}"
                """, RegexOptions.None, conversionFailureOk: false);
            Assert.False(token.IsMissing);
            Assert.False(chars.IsDefaultOrEmpty());
        }
    }

    [Fact]
    public void TestRegexCharClassCharacters()
    {
        foreach (var (charClass, _) in RegexCharClass.EscapeCategories)
        {
            foreach (var ch in charClass)
                Assert.True(RegexLexer.IsEscapeCategoryChar(new(new(ch, offset: 0, width: 1), tokenStart: 0)));
        }
    }
}