src/Compilers/VisualBasic/Portable/Parser/ParseScan.vb - main - githubmirror/roslyn

src/Compilers/VisualBasic/Portable/Parser/ParseScan.vb
430 строк19 KB
Youssef Victor
Remove unused private members (#61064)
12 июн 2022, 21:22
12 июн 2022, 21:22a528fe2
' Licensed to the .NET Foundation under one or more agreements.
' The .NET Foundation licenses this file to you under the MIT license.
' See the LICENSE file in the project root for more information.

Imports Microsoft.CodeAnalysis.Text
Imports Microsoft.CodeAnalysis.VisualBasic.Syntax
Imports System
Imports System.Runtime.InteropServices
Imports Microsoft.CodeAnalysis.Syntax.InternalSyntax

Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
    ' //
    ' //============ Methods to encapsulate scanning ========================
    ' //

    Partial Friend Class Parser

        ' File: Scanner.h
        ' Lines: 301 - 301
        ' Class: Token
        ' bool .::IsContinuableEOL( )
        Private Function IsContinuableEOL(Optional i As Integer = 0) As Boolean
            'TODO - This applies ot both IsContinuableEOL and PeekPastStatementTerminator and any code that peeks through an EOL.
            ' The scanner scans the fist token of a statement differently with regard to trivia.  It we peek past the EOL and then get the token.
            ' The token may not have the correct trivia attached to it when we get it.  The solution is to attach the state to the token so we
            ' know if we peeked the token in the first token of new statement state or the next token of a statement state.

            If PeekToken(i).Kind = SyntaxKind.StatementTerminatorToken Then
                If PeekToken(i + 1).Kind <> SyntaxKind.EmptyToken Then
                    Return True
                End If
            End If
            Return False
        End Function

        'TODO - This is really peekToken skipping optional statementterminator
        Private Function PeekPastStatementTerminator() As SyntaxToken
            Dim t1 = PeekToken(1)
            If t1.Kind = SyntaxKind.StatementTerminatorToken Then
                Dim t2 = PeekToken(2)
                If t2.Kind <> SyntaxKind.EmptyToken Then
                    Return t2
                End If
            End If
            Return t1
        End Function

        ' Is this token a valid end of statement?

        '// Parser::IsValidStatementTerminator: Is this token a valid "StatementTerminator"?
        '// NOTE: a compound statement e.g. "If b then : S1 : end if" has StatementTerminators inside it
        '// This function is used e.g. in parsing "If b Then <token>" to determine whether it's a line-if or a block-if.
        '// The statement terminators are newline, colon, comment. (Also due to a quirk in the way things flow together,
        '// "else" is considered a statement terminator if we're in a line-if.)

        ' File: Parser.cpp
        ' Lines: 18421 - 18421
        ' bool .Parser::IsValidStatementTerminator( [ _In_ Token* T ] )

        Friend Function IsValidStatementTerminator(t As SyntaxToken) As Boolean
            Debug.Assert(t IsNot Nothing)

            ' // The parser usually skips StatementTerminator tokens (if implicit line continuation is
            ' // enabled), and it isn't possible to look back past the first token of a
            ' // line, so test if this is the first token of the last read line.

            ' REM and XmlDocComment are now trivia so they have been removed from the test
            Return SyntaxFacts.IsTerminator(t.Kind)
        End Function

        ' // Parser::CanFollowStatement -- Can this token follow a complete statement?
        ' // NOTE: e.g. in "Dim x = Sub() S, y=3", if we're within a statement lambda, then the token tkCOMMA
        ' // can follow a complete statement
        ' // This function is used e.g. in parsing "End <token>" or "Call f() <token>" to determine whether
        ' // the token is part of the construct, or may be part of the following construct, or is just an error.

        ' bool .Parser::CanFollowStatement( [ _In_ Token* T ] )
        Private Function CanFollowStatement(T As SyntaxToken) As Boolean
            ' // Dev10#670492: in a single-line sub, e.g. "Dim x = (Sub() Stmt)", things like RParen are happy to end the statement
            Return If(Context.IsWithinSingleLineLambda, CanFollowExpression(T), IsValidStatementTerminator(T)) OrElse
                T.Kind = SyntaxKind.ElseKeyword
        End Function

        Friend Function CanFollowStatementButIsNotSelectFollowingExpression(nextToken As SyntaxToken) As Boolean
            ' Dev10#708061: Treat Select in "From x in Sub() End Select" as part of End statement.
            Return If(Context.IsWithinSingleLineLambda,
                      CanFollowExpression(nextToken) AndAlso Not nextToken.Kind = SyntaxKind.SelectKeyword,
                      IsValidStatementTerminator(nextToken)) OrElse (Context.IsLineIf AndAlso nextToken.Kind = SyntaxKind.ElseKeyword)
        End Function

        ' Is this token a valid end of executable statement?

        '// Parser::CanEndExecutableStatement -- Can this token follow a complete "executable" statement?
        '// WARNING: This function is *only* used when parsing a Call statement, to judge whether the tokens that follow
        '// it should produce an "obsolete: arguments must be enclosed in parentheses" message. It shouldn't be
        '// used by anything else.
        '// It's only difference from "CanFollowStatement" is that it returns true when given tkELSE: for all other
        '// inputs it's identical.

        ' File: Parser.cpp
        ' Lines: 18441 - 18441
        ' bool .Parser::CanEndExecutableStatement( [ _In_ Token* T ] )

        Private Function CanEndExecutableStatement(t As SyntaxToken) As Boolean
            Return CanFollowStatement(t) OrElse t.Kind = SyntaxKind.ElseKeyword
        End Function

        ' // Parser::CanFollowExpression -- Can this token follow a complete expression??
        ' // NOTE: a statement can end with an expression. Therefore, the set denoted by CanFollowExpression
        ' // is not smaller than that denoted by CanFollowStatement. (actually, the two sets are equal
        ' // if we happen to be parsing the statement body of a single-line sub lambda).

        ' bool .Parser::CanFollowExpression( [ _In_ Token* T ] )
        Private Function CanFollowExpression(t As SyntaxToken) As Boolean
            ' // e.g. "Aggregate" can end an expression
            Dim kind As SyntaxKind = Nothing

            If t.Kind = SyntaxKind.IdentifierToken AndAlso TryIdentifierAsContextualKeyword(t, kind) Then
                Return KeywordTable.CanFollowExpression(kind)
            End If
            Return KeywordTable.CanFollowExpression(t.Kind) OrElse IsValidStatementTerminator(t)
        End Function

        ' File: Parser.cpp
        ' Lines: 18561 - 18561

        'Token* // Returns the token following tkOF, or NULL if we aren't looking at generic type syntax 
        'Parser::BeginsGeneric // A generic is signified by '(' [tkStatementTerminator] tkOF

        Private Function BeginsGeneric(Optional nonArrayName As Boolean = False, Optional allowGenericsWithoutOf As Boolean = False) As Boolean

            If CurrentToken.Kind = SyntaxKind.OpenParenToken Then

                If nonArrayName Then
                    Return True
                End If

                Dim t = PeekPastStatementTerminator()

                If t.Kind = SyntaxKind.OfKeyword Then
                    Return True
                ElseIf allowGenericsWithoutOf Then
                    ' // To enable a better user experience in some common generics'
                    ' // error scenarios, we special case goo(Integer) and
                    ' // goo(Integer, garbage).
                    ' //
                    ' // "(Integer" indicates possibly type parameters with missing "of",
                    ' // but not "(Integer." and "Integer!" because they could possibly
                    ' // imply qualified names or expressions. Also note that "Integer :="
                    ' // could imply named arguments. Here "Integer" is just an example,
                    ' // it could be any intrinsic type.
                    ' //
                    If SyntaxFacts.IsPredefinedTypeOrVariant(t.Kind) Then
                        Select Case PeekToken(2).Kind
                            Case SyntaxKind.CloseParenToken, SyntaxKind.CommaToken
                                Return True
                        End Select
                    End If
                End If

            End If

            Return False
        End Function

        ' Does this token force the end of a statement?

        ' File: Parser.cpp
        ' Lines: 18635 - 18635
        ' bool .Parser::MustEndStatement( [ _In_ Token* T ] )

        Private Function MustEndStatement(t As SyntaxToken) As Boolean
            Debug.Assert(t IsNot Nothing)
            Return IsValidStatementTerminator(t)
        End Function

        ' /*********************************************************************
        ' *
        ' * Function:
        ' *     Parser::PeekAheadFor
        ' *
        ' * Purpose:
        ' *     Peek ahead the statement for the requested tokens. Return the
        ' *     token that was found. If the requested tokens are not encountered,
        ' *     return tkNone.
        ' *
        ' *     This routine does not consume the tokens. The current token is not
        ' *     advanced. To consume tokens, use ResyncAt().
        ' *
        ' **********************************************************************/

        ' // [in] count of tokens to look for
        ' // [in] var_list of tokens (tk[]) to look for

        ' File: Parser.cpp
        ' Lines: 18847 - 18847
        ' tokens __cdecl .Parser::PeekAheadFor( [ unsigned Count ] [ ...  ] )

        Private Function PeekAheadFor(ParamArray kinds As SyntaxKind()) As SyntaxKind
            Dim token As SyntaxToken = Nothing
            PeekAheadFor(s_isTokenOrKeywordFunc, kinds, token)
            Return If(token Is Nothing, Nothing, token.Kind)
        End Function

        Private Function PeekAheadFor(Of TArg)(predicate As Func(Of SyntaxToken, TArg, Boolean), arg As TArg, <Out()> ByRef token As SyntaxToken) As Integer
            Dim nextToken = CurrentToken

            Dim i As Integer = 0
            While Not IsValidStatementTerminator(nextToken)
                If predicate(nextToken, arg) Then
                    token = nextToken
                    Return i
                End If

                i += 1
                nextToken = PeekToken(i)
            End While

            token = Nothing
            Return 0
        End Function

        ' /*********************************************************************
        ' *
        ' * Function:
        ' *     Parser::ResyncAt
        ' *
        ' * Purpose:
        ' *     Consume tokens on the line until we encounter a token in the
        ' *     given list or EOS.
        ' *
        ' **********************************************************************/

        ' // [in] count of tokens to look for
        ' // [in] var_list of tokens to look for

        ' File: Parser.cpp
        ' Lines: 18939 - 18939
        ' tokens __cdecl .Parser::ResyncAt( [ unsigned Count ] [ ...  ] )

        ' TODO: note that queries sometimes use very large lists of resync tokens.
        ' linear search may not make much sense. Perhaps could use lazy-inited hashtables?
        Private Sub ResyncAt(skippedTokens As SyntaxListBuilder(Of SyntaxToken), state As ScannerState, resyncTokens As SyntaxKind())
            Debug.Assert(resyncTokens IsNot Nothing)

            While CurrentToken.Kind <> SyntaxKind.EndOfFileToken

                If state.IsVBState Then
                    If IsValidStatementTerminator(CurrentToken) OrElse CurrentToken.Kind = SyntaxKind.EmptyToken Then
                        Exit While
                    End If
                ElseIf CurrentToken.Kind = SyntaxKind.EndOfXmlToken OrElse CurrentToken.Kind = SyntaxKind.EndOfInterpolatedStringToken Then
                    Exit While
                End If

                If IsTokenOrKeyword(CurrentToken, resyncTokens) Then
                    Exit While
                End If

                skippedTokens.Add(CurrentToken)
                GetNextToken(state)
            End While
        End Sub

        Private Function ResyncAt(state As ScannerState, resyncTokens As SyntaxKind()) As CodeAnalysis.Syntax.InternalSyntax.SyntaxList(Of SyntaxToken)
            Dim skippedTokens = Me._pool.Allocate(Of SyntaxToken)()

            ResyncAt(skippedTokens, state, resyncTokens)

            Dim result = skippedTokens.ToList()
            Me._pool.Free(skippedTokens)

            Return result
        End Function

        ''' <summary>
        ''' Resyncs to next statement terminator. Used in Preprocessor
        ''' </summary>
        Private Function ResyncAndConsumeStatementTerminator() As CodeAnalysis.Syntax.InternalSyntax.SyntaxList(Of SyntaxToken)
            Dim skippedTokens = Me._pool.Allocate(Of SyntaxToken)()

            While CurrentToken.Kind <> SyntaxKind.EndOfFileToken AndAlso
                    CurrentToken.Kind <> SyntaxKind.StatementTerminatorToken

                skippedTokens.Add(CurrentToken)
                GetNextToken(ScannerState.VB)
            End While

            If CurrentToken.Kind = SyntaxKind.StatementTerminatorToken Then
                If CurrentToken.HasLeadingTrivia Then
                    skippedTokens.Add(CurrentToken)
                End If

                GetNextToken(ScannerState.VB)
            End If

            Dim result = skippedTokens.ToList()
            Me._pool.Free(skippedTokens)

            Return result
        End Function

        Friend Function ResyncAt() As CodeAnalysis.Syntax.InternalSyntax.SyntaxList(Of SyntaxToken)
            Return ResyncAt(ScannerState.VB, Array.Empty(Of SyntaxKind))
        End Function

        Friend Function ResyncAt(resyncTokens As SyntaxKind()) As CodeAnalysis.Syntax.InternalSyntax.SyntaxList(Of SyntaxToken)
            Debug.Assert(resyncTokens IsNot Nothing)
            Return ResyncAt(ScannerState.VB, resyncTokens)
        End Function

        Private Function ResyncAt(Of T As VisualBasicSyntaxNode)(syntax As T) As T
            Return syntax.AddTrailingSyntax(ResyncAt())
        End Function

        Private Function ResyncAt(Of T As GreenNode)(syntax As T, ParamArray resyncTokens As SyntaxKind()) As T
            Debug.Assert(resyncTokens IsNot Nothing)

            Return syntax.AddTrailingSyntax(ResyncAt(resyncTokens).Node)
        End Function

        ''' <summary>
        ''' If the current token is a newline statement terminator, then consume the token.
        ''' </summary>
        ''' <returns>True if the statement terminator was consumed</returns>
        Private Function TryEatNewLine(Optional state As ScannerState = ScannerState.VB) As Boolean

            '// Don't eat EOL's while evaluating conditional compilation constants. 

            If CurrentToken.Kind = SyntaxKind.StatementTerminatorToken AndAlso
                PeekEndStatement(1) = SyntaxKind.None AndAlso
                Not _evaluatingConditionCompilationExpression Then

                If Not NextLineStartsWithStatementTerminator() Then
                    _hadImplicitLineContinuation = True

                    If PrevToken.GetTrailingTrivia().ContainsCommentTrivia() Then
                        _hadLineContinuationComment = True
                    End If

                    GetNextToken(state)
                    Return True
                End If

            End If

            Return False
        End Function

        ' /*****************************************************************************************
        '  ;EatNewLineIfFollowedBy
        ' 
        '  If the current token is an EOL token, eat the EOL on the condition that the next line
        '  starts with the specified token
        ' ******************************************************************************************/
        ' the token we want to check for on start of the next line
        ' [in] whether the tokenType represents a query operator token
        ' .Parser::EatNewLineIfFollowedBy( [ tokens tokenType ] [ ParseTree::LineContinuationState& continuationState ] [ bool isQueryOp ] )

        Private Function TryEatNewLineIfFollowedBy(kind As SyntaxKind) As Boolean
            Debug.Assert(CanUseInTryGetToken(kind))

            If NextLineStartsWith(kind) Then
                'Add trivia to the token that has been peeked on next line
                Return TryEatNewLine()
            End If
            Return False
        End Function

        ' /*****************************************************************************************
        '  ;EatNewLineIfNotFollowedBy
        ' 
        '  If the current token is an EOL token, eat the EOL on the condition that the next line
        '  does not start with the specified token.
        ' ******************************************************************************************/
        ' the token we want to check for on start of the next line
        ' [in] whether the tokenType represents a query operator token
        ' .Parser::EatNewLineIfNotFollowedBy( [ tokens tokenType ] [ ParseTree::LineContinuationState& continuationState ] [ bool isQueryOp ] )
        Private Function TryEatNewLineIfNotFollowedBy(kind As SyntaxKind) As Boolean
            Debug.Assert(CanUseInTryGetToken(kind))

            If Not NextLineStartsWith(kind) Then
                Return TryEatNewLine()
            End If
            Return False
        End Function

        ' /*****************************************************************************************
        '  ;NextLineStartsWith
        ' 
        '  Determine if the next line starts with the specified token.
        ' ******************************************************************************************/
        ' the token we want to check for on start of the next line
        ' [out] we will update the HasExtraNewLine flag
        ' [in] whether the tokenType represents a query operator token
        ' bool .Parser::NextLineStartsWith( [ tokens tokenType ] [ ParseTree::LineContinuationState& continuationState ] [ bool isQueryOp ] )
        Private Function NextLineStartsWith(kind As SyntaxKind) As Boolean
            Debug.Assert(CanUseInTryGetToken(kind))

            If CurrentToken.IsEndOfLine Then
                Dim nextToken = PeekToken(1)

                If nextToken.Kind = kind Then
                    Return True

                ElseIf nextToken.Kind = SyntaxKind.IdentifierToken Then
                    Dim contextualKind As SyntaxKind = Nothing
                    If TryIdentifierAsContextualKeyword(nextToken, contextualKind) AndAlso contextualKind = kind Then
                        Return True
                    End If
                End If
            End If

            Return False
        End Function

        Private Function NextLineStartsWithStatementTerminator(Optional offset As Integer = 0) As Boolean
            Debug.Assert(If(offset = 0, CurrentToken, PeekToken(offset)).IsEndOfLine)

            Dim kind = PeekToken(offset + 1).Kind
            Return kind = SyntaxKind.EmptyToken OrElse kind = SyntaxKind.EndOfFileToken
        End Function

        Private Shared Function CanUseInTryGetToken(kind As SyntaxKind) As Boolean
            Return Not SyntaxFacts.IsTerminator(kind) AndAlso kind <> SyntaxKind.EmptyToken
        End Function

    End Class

End Namespace