src/Compilers/VisualBasic/Portable/Scanner/Blender.vb - main - githubmirror/roslyn

src/Compilers/VisualBasic/Portable/Scanner/Blender.vb
573 строки22 KB
Cyrus Najmabadi
Update our syntax list types to be "collection builder" types. (#71646)
25 янв 2024, 21:52
25 янв 2024, 21:52943351c
' Licensed to the .NET Foundation under one or more agreements.
' The .NET Foundation licenses this file to you under the MIT license.
' See the LICENSE file in the project root for more information.

'-----------------------------------------------------------------------------
' Contains the definition of the Scanner, which produces tokens from text 
'-----------------------------------------------------------------------------
Option Compare Binary
Option Strict On

Imports Microsoft.CodeAnalysis.Syntax.InternalSyntax
Imports Microsoft.CodeAnalysis.Text
Imports Microsoft.CodeAnalysis.VisualBasic

Namespace Microsoft.CodeAnalysis.VisualBasic.Syntax.InternalSyntax
    Friend NotInheritable Class Blender
        Inherits Scanner

        ''' <summary>
        ''' Candidate nodes that may be reused.
        ''' </summary>
        Private ReadOnly _nodeStack As New Stack(Of GreenNode)

        ''' <summary>
        ''' The text changes combined into a single region.
        ''' </summary>
        Private ReadOnly _change As TextChangeRange

        ''' <summary>
        ''' The range from which we cannot reuse nodes.
        ''' </summary>
        Private ReadOnly _affectedRange As TextChangeRange

        ''' <summary>
        ''' Current node. Not necessarily reusable or even a NonTerminal.
        ''' Can be null if we are out of nodes.
        ''' </summary>
        Private _currentNode As VisualBasicSyntaxNode
        Private _curNodeStart As Integer
        Private _curNodeLength As Integer

        Private ReadOnly _baseTreeRoot As VisualBasic.VisualBasicSyntaxNode

        ''' <summary>
        ''' preprocessor state before _currentNode
        ''' </summary>
        Private _currentPreprocessorState As PreprocessorState

        ''' <summary>
        ''' preprocessor state getter after _currentNode
        ''' </summary>
        Private _nextPreprocessorStateGetter As NextPreprocessorStateGetter

        Private Shared Sub PushReverseNonterminal(stack As Stack(Of GreenNode), nonterminal As GreenNode)
            Dim cnt = nonterminal.SlotCount
            For i As Integer = 1 To cnt
                Dim child = nonterminal.GetSlot(cnt - i)
                PushChildReverse(stack, child)
            Next
        End Sub

        Private Shared Sub PushReverseTerminal(stack As Stack(Of GreenNode), tk As SyntaxToken)
            Dim trivia = tk.GetTrailingTrivia

            If trivia IsNot Nothing Then
                PushChildReverse(stack, trivia)
            End If

            PushChildReverse(stack, DirectCast(tk.WithLeadingTrivia(Nothing).WithTrailingTrivia(Nothing), SyntaxToken))

            trivia = tk.GetLeadingTrivia

            If trivia IsNot Nothing Then
                PushChildReverse(stack, trivia)
            End If
        End Sub

        Private Shared Sub PushChildReverse(stack As Stack(Of GreenNode), child As GreenNode)
            If child IsNot Nothing Then
                If child.IsList Then
                    PushReverseNonterminal(stack, child)
                Else
                    stack.Push(child)
                End If
            End If
        End Sub

        ''' <summary>
        ''' Expand the span in the tree to encompass the
        ''' nearest statements that the span overlaps.
        ''' </summary>
        Private Shared Function ExpandToNearestStatements(root As VisualBasic.VisualBasicSyntaxNode, span As TextSpan) As TextSpan
            Dim fullSpan = New TextSpan(0, root.FullWidth)
            Dim start = NearestStatementThatContainsPosition(root, span.Start, fullSpan)
            Debug.Assert(start.Start <= span.Start)
            If span.Length = 0 Then
                Return start
            Else
                Dim [end] = NearestStatementThatContainsPosition(root, span.End - 1, fullSpan)
                Debug.Assert([end].End >= span.End)
                Return TextSpan.FromBounds(start.Start, [end].End)
            End If
        End Function

        ''' <remarks>
        ''' Not guaranteed to return the span of a StatementSyntax.
        ''' </remarks>
        Private Shared Function NearestStatementThatContainsPosition(
            node As SyntaxNode,
            position As Integer,
            rootFullSpan As TextSpan) As TextSpan

            If Not node.FullSpan.Contains(position) Then
                Debug.Assert(node.FullSpan.End = position)
                Return New TextSpan(position, 0)
            End If

            If node.Kind = SyntaxKind.CompilationUnit OrElse IsStatementLike(node) Then
                Do
                    Dim child = node.ChildThatContainsPosition(position).AsNode()
                    If child Is Nothing OrElse Not IsStatementLike(child) Then
                        Return node.FullSpan
                    End If
                    node = child
                Loop
            End If

            Return rootFullSpan
        End Function

        Private Shared Function IsStatementLike(node As SyntaxNode) As Boolean
            Select Case node.Kind
                Case SyntaxKind.ElseIfBlock,
                     SyntaxKind.ElseBlock,
                     SyntaxKind.CatchBlock,
                     SyntaxKind.FinallyBlock

                    Return node.GetTrailingTrivia().Any(SyntaxKind.EndOfLineTrivia)
                Case SyntaxKind.SingleLineIfStatement,
                     SyntaxKind.SingleLineElseClause
                    ' Steer clear of single-line if's because they have custom handling of statement 
                    ' terminators that may make it difficult to reuse sub-statements.
                    Return False
                Case Else
                    Return TypeOf node Is Syntax.StatementSyntax
            End Select
        End Function

        ''' <summary>
        ''' Expand the span in the tree by the maximum number
        ''' of tokens required for look ahead and the maximum
        ''' number of characters for look behind.
        ''' </summary>
        Private Shared Function ExpandByLookAheadAndBehind(root As VisualBasic.VisualBasicSyntaxNode, span As TextSpan) As TextSpan
            Dim fullWidth = root.FullWidth
            Dim start = Math.Min(span.Start, Math.Max(0, fullWidth - 1))
            Dim [end] = span.End

            If start > 0 Then
                ' Move to the left by the look ahead required by the Scanner.
                For i As Integer = 0 To Scanner.MaxTokensLookAheadBeyondEOL
                    Dim node = root.FindTokenInternal(start)
                    If node.Kind = SyntaxKind.None Then
                        Exit For
                    Else
                        start = node.Position
                        If start = 0 Then
                            Exit For
                        Else
                            start -= 1
                        End If
                    End If
                Next
            End If

            ' Allow for look behind of some number of characters.
            If [end] < fullWidth Then
                [end] += Scanner.MaxCharsLookBehind
            End If

            Return TextSpan.FromBounds(start, [end])
        End Function

        Friend Sub New(newText As SourceText,
                       changes As TextChangeRange(),
                       baseTreeRoot As SyntaxTree,
                       options As VisualBasicParseOptions)

            MyBase.New(newText, options)

            ' initially blend state and scanner state are same
            _currentPreprocessorState = _scannerPreprocessorState
            _nextPreprocessorStateGetter = Nothing

            _baseTreeRoot = baseTreeRoot.GetVisualBasicRoot()
            _currentNode = _baseTreeRoot.VbGreen
            _curNodeStart = 0
            _curNodeLength = 0

            TryCrumbleOnce()

            If _currentNode Is Nothing Then
                Return  ' tree seems to be empty
            End If

            _change = TextChangeRange.Collapse(changes)

#If DEBUG Then
            Dim start = _change.Span.Start
            Dim [end] = _change.Span.End
            Debug.Assert(start >= 0)
            Debug.Assert(start <= [end])
            Debug.Assert([end] <= _baseTreeRoot.FullWidth)
#End If

            ' Parser requires look ahead of some number of tokens
            ' beyond EOL and some number of characters back.
            ' Expand the change range to accommodate look ahead/behind.
            Dim span = ExpandToNearestStatements(
                _baseTreeRoot,
                ExpandByLookAheadAndBehind(_baseTreeRoot, _change.Span))
            _affectedRange = New TextChangeRange(span, span.Length - _change.Span.Length + _change.NewLength)
        End Sub

        Private Function MapNewPositionToOldTree(position As Integer) As Integer
            If position < _change.Span.Start Then
                Return position
            End If

            If position >= _change.Span.Start + _change.NewLength Then
                Return position - _change.NewLength + _change.Span.Length
            End If

            Return -1
        End Function

        ''' <summary>
        ''' Moving to the next node on the stack.
        ''' returns false if we are out of nodes.
        ''' </summary>
        Private Function TryPopNode() As Boolean
            If _nodeStack.Count > 0 Then
                Dim node = _nodeStack.Pop
                _currentNode = DirectCast(node, VisualBasicSyntaxNode)
                _curNodeStart = _curNodeStart + _curNodeLength
                _curNodeLength = node.FullWidth

                ' move blender preprocessor state forward if possible
                If _nextPreprocessorStateGetter.Valid Then
                    _currentPreprocessorState = _nextPreprocessorStateGetter.State()
                End If

                _nextPreprocessorStateGetter = New NextPreprocessorStateGetter(_currentPreprocessorState, DirectCast(node, VisualBasicSyntaxNode))

                Return True
            Else
                _currentNode = Nothing
                Return False
            End If
        End Function

        ''' <summary>
        ''' Crumbles current node onto the stack and pops one node into current.
        ''' Returns false if current node cannot be crumbled.
        ''' </summary>
        Friend Overrides Function TryCrumbleOnce() As Boolean
            If _currentNode Is Nothing Then
                Return False
            End If

            If _currentNode.SlotCount = 0 Then
                If Not _currentNode.ContainsStructuredTrivia Then
                    ' terminal with no structured trivia is not interesting
                    Return False
                End If

                ' try reusing structured trivia (in particular XML)
                PushReverseTerminal(_nodeStack, DirectCast(_currentNode, SyntaxToken))
            Else
                If Not ShouldCrumble(_currentNode) Then
                    Return False
                End If

                PushReverseNonterminal(_nodeStack, _currentNode)
            End If

            ' crumbling does not affect start, but length is set to 0 until we see a node
            _curNodeLength = 0

            ' crumbling doesn't move things forward. discard next preprocessor state we calculated before
            _nextPreprocessorStateGetter = Nothing

            Return TryPopNode()
        End Function

        ''' <summary>
        ''' Certain syntax node kinds should not be crumbled since
        ''' re-using individual child nodes may complicate parsing.
        ''' </summary>
        Private Shared Function ShouldCrumble(node As VisualBasicSyntaxNode) As Boolean
            If TypeOf node Is StructuredTriviaSyntax Then
                ' Do not crumble into structured trivia content.
                ' we will not use any of the parts anyways and 
                ' evaluation of directives may go out of sync.
                Return False
            End If

            Select Case node.Kind
                Case SyntaxKind.SingleLineIfStatement,
                    SyntaxKind.SingleLineElseClause
                    ' Parsing of single line If is particularly complicated
                    ' since the statement may contain colon separated or
                    ' multi-line statements. Avoid re-using child nodes.
                    Return False

                Case SyntaxKind.EnumBlock
                    ' Interpretation of other nodes within Enum block
                    ' may depend on the kind of this node.
                    Return False

                Case Else
                    Return True

            End Select
        End Function

        ''' <summary>
        ''' Advances to given position if needed (note: no way back)
        ''' Gets a nonterminal that can be used for incremental.
        ''' May return Nothing if such node is not available.
        ''' Typically it is _currentNode.
        ''' </summary>
        Private Function GetCurrentNode(position As Integer) As VisualBasicSyntaxNode
            Debug.Assert(_currentNode IsNot Nothing)

            Dim mappedPosition = MapNewPositionToOldTree(position)

            If mappedPosition = -1 Then
                Return Nothing
            End If

            Do
                ' too far ahead
                If _curNodeStart > mappedPosition Then
                    Return Nothing
                End If

                ' node ends before or on the mappedPosition
                ' whole node is unusable, move to the next node
                If (_curNodeStart + _curNodeLength) <= mappedPosition Then
                    If TryPopNode() Then
                        Continue Do
                    Else
                        Return Nothing
                    End If
                End If

                If _curNodeStart = mappedPosition AndAlso CanReuseNode(_currentNode) Then
                    ' have some node
                    Exit Do
                End If

                ' current node spans the position or node is not usable
                ' try crumbling and look through children
                If Not TryCrumbleOnce() Then
                    Return Nothing
                End If
            Loop

            ' zero-length nodes are ambiguous when given a particular position
            ' also the vast majority of such nodes are synthesized
            Debug.Assert(_currentNode.FullWidth > 0, "reusing zero-length nodes?")
            Return _currentNode
        End Function

        ''' <summary>
        ''' Returns current candidate for reuse if there is one.
        ''' </summary>
        Friend Overrides Function GetCurrentSyntaxNode() As VisualBasicSyntaxNode
            ' not going to get any nodes if there is no current node.
            If _currentNode Is Nothing Then
                Return Nothing
            End If

            ' node must start where the current token starts.
            Dim start = _currentToken.Position

            ' position is in affected range - no point trying.
            Dim range = New TextSpan(_affectedRange.Span.Start, _affectedRange.NewLength)
            If range.Contains(start) Then
                Return Nothing
            End If

            Dim nonterminal = GetCurrentNode(start)
            Return nonterminal
        End Function

        ''' <summary>
        ''' Checks if node is reusable.
        ''' The reasons for it not be usable are typically that it intersects affected range.
        ''' </summary>
        Private Function CanReuseNode(node As VisualBasicSyntaxNode) As Boolean
            If node Is Nothing Then
                Return False
            End If

            If node.SlotCount = 0 Then
                Return False
            End If

            ' TODO: This is a temporary measure to get around contextual errors.
            ' The problem is that some errors are contextual, but get attached to inner nodes.
            ' as a result in a case when an edit changes the context the error may be invalidated
            ' but since the node with actual error did not change the error will stay.
            If node.ContainsDiagnostics Then
                Return False
            End If

            ' As of 2013/03/14, the compiler never attempts to incrementally parse a tree containing
            ' annotations.  Our goal in instituting this restriction is to prevent API clients from
            ' taking a dependency on the survival of annotations.
            If node.ContainsAnnotations Then
                Return False
            End If

            ' If the node is an If statement, we need to determine whether it is a
            ' single-line or multi-line If. That requires the scanner to be positioned
            ' correctly relative to the end of line terminator if any, and currently we
            ' do not guarantee that. (See bug #16557.) For now, for simplicity, we
            ' do not reuse If statements.
            If node.Kind = SyntaxKind.IfStatement Then
                Return False
            End If

            Dim _curNodeSpan = New TextSpan(_curNodeStart, _curNodeLength)
            ' TextSpan.OverlapsWith does not handle empty spans so
            ' empty spans need to be handled explicitly.
            Debug.Assert(_curNodeSpan.Length > 0)
            If _affectedRange.Span.Length = 0 Then
                If _curNodeSpan.Contains(_affectedRange.Span.Start) Then
                    Return False
                End If
            Else
                If _curNodeSpan.OverlapsWith(_affectedRange.Span) Then
                    Return False
                End If
            End If

            ' we cannot use nodes that contain directives since we need to process
            ' directives individually.
            ' We however can use individual directives.
            If node.ContainsDirectives AndAlso Not TypeOf node Is DirectiveTriviaSyntax Then
                Return _scannerPreprocessorState.IsEquivalentTo(_currentPreprocessorState)
            End If

            ' sometimes nodes contain linebreaks in leading trivia
            ' if we are in VBAllowLeadingMultilineTrivia state (common case), it is ok.
            ' otherwise nodes with leading trivia containing linebreaks should be rejected.
            If Not Me._currentToken.State = ScannerState.VBAllowLeadingMultilineTrivia AndAlso
                ContainsLeadingLineBreaks(node) Then

                Return False
            End If

            If _currentNode.IsMissing Then
                Return Nothing
            End If

            Return True
        End Function

        Private Function ContainsLeadingLineBreaks(node As VisualBasicSyntaxNode) As Boolean
            Dim lt = node.GetLeadingTrivia
            If lt IsNot Nothing Then
                If lt.RawKind = SyntaxKind.EndOfLineTrivia Then
                    Return True
                End If

                Dim asList = TryCast(lt, CodeAnalysis.Syntax.InternalSyntax.SyntaxList)
                If asList IsNot Nothing Then
                    For i As Integer = 0 To asList.SlotCount - 1
                        If lt.GetSlot(i).RawKind = SyntaxKind.EndOfLineTrivia Then
                            Return True
                        End If
                    Next
                End If
            End If

            Return False
        End Function

        Friend Overrides Sub MoveToNextSyntaxNode()
            If _currentNode Is Nothing Then
                Return
            End If

            Debug.Assert(CanReuseNode(_currentNode), "this node could not have been used.")
            Debug.Assert(_nextPreprocessorStateGetter.Valid, "we should have _nextPreprocessorState")

            Dim nextPreprocessorState = _nextPreprocessorStateGetter.State()
            Debug.Assert(nextPreprocessorState.ConditionalStack.Count = 0 OrElse
                         nextPreprocessorState.ConditionalStack.Peek.BranchTaken = ConditionalState.BranchTakenState.Taken,
                        "how could a parser in taken PP state?")

            ' update buffer offset relative to current token
            _lineBufferOffset = _currentToken.Position + _curNodeLength

            ' sync current token's preprocessor state to blender preprocessor state
            ' it is safe to do so here since if we are here, it means we can reuse information in old tree up to this point
            ' including preprocessor state
            ' *NOTE* we are using _nextPreprocessorState instead of _currentPreprocessorState because
            ' we are actually moving things forward here. _nextPreprocessorState will become _currentPreprocessorState
            ' at the "TryPopNode" below.
            If _currentNode.ContainsDirectives Then
                _currentToken = _currentToken.With(nextPreprocessorState)
            End If

            ' this will discard any prefetched tokens, including current. 
            ' We do not need them since we moved to completely new node.
            MyBase.MoveToNextSyntaxNode()

            TryPopNode()

            ' at this point, all three pointers (position in old tree, position in text, position in parser)
            ' and all preprocessor state (_currentPreprocessorState, _scannerPreprocessorState, _currentToken.PreprocessorState) 
            ' should point to same position (in sync)
        End Sub

        Friend Overrides Sub MoveToNextSyntaxNodeInTrivia()
            If _currentNode Is Nothing Then
                Return
            End If

            Debug.Assert(CanReuseNode(_currentNode), "this node could not have been used.")

            ' just move forward
            _lineBufferOffset = _lineBufferOffset + _curNodeLength

            ' this will just verify that we do not have any prefetched tokens, including current. 
            ' otherwise advancing line buffer offset could go out of sync with token stream.
            MyBase.MoveToNextSyntaxNodeInTrivia()

            TryPopNode()
        End Sub

        Private Structure NextPreprocessorStateGetter
            Private ReadOnly _state As PreprocessorState
            Private ReadOnly _node As VisualBasicSyntaxNode

            Private _nextState As PreprocessorState

            Public Sub New(state As PreprocessorState, node As VisualBasicSyntaxNode)
                Me._state = state
                Me._node = node
                Me._nextState = Nothing
            End Sub

            Public ReadOnly Property Valid As Boolean
                Get
                    Return _node IsNot Nothing
                End Get
            End Property

            Public Function State() As PreprocessorState
                If _nextState Is Nothing Then
                    _nextState = ApplyDirectives(Me._state, Me._node)
                End If

                Return _nextState
            End Function
        End Structure
    End Class
End Namespace