Skip to content

Commit 4448112

Browse files
nosaminosami
committed
[F#] Fix parsing of long lines
Fixes VSTS #963356 Make Lexer.parseLine tail recursive and remove mutable state
1 parent 4c4c5ff commit 4448112

4 files changed

Lines changed: 101 additions & 75 deletions

File tree

‎main/external/fsharpbinding/MonoDevelop.FSharp.Shared/Lexer.fs‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -150,22 +150,28 @@ module Lexer =
150150
draftToken :: acc, Some draftToken
151151
) ([], None)
152152
|> fst
153-
154-
153+
154+
let rec parseLine (tokenizer:FSharpLineTokenizer) tokens state =
155+
match tokenizer.ScanToken(state) with
156+
| Some tok, state ->
157+
parseLine tokenizer (tok::tokens) state
158+
| None, state -> tokens, state
159+
160+
let rec parseLines (sourceTok:FSharpSourceTokenizer) tokens state lines filename defines =
161+
[ match lines with
162+
| line::lines ->
163+
// Create tokenizer & tokenize single line
164+
let tokenizer = sourceTok.CreateLineTokenizer(line)
165+
let tokens, state = parseLine tokenizer [] state
166+
yield tokens, line
167+
// Tokenize the rest of the lines using the new state
168+
yield! parseLines sourceTok tokens state lines filename defines
169+
| [] -> () ]
170+
155171
let getTokensWithInitialState state lines filename defines =
156-
[ let mutable state = state
157-
let sourceTok = FSharpSourceTokenizer(defines, filename)
158-
for lineText in lines do
159-
let tokenizer = sourceTok.CreateLineTokenizer(lineText)
160-
let rec parseLine() =
161-
[ match tokenizer.ScanToken(state) with
162-
| Some(tok), nstate ->
163-
state <- nstate
164-
yield tok
165-
yield! parseLine()
166-
| None, nstate -> state <- nstate ]
167-
yield parseLine(), lineText ]
168-
172+
let sourceTok = FSharpSourceTokenizer(defines, filename)
173+
parseLines sourceTok [] state lines filename defines
174+
169175
let findTokenAt col (tokens:FSharpTokenInfo list) =
170176
let isTokenAtOffset col (t:FSharpTokenInfo) = col-1 >= t.LeftColumn && col-1 <= t.RightColumn
171177
tokens |> List.tryFindBack (isTokenAtOffset col)

‎main/external/fsharpbinding/MonoDevelop.FSharp.Tests/LexerTests.fs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
namespace MonoDevelopTests
2+
3+
open System
4+
open NUnit.Framework
5+
open MonoDevelop.FSharp.Shared
6+
open FSharp.Compiler.SourceCodeServices
7+
8+
[<TestFixture>]
9+
module LexerTests =
10+
[<Test>]
11+
let ``can parse long line``() =
12+
let line = sprintf "let x = \"%s\"" (String('*', 10000000))
13+
let sourceTok = FSharpSourceTokenizer([], None)
14+
let tokenizer = sourceTok.CreateLineTokenizer line
15+
let tokens, state = Lexer.parseLine tokenizer [] FSharpTokenizerLexState.Initial
16+
Assert.AreNotEqual(state, FSharpTokenizerLexState.Initial)

‎main/external/fsharpbinding/MonoDevelop.FSharp.Tests/MonoDevelop.FSharp.Tests.fsproj‎

Lines changed: 57 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,21 @@
7373
</ItemGroup>
7474
<Import Project="$(CustomBeforeMicrosoftCommonTargets)" />
7575
<Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets" />
76+
<Import Project="..\.paket\paket.targets" />
77+
<ProjectExtensions>
78+
<MonoDevelop>
79+
<Properties>
80+
<Policies>
81+
<TextStylePolicy TabWidth="4" IndentWidth="4" RemoveTrailingWhitespace="True" NoTabsAfterNonTabs="False" EolMarker="Native" FileWidth="80" TabsToSpaces="True" scope="text/x-fsharp" />
82+
<FSharpFormattingPolicy scope="text/x-fsharp">
83+
<DefaultFormat IndentOnTryWith="False" ReorderOpenDeclaration="False" SpaceAfterComma="True" SpaceAfterSemicolon="True" SpaceAroundDelimiter="True" SpaceBeforeArgument="True" SpaceBeforeColon="True" __added="0" />
84+
</FSharpFormattingPolicy>
85+
<TextStylePolicy inheritsSet="null" scope="application/fsproject+xml" />
86+
<XmlFormattingPolicy inheritsSet="null" scope="application/fsproject+xml" />
87+
</Policies>
88+
</Properties>
89+
</MonoDevelop>
90+
</ProjectExtensions>
7691
<ItemGroup>
7792
<Compile Include="TestBase.fs" />
7893
<Compile Include="TestDocument.fs" />
@@ -105,86 +120,70 @@
105120
<Link>MonoDevelop.FSharpInteractive.Service.exe</Link>
106121
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
107122
</None>
108-
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.Debugger\MonoDevelop.Debugger.csproj">
109-
<Project>{2357AABD-08C7-4808-A495-8FF2D3CDFDB0}</Project>
110-
<Name>MonoDevelop.Debugger</Name>
123+
<IncludeCopyLocal Include="System.Reactive.dll" />
124+
<IncludeCopyLocal Include="Newtonsoft.Json.dll" />
125+
<IncludeCopyLocal Include="ExtCore.dll" />
126+
<IncludeCopyLocal Include="Fantomas.dll" />
127+
<IncludeCopyLocal Include="FSharp.Compiler.CodeDom.dll" />
128+
<IncludeCopyLocal Include="FSharp.Compiler.Service.dll" />
129+
<IncludeCopyLocal Include="FSharp.Core.dll" />
130+
<IncludeCopyLocal Include="MonoDevelop.FSharp.Shared.dll" />
131+
<IncludeCopyLocal Include="FSharp.Compiler.Interactive.Settings.dll" />
132+
<Compile Include="LexerTests.fs" />
133+
<ProjectReference Include="..\..\..\tests\UnitTests\UnitTests.csproj">
134+
<Project>{1497D0A8-AFF1-4938-BC22-BE79B358BA5B}</Project>
135+
<Name>UnitTests</Name>
111136
<Private>False</Private>
112137
</ProjectReference>
113-
<ProjectReference Include="..\..\..\src\core\MonoDevelop.Ide\MonoDevelop.Ide.csproj">
114-
<Project>{27096E7F-C91C-4AC6-B289-6897A701DF21}</Project>
115-
<Name>MonoDevelop.Ide</Name>
138+
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.Refactoring\MonoDevelop.Refactoring.csproj">
139+
<Project>{100568FC-F4E8-439B-94AD-41D11724E45B}</Project>
140+
<Name>MonoDevelop.Refactoring</Name>
116141
<Private>False</Private>
117142
</ProjectReference>
118-
<ProjectReference Include="..\MonoDevelop.FSharpBinding\MonoDevelop.FSharp.fsproj">
119-
<Project>{4C10F8F9-3816-4647-BA6E-85F5DE39883A}</Project>
120-
<Name>MonoDevelop.FSharp</Name>
143+
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.PackageManagement\MonoDevelop.PackageManagement.csproj">
144+
<Project>{F218643D-2E74-4309-820E-206A54B7133F}</Project>
145+
<Name>MonoDevelop.PackageManagement</Name>
121146
<Private>False</Private>
122147
</ProjectReference>
123-
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.UnitTesting\MonoDevelop.UnitTesting.csproj">
124-
<Project>{A7A4246D-CEC4-42DF-A3C1-C31B9F51C4EC}</Project>
125-
<Name>MonoDevelop.UnitTesting</Name>
126-
<Private>False</Private>
148+
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.PackageManagement\MonoDevelop.PackageManagement.Tests\MonoDevelop.PackageManagement.Tests.csproj">
149+
<Project>{2645C9F3-9ED5-4806-AB09-DAD9BE90C67B}</Project>
150+
<Name>MonoDevelop.PackageManagement.Tests</Name>
151+
</ProjectReference>
152+
<ProjectReference Include="..\MonoDevelop.FSharp.Shared\MonoDevelop.FSharp.Shared.fsproj">
153+
<Project>{AF5FEAD5-B50E-4F07-A274-32F23D5C504D}</Project>
154+
<Name>MonoDevelop.FSharp.Shared</Name>
127155
</ProjectReference>
128156
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.SourceEditor2\MonoDevelop.SourceEditor.csproj">
129157
<Project>{F8F92AA4-A376-4679-A9D4-60E7B7FBF477}</Project>
130158
<Name>MonoDevelop.SourceEditor</Name>
131159
<Private>False</Private>
132160
</ProjectReference>
133-
<ProjectReference Include="..\MonoDevelop.FSharp.Shared\MonoDevelop.FSharp.Shared.fsproj">
134-
<Project>{AF5FEAD5-B50E-4F07-A274-32F23D5C504D}</Project>
135-
<Name>MonoDevelop.FSharp.Shared</Name>
136-
</ProjectReference>
137-
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.PackageManagement\MonoDevelop.PackageManagement.csproj">
138-
<Project>{F218643D-2E74-4309-820E-206A54B7133F}</Project>
139-
<Name>MonoDevelop.PackageManagement</Name>
161+
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.UnitTesting\MonoDevelop.UnitTesting.csproj">
162+
<Project>{A7A4246D-CEC4-42DF-A3C1-C31B9F51C4EC}</Project>
163+
<Name>MonoDevelop.UnitTesting</Name>
140164
<Private>False</Private>
141165
</ProjectReference>
142-
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.PackageManagement\MonoDevelop.PackageManagement.Tests\MonoDevelop.PackageManagement.Tests.csproj">
143-
<Project>{2645C9F3-9ED5-4806-AB09-DAD9BE90C67B}</Project>
144-
<Name>MonoDevelop.PackageManagement.Tests</Name>
166+
<ProjectReference Include="..\MonoDevelop.FSharpBinding\MonoDevelop.FSharp.fsproj">
167+
<Project>{4C10F8F9-3816-4647-BA6E-85F5DE39883A}</Project>
168+
<Name>MonoDevelop.FSharp</Name>
169+
<Private>False</Private>
145170
</ProjectReference>
146-
<ProjectReference Include="..\..\..\src\core\MonoDevelop.Core\MonoDevelop.Core.csproj">
147-
<Project>{7525BB88-6142-4A26-93B9-A30C6983390A}</Project>
148-
<Name>MonoDevelop.Core</Name>
171+
<ProjectReference Include="..\..\..\src\core\MonoDevelop.Ide\MonoDevelop.Ide.csproj">
172+
<Project>{27096E7F-C91C-4AC6-B289-6897A701DF21}</Project>
173+
<Name>MonoDevelop.Ide</Name>
149174
<Private>False</Private>
150175
</ProjectReference>
151-
<ProjectReference Include="..\..\..\tests\UnitTests\UnitTests.csproj">
152-
<Project>{1497D0A8-AFF1-4938-BC22-BE79B358BA5B}</Project>
153-
<Name>UnitTests</Name>
176+
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.Debugger\MonoDevelop.Debugger.csproj">
177+
<Project>{2357AABD-08C7-4808-A495-8FF2D3CDFDB0}</Project>
178+
<Name>MonoDevelop.Debugger</Name>
154179
<Private>False</Private>
155180
</ProjectReference>
156-
<ProjectReference Include="..\..\..\src\addins\MonoDevelop.Refactoring\MonoDevelop.Refactoring.csproj">
157-
<Project>{100568FC-F4E8-439B-94AD-41D11724E45B}</Project>
158-
<Name>MonoDevelop.Refactoring</Name>
181+
<ProjectReference Include="..\..\..\src\core\MonoDevelop.Core\MonoDevelop.Core.csproj">
182+
<Project>{7525BB88-6142-4A26-93B9-A30C6983390A}</Project>
183+
<Name>MonoDevelop.Core</Name>
159184
<Private>False</Private>
160185
</ProjectReference>
161186
</ItemGroup>
162-
<Import Project="..\.paket\paket.targets" />
163-
<ProjectExtensions>
164-
<MonoDevelop>
165-
<Properties>
166-
<Policies>
167-
<TextStylePolicy TabWidth="4" IndentWidth="4" RemoveTrailingWhitespace="True" NoTabsAfterNonTabs="False" EolMarker="Native" FileWidth="80" TabsToSpaces="True" scope="text/x-fsharp" />
168-
<FSharpFormattingPolicy scope="text/x-fsharp">
169-
<DefaultFormat IndentOnTryWith="False" ReorderOpenDeclaration="False" SpaceAfterComma="True" SpaceAfterSemicolon="True" SpaceAroundDelimiter="True" SpaceBeforeArgument="True" SpaceBeforeColon="True" __added="0" />
170-
</FSharpFormattingPolicy>
171-
<TextStylePolicy inheritsSet="null" scope="application/fsproject+xml" />
172-
<XmlFormattingPolicy inheritsSet="null" scope="application/fsproject+xml" />
173-
</Policies>
174-
</Properties>
175-
</MonoDevelop>
176-
</ProjectExtensions>
177-
<ItemGroup>
178-
<IncludeCopyLocal Include="System.Reactive.dll" />
179-
<IncludeCopyLocal Include="Newtonsoft.Json.dll" />
180-
<IncludeCopyLocal Include="ExtCore.dll" />
181-
<IncludeCopyLocal Include="Fantomas.dll" />
182-
<IncludeCopyLocal Include="FSharp.Compiler.CodeDom.dll" />
183-
<IncludeCopyLocal Include="FSharp.Compiler.Service.dll" />
184-
<IncludeCopyLocal Include="FSharp.Core.dll" />
185-
<IncludeCopyLocal Include="MonoDevelop.FSharp.Shared.dll" />
186-
<IncludeCopyLocal Include="FSharp.Compiler.Interactive.Settings.dll" />
187-
</ItemGroup>
188187
<Choose>
189188
<When Condition="$(TargetFrameworkIdentifier) == '.NETFramework' And $(TargetFrameworkVersion) == 'v4.7.2'">
190189
<ItemGroup>

‎main/external/fsharpbinding/MonoDevelop.FSharpBinding/FSharpTokens.fs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,17 @@ module Tokens =
2222
match token with
2323
| Some token -> MonoDevelop.FSharp.Shared.Lexer.isNonTipToken token
2424
| None -> false
25-
25+
26+
27+
//(FSharpTokenInfo list * string) list option
2628
let tryGetTokens source defines fileName =
2729
try
2830
LoggingService.logDebug "FSharpParser: Processing tokens for %s" (Path.GetFileName fileName)
2931
let readOnlyDoc = TextEditorFactory.CreateNewReadonlyDocument (source, fileName)
30-
let lines = readOnlyDoc.GetLines() |> Seq.map readOnlyDoc.GetLineText
32+
let lines = readOnlyDoc.GetLines()
33+
|> Seq.map readOnlyDoc.GetLineText
34+
|> List.ofSeq
35+
3136
let tokens = MonoDevelop.FSharp.Shared.Lexer.getTokensWithInitialState FSharpTokenizerLexState.Initial lines (Some fileName) defines
3237
Some(tokens)
3338
with ex ->

0 commit comments

Comments
 (0)