Add initial lossless syntax tree implementation

Reqnroll.CodeAnalysis.Gherkin/.editorconfig

@@ -0,0 +1,10 @@
+[*]
+insert_final_newline = true
+indent_size = 4
+indent_style = space
+
+[*.cs]
+csharp_style_namespace_declarations = file_scoped
+
+[*.{csproj,targets,props}]
+indent_size = 2

Reqnroll.CodeAnalysis.Gherkin/DiagnosticDescriptors.cs

@@ -0,0 +1,21 @@
+using Microsoft.CodeAnalysis;
+using System.Resources;
+using System.Runtime.CompilerServices;
+
+namespace Reqnroll.CodeAnalysis.Gherkin;
+
+internal static partial class DiagnosticDescriptors
+{
+    private static ResourceManager ResourceManager { get; } = new ResourceManager(typeof(DiagnosticDescriptors));
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    internal static string? GetResourceString(string resourceKey) => ResourceManager.GetString(resourceKey);
+
+    public static readonly DiagnosticDescriptor ErrorExpectedFeatureOrTag = new(
+        id: DiagnosticIds.ErrorExpectedFeatureOrTag,
+        title: GetResourceString("ErrorExpectedFeatureOrTagTitle")!,
+        messageFormat: GetResourceString("ErrorExpectedFeatureOrTagMessage")!,
+        "Reqnroll.Gherkin",
+        DiagnosticSeverity.Error,
+        true);
+}

Reqnroll.CodeAnalysis.Gherkin/DiagnosticDescriptors.restext

@@ -0,0 +1,2 @@
+ErrorExpectedFeatureOrTagMessage = Feature or tag expected
+ErrorExpectedFeatureOrTagTitle = Feature or tag expected

Reqnroll.CodeAnalysis.Gherkin/DiagnosticIds.cs

@@ -0,0 +1,6 @@
+namespace Reqnroll.CodeAnalysis.Gherkin;
+
+internal static class DiagnosticIds
+{
+    public const string ErrorExpectedFeatureOrTag = "G1001";
+}

Reqnroll.CodeAnalysis.Gherkin/DialectProvider.cs

@@ -0,0 +1,47 @@
+using Gherkin;
+using Gherkin.Ast;
+
+namespace Reqnroll.CodeAnalysis.Gherkin;
+
+class DialectProvider(string defaultLanguage) : GherkinDialectProvider(defaultLanguage)
+{
+    public override GherkinDialect GetDialect(string language, Location location)
+    {
+        try
+        {
+            return base.GetDialect(language, location);
+        }
+        catch (NoSuchLanguageException)
+        {
+            var index = language.LastIndexOf('-');
+
+            if (index < 0)
+            {
+                throw;
+            }
+
+            var fallback = language.Substring(0, index);
+
+            return GetFallbackDialect(fallback, language, location);
+        }
+    }
+
+    private GherkinDialect GetFallbackDialect(string fallback, string language, Location location)
+    {
+        var dialect = GetDialect(fallback, location);
+
+        return new GherkinDialect(
+            language,
+            dialect.FeatureKeywords,
+            dialect.RuleKeywords,
+            dialect.BackgroundKeywords,
+            dialect.ScenarioKeywords,
+            dialect.ScenarioOutlineKeywords,
+            dialect.ExamplesKeywords,
+            dialect.GivenStepKeywords,
+            dialect.WhenStepKeywords,
+            dialect.ThenStepKeywords,
+            dialect.AndStepKeywords,
+            dialect.ButStepKeywords);
+    }
+}

Reqnroll.CodeAnalysis.Gherkin/GherkinParseOptions.cs

@@ -0,0 +1,43 @@
+using System.Globalization;
+
+namespace Reqnroll.CodeAnalysis.Gherkin;
+
+public sealed class GherkinParseOptions
+{
+    /// <summary>
+    /// The default parse options.
+    /// </summary>
+    public static GherkinParseOptions Default { get; } = new();
+
+    /// <summary>
+    /// Gets the culture to use when parsing text. This determines the language that will be used for the Gherkin keywords.
+    /// If the document contains a language directive, the directive takes precedent.
+    /// </summary>
+    public CultureInfo Culture { get; private set; }
+
+    internal GherkinParseOptions(CultureInfo? culture = null)
+    {
+        Culture = culture ?? CultureInfo.CurrentCulture;
+    }
+
+    private GherkinParseOptions(GherkinParseOptions other)
+    {
+        Culture = other.Culture;
+    }
+
+    /// <summary>
+    /// Returns a <see cref="GherkinParseOptions"/> instance for a specified culture.
+    /// </summary>
+    /// <param name="culture">The culture to use.</param>
+    /// <returns>If the culture is different, a new instance of <see cref="GherkinParseOptions"/> with the specified culture;
+    /// otherwise the current options instance.</returns>
+    public GherkinParseOptions WithCulture(CultureInfo culture)
+    {
+        if (culture == Culture)
+        {
+            return this;
+        }
+
+        return new GherkinParseOptions(this) { Culture = culture };
+    }
+}

Reqnroll.CodeAnalysis.Gherkin/GherkinSyntaxTree.cs

@@ -0,0 +1,133 @@
+using Gherkin;
+using System.Text;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Text;
+
+namespace Reqnroll.CodeAnalysis.Gherkin;
+
+using Reqnroll.CodeAnalysis.Gherkin.Syntax;
+
+/// <summary>
+/// A container of Gherkin syntax.
+/// </summary>
+/// <remarks>
+/// <para>This Roslyn-like representation enables:</para>
+/// <list type="bullet">
+///     <item>Simple syntactic equivalence comparison.</item>
+///     <item>Full mapping of Gherkin syntax positions to generated code because this model tracks whitespace.</item>
+///     <item>Future incremental parsing enhancements which rely on a model that can offset existing elements.</item>
+/// </list>
+/// </remarks>
+public class GherkinSyntaxTree
+{
+    private SourceText? _text;
+    private readonly SyntaxNode _root;
+
+    internal GherkinSyntaxTree(SourceText? text, SyntaxNode root, GherkinParseOptions options, string filePath)
+    {
+        _text = text;
+        _root = SyntaxNode.CloneAsRoot(root, this);
+        Options = options;
+        FilePath = filePath;
+    }
+
+    /// <summary>
+    /// Gets the options used by the parser to produce the syntax tree.
+    /// </summary>
+    public GherkinParseOptions Options { get; }
+
+    /// <summary>
+    /// Gets the path of the source document file.
+    /// </summary>
+    /// <value>The path of the source document file, or an empty string if the syntax tree is not associated with a file.</value>
+    public string FilePath { get; }
+
+    /// <summary>
+    /// Gets a list of all the diagnostics in the syntax tree.
+    /// </summary>
+    /// <param name="cancellationToken">A token used to signal when the operation should be canceled.</param>
+    /// <returns>A list of the <see cref="Diagnostic"/> instances associated with nodes in the tree.</returns>
+    public IEnumerable<Diagnostic> GetDiagnostics(CancellationToken cancellationToken = default) => 
+        GetDiagnostics(GetRoot(cancellationToken));
+
+    public IEnumerable<Diagnostic> GetDiagnostics(SyntaxToken token) => GetDiagnostics(token.RawNode, token.Position);
+
+    public IEnumerable<Diagnostic> GetDiagnostics(SyntaxTrivia trivia) => GetDiagnostics(trivia.RawNode, trivia.Position);
+
+    public IEnumerable<Diagnostic> GetDiagnostics(SyntaxNode node) => GetDiagnostics(node.RawNode, node.Position);
+
+    private IEnumerable<Diagnostic> GetDiagnostics(Syntax.Internal.RawNode? node, int position)
+    {
+        if (node != null && node.ContainsDiagnostics)
+        {
+            return EnumerateDiagnostics(node, position);
+        }
+
+        return [];
+    }
+
+    private IEnumerable<Diagnostic> EnumerateDiagnostics(Syntax.Internal.RawNode node, int position)
+    {
+        throw new NotImplementedException();
+    }
+
+    public SyntaxNode GetRoot(CancellationToken cancellationToken = default) => _root;
+
+    public SourceText GetSourceText(CancellationToken cancellationToken)
+    {
+        _text ??= GetRoot(cancellationToken).GetText();
+
+        return _text;
+    }
+
+    public static GherkinSyntaxTree ParseText(
+        string text,
+        GherkinParseOptions? options = null,
+        string path = "",
+        Encoding? encoding = null,
+        CancellationToken cancellationToken = default)
+    {
+        return ParseText(
+            SourceText.From(text, encoding, SourceHashAlgorithm.Sha1),
+            options,
+            path,
+            cancellationToken);
+    }
+
+    public static GherkinSyntaxTree ParseText(
+        SourceText text,
+        GherkinParseOptions? options = null,
+        string path = "",
+        CancellationToken cancellationToken = default)
+    {
+#if NET6_0_OR_GREATER
+        ArgumentNullException.ThrowIfNull(nameof(text));
+#else
+        if (text == null)
+        {
+            throw new ArgumentNullException(nameof(text));
+        }
+#endif
+
+        options ??= GherkinParseOptions.Default;
+
+        var matcher = new TokenMatcher(new DialectProvider(options.Culture.Name));
+        var builder = new Syntax.Internal.ParsedSyntaxTreeBuilder(options, text, path, cancellationToken);
+        var parser = new SyntaxParser(builder);
+        var tokens = new SourceTokenScanner(text);
+
+        return parser.Parse(tokens, matcher);
+    }
+
+    public Location GetLocation(TextSpan span) => 
+        Location.Create(FilePath, span, GetSourceText(CancellationToken.None).Lines.GetLinePositionSpan(span));
+
+    public SourceText GetText(CancellationToken cancellationToken)
+    {
+        _text ??= GetRoot(cancellationToken).GetText();
+
+        return _text;
+    }
+
+    public override string ToString() => GetText(CancellationToken.None).ToString();
+}

Reqnroll.CodeAnalysis.Gherkin/Hash.cs

@@ -0,0 +1,16 @@
+internal static class Hash
+{
+    private const int Prime1 = 79754099;
+    private const int Prime2 = 60214523;
+
+    public static int Combine<T1, T2, T3>(T1 value1, T2 value2, T3 value3)
+    {
+        var hash = Prime1;
+
+        hash = (Prime2 * hash) + value1?.GetHashCode() ?? 0;
+        hash = (Prime2 * hash) + value2?.GetHashCode() ?? 0;
+        hash = (Prime2 * hash) + value3?.GetHashCode() ?? 0;
+
+        return hash;
+    }
+}

Reqnroll.CodeAnalysis.Gherkin/IsExternalInit.cs

@@ -0,0 +1,22 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#if NETSTANDARD2_0 || NETSTANDARD2_1 || NETCOREAPP2_0 || NETCOREAPP2_1 || NETCOREAPP2_2 || NETCOREAPP3_0 || NETCOREAPP3_1 || NET45 || NET451 || NET452 || NET6 || NET461 || NET462 || NET47 || NET471 || NET472 || NET48
+
+using System.ComponentModel;
+
+// ReSharper disable once CheckNamespace
+namespace System.Runtime.CompilerServices
+{
+    /// <summary>
+    /// Reserved to be used by the compiler for tracking metadata.
+    /// This class should not be used by developers in source code.
+    /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    internal static class IsExternalInit
+    {
+    }
+}
+
+#endif