Incremental Parsing

package.json

@@ -207,6 +207,7 @@
     "command-line-usage": "^7.0.3",
     "commonmark": "^0.31.2",
     "dagre": "^0.8.5",
+    "diff": "^8.0.3",
     "gray-matter": "^4.0.3",
     "joi": "^18.0.1",
     "lz-string": "^1.5.0",

src/config.ts

@@ -250,7 +250,7 @@ export const FlowrConfig = {
 	 * The default configuration for flowR, used when no config file is found or when a config file is missing some options.
 	 * You can use this as a base for your own config and only specify the options you want to change.
 	 */
-	default(): FlowrConfig {
+	default(this: void): FlowrConfig {
 		return {
 			ignoreSourceCalls: false,
 			semantics:         {
@@ -364,7 +364,7 @@ export const FlowrConfig = {
 	/**
 	 * Parses the given JSON string as a flowR config file, returning the resulting config object if the parsing and validation were successful, or `undefined` if there was an error.
 	 */
-	parse(jsonString: string): FlowrConfig | undefined {
+	parse(this: void, jsonString: string): FlowrConfig | undefined {
 		try {
 			const parsed   = JSON.parse(jsonString) as FlowrConfig;
 			const validate = FlowrConfig.Schema.validate(parsed);
@@ -383,14 +383,14 @@ export const FlowrConfig = {
 	 * Creates a new flowr config that has the updated values.
 	 */
 	// eslint-disable-next-line @typescript-eslint/no-invalid-void-type
-	amend(config: FlowrConfig, amendmentFunc: (config: DeepWritable<FlowrConfig>) => FlowrConfig | void): FlowrConfig {
+	amend(this: void, config: FlowrConfig, amendmentFunc: (config: DeepWritable<FlowrConfig>) => FlowrConfig | void): FlowrConfig {
 		const newConfig = FlowrConfig.clone(config);
 		return amendmentFunc(newConfig as DeepWritable<FlowrConfig>) ?? newConfig;
 	},
 	/**
 	 * Clones the given flowr config object.
 	 */
-	clone(config: FlowrConfig): FlowrConfig {
+	clone(this: void, config: FlowrConfig): FlowrConfig {
 		return deepClonePreserveUnclonable(config);
 	},
 	/**
@@ -399,7 +399,7 @@ export const FlowrConfig = {
 	 * infer the config from flowR's default locations.
 	 * This is mostly useful for user-facing features.
 	 */
-	fromFile(configFile?: string, configWorkingDirectory = process.cwd()): FlowrConfig {
+	fromFile(this: void, configFile?: string, configWorkingDirectory = process.cwd()): FlowrConfig {
 		try {
 			return loadConfigFromFile(configFile, configWorkingDirectory);
 		} catch(e) {
@@ -410,7 +410,7 @@ export const FlowrConfig = {
 	/**
 	 * Gets the configuration for the given engine type from the config.
 	 */
-	getForEngine<T extends EngineConfig['type']>(config: FlowrConfig, engine: T): EngineConfig & { type: T } | undefined {
+	getForEngine<T extends EngineConfig['type']>(this: void, config: FlowrConfig, engine: T): EngineConfig & { type: T } | undefined {
 		const engines = config.engines;
 		if(engines.length > 0) {
 			return engines.find(e => e.type === engine) as EngineConfig & { type: T } | undefined;
@@ -429,7 +429,7 @@ export const FlowrConfig = {
 	 * console.log(newConfig.solver.variables); // Output: "builtin"
 	 * ```
 	 */
-	setInConfig<Path extends ValidFlowrConfigPaths>(config: FlowrConfig, key: Path, value: PathValue<FlowrConfig, Path>): FlowrConfig {
+	setInConfig<Path extends ValidFlowrConfigPaths>(this: void, config: FlowrConfig, key: Path, value: PathValue<FlowrConfig, Path>): FlowrConfig {
 		const clone = FlowrConfig.clone(config);
 		objectPath.set(clone, key, value);
 		return clone;
@@ -438,7 +438,7 @@ export const FlowrConfig = {
 	 * Modifies the given config object in place by setting the given value at the given key, where the key is a dot-separated path to the value in the config object.
 	 * @see {@link setInConfig} for a version that returns a new config object instead of modifying the given one in place.
 	 */
-	setInConfigInPlace<Path extends ValidFlowrConfigPaths>(config: FlowrConfig, key: Path, value: PathValue<FlowrConfig, Path>): void {
+	setInConfigInPlace<Path extends ValidFlowrConfigPaths>(this: void, config: FlowrConfig, key: Path, value: PathValue<FlowrConfig, Path>): void {
 		objectPath.set(config, key, value);
 	}
 } as const;

src/dataflow/internal/process/functions/call/built-in/built-in-source.ts

@@ -246,7 +246,7 @@ export function sourceRequest<OtherInfo>(rootId: NodeId, request: RParseRequest
 		} else {
 			guard(textRequest !== undefined, `Expected text request to be defined for sourced file ${JSON.stringify(request)}`);
 		}
-		const parsed = (!data.parser.async ? data.parser : new RShellExecutor()).parse(textRequest.r);
+		const parsed = (!data.parser.async ? data.parser : new RShellExecutor()).parse(textRequest.r, data.ctx);
 		const normalized = (typeof parsed !== 'string' ?
 			normalizeTreeSitter({ files: [{ parsed, filePath: textRequest.path }] }, getId, data.ctx.config)
 			: normalize({ files: [{ parsed, filePath: textRequest.path }] }, getId)) as NormalizedAst<OtherInfo & ParentInformation>;

src/documentation/wiki-analyzer.ts

@@ -39,7 +39,9 @@ import { FlowrAnalyzerPlugin } from '../project/plugins/flowr-analyzer-plugin';
 import { FlowrAnalyzerEnvironmentContext } from '../project/context/flowr-analyzer-environment-context';
 import { FlowrAnalyzerFunctionsContext } from '../project/context/flowr-analyzer-functions-context';
 import { FlowrAnalyzerMetaContext } from '../project/context/flowr-analyzer-meta-context';
+import { FlowrAnalyzerIncrementalAnalysisContext } from '../project/context/flowr-analyzer-incremental-analysis-context';
 import { FlowrConfig } from '../config';
+import { FlowrInlineTextFile } from '../project/context/flowr-file';
 
 async function analyzerQuickExample() {
 	const analyzer = await new FlowrAnalyzerBuilder()
@@ -99,11 +101,12 @@ ${
 			'How to add a new plugin': undefined,
 		},
 		'Context Information': {
-			'Files Context':         undefined,
-			'Loading Order Context': undefined,
-			'Dependencies Context':  undefined,
-			'Environment Context':   undefined,
-			'Meta Context':          undefined,
+			'Files Context':                undefined,
+			'Loading Order Context':        undefined,
+			'Dependencies Context':         undefined,
+			'Environment Context':          undefined,
+			'Meta Context':                 undefined,
+			'Incremental Analysis Context': undefined,
 		},
 		'Caching': undefined
 	})
@@ -478,6 +481,52 @@ and the project namespace via
 ${ctx.linkM(FlowrAnalyzerMetaContext, 'getNamespace', { codeFont: true, realNameWrapper: 'i' })}.
 
 
+${section('Incremental Analysis Context', 3)}
+
+The ${ctx.link(FlowrAnalyzerIncrementalAnalysisContext)} is a context that stores analysis information needed for making the next analysis run incremental by reusing the previous analysis results:
+
+${ctx.hierarchy(FlowrAnalyzerIncrementalAnalysisContext, { showImplSnippet: false })}
+
+This context is not an analysis-result cache by itself.
+Instead, it carries forward the minimal state needed by future incremental phases after an invalidation happened.
+At the moment, it is used for incremental parsing with Tree-sitter, but it is intended to become the shared context for additional incremental analysis stages as well.
+
+If the analyzer or context is reset, the incremental information is discarded via
+${ctx.linkM(FlowrAnalyzerIncrementalAnalysisContext, 'reset', { codeFont: true, realNameWrapper: 'i' })}.
+In other words, this context only transports incremental handoff state between analysis runs.
+
+${section('Incremental Parsing', 4)}
+
+This context is used to exploit Tree-sitter's incremental parsing feature.
+For one file, the incremental state follows a fixed lifecycle:
+
+1. After a successful parse-oriented analysis run, the analyzer cache stores the latest Tree-sitter parse tree via
+   ${ctx.linkM(FlowrAnalyzerIncrementalAnalysisContext, 'storeOldParseResults', { codeFont: true, realNameWrapper: 'i' })}.
+   This tree is the baseline for the next incremental parse of that file.
+2. When a mutable file provider such as ${ctx.link('FlowrInlineTextFile')} is invalidated via
+   ${ctx.linkM(FlowrInlineTextFile, 'invalidate', { codeFont: true, realNameWrapper: 'i' })},
+   the analyzer receives a file invalidation event and stores the file path together with the old source text.
+   If the same file is invalidated again before the next parse, this stored old text is intentionally **not** replaced:
+   the stored parse tree still belongs to the version from before the first invalidation, so the incremental parse must keep that matching old-content baseline.
+3. When parsing is requested again, flowR retrieves
+   * the previous parse tree from
+     ${ctx.linkM(FlowrAnalyzerIncrementalAnalysisContext, 'getOldParseResultOf', { codeFont: true, realNameWrapper: 'i' })}
+   * the stored old source text from
+     ${ctx.linkM(FlowrAnalyzerIncrementalAnalysisContext, 'getOldContentOf', { codeFont: true, realNameWrapper: 'i' })}
+
+   Using these together with the current file content, flowR computes a minimal ${ctx.link('Parser.Edit')} only when a new parse is actually requested.
+   If the file content did not change, the previous tree can be reused directly.
+   Otherwise, the edit is applied to the previous tree and Tree-sitter reparses incrementally instead of starting from scratch.
+4. The stored old-content entry is removed when it is used because it belongs only to that previous parse snapshot.
+   After the new parse succeeds, the analyzer stores a new parse tree baseline.
+   A later invalidation must then be able to record a fresh old-content value that matches this new tree.
+   If the old-content entry were kept, later invalidations of the same file would not replace it, and the next incremental parse could compare the current file content against stale old text that no longer matches the stored previous tree.
+
+${section('Incremental Dataflow', 4)}
+
+This context is planned to also support future incremental dataflow graph computation.
+
+
 ${section('Caching', 2)}
 
 To speed up analyses, flowR provides a caching mechanism that stores intermediate results of the analysis.

src/project/cache/flowr-analyzer-cache.ts

@@ -1,5 +1,5 @@
-import type { KnownParser } from '../../r-bridge/parser';
-import { type CacheInvalidationEvent, CacheInvalidationEventType, FlowrCache } from './flowr-cache';
+import type { KnownParser, ParseStepOutput } from '../../r-bridge/parser';
+import { type InvalidationEvent, InvalidationEventType, FlowrCache } from './flowr-cache';
 import {
 	createDataflowPipeline,
 	type DEFAULT_DATAFLOW_PIPELINE,
@@ -18,7 +18,7 @@ import type { FlowrAnalyzerContext } from '../context/flowr-analyzer-context';
 import { FlowrAnalyzerControlFlowCache } from './flowr-analyzer-controlflow-cache';
 import type { CallGraph } from '../../dataflow/graph/call-graph';
 import { computeCallGraph } from '../../dataflow/graph/call-graph';
-
+import type { Tree } from 'web-tree-sitter';
 interface FlowrAnalyzerCacheOptions<Parser extends KnownParser> {
 	parser:  Parser;
 	context: FlowrAnalyzerContext;
@@ -56,30 +56,33 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 		}) as AnalyzerPipelineExecutor<Parser>;
 		this.controlFlowCache = new FlowrAnalyzerControlFlowCache();
 		this.callGraphCache = undefined;
+		this.computeIfAbsent(true, () => this.pipeline?.getResults(true));
 	}
 
 	public static create<Parser extends KnownParser>(data: FlowrAnalyzerCacheOptions<Parser>): FlowrAnalyzerCache<Parser> {
 		return new FlowrAnalyzerCache<Parser>(data);
 	}
 
-	public override receive(event: CacheInvalidationEvent): void {
+	public override receive(event: InvalidationEvent): void {
 		super.receive(event);
-		switch(event.type) {
-			case CacheInvalidationEventType.Full:
+		const type = event.type;
+		switch(type) {
+			case InvalidationEventType.Full:
+			case InvalidationEventType.SingleFileInvalidate:
 				this.initCacheProviders();
 				break;
 			default:
-				assertUnreachable(event.type);
+				assertUnreachable(type);
 		}
 	}
 
 	private get(): AnalyzerCacheType<Parser> {
 		/* this will do a ref assignment, so indirect force */
-		return this.computeIfAbsent(false, () => this.pipeline.getResults(true));
+		return this.computeIfAbsent(false, () => this.pipeline?.getResults(true));
 	}
 
 	public reset() {
-		this.receive({ type: CacheInvalidationEventType.Full });
+		this.receive({ type: InvalidationEventType.Full });
 	}
 
 	private async runTapeUntil<T>(force: boolean | undefined, until: () => T | undefined): Promise<T> {
@@ -92,10 +95,26 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 		while((g = until()) === undefined && this.pipeline.hasNextStep()) {
 			await this.pipeline.nextStep();
 		}
+
+		this.storeIncrementalSnapshotIfAvailable();
+
 		guard(g !== undefined, 'Could not reach the desired pipeline step, invalid cache state(?)');
 		return g;
 	}
 
+	private storeIncrementalSnapshotIfAvailable(): void {
+		if(this.args.parser.name !== 'tree-sitter') {
+			return;
+		}
+
+		const parse = this.peekParse();
+		if(parse !== undefined) {
+			this.args.context.inc.storeOldParseResults(
+				parse as ParseStepOutput<Tree> // cast needed because of TypeScript's limited narrowing capabilities
+			);
+		}
+	}
+
 	/**
 	 * Get the parse output for the request, parsing if necessary.
 	 * @param force - Do not use the cache, instead force a new parse.
@@ -112,7 +131,7 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 	 * @see {@link FlowrAnalyzerCache#parse} - to get the parse output, parsing if necessary.
 	 */
 	public peekParse(): NonNullable<AnalyzerCacheType<Parser>['parse']> | undefined {
-		return this.get().parse;
+		return this.get()?.parse;
 	}
 
 	/**
@@ -131,7 +150,7 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 	 * @see {@link FlowrAnalyzerCache#normalize} - to get the normalized AST, normalizing if necessary.
 	 */
 	public peekNormalize(): NonNullable<AnalyzerCacheType<Parser>['normalize']> | undefined {
-		return this.get().normalize;
+		return this.get()?.normalize;
 	}
 
 	/**
@@ -150,7 +169,7 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 	 * @see {@link FlowrAnalyzerCache#dataflow} - to get the dataflow graph, computing if necessary.
 	 */
 	public peekDataflow(): NonNullable<AnalyzerCacheType<Parser>['dataflow']> | undefined {
-		return this.get().dataflow;
+		return this.get()?.dataflow;
 	}
 
 	/**