diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa28732a..54208846a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,14 @@ Here are some of the features that notebooks bring: - After each cell is executed we issue a `retall` command to make sure that we are at the top-level and not stopped in a weird state +## 3.2.2 September 2023 + +Major change to the language server and worker threads to implement a cancellation framework. The ability to cancel work happens automatically for PRO files and IDL Notebooks. + +This change will address performance issues where, if the code can not be parsed as quickly as you were typing, you would not get auto-complete, outlines, hover-help, semantic tokens, formatting on save, etc. + +In these cases it could take 15-30 seconds for the language server to respond while it worked through a backlog of processing that was no longer relevant. + ## 3.2.1 August 2023 Notebook key behavior change: If you are running one or more cells, and there is an error from IDL for any cell being executed, all pending cells are cleared and not executed. This makes sure that, if later cells depend on earlier ones, that you don't have cascading failures. diff --git a/apps/parsing-worker/src/main.ts b/apps/parsing-worker/src/main.ts index 8037b8bee..ae283dce1 100644 --- a/apps/parsing-worker/src/main.ts +++ b/apps/parsing-worker/src/main.ts @@ -98,9 +98,9 @@ client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.TRACK_GLOBAL, async (message) => { */ client.on( LSP_WORKER_THREAD_MESSAGE_LOOKUP.CHANGE_DETECTION, - async (message) => { + async (message, cancel) => { // run change detection! - const changed = ChangeDetection(WORKER_INDEX, message.changed); + const changed = ChangeDetection(WORKER_INDEX, cancel, message.changed); // get syntax problems const problems = WORKER_INDEX.getSyntaxProblems(); @@ -171,45 +171,51 @@ client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.GET_TOKEN_DEF, async (message) => { /** * Handle requests to parse and post process a file */ -client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_FILE, async (message) => { - // index the file - const parsed = await WORKER_INDEX.getParsedProCode( - message.file, - WORKER_INDEX.getFileStrings(message.file), - message - ); - - // make non-circular - RemoveScopeDetail(parsed); - - // return - return parsed; -}); +client.on( + LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_FILE, + async (message, cancel) => { + // index the file + const parsed = await WORKER_INDEX.getParsedProCode( + message.file, + WORKER_INDEX.getFileStrings(message.file), + message + ); + + // make non-circular + RemoveScopeDetail(parsed, cancel); + + // return + return parsed; + } +); /** * Handle requests to parse and post process code for a file */ -client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_CODE, async (message) => { - // index the file - const parsed = await WORKER_INDEX.getParsedProCode( - message.file, - message.code, - message - ); - - // make non-circular - RemoveScopeDetail(parsed); - - // return - return parsed; -}); +client.on( + LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_CODE, + async (message, cancel) => { + // index the file + const parsed = await WORKER_INDEX.getParsedProCode( + message.file, + message.code, + message + ); + + // make non-circular + RemoveScopeDetail(parsed, cancel); + + // return + return parsed; + } +); /** * Parse files quickly to get the basic overview and thats it */ client.on( LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_FILES_FAST, - async (message) => { + async (message, cancel) => { /** Get files to process */ const files = message.files; @@ -233,7 +239,7 @@ client.on( /** * Parse our file */ - const parsed = ParseFileSync(files[i], { full: false }); + const parsed = ParseFileSync(files[i], cancel, { full: false }); // track syntax problems WORKER_INDEX.trackSyntaxProblemsForFile(files[i], parsed.parseProblems); @@ -325,50 +331,54 @@ client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_FILES, async (message) => { /** * Parse notebooks */ -client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_NOTEBOOK, async (message) => { - /** - * Initialize our response - */ - const resp: ParseNotebookResponse = { - lines: 0, - globals: {}, - problems: {}, - }; +client.on( + LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_NOTEBOOK, + async (message, cancel) => { + /** + * Initialize our response + */ + const resp: ParseNotebookResponse = { + lines: 0, + globals: {}, + problems: {}, + }; - /** - * Index our notebook - */ - const byCell = await WORKER_INDEX.indexIDLNotebook( - message.file, - message.notebook - ); + /** + * Index our notebook + */ + const byCell = await WORKER_INDEX.getParsedNotebook( + message.file, + message.notebook, + cancel + ); - /** - * Get files for cells that we actually processed - */ - const files = Object.keys(byCell); + /** + * Get files for cells that we actually processed + */ + const files = Object.keys(byCell); - // process each cell and save information we need to return - for (let i = 0; i < files.length; i++) { - if (byCell[files[i]] === undefined) { - resp.globals[files[i]] = []; - resp.problems[files[i]] = []; - continue; + // process each cell and save information we need to return + for (let i = 0; i < files.length; i++) { + if (byCell[files[i]] === undefined) { + resp.globals[files[i]] = []; + resp.problems[files[i]] = []; + continue; + } + resp.globals[files[i]] = byCell[files[i]].global; + resp.problems[files[i]] = GetSyntaxProblems(byCell[files[i]]); } - resp.globals[files[i]] = byCell[files[i]].global; - resp.problems[files[i]] = GetSyntaxProblems(byCell[files[i]]); - } - // return each cell - return resp; -}); + // return each cell + return resp; + } +); /** * Get notebook cells */ client.on( LSP_WORKER_THREAD_MESSAGE_LOOKUP.GET_NOTEBOOK_CELL, - async (message) => { + async (message, cancel) => { // get parsed code and return const parsed = await WORKER_INDEX.getParsedProCode( message.file, @@ -377,7 +387,7 @@ client.on( // make non-circular if (parsed !== undefined) { - RemoveScopeDetail(parsed); + RemoveScopeDetail(parsed, cancel); } return parsed; @@ -389,14 +399,18 @@ client.on( */ client.on( LSP_WORKER_THREAD_MESSAGE_LOOKUP.POST_PROCESS_FILES, - async (message) => { + async (message, cancel) => { /** Get files */ const files = Array.isArray(message.files) ? message.files : WORKER_INDEX.tokensByFile.allFiles(); // post process, no change detection - const missing = await WORKER_INDEX.postProcessProFiles(files, false); + const missing = await WORKER_INDEX.postProcessProFiles( + files, + cancel, + false + ); // get syntax problems const problems = WORKER_INDEX.getSyntaxProblems(); @@ -440,46 +454,53 @@ client.on( * TODO: Correctly perform change detection from removing files instead of processing everything * we have which is brute force but works */ -client.on(LSP_WORKER_THREAD_MESSAGE_LOOKUP.REMOVE_FILES, async (message) => { - // remove all files - await WORKER_INDEX.removeWorkspaceFiles(message.files, false); - - /** Get files that we manage */ - const ourFiles = WORKER_INDEX.tokensByFile.allFiles(); +client.on( + LSP_WORKER_THREAD_MESSAGE_LOOKUP.REMOVE_FILES, + async (message, cancel) => { + // remove all files + await WORKER_INDEX.removeWorkspaceFiles(message.files, false); + + /** Get files that we manage */ + const ourFiles = WORKER_INDEX.tokensByFile.allFiles(); + + // post process all of our files again + const missing = await WORKER_INDEX.postProcessProFiles( + ourFiles, + cancel, + false + ); - // post process all of our files again - const missing = await WORKER_INDEX.postProcessProFiles(ourFiles, false); + // get syntax problems + const problems = WORKER_INDEX.getSyntaxProblems(); - // get syntax problems - const problems = WORKER_INDEX.getSyntaxProblems(); + // craft our response + const resp: RemoveFilesResponse = { + problems: {}, + missing, + }; - // craft our response - const resp: RemoveFilesResponse = { - problems: {}, - missing, - }; + // populate response + for (let i = 0; i < ourFiles.length; i++) { + if (global.gc) { + if (i % IDL_INDEX_OPTIONS.GC_FREQUENCY === 0) { + global.gc(); + } + } - // populate response - for (let i = 0; i < ourFiles.length; i++) { - if (global.gc) { - if (i % IDL_INDEX_OPTIONS.GC_FREQUENCY === 0) { - global.gc(); + /** + * Skip if we dont have a file. Could happen from parsing errors + */ + if (!WORKER_INDEX.tokensByFile.has(ourFiles[i])) { + continue; } - } - /** - * Skip if we dont have a file. Could happen from parsing errors - */ - if (!WORKER_INDEX.tokensByFile.has(ourFiles[i])) { - continue; + // populate problems + resp.problems[ourFiles[i]] = problems[ourFiles[i]] || []; } - // populate problems - resp.problems[ourFiles[i]] = problems[ourFiles[i]] || []; + return resp; } - - return resp; -}); +); /** * Listen for events from our main thread diff --git a/apps/test-tokenizer/src/parse-tests/format-test.ts b/apps/test-tokenizer/src/parse-tests/format-test.ts index 4ffd9243d..c151300c7 100644 --- a/apps/test-tokenizer/src/parse-tests/format-test.ts +++ b/apps/test-tokenizer/src/parse-tests/format-test.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { GetTokenNames, Parser } from '@idl/parser'; import { TimeIt } from '@idl/shared'; import { deepEqual } from 'fast-equals'; @@ -63,7 +64,7 @@ export async function FormatTest(folder: string): Promise { for (let i = 0; i < code.length; i++) { let canTick = true; // extract tokens - const tokenized = Parser(code[i]); + const tokenized = Parser(code[i], new CancellationToken()); // validate formatting for code // track time so we can more accurately represent true parsing time @@ -73,12 +74,14 @@ export async function FormatTest(folder: string): Promise { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { formatter: 'fiddle' }); + const formatted = Assembler(tokenized, new CancellationToken(), { + formatter: 'fiddle', + }); // verify that our tokens are the same as they were before if (formatted !== undefined) { // parse formatted code - const reParsed = Parser(formatted); + const reParsed = Parser(formatted, new CancellationToken()); // make sure things equal if (!deepEqual(GetTokenNames(reParsed), tokenizedNames)) { diff --git a/apps/test-tokenizer/src/parse-tests/routine-popularity.ts b/apps/test-tokenizer/src/parse-tests/routine-popularity.ts index 9329dcc71..c43e2a044 100644 --- a/apps/test-tokenizer/src/parse-tests/routine-popularity.ts +++ b/apps/test-tokenizer/src/parse-tests/routine-popularity.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { GLOBAL_TOKEN_TYPES } from '@idl/data-types/core'; import { GetRoutine, IDLIndex } from '@idl/parsing/index'; import { IParsed, TreeRecurserBasic } from '@idl/parsing/syntax-tree'; @@ -63,7 +64,7 @@ ROUTINES[TOKEN_NAMES.CALL_PROCEDURE_METHOD] = true; * Tracks the usage stats for routines */ export function TrackPopularity(index: IDLIndex, parsed: IParsed) { - TreeRecurserBasic(parsed.tree, { + TreeRecurserBasic(parsed.tree, new CancellationToken(), { onBranchToken: (token) => { if (token.name in ROUTINES) { const defs = GetRoutine(index, parsed, token); diff --git a/apps/test-tokenizer/src/playground.ts b/apps/test-tokenizer/src/playground.ts index df2b8ad0b..b6c198e89 100644 --- a/apps/test-tokenizer/src/playground.ts +++ b/apps/test-tokenizer/src/playground.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { IParsed, RemoveScopeDetail } from '@idl/parsing/syntax-tree'; import { TimeIt } from '@idl/shared'; @@ -32,7 +33,7 @@ export async function Playground() { console.log(); console.log('Warming up algorithms...'); for (let i = 0; i < WARM_UP; i++) { - Parser(PLAYGROUND_CODE, { cleanup: false }); + Parser(PLAYGROUND_CODE, new CancellationToken(), { cleanup: false }); await TextMateParse(PLAYGROUND_CODE); } console.log(' Ready to rock!'); @@ -42,7 +43,7 @@ export async function Playground() { */ let parsed: IParsed; const t1 = TimeIt(() => { - Parser(PLAYGROUND_CODE, { cleanup: false }); + Parser(PLAYGROUND_CODE, new CancellationToken(), { cleanup: false }); }); console.log(); console.log(`Parser time (ms): ${t1}`); @@ -73,7 +74,7 @@ export async function Playground() { let formatted: string | undefined; const t2 = TimeIt(() => { formatted = 'no'; - formatted = Assembler(formattedTokenized, { + formatted = Assembler(formattedTokenized, new CancellationToken(), { formatter: 'fiddle', autoDoc: true, style: {}, @@ -109,8 +110,8 @@ export async function Playground() { } forTests.push(`]`); - RemoveScopeDetail(parsed); - RemoveScopeDetail(formattedTokenized); + RemoveScopeDetail(parsed, new CancellationToken()); + RemoveScopeDetail(formattedTokenized, new CancellationToken()); // save outputs to disk writeFileSync( diff --git a/apps/test-tokenizer/src/test-maker/generate-automated-tests.ts b/apps/test-tokenizer/src/test-maker/generate-automated-tests.ts index 5b260d89d..bd9657913 100644 --- a/apps/test-tokenizer/src/test-maker/generate-automated-tests.ts +++ b/apps/test-tokenizer/src/test-maker/generate-automated-tests.ts @@ -91,7 +91,7 @@ export async function GenerateAutomatedTests(useCache: boolean) { ) ) { console.log(` Suite (${i}): ${AUTO_TOKEN_TESTS[i].suiteName}`); - TestsForTokenizer( + await TestsForTokenizer( AUTO_TOKEN_TESTS[i].suiteName, AUTO_TOKEN_TESTS[i].tests, join(outDirToken, AUTO_TOKEN_TESTS[i].fileName) @@ -149,7 +149,7 @@ export async function GenerateAutomatedTests(useCache: boolean) { ) ) { console.log(` Suite (${i}): ${AUTO_POST_PROCESSOR_TESTS[i].suiteName}`); - TestsForSyntaxPostProcessors( + await TestsForSyntaxPostProcessors( AUTO_POST_PROCESSOR_TESTS[i].suiteName, AUTO_POST_PROCESSOR_TESTS[i].tests, join(outDirPostProcess, AUTO_POST_PROCESSOR_TESTS[i].fileName) @@ -207,7 +207,7 @@ export async function GenerateAutomatedTests(useCache: boolean) { ) ) { console.log(` Suite (${i}): ${AUTO_SELECTED_TOKEN_TESTS[i].suiteName}`); - TestsForTokenAtCursor( + await TestsForTokenAtCursor( AUTO_SELECTED_TOKEN_TESTS[i].suiteName, AUTO_SELECTED_TOKEN_TESTS[i].tests, join(outDirSelectedToken, AUTO_SELECTED_TOKEN_TESTS[i].fileName) @@ -328,7 +328,7 @@ export async function GenerateAutomatedTests(useCache: boolean) { console.log( ` Suite (${i}): ${AUTO_LOCAL_GLOBAL_SCOPE_COMPILE_AND_TYPES_TESTS[i].suiteName}` ); - TestsForLocalGlobalScopeAndCompile( + await TestsForLocalGlobalScopeAndCompile( AUTO_LOCAL_GLOBAL_SCOPE_COMPILE_AND_TYPES_TESTS[i].suiteName, AUTO_LOCAL_GLOBAL_SCOPE_COMPILE_AND_TYPES_TESTS[i].tests, join( diff --git a/apps/test-tokenizer/src/test-maker/makers/tests-for-assembler.ts b/apps/test-tokenizer/src/test-maker/makers/tests-for-assembler.ts index 8185d7e07..54ad437fe 100644 --- a/apps/test-tokenizer/src/test-maker/makers/tests-for-assembler.ts +++ b/apps/test-tokenizer/src/test-maker/makers/tests-for-assembler.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDLIndex } from '@idl/parsing/index'; import { writeFileSync } from 'fs'; @@ -24,6 +25,7 @@ export async function TestsForAssembler( // add imports strings.push(`import { Assembler } from '@idl/assembler';`); + strings.push(`import { CancellationToken } from '@idl/cancellation-tokens';`); strings.push(`import { LogManager } from '@idl/logger';`); strings.push(`import { GetTokenNames } from '@idl/parser';`); strings.push( @@ -73,7 +75,11 @@ export async function TestsForAssembler( }; // format - const formatted = Assembler(tokenized, test.config); + const formatted = Assembler( + tokenized, + new CancellationToken(), + test.config + ); // add our tokens strings.push(` it(\`[auto generated] ${testName}\`, async () => {`); @@ -101,14 +107,14 @@ export async function TestsForAssembler( strings.push(` // format code`); if (test.config !== undefined) { strings.push( - ` const formatted = Assembler(tokenized, ${JSON.stringify( + ` const formatted = Assembler(tokenized, new CancellationToken(), ${JSON.stringify( test.config )});` ); strings.push(``); } else { strings.push( - ` const formatted = Assembler(tokenized, { formatter: 'fiddle' });` + ` const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle' });` ); strings.push(``); } diff --git a/apps/test-tokenizer/src/test-maker/makers/tests-for-auto-fixing.ts b/apps/test-tokenizer/src/test-maker/makers/tests-for-auto-fixing.ts index 610ad0d99..c9203061c 100644 --- a/apps/test-tokenizer/src/test-maker/makers/tests-for-auto-fixing.ts +++ b/apps/test-tokenizer/src/test-maker/makers/tests-for-auto-fixing.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDLIndex } from '@idl/parsing/index'; import { writeFileSync } from 'fs'; @@ -21,6 +22,7 @@ export async function TestForAutoFixing( // add imports strings.push(`import { Assembler } from '@idl/assembler';`); + strings.push(`import { CancellationToken } from '@idl/cancellation-tokens';`); strings.push(`import { LogManager } from '@idl/logger';`); strings.push( `import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index';` @@ -79,7 +81,11 @@ export async function TestForAutoFixing( }; // format - const formatted = Assembler(tokenized, test.config); + const formatted = Assembler( + tokenized, + new CancellationToken(), + test.config + ); // add our tokens strings.push(` it(\`[auto generated] ${testName}\`, async () => {`); @@ -106,14 +112,14 @@ export async function TestForAutoFixing( strings.push(` // format code`); if (test.config !== undefined) { strings.push( - ` const formatted = Assembler(tokenized, ${JSON.stringify( + ` const formatted = Assembler(tokenized, new CancellationToken(), ${JSON.stringify( test.config )});` ); strings.push(``); } else { strings.push( - ` const formatted = Assembler(tokenized, { formatter: 'fiddle' });` + ` const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle' });` ); strings.push(``); } diff --git a/apps/test-tokenizer/src/test-maker/makers/tests-for-syntax-post-processors.ts b/apps/test-tokenizer/src/test-maker/makers/tests-for-syntax-post-processors.ts index 08ad1d61b..5a9a147ce 100644 --- a/apps/test-tokenizer/src/test-maker/makers/tests-for-syntax-post-processors.ts +++ b/apps/test-tokenizer/src/test-maker/makers/tests-for-syntax-post-processors.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { writeFileSync } from 'fs'; import { join } from 'path'; @@ -19,6 +20,7 @@ export function TestsForSyntaxPostProcessors( const strings: string[] = []; // add imports + strings.push(`import { CancellationToken } from '@idl/cancellation-tokens';`); strings.push(`import { Parser } from '@idl/parser';`); strings.push(`import { SyntaxProblems } from '@idl/parsing/problem-codes';`); strings.push(`import { SyntaxTree } from '@idl/parsing/syntax-tree';`); @@ -38,7 +40,7 @@ export function TestsForSyntaxPostProcessors( const toProcess = ArrayifyCode(code); // extract our tokens from the cleaned code - const tokenized = Parser(toProcess); + const tokenized = Parser(toProcess, new CancellationToken()); // build our code string to insert into the automated test const codeStr = StringifyCode(toProcess); @@ -49,7 +51,9 @@ export function TestsForSyntaxPostProcessors( strings.push(` const code = ${codeStr}`); strings.push(``); strings.push(` // extract tokens`); - strings.push(` const tokenized = Parser(code);`); + strings.push( + ` const tokenized = Parser(code, new CancellationToken());` + ); strings.push(``); // add the start to our tokens diff --git a/apps/test-tokenizer/src/test-maker/makers/tests-for-token-at-cursor.ts b/apps/test-tokenizer/src/test-maker/makers/tests-for-token-at-cursor.ts index ea609c002..8a0997bf2 100644 --- a/apps/test-tokenizer/src/test-maker/makers/tests-for-token-at-cursor.ts +++ b/apps/test-tokenizer/src/test-maker/makers/tests-for-token-at-cursor.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { GetTokenAtCursor, RemoveScopeDetail } from '@idl/parsing/syntax-tree'; import { writeFileSync } from 'fs'; @@ -19,6 +20,7 @@ export function TestsForTokenAtCursor( const strings: string[] = []; // add imports + strings.push(`import { CancellationToken } from '@idl/cancellation-tokens';`); strings.push(`import { Parser } from '@idl/parser';`); strings.push( `import { GetTokenAtCursor, RemoveScopeDetail } from '@idl/parsing/syntax-tree';` @@ -41,7 +43,7 @@ export function TestsForTokenAtCursor( const toProcess = ArrayifyCode(code); // extract our tokens from the cleaned code - const tokenized = Parser(toProcess); + const tokenized = Parser(toProcess, new CancellationToken()); // build our code string to insert into the automated test const codeStr = StringifyCode(toProcess); @@ -52,7 +54,9 @@ export function TestsForTokenAtCursor( strings.push(` const code = ${codeStr}`); strings.push(``); strings.push(` // extract tokens`); - strings.push(` const tokenized = Parser(code);`); + strings.push( + ` const tokenized = Parser(code, new CancellationToken());` + ); strings.push(``); // process each location @@ -60,7 +64,7 @@ export function TestsForTokenAtCursor( const found = GetTokenAtCursor(tokenized, test.position[j]); // remove scope detail so we can stringify again - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // make a string of our token const str = @@ -89,7 +93,7 @@ export function TestsForTokenAtCursor( // add the start to our tokens strings.push(` // remove scope detail`); - strings.push(` RemoveScopeDetail(tokenized)`); + strings.push(` RemoveScopeDetail(tokenized, new CancellationToken())`); strings.push(''); // verify results diff --git a/apps/test-tokenizer/src/test-maker/makers/tests-for-tokenizer.ts b/apps/test-tokenizer/src/test-maker/makers/tests-for-tokenizer.ts index 853a02021..9c54a8fc2 100644 --- a/apps/test-tokenizer/src/test-maker/makers/tests-for-tokenizer.ts +++ b/apps/test-tokenizer/src/test-maker/makers/tests-for-tokenizer.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { TOKEN_NAMES, TOKEN_TYPES, Tokenizer } from '@idl/parsing/tokenizer'; import deepCopy from 'fast-copy'; import { writeFileSync } from 'fs'; @@ -19,6 +20,7 @@ export function TestsForTokenizer( const strings: string[] = []; // add imports + strings.push(`import { CancellationToken } from '@idl/cancellation-tokens';`); strings.push( `import { IBaseToken, StripIDs, TOKEN_NAMES, TOKEN_TYPES, Tokenizer,TokenName } from '@idl/parsing/tokenizer';` ); @@ -38,7 +40,7 @@ export function TestsForTokenizer( const toProcess = ArrayifyCode(code); // extract our tokens from the cleaned code - const tokenized = Tokenizer(toProcess); + const tokenized = Tokenizer(toProcess, new CancellationToken()); // build our code string to insert into the automated test const codeStr = StringifyCode(toProcess); @@ -49,7 +51,9 @@ export function TestsForTokenizer( strings.push(` const code = ${codeStr}`); strings.push(``); strings.push(` // extract tokens`); - strings.push(` const tokenized = Tokenizer(code)`); + strings.push( + ` const tokenized = Tokenizer(code, new CancellationToken())` + ); strings.push(``); // add the start to our tokens diff --git a/idl/vscode/notebooks/idlnotebook/idlnotebook__define.pro b/idl/vscode/notebooks/idlnotebook/idlnotebook__define.pro index 2ae0f937a..faa9c32fe 100644 --- a/idl/vscode/notebooks/idlnotebook/idlnotebook__define.pro +++ b/idl/vscode/notebooks/idlnotebook/idlnotebook__define.pro @@ -41,6 +41,7 @@ pro IDLNotebook::AddToNotebookMap, item, properties, $ ; Handle rasters ;- isa(item, 'ENVIRaster'): begin + if (item.spatialref eq !null) then message, 'Raster must have a spatial reference', level = -1 rasterForMap = {IDLNotebookMap_Image} rasterForMap.raster = item !idlnotebookmagic.mapitems.add, rasterForMap diff --git a/libs/assembling/assembler/src/lib/assembler.ts b/libs/assembling/assembler/src/lib/assembler.ts index 0ff60e337..4a443471a 100644 --- a/libs/assembling/assembler/src/lib/assembler.ts +++ b/libs/assembling/assembler/src/lib/assembler.ts @@ -6,6 +6,7 @@ import { import { ApplyFixers } from '@idl/assembling/fixers'; import { ApplyFormatter } from '@idl/assembling/formatters'; import { ApplyStyle, ReplaceRoutineDocs } from '@idl/assembling/styles'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { SaveGlobalDisplayNames } from '@idl/parsing/index'; import { IParsed } from '@idl/parsing/syntax-tree'; import { MergeConfig } from '@idl/schemas/idl.json'; @@ -29,6 +30,7 @@ import { CombinerBasic } from './combiner/combiner-basic'; */ export function Assembler( parsed: IParsed, + cancel: CancellationToken, options?: Partial> ): string | undefined { // verify that we can format @@ -44,16 +46,16 @@ export function Assembler( // if we can fix problems, lets fix them if (useOptions.autoFix) { - ApplyFixers(parsed); + ApplyFixers(parsed, cancel); } // style and format code if (useOptions.styleAndFormat) { // apply styling - ApplyStyle(parsed, useOptions.style); + ApplyStyle(parsed, cancel, useOptions.style); // apply formatting - ApplyFormatter(parsed, useOptions.formatter); + ApplyFormatter(parsed, cancel, useOptions.formatter); } // check if we want to format and create our docs diff --git a/libs/assembling/fixers/src/lib/apply-fixers.ts b/libs/assembling/fixers/src/lib/apply-fixers.ts index b17a6bcfa..eca39e611 100644 --- a/libs/assembling/fixers/src/lib/apply-fixers.ts +++ b/libs/assembling/fixers/src/lib/apply-fixers.ts @@ -1,11 +1,12 @@ import { ASSEMBLER_PROBLEM_FIXERS } from '@idl/assembling/tree-handlers'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { IParsed, PopulateIndex } from '@idl/parsing/syntax-tree'; /** * Applies problem fixers to code */ -export function ApplyFixers(parsed: IParsed) { - ASSEMBLER_PROBLEM_FIXERS.run(parsed, (token, meta) => meta); +export function ApplyFixers(parsed: IParsed, cancel: CancellationToken) { + ASSEMBLER_PROBLEM_FIXERS.run(parsed, cancel, (token, meta) => meta); // fix any index values in the tree PopulateIndex(parsed.tree); diff --git a/libs/assembling/formatters/src/lib/apply-formatter.ts b/libs/assembling/formatters/src/lib/apply-formatter.ts index 6a9d098b4..759f89507 100644 --- a/libs/assembling/formatters/src/lib/apply-formatter.ts +++ b/libs/assembling/formatters/src/lib/apply-formatter.ts @@ -2,6 +2,7 @@ import { ASSEMBLER_FORMATTER_LOOKUP, FormatterType, } from '@idl/assembling/config'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { IParsed, TreeRecurser } from '@idl/parsing/syntax-tree'; import { DefaultFormatter } from './default-formatter'; @@ -12,7 +13,11 @@ import { FormatterRuleSet } from './formatter-rule-set.interface'; /** * Applies formatter rule sets to code */ -export function ApplyFormatter(parsed: IParsed, formatter: FormatterType) { +export function ApplyFormatter( + parsed: IParsed, + cancel: CancellationToken, + formatter: FormatterType +) { // init value of rule set let ruleSet: FormatterRuleSet; @@ -26,7 +31,7 @@ export function ApplyFormatter(parsed: IParsed, formatter: FormatterType) { } // use our tree recurser to process what we parsed - TreeRecurser(parsed, { + TreeRecurser(parsed, cancel, { onBasicToken: (token, current) => { if (token.name in ruleSet) { // not sure why, but we need any here to have it work diff --git a/libs/assembling/shared/src/lib/add-code-to-syntax-tree.ts b/libs/assembling/shared/src/lib/add-code-to-syntax-tree.ts index b03c39e5b..15437a021 100644 --- a/libs/assembling/shared/src/lib/add-code-to-syntax-tree.ts +++ b/libs/assembling/shared/src/lib/add-code-to-syntax-tree.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { ConditionalLineNumberIncrement } from './conditional-line-number-increment'; @@ -29,7 +30,7 @@ export function GenerateCodeToInsert( lineStart: number ) { // parse code and get our syntax tree - const add = Parser(code).tree; + const add = Parser(code, new CancellationToken()).tree; // bump line numbers for new code to start at the right place IncrementLineNumbers(add, lineStart); diff --git a/libs/assembling/styles/src/lib/apply-style.ts b/libs/assembling/styles/src/lib/apply-style.ts index 3fd3bc6de..89331ffa6 100644 --- a/libs/assembling/styles/src/lib/apply-style.ts +++ b/libs/assembling/styles/src/lib/apply-style.ts @@ -4,6 +4,7 @@ import { ICodeStyle, } from '@idl/assembling/config'; import { ASSEMBLER_DEFAULT_STYLING } from '@idl/assembling/tree-handlers'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { IParsed, PopulateScopeDetail } from '@idl/parsing/syntax-tree'; /** @@ -11,16 +12,17 @@ import { IParsed, PopulateScopeDetail } from '@idl/parsing/syntax-tree'; */ export function ApplyStyle( parsed: IParsed, + cancel: CancellationToken, style: ICodeStyle, styler: AssemblerStyler = ASSEMBLER_STYLER_LOOKUP.DEFAULT ) { // make sure we have scope detail - PopulateScopeDetail(parsed); + PopulateScopeDetail(parsed, cancel); switch (styler) { case ASSEMBLER_STYLER_LOOKUP.DEFAULT: // apply styling - ASSEMBLER_DEFAULT_STYLING.run(parsed, (token, meta) => { + ASSEMBLER_DEFAULT_STYLING.run(parsed, cancel, (token, meta) => { return { ...meta, style }; }); break; diff --git a/libs/cancellation-tokens/.eslintrc.json b/libs/cancellation-tokens/.eslintrc.json new file mode 100644 index 000000000..9d9c0db55 --- /dev/null +++ b/libs/cancellation-tokens/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/libs/cancellation-tokens/README.md b/libs/cancellation-tokens/README.md new file mode 100644 index 000000000..a67de7981 --- /dev/null +++ b/libs/cancellation-tokens/README.md @@ -0,0 +1,3 @@ +# Cancellation Tokens + +A lib to develop cancellation tokens that can be passed around the tokenizer and allows us to cancel processing. diff --git a/libs/cancellation-tokens/jest.config.ts b/libs/cancellation-tokens/jest.config.ts new file mode 100644 index 000000000..2dac872b5 --- /dev/null +++ b/libs/cancellation-tokens/jest.config.ts @@ -0,0 +1,11 @@ +/* eslint-disable */ +export default { + displayName: 'cancellation-tokens', + preset: '../../jest.preset.js', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: '../../coverage/libs/cancellation-tokens', +}; diff --git a/libs/cancellation-tokens/project.json b/libs/cancellation-tokens/project.json new file mode 100644 index 000000000..311248434 --- /dev/null +++ b/libs/cancellation-tokens/project.json @@ -0,0 +1,30 @@ +{ + "name": "cancellation-tokens", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/cancellation-tokens/src", + "projectType": "library", + "targets": { + "lint": { + "executor": "@nx/linter:eslint", + "outputs": ["{options.outputFile}"], + "options": { + "lintFilePatterns": ["libs/cancellation-tokens/**/*.ts"] + } + }, + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "libs/cancellation-tokens/jest.config.ts", + "passWithNoTests": true + }, + "configurations": { + "ci": { + "ci": true, + "codeCoverage": true + } + } + } + }, + "tags": [] +} diff --git a/libs/cancellation-tokens/src/index.ts b/libs/cancellation-tokens/src/index.ts new file mode 100644 index 000000000..937a735df --- /dev/null +++ b/libs/cancellation-tokens/src/index.ts @@ -0,0 +1,2 @@ +export * from './lib/cancellation-token.class'; +export * from './lib/cancellation-token.interface'; diff --git a/libs/cancellation-tokens/src/lib/cancellation-token.class.ts b/libs/cancellation-tokens/src/lib/cancellation-token.class.ts new file mode 100644 index 000000000..2b1ea84b0 --- /dev/null +++ b/libs/cancellation-tokens/src/lib/cancellation-token.class.ts @@ -0,0 +1,34 @@ +import { CANCELLATION_MESSAGE } from './cancellation-token.interface'; + +/** + * Base class for cancellation tokens + */ +export class CancellationToken { + /** + * Track if we have been cancelled or not + */ + private cancelled: boolean; + + /** + * Sets the cancel flag for our token + */ + cancel() { + this.cancelled = true; + } + + /** + * Returns a flag if we are cancelled + */ + cancelRequested() { + return this.cancelled; + } + + /** + * Checks for cancellation and, if cancelled, throws an error to halt execution + */ + throwIfCancelled() { + if (this.cancelled) { + throw new Error(CANCELLATION_MESSAGE); + } + } +} diff --git a/libs/cancellation-tokens/src/lib/cancellation-token.interface.ts b/libs/cancellation-tokens/src/lib/cancellation-token.interface.ts new file mode 100644 index 000000000..0b5726211 --- /dev/null +++ b/libs/cancellation-tokens/src/lib/cancellation-token.interface.ts @@ -0,0 +1,4 @@ +/** + * Message used for cancellation + */ +export const CANCELLATION_MESSAGE = 'Cancelled via cancellation token'; diff --git a/libs/cancellation-tokens/tsconfig.json b/libs/cancellation-tokens/tsconfig.json new file mode 100644 index 000000000..19b9eece4 --- /dev/null +++ b/libs/cancellation-tokens/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs" + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/libs/cancellation-tokens/tsconfig.lib.json b/libs/cancellation-tokens/tsconfig.lib.json new file mode 100644 index 000000000..3f06e8028 --- /dev/null +++ b/libs/cancellation-tokens/tsconfig.lib.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "outDir": "../../dist/out-tsc", + "declaration": true, + "types": ["node"] + }, + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"], + "include": ["src/**/*.ts"] +} diff --git a/libs/cancellation-tokens/tsconfig.spec.json b/libs/cancellation-tokens/tsconfig.spec.json new file mode 100644 index 000000000..9b2a121d1 --- /dev/null +++ b/libs/cancellation-tokens/tsconfig.spec.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/libs/idl/src/lib/idl.class.ts b/libs/idl/src/lib/idl.class.ts index b7bbd4e89..73970d98b 100644 --- a/libs/idl/src/lib/idl.class.ts +++ b/libs/idl/src/lib/idl.class.ts @@ -180,7 +180,11 @@ export class IDL extends EventEmitter { }); // launch IDL with the environment from our parent process and in the specified folder - this.idl = spawn(cmd, null, { env: args.env, cwd: args.cwd }); + this.idl = spawn(cmd, null, { + env: args.env, + cwd: args.cwd, + stdio: ['pipe', 'pipe', 'pipe', 'pipe'], + }); // check for errors if (!this.idl.stdout || !this.idl.stderr || !this.idl.stdin) { @@ -188,6 +192,14 @@ export class IDL extends EventEmitter { return; } + // listen for standard out output from IDL + this.idl.stdio[3].on('data', (buff) => { + this.log.log({ + type: 'info', + content: `Stdout: ${JSON.stringify(buff.toString('utf8'))}`, + }); + }); + // write the IDL prompt if not windows so that we properly // detect start. for our "poor man's solution" this is the indicator // that we are ready to go again diff --git a/libs/logger/src/lib/log-manager.class.ts b/libs/logger/src/lib/log-manager.class.ts index 33881b3cc..7b961a2d1 100644 --- a/libs/logger/src/lib/log-manager.class.ts +++ b/libs/logger/src/lib/log-manager.class.ts @@ -1,4 +1,6 @@ +import { CANCELLATION_MESSAGE } from '@idl/cancellation-tokens'; import { DEFAULT_IDL_EXTENSION_CONFIG } from '@idl/vscode/extension-config'; +import { deepEqual } from 'fast-equals'; import { DEFAULT_LOGGER_OPTIONS, @@ -124,6 +126,25 @@ export class LogManager implements ILogManagerOptions { return this.logs[name]; } + /** + * Determines if we can log a message or not + */ + canLog(options: ILogOptions): boolean { + // filter debug + if (!this.debug && options.type === 'debug') { + return false; + } + + // check if we have an array of content + if (Array.isArray(options.content)) { + if (deepEqual(options.content[1]?.err?.message, [CANCELLATION_MESSAGE])) { + return false; + } + } + + return true; + } + /** * Log content for a specific log * @@ -135,7 +156,7 @@ export class LogManager implements ILogManagerOptions { */ log(options: ILogOptions) { // filter debug - if (!this.debug && options.type === 'debug') { + if (!this.canLog(options)) { return; } diff --git a/libs/notebooks/shared/src/lib/create-code-for-notebooks.ts b/libs/notebooks/shared/src/lib/create-code-for-notebooks.ts index 65116e7f1..8b6a7e786 100644 --- a/libs/notebooks/shared/src/lib/create-code-for-notebooks.ts +++ b/libs/notebooks/shared/src/lib/create-code-for-notebooks.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { IDL_PROBLEM_CODES } from '@idl/parsing/problem-codes'; @@ -43,7 +44,7 @@ export async function CreateCodeForNotebooks( .join('\n'); // parse - const parsed = Parser(combined); + const parsed = Parser(combined, new CancellationToken()); // check for problems for (let i = 0; i < parsed.parseProblems.length; i++) { diff --git a/libs/parsing/index/src/lib/change-detection/change-detection.ts b/libs/parsing/index/src/lib/change-detection/change-detection.ts index b2b966c69..6ed0d1a3b 100644 --- a/libs/parsing/index/src/lib/change-detection/change-detection.ts +++ b/libs/parsing/index/src/lib/change-detection/change-detection.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { GlobalTokens } from '@idl/data-types/core'; import { IDL_WORKER_THREAD_CONSOLE } from '@idl/logger'; import { IDL_TRANSLATION } from '@idl/translation'; @@ -22,6 +23,7 @@ const IS_CHANGING: { [key: string]: undefined } = {}; */ export function ChangeDetection( index: IDLIndex, + token: CancellationToken, changed: GlobalTokens ): IChangeDetection { /** Files that we will post-process again */ @@ -68,7 +70,8 @@ export function ChangeDetection( PostProcessParsed( index, postProcessThese[z], - index.tokensByFile.get(postProcessThese[z]) + index.tokensByFile.get(postProcessThese[z]), + token ); } catch (err) { // check if we have a "false" error because a file was deleted diff --git a/libs/parsing/index/src/lib/get-parsed-notebook.interface.ts b/libs/parsing/index/src/lib/get-parsed-notebook.interface.ts new file mode 100644 index 000000000..771e9b2a9 --- /dev/null +++ b/libs/parsing/index/src/lib/get-parsed-notebook.interface.ts @@ -0,0 +1,20 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; +import { IParsedIDLNotebook } from '@idl/notebooks/shared'; + +/** + * Data we track for pending PRO code + */ +export interface IGetParsedNotebookPending { + /** + * Promise that resolves to the parsed notebook + */ + promise: Promise; + /** + * Cancellation token that we can use + */ + token: CancellationToken; + /** + * The checksums for the code cells + */ + checksums: string[]; +} diff --git a/libs/parsing/index/src/lib/get-parsed-notebook.ts b/libs/parsing/index/src/lib/get-parsed-notebook.ts new file mode 100644 index 000000000..0252b483b --- /dev/null +++ b/libs/parsing/index/src/lib/get-parsed-notebook.ts @@ -0,0 +1,78 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; +import { IDLNotebookDocument, IParsedIDLNotebook } from '@idl/notebooks/shared'; +import { CodeChecksum } from '@idl/parser'; +import { deepEqual } from 'fast-equals'; +import { NotebookCellKind } from 'vscode-languageserver'; + +import { IGetParsedNotebookPending } from './get-parsed-notebook.interface'; +import { IDLIndex } from './idl-index.class'; + +/** + * Track the pending files + */ +export const PENDING_NOTEBOOK: { [key: string]: IGetParsedNotebookPending } = + {}; + +/** + * Handles getting a parsed notebook from the IDL index + */ +export async function GetParsedNotebook( + index: IDLIndex, + file: string, + notebook: IDLNotebookDocument, + token: CancellationToken +): Promise { + // track all checksums + const checksums: string[] = []; + + // get checksums + for (let i = 0; i < notebook.cells.length; i++) { + // skip if not code + if (notebook.cells[i].kind !== NotebookCellKind.Code) { + continue; + } + + // track checksums + checksums.push(CodeChecksum(notebook.cells[i].text)); + } + + // determine how to proceed + switch (true) { + case file in PENDING_NOTEBOOK: { + /** + * Get pending file + */ + const pending = PENDING_NOTEBOOK[file]; + + // check if our checksums are valid + if (deepEqual(pending.checksums, checksums)) { + return pending.promise; + } else { + // different checksums, so mark as cancelled + pending.token.cancel(); + } + break; + } + default: + break; + } + + // track pending notebook parsing + const newPending: IGetParsedNotebookPending = { + checksums, + token, + promise: index.indexIDLNotebook(file, notebook, token), + }; + + // mark as pending + PENDING_NOTEBOOK[file] = newPending; + + // wait for finish + const res = await newPending.promise; + + // clean up pending + delete PENDING_NOTEBOOK[file]; + + // return result + return res; +} diff --git a/libs/parsing/index/src/lib/get-parsed-pro-code.interface.ts b/libs/parsing/index/src/lib/get-parsed-pro-code.interface.ts new file mode 100644 index 000000000..8916d59dc --- /dev/null +++ b/libs/parsing/index/src/lib/get-parsed-pro-code.interface.ts @@ -0,0 +1,20 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; +import { IParsed } from '@idl/parsing/syntax-tree'; + +/** + * Data we track for pending PRO code + */ +export interface IGetParsedPROCodePending { + /** + * Promise that resolves to the parsed file + */ + promise: Promise; + /** + * Cancellation token that we can use + */ + token: CancellationToken; + /** + * The checksum for the pending code + */ + checksum: string; +} diff --git a/libs/parsing/index/src/lib/get-parsed-pro-code.ts b/libs/parsing/index/src/lib/get-parsed-pro-code.ts new file mode 100644 index 000000000..43edc1a95 --- /dev/null +++ b/libs/parsing/index/src/lib/get-parsed-pro-code.ts @@ -0,0 +1,200 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; +import { CodeChecksum } from '@idl/parser'; +import { IParsed } from '@idl/parsing/syntax-tree'; +import { LSP_WORKER_THREAD_MESSAGE_LOOKUP } from '@idl/workers/parsing'; +import copy from 'fast-copy'; + +import { PENDING_NOTEBOOK } from './get-parsed-notebook'; +import { IGetParsedPROCodePending } from './get-parsed-pro-code.interface'; +import { GetSyntaxProblems } from './helpers/get-syntax-problems'; +import { IDLIndex } from './idl-index.class'; +import { + DEFAULT_INDEX_PRO_CODE_OPTIONS, + IIndexProCodeOptions, +} from './idl-index.interface'; + +/** + * Track the pending files + */ +const PENDING_PRO_CODE: { [key: string]: IGetParsedPROCodePending } = {}; + +/** + * Get parsed PRO code + * + * This is where the logic lives to check pending files and cancel pending requests + */ +export async function GetParsedPROCode( + index: IDLIndex, + file: string, + code: string | string[], + options: Partial = {} +): Promise { + /** + * Get the checksum of our code + */ + const checksum = CodeChecksum(code); + + /** + * Make a cancellation token in case we need it + */ + const token = new CancellationToken(); + + switch (true) { + /** + * Check if we are a notebook cell + */ + case index.isIDLNotebookFile(file): { + /** + * If we have a pending notebook file, pause + * and wait for it to finish before we do anything + */ + const base = file.split('#')[0]; + if (base in PENDING_NOTEBOOK) { + await PENDING_NOTEBOOK[base]; + } + + /** + * Check if we are multi threaded and need to fetch our file from + * the worker + * + * TODO: Add in proper cancellation and logic for this + */ + if (index.isMultiThreaded()) { + return index.indexerPool.workerio.postAndReceiveMessage( + index.getWorkerID(base), + LSP_WORKER_THREAD_MESSAGE_LOOKUP.GET_NOTEBOOK_CELL, + { + file, + code, + } + ).response; + } else { + if (index.tokensByFile.has(file)) { + return index.tokensByFile.get(file); + } else { + // make new pending + const newPending: IGetParsedPROCodePending = { + checksum, + token, + promise: index.indexProCode(file, code, token, options), + }; + + // save + PENDING_PRO_CODE[file] = newPending; + + // wait for finish + const res = await newPending.promise; + + // clean up + delete PENDING_PRO_CODE[file]; + + // return + return res; + } + } + } + + /** + * Check for a pending file + */ + case file in PENDING_PRO_CODE: { + /** + * Get pending file + */ + const pending = PENDING_PRO_CODE[file]; + + // check if our checksums are valid + if (pending.checksum === checksum) { + return pending.promise; + } else { + // different checksums, so mark as cancelled + pending.token.cancel(); + } + break; + } + + /** + * If multi-threaded, check if we have a copy of our parsed code + * + * Some duplication with indexProCode, but special case if we are multi threaded + */ + case index.isMultiThreaded(): { + const resp = index.indexerPool.workerio.postAndReceiveMessage( + index.getWorkerID(file), + LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_CODE, + { + file, + code, + ...Object.assign(copy(DEFAULT_INDEX_PRO_CODE_OPTIONS), options), + } + ); + + const newPending: IGetParsedPROCodePending = { + checksum, + token: resp.token, + promise: resp.response, + }; + + // save new pending file + PENDING_PRO_CODE[file] = newPending; + + // get the latest - cache busting happens in the worker + const current = await newPending.promise; + + // remove from local cache + delete PENDING_PRO_CODE[file]; + + // get current global tokens + const oldGlobals = index.getGlobalsForFile(file); + + // save and sync global tokens + await index.saveGlobalTokens(file, current.global); + + // track syntax problems + index.trackSyntaxProblemsForFile(file, GetSyntaxProblems(current)); + + // do change detection + if (options.postProcess) { + await index.changeDetection(token, current.global, oldGlobals); + } + + // make sure that we never cache in memory if we are the main thread + index.tokensByFile.remove(file); + + // return the tokens from the worker + return current; + } + + /** + * Check if we have it stored locally + */ + case index.tokensByFile.has(file): { + if (index.tokensByFile.checksumMatches(file, checksum)) { + return index.tokensByFile.get(file); + } + break; + } + default: + } + + /** + * Re-index our code + */ + const newPending: IGetParsedPROCodePending = { + checksum, + token, + promise: index.indexProCode(file, code, token, options), + }; + + // mark as pending + PENDING_PRO_CODE[file] = newPending; + + // wait for finish + const res = await newPending.promise; + + // clean up pending + delete PENDING_PRO_CODE[file]; + + // return result + return res; +} diff --git a/libs/parsing/index/src/lib/idl-index.class.ts b/libs/parsing/index/src/lib/idl-index.class.ts index d06292f7e..f76891a63 100644 --- a/libs/parsing/index/src/lib/idl-index.class.ts +++ b/libs/parsing/index/src/lib/idl-index.class.ts @@ -3,6 +3,7 @@ import { FormatterType, IAssemblerOptions, } from '@idl/assembling/config'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { GLOBAL_TOKEN_SOURCE_LOOKUP, GlobalTokens, @@ -15,7 +16,7 @@ import { LogManager, } from '@idl/logger'; import { IDLNotebookDocument, IParsedIDLNotebook } from '@idl/notebooks/shared'; -import { CodeChecksum, Parser } from '@idl/parser'; +import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { GetSemanticTokens } from '@idl/parsing/semantic-tokens'; import { IParsed, TreeToken } from '@idl/parsing/syntax-tree'; @@ -71,6 +72,8 @@ import { Worker } from 'worker_threads'; import { GetAutoComplete } from './auto-complete/get-auto-complete'; import { CanChangeDetection } from './change-detection/can-change-detection'; import { ChangeDetection } from './change-detection/change-detection'; +import { GetParsedNotebook } from './get-parsed-notebook'; +import { GetParsedPROCode } from './get-parsed-pro-code'; import { GlobalIndex } from './global-index.class'; import { GetSyntaxProblems } from './helpers/get-syntax-problems'; import { PopulateNotebookVariables } from './helpers/populate-notebook-variables'; @@ -93,6 +96,7 @@ import { OUTLINE_THESE_TOKENS, OUTLINE_TOKEN_KIND_MAP, } from './outline.interface'; +import { ParseNotebook } from './parse-notebook'; import { PostProcessParsed } from './post-process/post-process-parsed'; import { GetTokenDefinition } from './token-definiton/get-token-definition'; @@ -114,7 +118,7 @@ export class IDLIndex { /** * Our worker pool to share the load of parsing code */ - private indexerPool: ILSPWorkerThreadPool; + indexerPool: ILSPWorkerThreadPool; /** * NUmber of workers that we populate our indexer pool with @@ -154,17 +158,6 @@ export class IDLIndex { */ tokensByFile = new IDLParsedCache(); - /** - * Track files that we are currently processing so that we can't accidentally - * re-process files that we are already processing - */ - pendingFiles: { [key: string]: Promise } = {}; - - /** - * Track pending notebooks - */ - pendingNotebooks: { [key: string]: Promise } = {}; - /** * Track the workers that own each file * @@ -380,7 +373,7 @@ export class IDLIndex { ids[i], LSP_WORKER_THREAD_MESSAGE_LOOKUP.CLEAN_UP, undefined - ) + ).response ); } @@ -506,7 +499,7 @@ export class IDLIndex { this.getWorkerID(file), LSP_WORKER_THREAD_MESSAGE_LOOKUP.GET_SEMANTIC_TOKENS, { file, code } - ); + ).response; } else { return GetSemanticTokens( await this.getParsedProCode(file, code, { @@ -600,7 +593,7 @@ export class IDLIndex { ids[i], LSP_WORKER_THREAD_MESSAGE_LOOKUP.TRACK_GLOBAL, msg - ) + ).response ); } @@ -642,7 +635,7 @@ export class IDLIndex { this.getWorkerID(file), LSP_WORKER_THREAD_MESSAGE_LOOKUP.GET_OUTLINE, { file, code } - ); + ).response; } // get tokens for our file @@ -724,7 +717,7 @@ export class IDLIndex { // check if we should do change detection because it has been removed if (changeDetection) { - await this.changeDetection([], global); + await this.changeDetection(new CancellationToken(), [], global); } // return removed globals @@ -1027,6 +1020,7 @@ export class IDLIndex { async indexProCode( file: string, code: string | string[], + token: CancellationToken, inOptions: Partial = {} ): Promise { try { @@ -1060,38 +1054,8 @@ export class IDLIndex { } /** Init value of parsed */ - let parsed: IParsed; - - /** - * Flag if we have post processed already or not - */ - let postProcessed = false; - - // determine how to proceed - switch (true) { - /** - * Check if we farm work to a thread - */ - case this.isMultiThreaded(): - postProcessed = options.postProcess; - parsed = await this.indexerPool.workerio.postAndReceiveMessage( - this.getWorkerID(file), - LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_CODE, - { - file, - code, - ...options, - } - ); - break; - /** - * Default to doing the work here - */ - default: - parsed = Parser(code, options); - this.workerIDsByFile[file] = undefined; - break; - } + const parsed = Parser(code, token, options); + this.workerIDsByFile[file] = undefined; // add to our global index - do this before we post-process await this.saveGlobalTokens(file, parsed.global); @@ -1101,13 +1065,11 @@ export class IDLIndex { case !options.full: // do nothing if not full parse break; - case postProcessed: - await this.changeDetection(parsed.global, oldGlobals); - break; case options.postProcess: this.postProcessProFile( file, parsed, + token, oldGlobals, options.changeDetection ); @@ -1143,128 +1105,13 @@ export class IDLIndex { * * Assumes that code is more up-to-date than whatever is on disk, so we wait for any * pending processes for the file to finish before we index it again - * - * TODO: Cancel pending requests, or find a way to update after they have been triggered? */ async getParsedProCode( file: string, code: string | string[], options: Partial = {} ): Promise { - switch (true) { - /** - * Check if we are a notebook - */ - case this.isIDLNotebookFile(file): { - /** - * If we have a pending notebook file, pause - * and wait for it to finish before we do anything - */ - const base = file.split('#')[0]; - if (base in this.pendingNotebooks) { - await this.pendingNotebooks[base]; - } - - /** - * Check if we are multi threaded and need to fetch our file from - * the worker - */ - if (this.isMultiThreaded()) { - return this.indexerPool.workerio.postAndReceiveMessage( - this.getWorkerID(base), - LSP_WORKER_THREAD_MESSAGE_LOOKUP.GET_NOTEBOOK_CELL, - { - file, - code, - } - ); - } else { - if (this.tokensByFile.has(file)) { - return this.tokensByFile.get(file); - } else { - this.pendingFiles[file] = this.indexProCode(file, code, options); - const res = await this.pendingFiles[file]; - delete this.pendingFiles[file]; - return res; - } - } - } - /** - * Check for a pending file - */ - case file in this.pendingFiles: { - await this.pendingFiles[file]; - if (this.tokensByFile.has(file)) { - if (this.tokensByFile.checksumMatches(file, CodeChecksum(code))) { - return this.tokensByFile.get(file); - } - } - break; - } - - /** - * If multi-threaded, check if we have a copy of our parsed code - * - * Some duplication with indexProCode, but special case if we are multi threaded - */ - case this.isMultiThreaded(): { - /** - * Fetch from worker (it validates cache) - */ - this.pendingFiles[file] = - this.indexerPool.workerio.postAndReceiveMessage( - this.getWorkerID(file), - LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_CODE, - { - file, - code, - ...Object.assign(copy(DEFAULT_INDEX_PRO_CODE_OPTIONS), options), - } - ); - - // get the latest - cache busting happens in the worker - const current = await this.pendingFiles[file]; - delete this.pendingFiles[file]; - - // get current global tokens - const oldGlobals = this.getGlobalsForFile(file); - - // save and sync global tokens - await this.saveGlobalTokens(file, current.global); - - // track syntax problems - this.trackSyntaxProblemsForFile(file, GetSyntaxProblems(current)); - - // do change detection - if (options.postProcess) { - await this.changeDetection(current.global, oldGlobals); - } - - // make sure that we never cache in memory if we are the main thread - this.tokensByFile.remove(file); - - return current; - } - - /** - * Check if we have it stored locally - */ - case this.tokensByFile.has(file): { - if (this.tokensByFile.checksumMatches(file, CodeChecksum(code))) { - return this.tokensByFile.get(file); - } - break; - } - default: - } - - /** - * Re-index our code - */ - this.pendingFiles[file] = this.indexProCode(file, code, options); - const res = await this.pendingFiles[file]; - delete this.pendingFiles[file]; - return res; + return GetParsedPROCode(this, file, code, options); } /** @@ -1272,7 +1119,8 @@ export class IDLIndex { */ async indexIDLNotebook( file: string, - notebook: IDLNotebookDocument + notebook: IDLNotebookDocument, + token: CancellationToken ): Promise { // remove notebook await this.removeNotebook(file); @@ -1281,103 +1129,99 @@ export class IDLIndex { this.knownFiles[file] = undefined; this.fileTypes['idl-notebook'].add(file); - try { - /** - * Resolver for our work being done - */ - let resolver: () => void; - - // make a promise for panding - const pending = new Promise((res) => { - resolver = res; - }); - - // track that we have a pending notebook parse - this.pendingNotebooks[file] = pending; - - /** - * Track parsed code by cell - */ - const byCell: IParsedIDLNotebook = {}; + /** + * Track parsed code by cell + */ + const byCell: IParsedIDLNotebook = {}; - // process each cell - for (let i = 0; i < notebook.cells.length; i++) { - /** Get notebook cell */ - const cell = notebook.cells[i]; + // process each cell + for (let i = 0; i < notebook.cells.length; i++) { + // check for cancellation + token.throwIfCancelled(); - // make file for our cell - const cellFSPath = `${file}#${i}`; + /** Get notebook cell */ + const cell = notebook.cells[i]; - // skip if no cells - if (cell.kind !== NotebookCellKind.Code) { - byCell[cellFSPath] = undefined; - continue; - } + // make file for our cell + const cellFSPath = `${file}#${i}`; - // process the cell - byCell[cellFSPath] = Parser(cell.text, { - isNotebook: true, - }); + // skip if no cells + if (cell.kind !== NotebookCellKind.Code) { + byCell[cellFSPath] = undefined; + continue; } - /** - * Get files for cells that we actually processed - */ - const files = Object.keys(byCell); + // process the cell + byCell[cellFSPath] = Parser(cell.text, token, { + isNotebook: true, + }); + } - // share variable usage - for (let i = 0; i < files.length; i++) { - if (byCell[files[i]] === undefined) { - continue; - } + /** + * Get files for cells that we actually processed + */ + const files = Object.keys(byCell); - PopulateNotebookVariables(files[i], byCell, true); + // share variable usage + for (let i = 0; i < files.length; i++) { + // check for cancellation + token.throwIfCancelled(); + + if (byCell[files[i]] === undefined) { + continue; } - // process each file - for (let i = 0; i < files.length; i++) { - if (byCell[files[i]] === undefined) { - continue; - } + PopulateNotebookVariables(files[i], byCell, true); + } - // inherit data types from cells above us - if (i > 0) { - PopulateNotebookVariables(files[i], byCell, false); - } + // process each file + for (let i = 0; i < files.length; i++) { + // check for cancellation + token.throwIfCancelled(); - // post process cell - await this.postProcessProFile(files[i], byCell[files[i]], [], false); + if (byCell[files[i]] === undefined) { + continue; + } - // update stored token - this.tokensByFile.add(files[i], byCell[files[i]]); + // inherit data types from cells above us + if (i > 0) { + PopulateNotebookVariables(files[i], byCell, false); } - // indicate that we have finished and clean up - resolver(); - delete this.pendingNotebooks[file]; + // post process cell + await this.postProcessProFile( + files[i], + byCell[files[i]], + token, + [], + false + ); - return byCell; - } catch (err) { - this.log.log({ - log: IDL_LSP_LOG, - type: 'error', - content: [ - `${IDL_TRANSLATION.lsp.index.failedParseNotebook}: "${file}"`, - err, - ], - alert: `${IDL_TRANSLATION.lsp.index.failedParseNotebook}: "${file}"`, - alertMeta: { - file, - }, - }); - return undefined; + // update stored token + this.tokensByFile.add(files[i], byCell[files[i]]); } + + return byCell; } /** - * Indexes an IDL notebook file + * Gets a parsed notebook */ async getParsedNotebook( + file: string, + notebook: IDLNotebookDocument, + cancel: CancellationToken + ): Promise { + return GetParsedNotebook(this, file, notebook, cancel); + } + + /** + * Parses an IDL notebook using a worker thread + * + * The notebook parsing response just includes information that should be synced + * amongst threads + */ + async parseAndTrackNotebook( file: string, notebook: IDLNotebookDocument ): Promise { @@ -1404,11 +1248,7 @@ export class IDLIndex { const id = this.getWorkerID(file); // parse our notebook - const resp = await this.indexerPool.workerio.postAndReceiveMessage( - id, - LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_NOTEBOOK, - { file, notebook } - ); + const resp = await ParseNotebook(this, file, notebook); // track cells as known files so we can clean up correctly next time this.trackFiles(Object.keys(resp.globals)); @@ -1431,7 +1271,7 @@ export class IDLIndex { ids[j], LSP_WORKER_THREAD_MESSAGE_LOOKUP.TRACK_GLOBAL, resp.globals - ) + ).response ); } @@ -1611,7 +1451,7 @@ export class IDLIndex { { files, } - ) + ).response ); } @@ -1749,7 +1589,11 @@ export class IDLIndex { * Performs change detection and triggers post-processing of all files * that use our affected global tokens */ - async changeDetection(newGlobals: GlobalTokens, oldGlobals: GlobalTokens) { + async changeDetection( + token: CancellationToken, + newGlobals: GlobalTokens, + oldGlobals: GlobalTokens + ) { // verify we have changes in our global tokens if (CanChangeDetection(newGlobals, oldGlobals)) { /** @@ -1779,7 +1623,7 @@ export class IDLIndex { ids[i], LSP_WORKER_THREAD_MESSAGE_LOOKUP.CHANGE_DETECTION, { changed } - ) + ).response ); } @@ -1812,7 +1656,7 @@ export class IDLIndex { /** * Run locally */ - const change = ChangeDetection(this, changed); + const change = ChangeDetection(this, token, changed); // check for missing files if (change.missing.length > 0) { @@ -1833,15 +1677,16 @@ export class IDLIndex { async postProcessProFile( file: string, parsed: IParsed, + token: CancellationToken, oldGlobals: GlobalTokens, changeDetection = false ) { try { - PostProcessParsed(this, file, parsed); + PostProcessParsed(this, file, parsed, token); // check if we need to do change detection if (changeDetection) { - await this.changeDetection(parsed.global, oldGlobals); + await this.changeDetection(token, parsed.global, oldGlobals); } } catch (err) { this.log.log({ @@ -1869,6 +1714,7 @@ export class IDLIndex { */ async postProcessProFiles( files: string[], + token: CancellationToken, changeDetection = false ): Promise { /** Track any files that are missing and should no longer be tracked */ @@ -1891,6 +1737,7 @@ export class IDLIndex { await this.postProcessProFile( files[i], this.tokensByFile.get(files[i]), + token, [], changeDetection ); @@ -1987,7 +1834,7 @@ export class IDLIndex { { files: buckets[i], } - ) + ).response ); } @@ -2040,7 +1887,7 @@ export class IDLIndex { ids[j], LSP_WORKER_THREAD_MESSAGE_LOOKUP.TRACK_GLOBAL, res.globals - ) + ).response ); } } @@ -2076,7 +1923,7 @@ export class IDLIndex { */ // files: buckets[i] } - ) + ).response ); } @@ -2176,7 +2023,7 @@ export class IDLIndex { { files: buckets[i], } - ) + ).response ); } @@ -2239,7 +2086,7 @@ export class IDLIndex { ids[j], LSP_WORKER_THREAD_MESSAGE_LOOKUP.TRACK_GLOBAL, res.globals - ) + ).response ); } } @@ -2277,6 +2124,7 @@ export class IDLIndex { */ private async indexWorkspaceProFiles( files: string[], + token: CancellationToken, full: boolean ): Promise { // process in this thread if we don't have any workers @@ -2285,7 +2133,7 @@ export class IDLIndex { const missing1 = await this.indexProFiles(files, false); // post-process all of our files - const missing2 = await this.postProcessProFiles(files); + const missing2 = await this.postProcessProFiles(files, token); // remove missing files if we have them const allMissing = missing1.concat(missing2); @@ -2371,6 +2219,8 @@ export class IDLIndex { */ const buckets = this.bucketFiles(files); + const token = new CancellationToken(); + // save discovery time this.lastWorkspaceIndexStats.timeSearch = Math.floor( performance.now() - t0 @@ -2396,7 +2246,7 @@ export class IDLIndex { await this.indexNotebookFiles(buckets.notebookFiles); // do the indexing - await this.indexWorkspaceProFiles(buckets.proFiles, full); + await this.indexWorkspaceProFiles(buckets.proFiles, token, full); // save total time this.lastWorkspaceIndexStats.timeTotal = Math.floor(performance.now() - t0); diff --git a/libs/parsing/index/src/lib/idl-parsed-cache.class.ts b/libs/parsing/index/src/lib/idl-parsed-cache.class.ts index 5c52a1082..e2e2bee96 100644 --- a/libs/parsing/index/src/lib/idl-parsed-cache.class.ts +++ b/libs/parsing/index/src/lib/idl-parsed-cache.class.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IParsed, RemoveScopeDetailAndResetTokenCache, @@ -48,7 +49,7 @@ export class IDLParsedCache { const parsed = copy(orig); // clean up and make non-circular - RemoveScopeDetailAndResetTokenCache(parsed); + RemoveScopeDetailAndResetTokenCache(parsed, new CancellationToken()); // compress the keys for (let i = 0; i < COMPRESS_THESE.length; i++) { diff --git a/libs/parsing/index/src/lib/parse-notebook.interface.ts b/libs/parsing/index/src/lib/parse-notebook.interface.ts new file mode 100644 index 000000000..00aa49462 --- /dev/null +++ b/libs/parsing/index/src/lib/parse-notebook.interface.ts @@ -0,0 +1,20 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; +import { ParseNotebookResponse } from '@idl/workers/parsing'; + +/** + * Data we track for pending PRO code + */ +export interface IParsedNotebookPending { + /** + * Promise that resolves to the parsed notebook + */ + promise: Promise; + /** + * Cancellation token that we can use + */ + token: CancellationToken; + /** + * The checksums for the code cells + */ + checksums: string[]; +} diff --git a/libs/parsing/index/src/lib/parse-notebook.ts b/libs/parsing/index/src/lib/parse-notebook.ts new file mode 100644 index 000000000..635f9477d --- /dev/null +++ b/libs/parsing/index/src/lib/parse-notebook.ts @@ -0,0 +1,88 @@ +import { IDLNotebookDocument } from '@idl/notebooks/shared'; +import { CodeChecksum } from '@idl/parser'; +import { + LSP_WORKER_THREAD_MESSAGE_LOOKUP, + ParseNotebookResponse, +} from '@idl/workers/parsing'; +import { deepEqual } from 'fast-equals'; +import { NotebookCellKind } from 'vscode-languageserver'; + +import { IDLIndex } from './idl-index.class'; +import { IParsedNotebookPending } from './parse-notebook.interface'; + +/** + * Track the pending files + */ +const PENDING_NOTEBOOK: { [key: string]: IParsedNotebookPending } = {}; + +/** + * Handles getting a parsed notebook from the IDL index + */ +export async function ParseNotebook( + index: IDLIndex, + file: string, + notebook: IDLNotebookDocument +): Promise { + // track all checksums + const checksums: string[] = []; + + // get checksums + for (let i = 0; i < notebook.cells.length; i++) { + // skip if not code + if (notebook.cells[i].kind !== NotebookCellKind.Code) { + continue; + } + + // track checksums + checksums.push(CodeChecksum(notebook.cells[i].text)); + } + + // determine how to proceed + switch (true) { + case file in PENDING_NOTEBOOK: { + /** + * Get pending file + */ + const pending = PENDING_NOTEBOOK[file]; + + // check if our checksums are valid + if (deepEqual(pending.checksums, checksums)) { + return pending.promise; + } else { + // different checksums, so mark as cancelled + pending.token.cancel(); + } + break; + } + default: + break; + } + + /** + * Get ID of our worker + */ + const id = index.getWorkerID(file); + + // parse our notebook + const resp = index.indexerPool.workerio.postAndReceiveMessage( + id, + LSP_WORKER_THREAD_MESSAGE_LOOKUP.PARSE_NOTEBOOK, + { file, notebook } + ); + + // mark as pending + PENDING_NOTEBOOK[file] = { + checksums, + token: resp.token, + promise: resp.response, + }; + + // wait for finish + const res = await resp.response; + + // clean up pending + delete PENDING_NOTEBOOK[file]; + + // return result + return res; +} diff --git a/libs/parsing/index/src/lib/post-process/populate-and-validate-type.ts b/libs/parsing/index/src/lib/post-process/populate-and-validate-type.ts index 16a86be95..0ee1372f6 100644 --- a/libs/parsing/index/src/lib/post-process/populate-and-validate-type.ts +++ b/libs/parsing/index/src/lib/post-process/populate-and-validate-type.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IParsed } from '@idl/parsing/syntax-tree'; import { IDLIndex } from '../idl-index.class'; @@ -10,11 +11,12 @@ import { ValidateType } from './tree-handlers/validate-type'; export function PopulateAndValidateType( index: IDLIndex, file: string, - parsed: IParsed + parsed: IParsed, + cancel: CancellationToken ) { // populate types - PopulateType(index, file, parsed); + PopulateType(index, file, parsed, cancel); // validate types - ValidateType(index, file, parsed); + ValidateType(index, file, parsed, cancel); } diff --git a/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-array-creation.ts b/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-array-creation.ts index 627439eb1..13ccc869d 100644 --- a/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-array-creation.ts +++ b/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-array-creation.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IDL_ARRAY_TYPE, IDL_TYPE_LOOKUP, @@ -39,7 +40,10 @@ export function TypeFromArrayCreation( // process each split for (let i = 0; i < splitCommas.length; i++) { // sub split on operators to get types for each element - const operatorSplit = SplitTreeOnOperators(splitCommas[i]); + const operatorSplit = SplitTreeOnOperators( + splitCommas[i], + new CancellationToken() + ); // get the children and filter empty elements const operatorSplitTrees = operatorSplit.children.filter( diff --git a/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-indexing.ts b/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-indexing.ts index 0f2b79611..783d5e31c 100644 --- a/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-indexing.ts +++ b/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-indexing.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IDL_ANY_TYPE, IDLDataType, @@ -46,6 +47,7 @@ export function TypeFromIndexing( type: 'function', // doesn't matter, name: '', // doesnt matter }, + cancel: new CancellationToken(), }, parsed ); diff --git a/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-multiple-tokens.ts b/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-multiple-tokens.ts index 9cf2de09d..e66b65359 100644 --- a/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-multiple-tokens.ts +++ b/libs/parsing/index/src/lib/post-process/populate-type/from/type-from-multiple-tokens.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IDL_ANY_TYPE, IDLDataType } from '@idl/data-types/core'; import { IParsed, @@ -163,8 +164,8 @@ export function TypeFromMultipleTokens( /** Data types that we have found */ let foundTypes: IDLDataType[] = []; - // split on operators - const split = SplitTreeOnOperators(children); + // split on operators - make cancel token because this should be fast + const split = SplitTreeOnOperators(children, new CancellationToken()); // get the children and filter empty elements const splitTrees = split.children.filter((tree) => tree.length > 0); diff --git a/libs/parsing/index/src/lib/post-process/populate-uses-these.ts b/libs/parsing/index/src/lib/post-process/populate-uses-these.ts index 04660ca86..e123e47a0 100644 --- a/libs/parsing/index/src/lib/post-process/populate-uses-these.ts +++ b/libs/parsing/index/src/lib/post-process/populate-uses-these.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { GLOBAL_TOKEN_TYPES } from '@idl/data-types/core'; import { FindDirectBranchChildren, @@ -16,11 +17,15 @@ import { IDLIndex } from '../idl-index.class'; * We do this AFTER we populate types because we need to have * type detail for methods */ -export function PopulateUsesThese(index: IDLIndex, parsed: IParsed) { +export function PopulateUsesThese( + index: IDLIndex, + parsed: IParsed, + cancel: CancellationToken +) { // populate scope detail - PopulateScopeDetail(parsed); + PopulateScopeDetail(parsed, cancel); - TreeRecurserBasic(parsed.tree, { + TreeRecurserBasic(parsed.tree, cancel, { onBasicToken: (token) => { switch (token.name) { case TOKEN_NAMES.SYSTEM_VARIABLE: diff --git a/libs/parsing/index/src/lib/post-process/post-process-parsed.ts b/libs/parsing/index/src/lib/post-process/post-process-parsed.ts index 952318154..438324d95 100644 --- a/libs/parsing/index/src/lib/post-process/post-process-parsed.ts +++ b/libs/parsing/index/src/lib/post-process/post-process-parsed.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IParsed, PopulateScopeDetailAndResetTokenCache, @@ -15,7 +16,8 @@ import { ValidateVariableUsage } from './tree-handlers/validate-variable-usage'; export function PostProcessParsed( index: IDLIndex, file: string, - parsed: IParsed + parsed: IParsed, + cancel: CancellationToken ) { // clear secondary problems parsed.postProcessProblems = []; @@ -23,12 +25,12 @@ export function PostProcessParsed( /** * Reset cache and set scope detail if we need it */ - PopulateScopeDetailAndResetTokenCache(parsed); + PopulateScopeDetailAndResetTokenCache(parsed, cancel); /** * Populate types of local variables */ - PopulateAndValidateType(index, file, parsed); + PopulateAndValidateType(index, file, parsed, cancel); /** * Validate variables @@ -38,7 +40,7 @@ export function PostProcessParsed( /** * Populate the global tokens that we use */ - PopulateUsesThese(index, parsed); + PopulateUsesThese(index, parsed, cancel); // update problems for our file index.trackSyntaxProblemsForFile(file, GetSyntaxProblems(parsed)); diff --git a/libs/parsing/index/src/lib/post-process/tree-handlers/helpers/include-cache.ts b/libs/parsing/index/src/lib/post-process/tree-handlers/helpers/include-cache.ts index a8150c681..e8a28136a 100644 --- a/libs/parsing/index/src/lib/post-process/tree-handlers/helpers/include-cache.ts +++ b/libs/parsing/index/src/lib/post-process/tree-handlers/helpers/include-cache.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { CodeChecksum, Parser } from '@idl/parser'; import { IParsed, @@ -18,7 +19,11 @@ const CACHE: { [key: string]: IParsed } = {}; * Manges a basic cache for include statements so we don't read the same * file over and over */ -export function IncludeCache(index: IDLIndex, file: string) { +export function IncludeCache( + index: IDLIndex, + file: string, + cancel: CancellationToken +) { const strings = readFileSync(file, 'utf-8'); // check for and validate cache @@ -29,14 +34,14 @@ export function IncludeCache(index: IDLIndex, file: string) { } // parse - const parsed = Parser(strings); + const parsed = Parser(strings, cancel); // post-process and do some type defining - PostProcessParsed(index, file, parsed); + PostProcessParsed(index, file, parsed, cancel); // clean up - RemoveScopeDetail(parsed); - ResetTokenCache(parsed); + RemoveScopeDetail(parsed, cancel); + ResetTokenCache(parsed, cancel); // remove globals to save memory parsed.global = []; diff --git a/libs/parsing/index/src/lib/post-process/tree-handlers/populate-type.ts b/libs/parsing/index/src/lib/post-process/tree-handlers/populate-type.ts index 46fa3b430..f7965904f 100644 --- a/libs/parsing/index/src/lib/post-process/tree-handlers/populate-type.ts +++ b/libs/parsing/index/src/lib/post-process/tree-handlers/populate-type.ts @@ -4,6 +4,7 @@ import './populators/type-from-foreach'; import './populators/type-from-assignment'; import './populators/type-from-arguments'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { ILocalTokenLookup, IParsed } from '@idl/parsing/syntax-tree'; import { IDLIndex } from '../../idl-index.class'; @@ -13,7 +14,12 @@ import { SetVariables } from './set-variables'; /** * Applies styles from a tree handler to our parsed code */ -export function PopulateType(index: IDLIndex, file: string, parsed: IParsed) { +export function PopulateType( + index: IDLIndex, + file: string, + parsed: IParsed, + cancel: CancellationToken +) { // init variables const variables: ILocalTokenLookup = {}; @@ -24,7 +30,7 @@ export function PopulateType(index: IDLIndex, file: string, parsed: IParsed) { const baseMeta = { variables, index, file }; // process our tree - POPULATE_TYPE_HANDLER.run(parsed, (token, meta) => { + POPULATE_TYPE_HANDLER.run(parsed, cancel, (token, meta) => { // make sure we have a token, undefined for full-tree if (token !== undefined) { const use = token.scopeTokens[0] || token; diff --git a/libs/parsing/index/src/lib/post-process/tree-handlers/populators/type-from-include.ts b/libs/parsing/index/src/lib/post-process/tree-handlers/populators/type-from-include.ts index 3d82757e0..18de8d2eb 100644 --- a/libs/parsing/index/src/lib/post-process/tree-handlers/populators/type-from-include.ts +++ b/libs/parsing/index/src/lib/post-process/tree-handlers/populators/type-from-include.ts @@ -75,7 +75,7 @@ const cb: BasicCallback = ( */ const includeParsed = meta.index.tokensByFile.has(foundFile) ? meta.index.tokensByFile.get(foundFile) - : IncludeCache(meta.index, foundFile); + : IncludeCache(meta.index, foundFile, meta.cancel); // remove from cache as trying to include delete IS_INCLUDING[foundFile]; diff --git a/libs/parsing/index/src/lib/post-process/tree-handlers/validate-type.ts b/libs/parsing/index/src/lib/post-process/tree-handlers/validate-type.ts index 77e3e5d9a..f329fefbf 100644 --- a/libs/parsing/index/src/lib/post-process/tree-handlers/validate-type.ts +++ b/libs/parsing/index/src/lib/post-process/tree-handlers/validate-type.ts @@ -4,6 +4,7 @@ import './validators/validate-assignment'; import './validators/validate-arguments'; import './validators/ambiguous-keyword-usage'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { ILocalTokenLookup, IParsed } from '@idl/parsing/syntax-tree'; import { IDLIndex } from '../../idl-index.class'; @@ -13,7 +14,12 @@ import { VALIDATE_TYPE_HANDLER } from './validate-type-handler'; /** * Applies styles from a tree handler to our parsed code */ -export function ValidateType(index: IDLIndex, file: string, parsed: IParsed) { +export function ValidateType( + index: IDLIndex, + file: string, + parsed: IParsed, + cancel: CancellationToken +) { // init variables const variables: ILocalTokenLookup = {}; @@ -24,7 +30,7 @@ export function ValidateType(index: IDLIndex, file: string, parsed: IParsed) { const baseMeta = { variables, index, file }; // process our tree - VALIDATE_TYPE_HANDLER.run(parsed, (token, meta) => { + VALIDATE_TYPE_HANDLER.run(parsed, cancel, (token, meta) => { // make sure we have a token, undefined for full-tree if (token !== undefined) { const use = token.scopeTokens[0] || token; diff --git a/libs/parsing/parser/src/lib/parser.ts b/libs/parsing/parser/src/lib/parser.ts index fccb65746..feaebde4b 100644 --- a/libs/parsing/parser/src/lib/parser.ts +++ b/libs/parsing/parser/src/lib/parser.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { ActivateDefaultSyntaxPostProcessors } from '@idl/parsing/syntax-post-processors'; import { BuildSyntaxTree, @@ -25,6 +26,7 @@ ActivateDefaultSyntaxPostProcessors(); export function ParserTokenize( code: string | string[], tokenized: IParsed, + cancel: CancellationToken, full = true ) { /** @@ -35,7 +37,9 @@ export function ParserTokenize( // try { Object.assign( tokenized, - full ? Tokenizer(code) : Tokenizer(code, FAST_FIND_TOKEN_OPTIONS) + full + ? Tokenizer(code, cancel) + : Tokenizer(code, cancel, FAST_FIND_TOKEN_OPTIONS) ); // } catch (err) { @@ -52,31 +56,6 @@ export function ParserTokenize( // } } -/** - * Creates a syntax tree and does checking for syntax errors - */ -export function ParserBuildTree( - tokenized: IParsed, - full: boolean, - isNotebook: boolean -) { - // try { - // build tree - updates property for tokenized - BuildSyntaxTree(tokenized, full, isNotebook); - // } catch (err) { - // console.error(err); - - // // track problem that will get reported to user - // tokenized.parseProblems.push( - // ProblemWithTranslation( - // IDL_PROBLEM_CODES.EMBARRASSING_FILE, - // [0, 0, 0], - // [0, 0, 0] - // ) - // ); - // } -} - /** * Parses IDL code into tokens for linting, formatting, and error checking. * @param {(string | string[])} code The Code to parse @@ -86,6 +65,7 @@ export function ParserBuildTree( */ export function Parser( code: string | string[], + cancel: CancellationToken, inOptions: Partial = {} ): IParsed { /** @@ -119,10 +99,10 @@ export function Parser( }; // extract tokens - ParserTokenize(code, tokenized, options.full); + ParserTokenize(code, tokenized, cancel, options.full); // build the syntax tree and detect syntax problems - ParserBuildTree(tokenized, options.full, options.isNotebook); + BuildSyntaxTree(tokenized, cancel, options.full, options.isNotebook); /** * Populate our global, local (variables), and compile-opts @@ -132,7 +112,7 @@ export function Parser( * * If it is off, we dont get hover help or useful auto-complete */ - PopulateGlobalLocalCompileOpts(tokenized, true, options.isNotebook); + PopulateGlobalLocalCompileOpts(tokenized, cancel, true, options.isNotebook); // remove all problems if fast parse if (!options.full) { @@ -168,6 +148,7 @@ export function Parser( */ export async function ParseFile( file: string, + cancel: CancellationToken, options: Partial = {} ): Promise { // make sure that our file exists @@ -179,7 +160,7 @@ export async function ParseFile( const code = await readFile(file, 'utf-8'); // parse and return - return Parser(code, options); + return Parser(code, cancel, options); } /** @@ -190,6 +171,7 @@ export async function ParseFile( */ export function ParseFileSync( file: string, + cancel: CancellationToken, options: Partial = {} ): IParsed { // make sure that our file exists @@ -201,5 +183,5 @@ export function ParseFileSync( const code = readFileSync(file, 'utf-8'); // parse and return - return Parser(code, options); + return Parser(code, cancel, options); } diff --git a/libs/parsing/syntax-tree/src/index.ts b/libs/parsing/syntax-tree/src/index.ts index 129787f38..8dcd73dd0 100644 --- a/libs/parsing/syntax-tree/src/index.ts +++ b/libs/parsing/syntax-tree/src/index.ts @@ -1,6 +1,6 @@ export * from './lib/branches.interface'; -export * from './lib/build-tree'; -export * from './lib/build-tree.interface'; +export * from './lib/build-syntax-tree'; +export * from './lib/build-syntax-tree.interface'; export * from './lib/docs/docs.interface'; export * from './lib/docs/docs.regex.interface'; export * from './lib/docs/extract-docs.interface'; diff --git a/libs/parsing/syntax-tree/src/lib/build-tree.interface.ts b/libs/parsing/syntax-tree/src/lib/build-syntax-tree.interface.ts similarity index 100% rename from libs/parsing/syntax-tree/src/lib/build-tree.interface.ts rename to libs/parsing/syntax-tree/src/lib/build-syntax-tree.interface.ts diff --git a/libs/parsing/syntax-tree/src/lib/build-tree.ts b/libs/parsing/syntax-tree/src/lib/build-syntax-tree.ts similarity index 97% rename from libs/parsing/syntax-tree/src/lib/build-tree.ts rename to libs/parsing/syntax-tree/src/lib/build-syntax-tree.ts index 5e5f0114c..05656f074 100644 --- a/libs/parsing/syntax-tree/src/lib/build-tree.ts +++ b/libs/parsing/syntax-tree/src/lib/build-syntax-tree.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IDL_PROBLEM_CODES, IDLProblemCode } from '@idl/parsing/problem-codes'; import { IBasicToken, @@ -21,7 +22,7 @@ import { IParsed, IRecurserCloseOptions, IRecurserOptions, -} from './build-tree.interface'; +} from './build-syntax-tree.interface'; import { PopulateIndex } from './populate-index'; import { PopulateScope } from './populate-scope'; import { GetUniqueVariables } from './populators/get-unique-variables'; @@ -265,6 +266,7 @@ function BuildTreeRecurser( */ export function BuildSyntaxTree( parsed: IParsed, + cancel: CancellationToken, full: boolean, isNotebook: boolean ) { @@ -282,7 +284,7 @@ export function BuildSyntaxTree( PopulateIndex(parsed.tree); // populate the scope - PopulateScope(parsed); + PopulateScope(parsed, cancel); // perform post-processing IDL_SYNTAX_TREE_POST_PROCESSOR.processTree( @@ -297,7 +299,7 @@ export function BuildSyntaxTree( PopulateIndex(parsed.tree); // populate the scope again in case our tree changed - PopulateScope(parsed); + PopulateScope(parsed, cancel); // create metadata for our syntax validator // leave this for type checks even though unused @@ -307,7 +309,7 @@ export function BuildSyntaxTree( }; // run our syntax validation - IDL_SYNTAX_TREE_VALIDATOR.run(parsed, (token, meta) => { + IDL_SYNTAX_TREE_VALIDATOR.run(parsed, cancel, (token, meta) => { Object.assign(meta, { isNotebook }); return meta as any as IDLSyntaxValidatorMeta; }); diff --git a/libs/parsing/syntax-tree/src/lib/helpers/get-local-token-lookup.ts b/libs/parsing/syntax-tree/src/lib/helpers/get-local-token-lookup.ts index 322e8c3d5..cb423d216 100644 --- a/libs/parsing/syntax-tree/src/lib/helpers/get-local-token-lookup.ts +++ b/libs/parsing/syntax-tree/src/lib/helpers/get-local-token-lookup.ts @@ -1,7 +1,7 @@ import { TOKEN_NAMES } from '@idl/parsing/tokenizer'; import { TreeBranchToken } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { ILocalTokenLookup } from '../populators/populate-local.interface'; import { GetRoutineName } from './get-routine-name'; diff --git a/libs/parsing/syntax-tree/src/lib/helpers/get-token-at-cursor.ts b/libs/parsing/syntax-tree/src/lib/helpers/get-token-at-cursor.ts index ac4797a8b..bff79d7d9 100644 --- a/libs/parsing/syntax-tree/src/lib/helpers/get-token-at-cursor.ts +++ b/libs/parsing/syntax-tree/src/lib/helpers/get-token-at-cursor.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { MainLevelToken, RoutineFunctionToken, @@ -7,7 +8,7 @@ import { import { Position } from 'vscode-languageserver/node'; import { IBranch, TreeToken } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { PopulateScopeDetail } from '../populate-scope-detail'; import { TreeRecurser } from '../recursion-and-callbacks/tree-recurser'; import { @@ -32,22 +33,26 @@ export function GetTokenAtCursor( pos: Position, onlyStart = false ): ISelectedToken { + // make a cancellation token + const cancel = new CancellationToken(); + /** Initialize result */ let result: ISelectedToken = { accessTokens: [], scope: [], scopeTokens: [], + cancel, }; // make sure we have scope detail populated in our tree - SetTokenCache(parsed); - PopulateScopeDetail(parsed); + SetTokenCache(parsed, cancel); + PopulateScopeDetail(parsed, cancel); /** Init token that we found */ let foundToken: TreeToken = undefined; // recurse through the tree to find a match - TreeRecurser(parsed, { + TreeRecurser(parsed, cancel, { processBranchFirst: true, onBasicToken: (token, current) => { // return if our token is before our line diff --git a/libs/parsing/syntax-tree/src/lib/helpers/get-variable-token-def.ts b/libs/parsing/syntax-tree/src/lib/helpers/get-variable-token-def.ts index ef4444e0a..4657ad7b2 100644 --- a/libs/parsing/syntax-tree/src/lib/helpers/get-variable-token-def.ts +++ b/libs/parsing/syntax-tree/src/lib/helpers/get-variable-token-def.ts @@ -2,7 +2,7 @@ import { TokenName } from '@idl/parsing/tokenizer'; import copy from 'fast-copy'; import { TreeToken } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { ILocalIndexedToken, ILocalTokenLookup, diff --git a/libs/parsing/syntax-tree/src/lib/helpers/searching/split-tree-on-operators.ts b/libs/parsing/syntax-tree/src/lib/helpers/searching/split-tree-on-operators.ts index 11e9a0bc3..84ea667f4 100644 --- a/libs/parsing/syntax-tree/src/lib/helpers/searching/split-tree-on-operators.ts +++ b/libs/parsing/syntax-tree/src/lib/helpers/searching/split-tree-on-operators.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { OperatorIncrementDecrementToken, OperatorLogicalToken, @@ -39,7 +40,10 @@ export type SplitOperatorToken = TreeToken< * Takes an array of tokens (i.e. syntax tree) and recursively splits them * by operators */ -export function SplitTreeOnOperators(tree: SyntaxTree) { +export function SplitTreeOnOperators( + tree: SyntaxTree, + cancel: CancellationToken +) { /** Commas we split by */ const operators: SplitOperatorToken[] = []; @@ -58,7 +62,7 @@ export function SplitTreeOnOperators(tree: SyntaxTree) { /** * Recurse through our tree */ - TreeRecurserBasic(tree, { + TreeRecurserBasic(tree, cancel, { recursionFilter: OPERATORS, onBranchToken: (token) => { // save starting position diff --git a/libs/parsing/syntax-tree/src/lib/populate-scope-detail-and-reset-token-cache.ts b/libs/parsing/syntax-tree/src/lib/populate-scope-detail-and-reset-token-cache.ts index 361071a91..bc24c12fe 100644 --- a/libs/parsing/syntax-tree/src/lib/populate-scope-detail-and-reset-token-cache.ts +++ b/libs/parsing/syntax-tree/src/lib/populate-scope-detail-and-reset-token-cache.ts @@ -1,4 +1,6 @@ -import { IParsed } from './build-tree.interface'; +import { CancellationToken } from '@idl/cancellation-tokens'; + +import { IParsed } from './build-syntax-tree.interface'; import { TreeRecurser } from './recursion-and-callbacks/tree-recurser'; import { ResetTokenCache } from './reset-token-cache'; @@ -12,10 +14,13 @@ import { ResetTokenCache } from './reset-token-cache'; * * Use `RemoveScopeDetail` to get rid of this information */ -export function PopulateScopeDetailAndResetTokenCache(parsed: IParsed): void { +export function PopulateScopeDetailAndResetTokenCache( + parsed: IParsed, + cancel: CancellationToken +): void { switch (true) { case !parsed.hasDetail: - TreeRecurser(parsed, { + TreeRecurser(parsed, cancel, { onBasicToken: (token, current) => { token.scopeTokens = current.scopeTokens.slice(); token.accessTokens = current.accessTokens.slice(); @@ -31,7 +36,7 @@ export function PopulateScopeDetailAndResetTokenCache(parsed: IParsed): void { parsed.hasDetail = true; break; default: - ResetTokenCache(parsed); + ResetTokenCache(parsed, cancel); break; } } diff --git a/libs/parsing/syntax-tree/src/lib/populate-scope-detail.ts b/libs/parsing/syntax-tree/src/lib/populate-scope-detail.ts index 6f24f0f9c..0a29a5634 100644 --- a/libs/parsing/syntax-tree/src/lib/populate-scope-detail.ts +++ b/libs/parsing/syntax-tree/src/lib/populate-scope-detail.ts @@ -1,4 +1,6 @@ -import { IParsed } from './build-tree.interface'; +import { CancellationToken } from '@idl/cancellation-tokens'; + +import { IParsed } from './build-syntax-tree.interface'; import { TreeRecurser } from './recursion-and-callbacks/tree-recurser'; /** @@ -8,10 +10,13 @@ import { TreeRecurser } from './recursion-and-callbacks/tree-recurser'; * * Use `RemoveScopeDetail` to get rid of this information */ -export function PopulateScopeDetail(parsed: IParsed): void { +export function PopulateScopeDetail( + parsed: IParsed, + cancel: CancellationToken +): void { // only process if we dont have scope detail if (!parsed.hasDetail) { - TreeRecurser(parsed, { + TreeRecurser(parsed, cancel, { onBasicToken: (token, current) => { token.scopeTokens = current.scopeTokens.slice(); token.accessTokens = current.accessTokens.slice(); diff --git a/libs/parsing/syntax-tree/src/lib/populate-scope.ts b/libs/parsing/syntax-tree/src/lib/populate-scope.ts index b1e397bc6..5d935e2b2 100644 --- a/libs/parsing/syntax-tree/src/lib/populate-scope.ts +++ b/libs/parsing/syntax-tree/src/lib/populate-scope.ts @@ -1,4 +1,6 @@ -import { IParsed } from './build-tree.interface'; +import { CancellationToken } from '@idl/cancellation-tokens'; + +import { IParsed } from './build-syntax-tree.interface'; import { TreeRecurser } from './recursion-and-callbacks/tree-recurser'; /** @@ -6,8 +8,11 @@ import { TreeRecurser } from './recursion-and-callbacks/tree-recurser'; * present and this step is executed shortly before the syntax tree * is actually returned */ -export function PopulateScope(parsed: IParsed): void { - TreeRecurser(parsed, { +export function PopulateScope( + parsed: IParsed, + cancel: CancellationToken +): void { + TreeRecurser(parsed, cancel, { onBasicToken: (token, current) => { token.scope = current.scope.slice(0); }, diff --git a/libs/parsing/syntax-tree/src/lib/populators/get-unique-variables.ts b/libs/parsing/syntax-tree/src/lib/populators/get-unique-variables.ts index 7c43c9aa2..8ed699c7a 100644 --- a/libs/parsing/syntax-tree/src/lib/populators/get-unique-variables.ts +++ b/libs/parsing/syntax-tree/src/lib/populators/get-unique-variables.ts @@ -15,7 +15,7 @@ import { } from '@idl/parsing/tokenizer'; import { IBasicBranch, IBranch, TreeToken } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { IDL_DOCS_HEADERS } from '../docs/docs.interface'; import { ExtractDocs } from '../docs/extract-docs'; import { JoinDocs } from '../docs/join-docs'; diff --git a/libs/parsing/syntax-tree/src/lib/populators/populate-global.ts b/libs/parsing/syntax-tree/src/lib/populators/populate-global.ts index 8b9b97fa8..5fb6ebfb9 100644 --- a/libs/parsing/syntax-tree/src/lib/populators/populate-global.ts +++ b/libs/parsing/syntax-tree/src/lib/populators/populate-global.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { GLOBAL_TOKEN_SOURCE_LOOKUP, GLOBAL_TOKEN_TYPES, @@ -23,7 +24,7 @@ import { IDL_TRANSLATION } from '@idl/translation'; import copy from 'fast-copy'; import { IBranch } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { GenerateRoutineDocsAndMetadata } from '../docs/generate-routine-docs-and-metadata'; import { GenerateRoutineMetadataFast } from '../docs/generate-routine-metadata-fast'; import { FindStructureDefs } from './find-structure-defs'; @@ -38,6 +39,7 @@ import { PopulateLocalForMain } from './populate-local-for-main'; */ export function PopulateGlobalLocalCompileOpts( parsed: IParsed, + cancel: CancellationToken, full: boolean, isNotebook: boolean ) { @@ -58,6 +60,9 @@ export function PopulateGlobalLocalCompileOpts( // process all of the direct children in our tree for (let i = 0; i < tree.length; i++) { + // check for cancel + cancel.throwIfCancelled(); + // extract our branch const branch = tree[i]; diff --git a/libs/parsing/syntax-tree/src/lib/populators/populate-local-for-main.ts b/libs/parsing/syntax-tree/src/lib/populators/populate-local-for-main.ts index 5ee33ecd6..16f463859 100644 --- a/libs/parsing/syntax-tree/src/lib/populators/populate-local-for-main.ts +++ b/libs/parsing/syntax-tree/src/lib/populators/populate-local-for-main.ts @@ -1,7 +1,7 @@ import { MainLevelToken, TOKEN_NAMES } from '@idl/parsing/tokenizer'; import { IBranch } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { GetUniqueVariables } from './get-unique-variables'; /** diff --git a/libs/parsing/syntax-tree/src/lib/populators/replace-functions-as-variables.ts b/libs/parsing/syntax-tree/src/lib/populators/replace-functions-as-variables.ts index b21eca79b..8cebcdf25 100644 --- a/libs/parsing/syntax-tree/src/lib/populators/replace-functions-as-variables.ts +++ b/libs/parsing/syntax-tree/src/lib/populators/replace-functions-as-variables.ts @@ -14,7 +14,7 @@ import { TreeBranchToken, TreeToken, } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { SyntaxProblemWithTranslation } from '../syntax-problem-with'; import { ILocalTokenLookup } from './populate-local.interface'; diff --git a/libs/parsing/syntax-tree/src/lib/post-process-problems.ts b/libs/parsing/syntax-tree/src/lib/post-process-problems.ts index 662db1719..e1c325a62 100644 --- a/libs/parsing/syntax-tree/src/lib/post-process-problems.ts +++ b/libs/parsing/syntax-tree/src/lib/post-process-problems.ts @@ -1,6 +1,6 @@ import { IDL_PROBLEM_CODES } from '@idl/parsing/problem-codes'; -import { IParsed } from './build-tree.interface'; +import { IParsed } from './build-syntax-tree.interface'; /** * Post process syntax problems to clean up reporting. diff --git a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-handler.class.ts b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-handler.class.ts index a93073e4b..98d7c12a9 100644 --- a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-handler.class.ts +++ b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-handler.class.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { BasicTokenNames, NonBasicTokenNames, @@ -5,7 +6,7 @@ import { } from '@idl/parsing/tokenizer'; import { ITreeRecurserOptions, TreeToken } from '../..'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { BasicCallback, BasicCallbackLookup, @@ -139,8 +140,9 @@ export class TreeCallbackHandler { */ run( parsed: IParsed, + cancel: CancellationToken, cb: (token: TreeToken, meta: IHandlerCallbackMetadata) => TMeta ) { - TreeCallbackRunner(this, parsed, cb); + TreeCallbackRunner(this, parsed, cancel, cb); } } diff --git a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-handler.interface.ts b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-handler.interface.ts index dfcf7d7b6..2b96c7a6f 100644 --- a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-handler.interface.ts +++ b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-handler.interface.ts @@ -1,7 +1,7 @@ import { BasicTokenNames, NonBasicTokenNames } from '@idl/parsing/tokenizer'; import { SyntaxTree, TreeToken } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { ITreeRecurserCurrent } from './tree-recurser.interface'; /** diff --git a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-runner.ts b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-runner.ts index 9c4b32a73..36ded5232 100644 --- a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-runner.ts +++ b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-callback-runner.ts @@ -1,7 +1,8 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { TokenName } from '@idl/parsing/tokenizer'; import { TreeToken } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { TreeCallbackHandler } from './tree-callback-handler.class'; import { IHandlerCallbackMetadata } from './tree-callback-handler.interface'; import { TreeRecurser } from './tree-recurser'; @@ -13,6 +14,7 @@ import { DEFAULT_CURRENT } from './tree-recurser.interface'; export function TreeCallbackRunner( handler: TreeCallbackHandler, parsed: IParsed, + cancel: CancellationToken, metaResolver: ( token: TreeToken, meta: IHandlerCallbackMetadata @@ -21,6 +23,7 @@ export function TreeCallbackRunner( // use our tree recurser to process what we parsed TreeRecurser( parsed, + cancel, Object.assign(handler.recursionOptions, { onBasicToken: (token, current) => { handler.processBasicToken(token, parsed, () => diff --git a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser-basic.interface.ts b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser-basic.interface.ts index f0aa58736..0524f1fc0 100644 --- a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser-basic.interface.ts +++ b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser-basic.interface.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { BasicTokenNames, NonBasicTokenNames } from '@idl/parsing/tokenizer'; import { TreeToken } from '../branches.interface'; @@ -7,7 +8,8 @@ import { TreeToken } from '../branches.interface'; * to stop iterating through the tree */ export type BasicTokenBasicCallback = ( - token: TreeToken + token: TreeToken, + cancel: CancellationToken ) => boolean | void; /** @@ -15,7 +17,8 @@ export type BasicTokenBasicCallback = ( * to stop iterating through the tree */ export type BranchTokenBasicCallback = ( - token: TreeToken + token: TreeToken, + cancel: CancellationToken ) => boolean | void; /** @@ -52,6 +55,10 @@ export interface ITreeRecurserBasicRecursionOptions extends ITreeRecurserBasicOptions { /** Flag if we need to exit */ exit?: boolean | void; + /** + * Cancellation token + */ + cancel: CancellationToken; } /** @@ -65,4 +72,5 @@ export const BASE_TREE_RECURSER_BASIC_OPTIONS: ITreeRecurserBasicRecursionOption onBranchToken: () => { return false; }, + cancel: new CancellationToken(), }; diff --git a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser-basic.ts b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser-basic.ts index 41accb2fe..65890601c 100644 --- a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser-basic.ts +++ b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser-basic.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import copy from 'fast-copy'; import { @@ -21,10 +22,16 @@ function _Recurser( ): void { // process every token for (let i = 0; i < tree.length; i++) { + // check for cancel + current.cancel.throwIfCancelled(); + // skip if before our line and we if (tree[i].type === BRANCH_TYPES.BRANCH) { // handle branch token - current.exit = current.onBranchToken(tree[i] as TreeBranchToken); + current.exit = current.onBranchToken( + tree[i] as TreeBranchToken, + current.cancel + ); // check for exit if (current.exit) { @@ -41,7 +48,10 @@ function _Recurser( } } else { // handle branch token - current.exit = current.onBasicToken(tree[i] as TreeBasicToken); + current.exit = current.onBasicToken( + tree[i] as TreeBasicToken, + current.cancel + ); } // check for exit @@ -60,11 +70,12 @@ function _Recurser( */ export function TreeRecurserBasic( tree: SyntaxTree, + cancel: CancellationToken, options: Partial ) { // recurse through the tree _Recurser( tree, - Object.assign(copy(BASE_TREE_RECURSER_BASIC_OPTIONS), options) + Object.assign(copy(BASE_TREE_RECURSER_BASIC_OPTIONS), options, { cancel }) ); } diff --git a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser.interface.ts b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser.interface.ts index 850ba8690..ba3ca2285 100644 --- a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser.interface.ts +++ b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser.interface.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { BasicTokenNames, NonBasicTokenNames, @@ -106,6 +107,10 @@ export interface ITreeRecurserCurrent { * Any tokens used for access directly before our current token */ accessTokens: TreeToken[]; + /** + * Token to cancel work + */ + cancel: CancellationToken; } /** @@ -116,6 +121,7 @@ export const DEFAULT_CURRENT: ITreeRecurserCurrent = { scope: [], scopeTokens: [], accessTokens: [], + cancel: new CancellationToken(), }; /** diff --git a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser.ts b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser.ts index da243a1ca..49ec86e35 100644 --- a/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser.ts +++ b/libs/parsing/syntax-tree/src/lib/recursion-and-callbacks/tree-recurser.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { MainLevelToken, NonBasicTokenNames, @@ -14,7 +15,7 @@ import { TreeBranchToken, TreeToken, } from '../branches.interface'; -import { IParsed } from '../build-tree.interface'; +import { IParsed } from '../build-syntax-tree.interface'; import { GetRoutineName } from '../helpers/get-routine-name'; import { BASE_TREE_RECURSER_OPTIONS, @@ -37,6 +38,9 @@ function _Recurser( continue; } + // check for cancel + current.cancel.throwIfCancelled(); + // check if we can populate our parent if (tree[i].name in current.allowedGlobalParents) { current.globalParent = { @@ -149,6 +153,7 @@ function _Recurser( */ export function TreeRecurser( parsed: IParsed, + cancel: CancellationToken, options: Partial ) { // recurse through the tree @@ -157,7 +162,8 @@ export function TreeRecurser( Object.assign( copy(BASE_TREE_RECURSER_OPTIONS), copy(DEFAULT_CURRENT), - options + options, + { cancel } ) ); } diff --git a/libs/parsing/syntax-tree/src/lib/remove-scope-detail-and-reset-token-cache.ts b/libs/parsing/syntax-tree/src/lib/remove-scope-detail-and-reset-token-cache.ts index 43b13c877..5b8cd2b6e 100644 --- a/libs/parsing/syntax-tree/src/lib/remove-scope-detail-and-reset-token-cache.ts +++ b/libs/parsing/syntax-tree/src/lib/remove-scope-detail-and-reset-token-cache.ts @@ -1,14 +1,19 @@ -import { IParsed } from './build-tree.interface'; +import { CancellationToken } from '@idl/cancellation-tokens'; + +import { IParsed } from './build-syntax-tree.interface'; import { TreeRecurserBasic } from './recursion-and-callbacks/tree-recurser-basic'; import { ResetTokenCache } from './reset-token-cache'; /** * Removes scope detail and resets our token cache */ -export function RemoveScopeDetailAndResetTokenCache(parsed: IParsed): void { +export function RemoveScopeDetailAndResetTokenCache( + parsed: IParsed, + cancel: CancellationToken +): void { switch (true) { case parsed.hasDetail: - TreeRecurserBasic(parsed.tree, { + TreeRecurserBasic(parsed.tree, cancel, { onBasicToken: (token) => { delete token.scopeTokens; delete token.accessTokens; @@ -24,7 +29,7 @@ export function RemoveScopeDetailAndResetTokenCache(parsed: IParsed): void { parsed.hasDetail = false; break; default: - ResetTokenCache(parsed); + ResetTokenCache(parsed, cancel); break; } } diff --git a/libs/parsing/syntax-tree/src/lib/remove-scope-detail.ts b/libs/parsing/syntax-tree/src/lib/remove-scope-detail.ts index 09435d7f5..a3d7ac619 100644 --- a/libs/parsing/syntax-tree/src/lib/remove-scope-detail.ts +++ b/libs/parsing/syntax-tree/src/lib/remove-scope-detail.ts @@ -1,13 +1,18 @@ -import { IParsed } from './build-tree.interface'; +import { CancellationToken } from '@idl/cancellation-tokens'; + +import { IParsed } from './build-syntax-tree.interface'; import { TreeRecurserBasic } from './recursion-and-callbacks/tree-recurser-basic'; /** * Removes scope detail from our object to force our tree to no * longer be circular. */ -export function RemoveScopeDetail(parsed: IParsed): void { +export function RemoveScopeDetail( + parsed: IParsed, + cancel: CancellationToken +): void { if (parsed.hasDetail) { - TreeRecurserBasic(parsed.tree, { + TreeRecurserBasic(parsed.tree, cancel, { onBasicToken: (token) => { delete token.scopeTokens; delete token.accessTokens; diff --git a/libs/parsing/syntax-tree/src/lib/reset-token-cache.ts b/libs/parsing/syntax-tree/src/lib/reset-token-cache.ts index fbfbac0ff..05370bede 100644 --- a/libs/parsing/syntax-tree/src/lib/reset-token-cache.ts +++ b/libs/parsing/syntax-tree/src/lib/reset-token-cache.ts @@ -1,12 +1,17 @@ -import { IParsed } from './build-tree.interface'; +import { CancellationToken } from '@idl/cancellation-tokens'; + +import { IParsed } from './build-syntax-tree.interface'; import { TreeRecurserBasic } from './recursion-and-callbacks/tree-recurser-basic'; /** * Resets token cache to clear anything that was previously set so that * we can re-validate our tokens */ -export function ResetTokenCache(parsed: IParsed): void { - TreeRecurserBasic(parsed.tree, { +export function ResetTokenCache( + parsed: IParsed, + cancel: CancellationToken +): void { + TreeRecurserBasic(parsed.tree, cancel, { onBasicToken: (token) => { token.cache = {}; }, diff --git a/libs/parsing/syntax-tree/src/lib/set-token-cache.ts b/libs/parsing/syntax-tree/src/lib/set-token-cache.ts index 9db0a085a..ba5e11b2f 100644 --- a/libs/parsing/syntax-tree/src/lib/set-token-cache.ts +++ b/libs/parsing/syntax-tree/src/lib/set-token-cache.ts @@ -1,12 +1,17 @@ -import { IParsed } from './build-tree.interface'; +import { CancellationToken } from '@idl/cancellation-tokens'; + +import { IParsed } from './build-syntax-tree.interface'; import { TreeRecurserBasic } from './recursion-and-callbacks/tree-recurser-basic'; /** * Set the cache to make sure it is present */ -export function SetTokenCache(parsed: IParsed): void { +export function SetTokenCache( + parsed: IParsed, + cancel: CancellationToken +): void { if (!parsed.hasCache) { - TreeRecurserBasic(parsed.tree, { + TreeRecurserBasic(parsed.tree, cancel, { onBasicToken: (token) => { token.cache = {}; }, diff --git a/libs/parsing/tokenizer/src/lib/iterator.class.ts b/libs/parsing/tokenizer/src/lib/iterator.class.ts index eb915ec7a..8f156facd 100644 --- a/libs/parsing/tokenizer/src/lib/iterator.class.ts +++ b/libs/parsing/tokenizer/src/lib/iterator.class.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { PositionArray } from '@idl/parsing/tokenizer-types'; import { GetMatchesArray } from './helpers/get-matches-array'; @@ -55,11 +56,15 @@ export class Iterator { /** Are we doing a full parse or not */ full: boolean; - constructor(code: string | string[], full = true) { + /** Cancellation token */ + cancel: CancellationToken; + + constructor(code: string | string[], cancel: CancellationToken, full = true) { /** Split our strings */ this.split = Split(code); this.processedLeftovers = new Array(this.split.length).fill(false); this.full = full; + this.cancel = cancel; // initialize current this.current = { @@ -200,6 +205,9 @@ export class Iterator { // return true; // } + // check for cancellation + this.cancel.throwIfCancelled(); + // get the text before token start that we are shifting for if (tokenStart > 0 && this.full) { const before = this.current.sub.substring(0, tokenStart); @@ -333,6 +341,9 @@ export class Iterator { * complicated empty logic */ nextLine(isContinuedLine = false): boolean { + // check for cancellation + this.cancel.throwIfCancelled(); + // check our current line for leftovers this.findLeftovers(); diff --git a/libs/parsing/tokenizer/src/lib/tests/multi-line-consistency.spec.ts b/libs/parsing/tokenizer/src/lib/tests/multi-line-consistency.spec.ts index 53e855977..384095061 100644 --- a/libs/parsing/tokenizer/src/lib/tests/multi-line-consistency.spec.ts +++ b/libs/parsing/tokenizer/src/lib/tests/multi-line-consistency.spec.ts @@ -1,3 +1,5 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; + import { StripIDs } from '../helpers/strip-ids'; import { Tokenizer } from '../tokenizer'; @@ -9,7 +11,7 @@ describe('Validates processing the same line multiple times', () => { ]; // get our tokens - const tokens = Tokenizer(code).tokens; + const tokens = Tokenizer(code, new CancellationToken()).tokens; // extract line-specific tokens const line1 = tokens.filter((t) => t.pos[0] === 0); diff --git a/libs/parsing/tokenizer/src/lib/tests/number-string.spec.ts b/libs/parsing/tokenizer/src/lib/tests/number-string.spec.ts index cf3d0d946..73dd6df8c 100644 --- a/libs/parsing/tokenizer/src/lib/tests/number-string.spec.ts +++ b/libs/parsing/tokenizer/src/lib/tests/number-string.spec.ts @@ -1,3 +1,5 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; + import { StripIDs } from '../helpers/strip-ids'; import { Tokenizer } from '../tokenizer'; import { @@ -16,7 +18,10 @@ describe('Validates quote parsing', () => { it('single quote binary data', () => { for (let i = 0; i < types.length; i++) { - const tokens = Tokenizer(`'1001'b${types[i]}`).tokens; + const tokens = Tokenizer( + `'1001'b${types[i]}`, + new CancellationToken() + ).tokens; // make sure we have two tokens expect(tokens.length).toBe(1); @@ -42,7 +47,10 @@ describe('Validates quote parsing', () => { it('single quote hex data', () => { for (let i = 0; i < types.length; i++) { - const tokens = Tokenizer(`'1001'x${types[i]}`).tokens; + const tokens = Tokenizer( + `'1001'x${types[i]}`, + new CancellationToken() + ).tokens; // make sure we have two tokens expect(tokens.length).toBe(1); @@ -68,7 +76,10 @@ describe('Validates quote parsing', () => { it('single quote hex octal', () => { for (let i = 0; i < types.length; i++) { - const tokens = Tokenizer(`'1001'o${types[i]}`).tokens; + const tokens = Tokenizer( + `'1001'o${types[i]}`, + new CancellationToken() + ).tokens; // make sure we have two tokens expect(tokens.length).toBe(1); @@ -94,7 +105,10 @@ describe('Validates quote parsing', () => { it('double quote binary data', () => { for (let i = 0; i < types.length; i++) { - const tokens = Tokenizer(`"1001"b${types[i]}`).tokens; + const tokens = Tokenizer( + `"1001"b${types[i]}`, + new CancellationToken() + ).tokens; // make sure we have two tokens expect(tokens.length).toBe(1); @@ -120,7 +134,10 @@ describe('Validates quote parsing', () => { it('double quote hex data', () => { for (let i = 0; i < types.length; i++) { - const tokens = Tokenizer(`"1001"x${types[i]}`).tokens; + const tokens = Tokenizer( + `"1001"x${types[i]}`, + new CancellationToken() + ).tokens; // make sure we have two tokens expect(tokens.length).toBe(1); @@ -146,7 +163,10 @@ describe('Validates quote parsing', () => { it('double quote octal data', () => { for (let i = 0; i < types.length; i++) { - const tokens = Tokenizer(`"1001"o${types[i]}`).tokens; + const tokens = Tokenizer( + `"1001"o${types[i]}`, + new CancellationToken() + ).tokens; // make sure we have two tokens expect(tokens.length).toBe(1); diff --git a/libs/parsing/tokenizer/src/lib/tests/numbers.spec.ts b/libs/parsing/tokenizer/src/lib/tests/numbers.spec.ts index 1ee0efdce..4f385185c 100644 --- a/libs/parsing/tokenizer/src/lib/tests/numbers.spec.ts +++ b/libs/parsing/tokenizer/src/lib/tests/numbers.spec.ts @@ -1,3 +1,5 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; + import { StripIDs } from '../helpers/strip-ids'; import { Tokenizer } from '../tokenizer'; import { IBaseTokenWithoutMatches, TOKEN_TYPES } from '../tokenizer.interface'; @@ -47,7 +49,7 @@ describe('Validates number parsing', () => { const number = numbers[i]; // get tokens - const tokens = Tokenizer(number).tokens; + const tokens = Tokenizer(number, new CancellationToken()).tokens; // make sure we only found one token expect(tokens.length).toBe(1); diff --git a/libs/parsing/tokenizer/src/lib/tests/operators.compound.spec.ts b/libs/parsing/tokenizer/src/lib/tests/operators.compound.spec.ts index db193065d..d37753b00 100644 --- a/libs/parsing/tokenizer/src/lib/tests/operators.compound.spec.ts +++ b/libs/parsing/tokenizer/src/lib/tests/operators.compound.spec.ts @@ -1,3 +1,5 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; + import { StripIDs } from '../helpers/strip-ids'; import { Tokenizer } from '../tokenizer'; import { IBaseTokenWithoutMatches, TOKEN_TYPES } from '../tokenizer.interface'; @@ -33,7 +35,10 @@ describe('Validates compound operator parsing', () => { const operator = operators[i]; // get tokens - const tokens = Tokenizer(`a ${operator} b`).tokens; + const tokens = Tokenizer( + `a ${operator} b`, + new CancellationToken() + ).tokens; // make sure we only found one token expect(tokens.length).toBe(4); diff --git a/libs/parsing/tokenizer/src/lib/tests/operators.spec.ts b/libs/parsing/tokenizer/src/lib/tests/operators.spec.ts index 96c350009..9672f47be 100644 --- a/libs/parsing/tokenizer/src/lib/tests/operators.spec.ts +++ b/libs/parsing/tokenizer/src/lib/tests/operators.spec.ts @@ -1,3 +1,5 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; + import { StripIDs } from '../helpers/strip-ids'; import { TestGlobal } from '../helpers/test-global'; import { Tokenizer } from '../tokenizer'; @@ -27,7 +29,10 @@ describe('Validates operator parsing', () => { const operator = operators[i]; // get tokens - const tokens = Tokenizer(`a ${operator} b`).tokens; + const tokens = Tokenizer( + `a ${operator} b`, + new CancellationToken() + ).tokens; // make sure we only found one token expect(tokens.length).toBe(4); @@ -82,7 +87,10 @@ describe('Validates operator parsing', () => { const operator = operators[i]; // get tokens - const tokens = Tokenizer(`a ${operator} b`).tokens; + const tokens = Tokenizer( + `a ${operator} b`, + new CancellationToken() + ).tokens; // make sure we only found one token expect(tokens.length).toBe(4); @@ -124,7 +132,7 @@ describe('Validates operator parsing', () => { const operator = operators[i]; // get tokens - const tokens = Tokenizer(`${operator}a`).tokens; + const tokens = Tokenizer(`${operator}a`, new CancellationToken()).tokens; // make sure we only found one token expect(tokens.length).toBe(3); @@ -161,7 +169,7 @@ describe('Validates operator parsing', () => { const operator = operators[i]; // get tokens - const tokens = Tokenizer(`a${operator}`).tokens; + const tokens = Tokenizer(`a${operator}`, new CancellationToken()).tokens; // make sure we only found one token expect(tokens.length).toBe(3); diff --git a/libs/parsing/tokenizer/src/lib/tests/variables.spec.ts b/libs/parsing/tokenizer/src/lib/tests/variables.spec.ts index fbc532599..3963f1e6a 100644 --- a/libs/parsing/tokenizer/src/lib/tests/variables.spec.ts +++ b/libs/parsing/tokenizer/src/lib/tests/variables.spec.ts @@ -1,3 +1,5 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; + import { StripIDs } from '../helpers/strip-ids'; import { Tokenizer } from '../tokenizer'; import { IBaseTokenWithoutMatches, TOKEN_TYPES } from '../tokenizer.interface'; @@ -14,7 +16,10 @@ describe('Validates variable parsing (mostly done elsewhere)', () => { const variable = variables[i]; // get tokens - const tokens = Tokenizer(`a = ${variable}`).tokens; + const tokens = Tokenizer( + `a = ${variable}`, + new CancellationToken() + ).tokens; // make sure we only found one token expect(tokens.length).toBe(4); @@ -56,7 +61,7 @@ describe('Validates variable parsing (mostly done elsewhere)', () => { const variable = variables[i]; // get tokens - const tokens = Tokenizer(`${variable}`).tokens; + const tokens = Tokenizer(`${variable}`, new CancellationToken()).tokens; // make sure we only found one token expect(tokens.length).toBe(2); diff --git a/libs/parsing/tokenizer/src/lib/tokenizer.ts b/libs/parsing/tokenizer/src/lib/tokenizer.ts index 7e079c143..7e871327e 100644 --- a/libs/parsing/tokenizer/src/lib/tokenizer.ts +++ b/libs/parsing/tokenizer/src/lib/tokenizer.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { nanoid } from 'nanoid'; import { FindMatches } from './helpers/find-matches'; @@ -358,10 +359,11 @@ export function TokenizerRecurser( */ export function Tokenizer( code: string | string[], + cancel: CancellationToken, options: IFindTokensOptions = DEFAULT_FIND_TOKEN_OPTIONS ): IFoundTokens { // create our iterator - const iterator = new Iterator(code, options.full); + const iterator = new Iterator(code, cancel, options.full); // find our tokens if we can if (iterator.canProcess) { diff --git a/libs/shared/src/lib/version.interface.ts b/libs/shared/src/lib/version.interface.ts index 7c2405330..6c0c1f76c 100644 --- a/libs/shared/src/lib/version.interface.ts +++ b/libs/shared/src/lib/version.interface.ts @@ -1,4 +1,4 @@ /** * Version of the extension */ -export const VERSION = '3.2.1'; +export const VERSION = '3.2.2'; diff --git a/libs/tests/assembler/src/lib/auto-doc.args.spec.ts b/libs/tests/assembler/src/lib/auto-doc.args.spec.ts index 4acd3f869..794793e62 100644 --- a/libs/tests/assembler/src/lib/auto-doc.args.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.args.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -45,7 +46,7 @@ describe(`[auto generated] Verify arg formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.basic.spec.ts b/libs/tests/assembler/src/lib/auto-doc.basic.spec.ts index c737dfddd..7ef22f39b 100644 --- a/libs/tests/assembler/src/lib/auto-doc.basic.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.basic.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -194,7 +195,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -268,7 +269,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -384,7 +385,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -502,7 +503,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -658,7 +659,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -742,7 +743,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -834,7 +835,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -921,7 +922,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.complex2.spec.ts b/libs/tests/assembler/src/lib/auto-doc.complex2.spec.ts index ceccd7726..12cd704f4 100644 --- a/libs/tests/assembler/src/lib/auto-doc.complex2.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.complex2.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -41,7 +42,7 @@ describe(`[auto generated] Verify complex formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.complex3.spec.ts b/libs/tests/assembler/src/lib/auto-doc.complex3.spec.ts index 52e4baef5..c07f97acc 100644 --- a/libs/tests/assembler/src/lib/auto-doc.complex3.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.complex3.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Verify complex formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.complex4.spec.ts b/libs/tests/assembler/src/lib/auto-doc.complex4.spec.ts index 978a9db70..a8d537df3 100644 --- a/libs/tests/assembler/src/lib/auto-doc.complex4.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.complex4.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -67,7 +68,7 @@ describe(`[auto generated] Verify complex formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.example-code.spec.ts b/libs/tests/assembler/src/lib/auto-doc.example-code.spec.ts index a7a9c0b6a..ed2f3ddc9 100644 --- a/libs/tests/assembler/src/lib/auto-doc.example-code.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.example-code.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -62,7 +63,7 @@ describe(`[auto generated] Verify formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.format-keywords.spec.ts b/libs/tests/assembler/src/lib/auto-doc.format-keywords.spec.ts index e2020bf9a..bc6fce76a 100644 --- a/libs/tests/assembler/src/lib/auto-doc.format-keywords.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.format-keywords.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -71,7 +72,7 @@ describe(`[auto generated] Apply keyword formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, style: { keywords: 'lower' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/auto-doc.non-idl-display-names.spec.ts b/libs/tests/assembler/src/lib/auto-doc.non-idl-display-names.spec.ts index 79039e89e..de4e418d0 100644 --- a/libs/tests/assembler/src/lib/auto-doc.non-idl-display-names.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.non-idl-display-names.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -45,7 +46,7 @@ describe(`[auto generated] Verify type formatting uses display names`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.non-idl-doc.spec.ts b/libs/tests/assembler/src/lib/auto-doc.non-idl-doc.spec.ts index bb049ad75..992e0f6ea 100644 --- a/libs/tests/assembler/src/lib/auto-doc.non-idl-doc.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.non-idl-doc.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -97,7 +98,7 @@ describe(`[auto generated] Verify doc formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.only.spec.ts b/libs/tests/assembler/src/lib/auto-doc.only.spec.ts index b6e85f7e3..d7b77e59c 100644 --- a/libs/tests/assembler/src/lib/auto-doc.only.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.only.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -51,7 +52,7 @@ describe(`[auto generated] Only use AutoDoc`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, styleAndFormat: false, autoFix: false, diff --git a/libs/tests/assembler/src/lib/auto-doc.only2.spec.ts b/libs/tests/assembler/src/lib/auto-doc.only2.spec.ts index 14457be66..0df4ecc55 100644 --- a/libs/tests/assembler/src/lib/auto-doc.only2.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.only2.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -30,7 +31,7 @@ describe(`[auto generated] Only use AutoDoc`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, styleAndFormat: false, autoFix: false, diff --git a/libs/tests/assembler/src/lib/auto-doc.order-args.spec.ts b/libs/tests/assembler/src/lib/auto-doc.order-args.spec.ts index 6af4e70de..7ca9c767e 100644 --- a/libs/tests/assembler/src/lib/auto-doc.order-args.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.order-args.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -48,7 +49,7 @@ describe(`[auto generated] Verify arg ordering`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.parameters.spec.ts b/libs/tests/assembler/src/lib/auto-doc.parameters.spec.ts index 57457e2d7..073d23553 100644 --- a/libs/tests/assembler/src/lib/auto-doc.parameters.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.parameters.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -45,7 +46,7 @@ describe(`[auto generated] Verify parameter formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -153,7 +154,7 @@ describe(`[auto generated] Verify parameter formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.structures.2.spec.ts b/libs/tests/assembler/src/lib/auto-doc.structures.2.spec.ts index 371828cd2..8be535353 100644 --- a/libs/tests/assembler/src/lib/auto-doc.structures.2.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.structures.2.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -41,7 +42,7 @@ describe(`[auto generated] Generate structure docs`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.structures.spec.ts b/libs/tests/assembler/src/lib/auto-doc.structures.spec.ts index e754f11e4..ab105e4f9 100644 --- a/libs/tests/assembler/src/lib/auto-doc.structures.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.structures.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Generate structure docs`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -127,7 +128,7 @@ describe(`[auto generated] Generate structure docs`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -222,7 +223,7 @@ describe(`[auto generated] Generate structure docs`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', @@ -315,7 +316,7 @@ describe(`[auto generated] Generate structure docs`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.task-types.spec.ts b/libs/tests/assembler/src/lib/auto-doc.task-types.spec.ts index 7738f0ef2..98c0f0826 100644 --- a/libs/tests/assembler/src/lib/auto-doc.task-types.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.task-types.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -49,7 +50,7 @@ describe(`[auto generated] Verify type formatting for`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/auto-doc.types.spec.ts b/libs/tests/assembler/src/lib/auto-doc.types.spec.ts index fc762dccb..f06485b40 100644 --- a/libs/tests/assembler/src/lib/auto-doc.types.spec.ts +++ b/libs/tests/assembler/src/lib/auto-doc.types.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -123,7 +124,7 @@ describe(`[auto generated] Verify type formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoDoc: true, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/format.after-line-continuation.spec.ts b/libs/tests/assembler/src/lib/format.after-line-continuation.spec.ts index 4a8d90c5e..2e9fff620 100644 --- a/libs/tests/assembler/src/lib/format.after-line-continuation.spec.ts +++ b/libs/tests/assembler/src/lib/format.after-line-continuation.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -37,7 +38,7 @@ describe(`[auto generated] Keep tokens after line continuations`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.after-main.spec.ts b/libs/tests/assembler/src/lib/format.after-main.spec.ts index 4b9505c83..79af4e5bb 100644 --- a/libs/tests/assembler/src/lib/format.after-main.spec.ts +++ b/libs/tests/assembler/src/lib/format.after-main.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -37,7 +38,7 @@ describe(`[auto generated] Keep tokens after main level programs`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.arrays.1.spec.ts b/libs/tests/assembler/src/lib/format.arrays.1.spec.ts index 4d2ac059d..b4c35cd2c 100644 --- a/libs/tests/assembler/src/lib/format.arrays.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.arrays.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -30,7 +31,7 @@ describe(`[auto generated] Verify array formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -109,7 +110,7 @@ describe(`[auto generated] Verify array formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -200,7 +201,7 @@ describe(`[auto generated] Verify array formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -402,7 +403,7 @@ describe(`[auto generated] Verify array formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -497,7 +498,7 @@ describe(`[auto generated] Verify array formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -569,7 +570,7 @@ describe(`[auto generated] Verify array formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.assignment.1.spec.ts b/libs/tests/assembler/src/lib/format.assignment.1.spec.ts index 465c960b6..498cd9a6f 100644 --- a/libs/tests/assembler/src/lib/format.assignment.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.assignment.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Verify assignment formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.comments.1.spec.ts b/libs/tests/assembler/src/lib/format.comments.1.spec.ts index d4671c599..924d33f02 100644 --- a/libs/tests/assembler/src/lib/format.comments.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.comments.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -37,7 +38,7 @@ describe(`[auto generated] Verify comment`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -134,7 +135,7 @@ describe(`[auto generated] Verify comment`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -262,7 +263,7 @@ describe(`[auto generated] Verify comment`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -353,7 +354,7 @@ describe(`[auto generated] Verify comment`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.executive-commands.1.spec.ts b/libs/tests/assembler/src/lib/format.executive-commands.1.spec.ts index e9d4cbebc..9ef24d5ce 100644 --- a/libs/tests/assembler/src/lib/format.executive-commands.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.executive-commands.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -37,7 +38,7 @@ describe(`[auto generated] Executive command formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.indent.1.spec.ts b/libs/tests/assembler/src/lib/format.indent.1.spec.ts index ff46f89a6..14a3ab695 100644 --- a/libs/tests/assembler/src/lib/format.indent.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.indent.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -43,7 +44,7 @@ describe(`[auto generated] Verify adjusting indent adjusts spacing`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', tabWidth: 3, style: { quotes: 'single' }, diff --git a/libs/tests/assembler/src/lib/format.line-separators.1.spec.ts b/libs/tests/assembler/src/lib/format.line-separators.1.spec.ts index 80e3d61a0..7e4de00f6 100644 --- a/libs/tests/assembler/src/lib/format.line-separators.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.line-separators.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Line separator formatting`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.line-separators.2.spec.ts b/libs/tests/assembler/src/lib/format.line-separators.2.spec.ts index c790545f2..27473337f 100644 --- a/libs/tests/assembler/src/lib/format.line-separators.2.spec.ts +++ b/libs/tests/assembler/src/lib/format.line-separators.2.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -40,7 +41,7 @@ describe(`[auto generated] Line separators (&)`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.new-lines.1.spec.ts b/libs/tests/assembler/src/lib/format.new-lines.1.spec.ts index 32294bfe4..557c175d0 100644 --- a/libs/tests/assembler/src/lib/format.new-lines.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.new-lines.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -41,7 +42,7 @@ describe(`[auto generated] Verify new lines`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.python.1.spec.ts b/libs/tests/assembler/src/lib/format.python.1.spec.ts index 0680337ec..f687966e5 100644 --- a/libs/tests/assembler/src/lib/format.python.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.python.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -38,7 +39,7 @@ describe(`[auto generated] Format python code`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.respect-errors.1.spec.ts b/libs/tests/assembler/src/lib/format.respect-errors.1.spec.ts index 8314a4b53..989953cd2 100644 --- a/libs/tests/assembler/src/lib/format.respect-errors.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.respect-errors.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -37,7 +38,7 @@ describe(`[auto generated] Verify we do not format when we have bad syntax error const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -142,7 +143,7 @@ describe(`[auto generated] Verify we do not format when we have bad syntax error const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.routines.1.spec.ts b/libs/tests/assembler/src/lib/format.routines.1.spec.ts index d87c26fd3..730cbab65 100644 --- a/libs/tests/assembler/src/lib/format.routines.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.routines.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -36,7 +37,7 @@ describe(`[auto generated] Verify we format routines`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -141,7 +142,7 @@ describe(`[auto generated] Verify we format routines`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.snap.1.spec.ts b/libs/tests/assembler/src/lib/format.snap.1.spec.ts index b73acbfd6..da6618ee8 100644 --- a/libs/tests/assembler/src/lib/format.snap.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.snap.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -103,7 +104,7 @@ describe(`[auto generated] Verify snapping branches to remove leading and traili const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/format.trimming.1.spec.ts b/libs/tests/assembler/src/lib/format.trimming.1.spec.ts index 0a811955e..dcaacacdb 100644 --- a/libs/tests/assembler/src/lib/format.trimming.1.spec.ts +++ b/libs/tests/assembler/src/lib/format.trimming.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -36,7 +37,7 @@ describe(`[auto generated] Verify trimming lines`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/lib-examples.1.spec.ts b/libs/tests/assembler/src/lib/lib-examples.1.spec.ts index 62b5f7de7..f4c5ab5c6 100644 --- a/libs/tests/assembler/src/lib/lib-examples.1.spec.ts +++ b/libs/tests/assembler/src/lib/lib-examples.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -48,7 +49,7 @@ describe(`[auto generated] Verify code snippets from the lib folder`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -217,7 +218,7 @@ describe(`[auto generated] Verify code snippets from the lib folder`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/lib-examples.2.spec.ts b/libs/tests/assembler/src/lib/lib-examples.2.spec.ts index ee80e627a..c94920ed7 100644 --- a/libs/tests/assembler/src/lib/lib-examples.2.spec.ts +++ b/libs/tests/assembler/src/lib/lib-examples.2.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -51,7 +52,7 @@ describe(`[auto generated] Lib examples 2`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.binary.spec.ts b/libs/tests/assembler/src/lib/style.binary.spec.ts index 47124ed68..ecee82fbd 100644 --- a/libs/tests/assembler/src/lib/style.binary.spec.ts +++ b/libs/tests/assembler/src/lib/style.binary.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -30,7 +31,7 @@ describe(`[auto generated] Binary number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { binary: 'lower' }, autoFix: false, @@ -100,7 +101,7 @@ describe(`[auto generated] Binary number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { binary: 'upper' }, autoFix: false, @@ -170,7 +171,7 @@ describe(`[auto generated] Binary number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { binary: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.control.spec.ts b/libs/tests/assembler/src/lib/style.control.spec.ts index 5f7e664f9..bc81604f7 100644 --- a/libs/tests/assembler/src/lib/style.control.spec.ts +++ b/libs/tests/assembler/src/lib/style.control.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -90,7 +91,7 @@ describe(`[auto generated] Control statement styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { control: 'lower' }, autoFix: false, @@ -334,7 +335,7 @@ describe(`[auto generated] Control statement styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { control: 'upper' }, autoFix: false, @@ -578,7 +579,7 @@ describe(`[auto generated] Control statement styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { control: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.hex.spec.ts b/libs/tests/assembler/src/lib/style.hex.spec.ts index 2ed8192d7..5458c7bae 100644 --- a/libs/tests/assembler/src/lib/style.hex.spec.ts +++ b/libs/tests/assembler/src/lib/style.hex.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -30,7 +31,7 @@ describe(`[auto generated] Hex number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { hex: 'lower' }, autoFix: false, @@ -100,7 +101,7 @@ describe(`[auto generated] Hex number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { hex: 'upper' }, autoFix: false, @@ -170,7 +171,7 @@ describe(`[auto generated] Hex number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { hex: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.internal-routines.spec.ts b/libs/tests/assembler/src/lib/style.internal-routines.spec.ts index 637a03e24..7411ab818 100644 --- a/libs/tests/assembler/src/lib/style.internal-routines.spec.ts +++ b/libs/tests/assembler/src/lib/style.internal-routines.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -52,7 +53,7 @@ describe(`[auto generated] Style internal routines`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { routines: 'match' }, autoFix: false, @@ -178,7 +179,7 @@ describe(`[auto generated] Style internal routines`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { routines: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.keywords.1.spec.ts b/libs/tests/assembler/src/lib/style.keywords.1.spec.ts index dce22a8cc..92b6455bb 100644 --- a/libs/tests/assembler/src/lib/style.keywords.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.keywords.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -30,7 +31,7 @@ describe(`[auto generated] Verify keywords`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -113,7 +114,7 @@ describe(`[auto generated] Verify keywords`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -245,7 +246,7 @@ describe(`[auto generated] Verify keywords`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -319,7 +320,7 @@ describe(`[auto generated] Verify keywords`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -393,7 +394,7 @@ describe(`[auto generated] Verify keywords`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.logic-case.1.spec.ts b/libs/tests/assembler/src/lib/style.logic-case.1.spec.ts index 305290c52..b2ae603d4 100644 --- a/libs/tests/assembler/src/lib/style.logic-case.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.logic-case.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -45,7 +46,7 @@ describe(`[auto generated] Verify we style case`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -153,7 +154,7 @@ describe(`[auto generated] Verify we style case`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -264,7 +265,7 @@ describe(`[auto generated] Verify we style case`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -423,7 +424,7 @@ describe(`[auto generated] Verify we style case`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -517,7 +518,7 @@ describe(`[auto generated] Verify we style case`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.logic-if-then.1.spec.ts b/libs/tests/assembler/src/lib/style.logic-if-then.1.spec.ts index 589982228..8c271e0ff 100644 --- a/libs/tests/assembler/src/lib/style.logic-if-then.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.logic-if-then.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -53,7 +54,7 @@ describe(`[auto generated] Verify we style if-then`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -153,7 +154,7 @@ describe(`[auto generated] Verify we style if-then`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -242,7 +243,7 @@ describe(`[auto generated] Verify we style if-then`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -429,7 +430,7 @@ describe(`[auto generated] Verify we style if-then`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.logic-switch.1.spec.ts b/libs/tests/assembler/src/lib/style.logic-switch.1.spec.ts index 02058f479..2acf39a4a 100644 --- a/libs/tests/assembler/src/lib/style.logic-switch.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.logic-switch.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -45,7 +46,7 @@ describe(`[auto generated] Verify we style switch`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -153,7 +154,7 @@ describe(`[auto generated] Verify we style switch`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.logic-ternary.1.spec.ts b/libs/tests/assembler/src/lib/style.logic-ternary.1.spec.ts index 7c579a3d5..06711c202 100644 --- a/libs/tests/assembler/src/lib/style.logic-ternary.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.logic-ternary.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -30,7 +31,7 @@ describe(`[auto generated] Verify we style ternary operators well`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -107,7 +108,7 @@ describe(`[auto generated] Verify we style ternary operators well`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.methods-call.spec.ts b/libs/tests/assembler/src/lib/style.methods-call.spec.ts index 8259978db..35a56e280 100644 --- a/libs/tests/assembler/src/lib/style.methods-call.spec.ts +++ b/libs/tests/assembler/src/lib/style.methods-call.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -52,7 +53,7 @@ describe(`[auto generated] Method styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { routines: 'match' }, autoFix: false, @@ -153,7 +154,7 @@ describe(`[auto generated] Method styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { routines: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.methods.1.spec.ts b/libs/tests/assembler/src/lib/style.methods.1.spec.ts index 623830486..cde770ee5 100644 --- a/libs/tests/assembler/src/lib/style.methods.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.methods.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Verify style for methods`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.methods.spec.ts b/libs/tests/assembler/src/lib/style.methods.spec.ts index 2a13ce687..55d7bdad4 100644 --- a/libs/tests/assembler/src/lib/style.methods.spec.ts +++ b/libs/tests/assembler/src/lib/style.methods.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Method styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { methods: 'dot' }, autoFix: false, @@ -139,7 +140,7 @@ describe(`[auto generated] Method styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { methods: 'arrow' }, autoFix: false, @@ -239,7 +240,7 @@ describe(`[auto generated] Method styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { methods: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.numbers.spec.ts b/libs/tests/assembler/src/lib/style.numbers.spec.ts index 9fbac6f33..694f92976 100644 --- a/libs/tests/assembler/src/lib/style.numbers.spec.ts +++ b/libs/tests/assembler/src/lib/style.numbers.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -48,7 +49,7 @@ describe(`[auto generated] Number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { numbers: 'lower' }, autoFix: false, @@ -141,7 +142,7 @@ describe(`[auto generated] Number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { numbers: 'upper' }, autoFix: false, @@ -234,7 +235,7 @@ describe(`[auto generated] Number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { numbers: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.octal.spec.ts b/libs/tests/assembler/src/lib/style.octal.spec.ts index d42079a37..2c7bc6f3e 100644 --- a/libs/tests/assembler/src/lib/style.octal.spec.ts +++ b/libs/tests/assembler/src/lib/style.octal.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -30,7 +31,7 @@ describe(`[auto generated] Octal number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { octal: 'lower' }, autoFix: false, @@ -100,7 +101,7 @@ describe(`[auto generated] Octal number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { octal: 'upper' }, autoFix: false, @@ -170,7 +171,7 @@ describe(`[auto generated] Octal number styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { octal: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.operators.1.spec.ts b/libs/tests/assembler/src/lib/style.operators.1.spec.ts index 9b8b7d334..af335824c 100644 --- a/libs/tests/assembler/src/lib/style.operators.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.operators.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -37,7 +38,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -131,7 +132,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -211,7 +212,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -324,7 +325,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -402,7 +403,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -527,7 +528,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -633,7 +634,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -704,7 +705,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -789,7 +790,7 @@ describe(`[auto generated] Verify operators`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.prompts.spec.ts b/libs/tests/assembler/src/lib/style.prompts.spec.ts index a7a88c4eb..063bfd0bb 100644 --- a/libs/tests/assembler/src/lib/style.prompts.spec.ts +++ b/libs/tests/assembler/src/lib/style.prompts.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -41,7 +42,7 @@ describe(`[auto generated] Prompt styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.properties.spec.ts b/libs/tests/assembler/src/lib/style.properties.spec.ts index 266fb72f3..1c36ea7bb 100644 --- a/libs/tests/assembler/src/lib/style.properties.spec.ts +++ b/libs/tests/assembler/src/lib/style.properties.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -38,7 +39,7 @@ describe(`[auto generated] Property styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { properties: 'match' }, autoFix: false, @@ -124,7 +125,7 @@ describe(`[auto generated] Property styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { properties: 'lower' }, autoFix: false, @@ -210,7 +211,7 @@ describe(`[auto generated] Property styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { properties: 'upper' }, autoFix: false, @@ -296,7 +297,7 @@ describe(`[auto generated] Property styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { properties: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.quotes-double.1.spec.ts b/libs/tests/assembler/src/lib/style.quotes-double.1.spec.ts index 8b69aafa9..7a09d7424 100644 --- a/libs/tests/assembler/src/lib/style.quotes-double.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.quotes-double.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -45,7 +46,7 @@ describe(`[auto generated] Verify double quote styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { quotes: 'single' }, autoFix: false, @@ -132,7 +133,7 @@ describe(`[auto generated] Verify double quote styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { quotes: 'double' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.quotes-nested.1.spec.ts b/libs/tests/assembler/src/lib/style.quotes-nested.1.spec.ts index fbd78ca73..5555224a2 100644 --- a/libs/tests/assembler/src/lib/style.quotes-nested.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.quotes-nested.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -36,7 +37,7 @@ describe(`[auto generated] Verify double quote parsing`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { quotes: 'single' }, autoFix: false, @@ -105,7 +106,7 @@ describe(`[auto generated] Verify double quote parsing`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { quotes: 'double' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.quotes-none.1.spec.ts b/libs/tests/assembler/src/lib/style.quotes-none.1.spec.ts index 0b6c71400..db9be2920 100644 --- a/libs/tests/assembler/src/lib/style.quotes-none.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.quotes-none.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -46,7 +47,7 @@ describe(`[auto generated] Verify no formatting of quotes`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { quotes: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.quotes-single.1.spec.ts b/libs/tests/assembler/src/lib/style.quotes-single.1.spec.ts index fa3da67d4..38078823c 100644 --- a/libs/tests/assembler/src/lib/style.quotes-single.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.quotes-single.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -45,7 +46,7 @@ describe(`[auto generated] Verify single quote parsing`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { quotes: 'double' }, autoFix: false, @@ -132,7 +133,7 @@ describe(`[auto generated] Verify single quote parsing`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { quotes: 'single' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.string-literal.1.spec.ts b/libs/tests/assembler/src/lib/style.string-literal.1.spec.ts index 2c656a642..1c9e98aa8 100644 --- a/libs/tests/assembler/src/lib/style.string-literal.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.string-literal.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Verify string literal styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -131,7 +132,7 @@ describe(`[auto generated] Verify string literal styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.structures.1.spec.ts b/libs/tests/assembler/src/lib/style.structures.1.spec.ts index 4d1022776..1bf41add3 100644 --- a/libs/tests/assembler/src/lib/style.structures.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.structures.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -35,7 +36,7 @@ describe(`[auto generated] Verify structures`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -132,7 +133,7 @@ describe(`[auto generated] Verify structures`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -228,7 +229,7 @@ describe(`[auto generated] Verify structures`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); @@ -323,7 +324,7 @@ describe(`[auto generated] Verify structures`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: false, formatter: 'fiddle', }); diff --git a/libs/tests/assembler/src/lib/style.system-variables.spec.ts b/libs/tests/assembler/src/lib/style.system-variables.spec.ts index 3043c2596..2b64989a1 100644 --- a/libs/tests/assembler/src/lib/style.system-variables.spec.ts +++ b/libs/tests/assembler/src/lib/style.system-variables.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] System variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { systemVariables: 'lower' }, autoFix: false, @@ -127,7 +128,7 @@ describe(`[auto generated] System variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { systemVariables: 'upper' }, autoFix: false, @@ -215,7 +216,7 @@ describe(`[auto generated] System variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { systemVariables: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.template-escape.spec.ts b/libs/tests/assembler/src/lib/style.template-escape.spec.ts index 58ad16d07..a8692a75a 100644 --- a/libs/tests/assembler/src/lib/style.template-escape.spec.ts +++ b/libs/tests/assembler/src/lib/style.template-escape.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -43,7 +44,7 @@ describe(`[auto generated] Verify auto-fix/format of template escape characters` const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { hex: 'lower' }, autoFix: false, formatter: 'fiddle', @@ -133,7 +134,7 @@ describe(`[auto generated] Verify auto-fix/format of template escape characters` const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { hex: 'upper' }, autoFix: false, formatter: 'fiddle', @@ -223,7 +224,7 @@ describe(`[auto generated] Verify auto-fix/format of template escape characters` const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { hex: 'none' }, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/assembler/src/lib/style.user-routines.spec.ts b/libs/tests/assembler/src/lib/style.user-routines.spec.ts index f03e75aaa..de2cb71ca 100644 --- a/libs/tests/assembler/src/lib/style.user-routines.spec.ts +++ b/libs/tests/assembler/src/lib/style.user-routines.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -45,7 +46,7 @@ describe(`[auto generated] Style user routines`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { routines: 'match' }, autoFix: false, @@ -132,7 +133,7 @@ describe(`[auto generated] Style user routines`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { formatter: 'fiddle', style: { routines: 'none' }, autoFix: false, diff --git a/libs/tests/assembler/src/lib/style.variables.1.spec.ts b/libs/tests/assembler/src/lib/style.variables.1.spec.ts index 379a7b2c1..1f7be891f 100644 --- a/libs/tests/assembler/src/lib/style.variables.1.spec.ts +++ b/libs/tests/assembler/src/lib/style.variables.1.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { GetTokenNames } from '@idl/parser'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; @@ -39,7 +40,7 @@ describe(`[auto generated] Verify variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { localVariables: 'match' }, autoFix: false, formatter: 'fiddle', @@ -114,7 +115,7 @@ describe(`[auto generated] Verify variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { localVariables: 'none' }, autoFix: false, formatter: 'fiddle', @@ -190,7 +191,7 @@ describe(`[auto generated] Verify variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { localVariables: 'match' }, autoFix: false, formatter: 'fiddle', @@ -267,7 +268,7 @@ describe(`[auto generated] Verify variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { localVariables: 'none' }, autoFix: false, formatter: 'fiddle', @@ -342,7 +343,7 @@ describe(`[auto generated] Verify variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { localVariables: 'match' }, autoFix: false, formatter: 'fiddle', @@ -440,7 +441,7 @@ describe(`[auto generated] Verify variable styling`, () => { const tokenizedNames = GetTokenNames(tokenized); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { style: { localVariables: 'none' }, autoFix: false, formatter: 'fiddle', diff --git a/libs/tests/auto-fixing/src/lib/code.105.illegal-var-index.spec.ts b/libs/tests/auto-fixing/src/lib/code.105.illegal-var-index.spec.ts index 92fabf65f..7cf603737 100644 --- a/libs/tests/auto-fixing/src/lib/code.105.illegal-var-index.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.105.illegal-var-index.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -26,7 +27,7 @@ describe(`[auto generated] Verify we correctly fix brackets for indexing`, () => }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -96,7 +97,7 @@ describe(`[auto generated] Verify we correctly fix brackets for indexing`, () => }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -164,7 +165,7 @@ describe(`[auto generated] Verify we correctly fix brackets for indexing`, () => }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -220,7 +221,7 @@ describe(`[auto generated] Verify we correctly fix brackets for indexing`, () => }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.14.colon-in-func.spec.ts b/libs/tests/auto-fixing/src/lib/code.14.colon-in-func.spec.ts index 34434414e..82742d440 100644 --- a/libs/tests/auto-fixing/src/lib/code.14.colon-in-func.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.14.colon-in-func.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -30,7 +31,7 @@ describe(`[auto generated] Verify function to array for`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.15.colon-in-func-method.spec.ts b/libs/tests/auto-fixing/src/lib/code.15.colon-in-func-method.spec.ts index 45ce10a9e..e364f2169 100644 --- a/libs/tests/auto-fixing/src/lib/code.15.colon-in-func-method.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.15.colon-in-func-method.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -26,7 +27,7 @@ describe(`[auto generated] Verify function method to array for`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.20.return-vals-pro.spec.ts b/libs/tests/auto-fixing/src/lib/code.20.return-vals-pro.spec.ts index ce9b7ed07..828c12137 100644 --- a/libs/tests/auto-fixing/src/lib/code.20.return-vals-pro.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.20.return-vals-pro.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -36,7 +37,7 @@ describe(`[auto generated] Verify we remove excess args`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -103,7 +104,7 @@ describe(`[auto generated] Verify we remove excess args`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -167,7 +168,7 @@ describe(`[auto generated] Verify we remove excess args`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.21.return-vals-func.spec.ts b/libs/tests/auto-fixing/src/lib/code.21.return-vals-func.spec.ts index 2e448ff99..268515a08 100644 --- a/libs/tests/auto-fixing/src/lib/code.21.return-vals-func.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.21.return-vals-func.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -40,7 +41,7 @@ describe(`[auto generated] Verify we remove excess args`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -115,7 +116,7 @@ describe(`[auto generated] Verify we remove excess args`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.3.after-main.spec.ts b/libs/tests/auto-fixing/src/lib/code.3.after-main.spec.ts index b54a452c3..f2cc526ff 100644 --- a/libs/tests/auto-fixing/src/lib/code.3.after-main.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.3.after-main.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -34,7 +35,7 @@ describe(`[auto generated] Verify tokens after main get removed on formatting`, }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.31.return-missing.spec.ts b/libs/tests/auto-fixing/src/lib/code.31.return-missing.spec.ts index e9c211604..ce4736a65 100644 --- a/libs/tests/auto-fixing/src/lib/code.31.return-missing.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.31.return-missing.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -37,7 +38,7 @@ describe(`[auto generated] Verify we add missing return statement`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -109,7 +110,7 @@ describe(`[auto generated] Verify we add missing return statement`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.35.after-continuation.spec.ts b/libs/tests/auto-fixing/src/lib/code.35.after-continuation.spec.ts index 12675701b..85f470a31 100644 --- a/libs/tests/auto-fixing/src/lib/code.35.after-continuation.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.35.after-continuation.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -26,7 +27,7 @@ describe(`[auto generated] Verify tokens after line continuation get removed on }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.edge-cases.spec.ts b/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.edge-cases.spec.ts index b30a2da5f..d1a554cea 100644 --- a/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.edge-cases.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.edge-cases.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -35,7 +36,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -94,7 +95,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.notebooks.spec.ts b/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.notebooks.spec.ts index 308ee6233..5876481ad 100644 --- a/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.notebooks.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.notebooks.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -37,7 +38,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -108,7 +109,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -169,7 +170,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -225,7 +226,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -281,7 +282,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -331,7 +332,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -381,7 +382,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -431,7 +432,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -488,7 +489,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.spec.ts b/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.spec.ts index 8ce54caf8..cb1ce0d8b 100644 --- a/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.38.no-comp-opt.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -36,7 +37,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -106,7 +107,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -166,7 +167,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -221,7 +222,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -276,7 +277,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -336,7 +337,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -396,7 +397,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -458,7 +459,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -527,7 +528,7 @@ describe(`[auto generated] Verify we add compile opt idl2`, () => { }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/auto-fixing/src/lib/code.76.init-method-pro.spec.ts b/libs/tests/auto-fixing/src/lib/code.76.init-method-pro.spec.ts index c9df44929..29cf18fa9 100644 --- a/libs/tests/auto-fixing/src/lib/code.76.init-method-pro.spec.ts +++ b/libs/tests/auto-fixing/src/lib/code.76.init-method-pro.spec.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { LogManager } from '@idl/logger'; import { IDL_INDEX_OPTIONS, IDLIndex } from '@idl/parsing/index'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; @@ -38,7 +39,7 @@ describe(`[auto generated] Verify we change procedure init methods to function m }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); @@ -124,7 +125,7 @@ describe(`[auto generated] Verify we change procedure init methods to function m }); // format code - const formatted = Assembler(tokenized, { + const formatted = Assembler(tokenized, new CancellationToken(), { autoFix: true, formatter: 'fiddle', }); diff --git a/libs/tests/syntax-post-processors/src/lib/arg-defs.spec.ts b/libs/tests/syntax-post-processors/src/lib/arg-defs.spec.ts index 1299f8e7f..912f60102 100644 --- a/libs/tests/syntax-post-processors/src/lib/arg-defs.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/arg-defs.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -14,7 +15,7 @@ describe(`[auto generated] Correctly extract argument definitions from code`, () ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -280,7 +281,7 @@ describe(`[auto generated] Correctly extract argument definitions from code`, () ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/arrows.spec.ts b/libs/tests/syntax-post-processors/src/lib/arrows.spec.ts index 318258a95..c5236e4d2 100644 --- a/libs/tests/syntax-post-processors/src/lib/arrows.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/arrows.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly map arrows`, () => { const code = [`compile_opt idl2`, `a->`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -89,7 +90,7 @@ describe(`[auto generated] Correctly map arrows`, () => { const code = [`compile_opt idl2`, `a = b->`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/comment-blocks.1.spec.ts b/libs/tests/syntax-post-processors/src/lib/comment-blocks.1.spec.ts index 5914c0115..a63ad7105 100644 --- a/libs/tests/syntax-post-processors/src/lib/comment-blocks.1.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/comment-blocks.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -13,7 +14,7 @@ describe(`[auto generated] Correctly map comments to comment blocks`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -116,7 +117,7 @@ describe(`[auto generated] Correctly map comments to comment blocks`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -233,7 +234,7 @@ describe(`[auto generated] Correctly map comments to comment blocks`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -372,7 +373,7 @@ describe(`[auto generated] Correctly map comments to comment blocks`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -514,7 +515,7 @@ describe(`[auto generated] Correctly map comments to comment blocks`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/comment-blocks.2.spec.ts b/libs/tests/syntax-post-processors/src/lib/comment-blocks.2.spec.ts index 834170fc7..e57ed5c4d 100644 --- a/libs/tests/syntax-post-processors/src/lib/comment-blocks.2.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/comment-blocks.2.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -16,7 +17,7 @@ describe(`[auto generated] Advanced comment block cases`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -163,7 +164,7 @@ describe(`[auto generated] Advanced comment block cases`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -341,7 +342,7 @@ describe(`[auto generated] Advanced comment block cases`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/comment-blocks.3.spec.ts b/libs/tests/syntax-post-processors/src/lib/comment-blocks.3.spec.ts index 609c4ef8a..3cb1aa700 100644 --- a/libs/tests/syntax-post-processors/src/lib/comment-blocks.3.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/comment-blocks.3.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -17,7 +18,7 @@ describe(`[auto generated] Confusing comment blocks `, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/control-options.spec.ts b/libs/tests/syntax-post-processors/src/lib/control-options.spec.ts index 9ee8d97c2..324505681 100644 --- a/libs/tests/syntax-post-processors/src/lib/control-options.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/control-options.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -16,7 +17,7 @@ describe(`[auto generated] Correctly map options for compound control statements ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/dot.spec.ts b/libs/tests/syntax-post-processors/src/lib/dot.spec.ts index df5e44806..e5d23bdb3 100644 --- a/libs/tests/syntax-post-processors/src/lib/dot.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/dot.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly map periods to dots`, () => { const code = [`compile_opt idl2`, `a.`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -89,7 +90,7 @@ describe(`[auto generated] Correctly map periods to dots`, () => { const code = [`compile_opt idl2`, `a = b.`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -191,7 +192,7 @@ describe(`[auto generated] Correctly map periods to dots`, () => { const code = [`compile_opt idl2`, `a = .`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -284,7 +285,7 @@ describe(`[auto generated] Correctly map periods to dots`, () => { const code = [`compile_opt idl2`, `.`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/keyword-defs.spec.ts b/libs/tests/syntax-post-processors/src/lib/keyword-defs.spec.ts index 8aeeddbad..aef942223 100644 --- a/libs/tests/syntax-post-processors/src/lib/keyword-defs.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/keyword-defs.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -14,7 +15,7 @@ describe(`[auto generated] Correctly extract keyword definitions`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -280,7 +281,7 @@ describe(`[auto generated] Correctly extract keyword definitions`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -564,7 +565,7 @@ describe(`[auto generated] Correctly extract keyword definitions`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/keywords-binary.spec.ts b/libs/tests/syntax-post-processors/src/lib/keywords-binary.spec.ts index 37c265bec..9032356be 100644 --- a/libs/tests/syntax-post-processors/src/lib/keywords-binary.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/keywords-binary.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly detect binary keywords`, () => { const code = [`mypro, /kw1, /KW2`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -90,7 +91,7 @@ describe(`[auto generated] Correctly detect binary keywords`, () => { const code = [`myclass.method, $`, ` /KW3, KW=!true`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -226,7 +227,7 @@ describe(`[auto generated] Correctly detect binary keywords`, () => { const code = [`a = myfunc(/KW1, /KW2)`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -320,7 +321,7 @@ describe(`[auto generated] Correctly detect binary keywords`, () => { const code = [`ZACH = AWESOME.SAUCE(/kw3, $`, `/KW17, KW18 = !false)`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -492,7 +493,7 @@ describe(`[auto generated] Correctly detect binary keywords`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -672,7 +673,7 @@ describe(`[auto generated] Correctly detect binary keywords`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/keywords.spec.ts b/libs/tests/syntax-post-processors/src/lib/keywords.spec.ts index 57e33bd49..3a5f15304 100644 --- a/libs/tests/syntax-post-processors/src/lib/keywords.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/keywords.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -14,7 +15,7 @@ describe(`[auto generated] Correctly extract keyword names from routine calls`, ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -280,7 +281,7 @@ describe(`[auto generated] Correctly extract keyword names from routine calls`, ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -555,7 +556,7 @@ describe(`[auto generated] Correctly extract keyword names from routine calls`, ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -833,7 +834,7 @@ describe(`[auto generated] Correctly extract keyword names from routine calls`, ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/main.spec.ts b/libs/tests/syntax-post-processors/src/lib/main.spec.ts index 644682f2f..94520f93c 100644 --- a/libs/tests/syntax-post-processors/src/lib/main.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/main.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly maps main level tokens`, () => { const code = [`function myfunc`, `compile_opt idl2`, ` return,1`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -105,7 +106,7 @@ describe(`[auto generated] Correctly maps main level tokens`, () => { const code = [`a = plot(/TEST)`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -181,7 +182,7 @@ describe(`[auto generated] Correctly maps main level tokens`, () => { const code = [`a = \`\${42, '42'}\${42, '42'}\``]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -336,7 +337,7 @@ describe(`[auto generated] Correctly maps main level tokens`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -514,7 +515,7 @@ describe(`[auto generated] Correctly maps main level tokens`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -701,7 +702,7 @@ describe(`[auto generated] Correctly maps main level tokens`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -956,7 +957,7 @@ describe(`[auto generated] Correctly maps main level tokens`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -1117,7 +1118,7 @@ describe(`[auto generated] Correctly maps main level tokens`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.indexing.array.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.indexing.array.spec.ts index f8e7d5e00..966b28afa 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.indexing.array.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.indexing.array.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify array indexing`, () => { const code = [`compile_opt idl2`, `subsel = sel[*, 1:*val]`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.basic.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.basic.spec.ts index cb877e45f..414861e73 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.basic.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.basic.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = 6 * 7`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -93,7 +94,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = (6) * (7)`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -202,7 +203,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = *var`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -278,7 +279,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = 5 + *var`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -375,7 +376,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = mod *var`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -463,7 +464,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = 5 le *var`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -560,7 +561,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`*ptr = 42`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.case.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.case.spec.ts index 4463f1a33..cda8238cd 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.case.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.case.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -15,7 +16,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.if.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.if.spec.ts index 1a077534c..e28ae5799 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.if.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.if.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`if *val then *var = 42 else *var = 84`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.loops.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.loops.spec.ts index 5e3d69b19..5f0b801f2 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.loops.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.loops.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`for i=*var, *other do *val = 42`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -180,7 +181,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`foreach val, *thing, key do *val = 42`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -337,7 +338,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`while *var do *val = 42`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -458,7 +459,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`repeat *val = 42 until *var`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.paren.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.paren.spec.ts index caa69accf..34ade121c 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.paren.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.paren.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = (*var)`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.regression.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.regression.spec.ts index 7ecf83ee0..870f92397 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.regression.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.regression.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -12,7 +13,7 @@ describe(`[auto generated] Correctly identify array indexing`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.routines.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.routines.spec.ts index cb93257c5..9184833df 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.routines.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.routines.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = func(*val, *other, kw=*last)`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -177,7 +178,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = var.func(*val, *other, kw=*last)`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -362,7 +363,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -543,7 +544,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`mypro, *val, *other, kw=*last`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -700,7 +701,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`var.mypro, *val, *other, kw=*last`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -866,7 +867,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`mypro,$`, ` *val`, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -968,7 +969,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.struct.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.struct.spec.ts index f7a8c8a27..4c7ab01ba 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.struct.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.struct.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = {prop: *ptr}`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -108,7 +109,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = var.(*thing)`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.switch.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.switch.spec.ts index 497863fee..d2e659be7 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.switch.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.switch.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -17,7 +18,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.ternary.spec.ts b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.ternary.spec.ts index 9336e365c..18aeb21a3 100644 --- a/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.ternary.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/operator.pointer-deref.ternary.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify pointer dereferencing`, () => { const code = [`a = *val ? *truthy : *falsy`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/strings.spec.ts b/libs/tests/syntax-post-processors/src/lib/strings.spec.ts index ba612801d..7f0f2cbc3 100644 --- a/libs/tests/syntax-post-processors/src/lib/strings.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/strings.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly merge strings together`, () => { const code = [`a = 'string''escaped'`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -72,7 +73,7 @@ describe(`[auto generated] Correctly merge strings together`, () => { const code = [`a = "string""escaped"`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -136,7 +137,7 @@ describe(`[auto generated] Correctly merge strings together`, () => { const code = [`a = 'string' 'escaped'`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -216,7 +217,7 @@ describe(`[auto generated] Correctly merge strings together`, () => { const code = [`a = "string" "escaped"`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/syntax-post-processors/src/lib/var-not-function.spec.ts b/libs/tests/syntax-post-processors/src/lib/var-not-function.spec.ts index a15304ec5..b484b8bd5 100644 --- a/libs/tests/syntax-post-processors/src/lib/var-not-function.spec.ts +++ b/libs/tests/syntax-post-processors/src/lib/var-not-function.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { SyntaxProblems } from '@idl/parsing/problem-codes'; import { SyntaxTree } from '@idl/parsing/syntax-tree'; @@ -8,7 +9,7 @@ describe(`[auto generated] Correctly identify variables instead of function call const code = [`;+ my var`, `a = 5`, ``, `!null = a()`, ``, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -155,7 +156,7 @@ describe(`[auto generated] Correctly identify variables instead of function call ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -286,7 +287,7 @@ describe(`[auto generated] Correctly identify variables instead of function call const code = [`compile_opt idl2`, `a = 5`, ``, `!null = a()`, ``, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ @@ -404,7 +405,7 @@ describe(`[auto generated] Correctly identify variables instead of function call const code = [`compile_opt idl3`, `a = 5`, ``, `!null = a()`, ``, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define expected syntax tree const expectedTree: SyntaxTree = [ diff --git a/libs/tests/token-at-cursor/src/lib/basic.spec.ts b/libs/tests/token-at-cursor/src/lib/basic.spec.ts index 8da901b62..e65b660c8 100644 --- a/libs/tests/token-at-cursor/src/lib/basic.spec.ts +++ b/libs/tests/token-at-cursor/src/lib/basic.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { GetTokenAtCursor, RemoveScopeDetail } from '@idl/parsing/syntax-tree'; import { Position } from 'vscode-languageserver/node'; @@ -16,7 +17,7 @@ describe(`[auto generated] Correctly identifies search terms from syntax tree`, ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define position const position_0: Position = { line: 0, character: 0 }; @@ -85,7 +86,7 @@ describe(`[auto generated] Correctly identifies search terms from syntax tree`, const selected_0 = GetTokenAtCursor(tokenized, position_0); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_0.token).toEqual(expectedFound_0); @@ -100,7 +101,7 @@ describe(`[auto generated] Correctly identifies search terms from syntax tree`, const selected_1 = GetTokenAtCursor(tokenized, position_1); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_1.token).toEqual(expectedFound_1); diff --git a/libs/tests/token-at-cursor/src/lib/keywords.spec.ts b/libs/tests/token-at-cursor/src/lib/keywords.spec.ts index 64687388d..93d3a1d8d 100644 --- a/libs/tests/token-at-cursor/src/lib/keywords.spec.ts +++ b/libs/tests/token-at-cursor/src/lib/keywords.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { GetTokenAtCursor, RemoveScopeDetail } from '@idl/parsing/syntax-tree'; import { Position } from 'vscode-languageserver/node'; @@ -12,7 +13,7 @@ describe(`[auto generated] Correctly identifies keywords from routine calls`, () ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define position const position_0: Position = { line: 1, character: 20 }; @@ -33,7 +34,7 @@ describe(`[auto generated] Correctly identifies keywords from routine calls`, () const selected_0 = GetTokenAtCursor(tokenized, position_0); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_0.token).toEqual(expectedFound_0); @@ -57,7 +58,7 @@ describe(`[auto generated] Correctly identifies keywords from routine calls`, () const selected_1 = GetTokenAtCursor(tokenized, position_1); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_1.token).toEqual(expectedFound_1); diff --git a/libs/tests/token-at-cursor/src/lib/proper-parent.spec.ts b/libs/tests/token-at-cursor/src/lib/proper-parent.spec.ts index 9c1387fb2..d83f8c006 100644 --- a/libs/tests/token-at-cursor/src/lib/proper-parent.spec.ts +++ b/libs/tests/token-at-cursor/src/lib/proper-parent.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { GetTokenAtCursor, RemoveScopeDetail } from '@idl/parsing/syntax-tree'; import { Position } from 'vscode-languageserver/node'; @@ -20,7 +21,7 @@ describe(`[auto generated] Correctly identifies search terms from syntax tree`, ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define position const position_0: Position = { line: 2, character: 2 }; @@ -41,7 +42,7 @@ describe(`[auto generated] Correctly identifies search terms from syntax tree`, const selected_0 = GetTokenAtCursor(tokenized, position_0); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_0.token).toEqual(expectedFound_0); @@ -65,7 +66,7 @@ describe(`[auto generated] Correctly identifies search terms from syntax tree`, const selected_1 = GetTokenAtCursor(tokenized, position_1); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_1.token).toEqual(expectedFound_1); diff --git a/libs/tests/token-at-cursor/src/lib/relaxed.spec.ts b/libs/tests/token-at-cursor/src/lib/relaxed.spec.ts index 879c76b10..b8795adda 100644 --- a/libs/tests/token-at-cursor/src/lib/relaxed.spec.ts +++ b/libs/tests/token-at-cursor/src/lib/relaxed.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { GetTokenAtCursor, RemoveScopeDetail } from '@idl/parsing/syntax-tree'; import { Position } from 'vscode-languageserver/node'; @@ -14,7 +15,7 @@ describe(`[auto generated] Correctly use relaxed options for hover help`, () => ]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define position const position_0: Position = { line: 2, character: 5 }; @@ -37,7 +38,7 @@ describe(`[auto generated] Correctly use relaxed options for hover help`, () => const selected_0 = GetTokenAtCursor(tokenized, position_0); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_0.token).toEqual(expectedFound_0); @@ -52,7 +53,7 @@ describe(`[auto generated] Correctly use relaxed options for hover help`, () => const selected_1 = GetTokenAtCursor(tokenized, position_1); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_1.token).toEqual(expectedFound_1); diff --git a/libs/tests/token-at-cursor/src/lib/within-parent.spec.ts b/libs/tests/token-at-cursor/src/lib/within-parent.spec.ts index 8a05dd345..308a4a179 100644 --- a/libs/tests/token-at-cursor/src/lib/within-parent.spec.ts +++ b/libs/tests/token-at-cursor/src/lib/within-parent.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { Parser } from '@idl/parser'; import { GetTokenAtCursor, RemoveScopeDetail } from '@idl/parsing/syntax-tree'; import { Position } from 'vscode-languageserver/node'; @@ -8,7 +9,7 @@ describe(`[auto generated] Find the right token when we do/don't have anything s const code = [`args.setData, `, `end`]; // extract tokens - const tokenized = Parser(code); + const tokenized = Parser(code, new CancellationToken()); // define position const position_0: Position = { line: 0, character: 12 }; @@ -29,7 +30,7 @@ describe(`[auto generated] Find the right token when we do/don't have anything s const selected_0 = GetTokenAtCursor(tokenized, position_0); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_0.token).toEqual(expectedFound_0); @@ -53,7 +54,7 @@ describe(`[auto generated] Find the right token when we do/don't have anything s const selected_1 = GetTokenAtCursor(tokenized, position_1); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_1.token).toEqual(expectedFound_1); @@ -90,7 +91,7 @@ describe(`[auto generated] Find the right token when we do/don't have anything s const selected_2 = GetTokenAtCursor(tokenized, position_2); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_2.token).toEqual(expectedFound_2); @@ -127,7 +128,7 @@ describe(`[auto generated] Find the right token when we do/don't have anything s const selected_3 = GetTokenAtCursor(tokenized, position_3); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_3.token).toEqual(expectedFound_3); @@ -164,7 +165,7 @@ describe(`[auto generated] Find the right token when we do/don't have anything s const selected_4 = GetTokenAtCursor(tokenized, position_4); // remove scope detail - RemoveScopeDetail(tokenized); + RemoveScopeDetail(tokenized, new CancellationToken()); // verify results expect(selected_4.token).toEqual(expectedFound_4); diff --git a/libs/tests/tokenizer/src/lib/assignment.spec.ts b/libs/tests/tokenizer/src/lib/assignment.spec.ts index c752c317e..2962b00a2 100644 --- a/libs/tests/tokenizer/src/lib/assignment.spec.ts +++ b/libs/tests/tokenizer/src/lib/assignment.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates assignment parsing`, () => { const code = [`a = 5`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -51,7 +52,7 @@ describe(`[auto generated] Validates assignment parsing`, () => { const code = [`!null = 5`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -89,7 +90,7 @@ describe(`[auto generated] Validates assignment parsing`, () => { const code = [`a[i] = b`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -145,7 +146,7 @@ describe(`[auto generated] Validates assignment parsing`, () => { const code = [`z = $`, ` 5`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -195,7 +196,7 @@ describe(`[auto generated] Validates assignment parsing`, () => { const code = [`(b) = 15`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -245,7 +246,7 @@ describe(`[auto generated] Validates assignment parsing`, () => { const code = [`for i=0, myFunc(a=42) do print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/blocks.spec.ts b/libs/tests/tokenizer/src/lib/blocks.spec.ts index 417ea4e93..f2f06e4a4 100644 --- a/libs/tests/tokenizer/src/lib/blocks.spec.ts +++ b/libs/tests/tokenizer/src/lib/blocks.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -23,7 +24,7 @@ describe(`[auto generated] Validates block parsing auto-closes`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/brackets.spec.ts b/libs/tests/tokenizer/src/lib/brackets.spec.ts index cc6b9adc9..79e2a317b 100644 --- a/libs/tests/tokenizer/src/lib/brackets.spec.ts +++ b/libs/tests/tokenizer/src/lib/brackets.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates bracket parsing`, () => { const code = [`[1 + 2]`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -63,7 +64,7 @@ describe(`[auto generated] Validates bracket parsing`, () => { const code = [`[1 + $`, ` 2]`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -125,7 +126,7 @@ describe(`[auto generated] Validates bracket parsing`, () => { const code = [`array1[1 + 2] * (1 + 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -229,7 +230,7 @@ describe(`[auto generated] Validates bracket parsing`, () => { const code = [`_a[i] = 5 * b`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -303,7 +304,7 @@ describe(`[auto generated] Validates bracket parsing`, () => { const code = [`_aA$[i] *= b`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/colon.spec.ts b/libs/tests/tokenizer/src/lib/colon.spec.ts index 22febb3f0..01fb7eb9d 100644 --- a/libs/tests/tokenizer/src/lib/colon.spec.ts +++ b/libs/tests/tokenizer/src/lib/colon.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates colon parsing`, () => { const code = [`[:]`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -45,7 +46,7 @@ describe(`[auto generated] Validates colon parsing`, () => { const code = [`a[0:I] = 42`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/commas.spec.ts b/libs/tests/tokenizer/src/lib/commas.spec.ts index 316da74b3..bb680f749 100644 --- a/libs/tests/tokenizer/src/lib/commas.spec.ts +++ b/libs/tests/tokenizer/src/lib/commas.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates comma parsing (mostly covered elsewhere)`, const code = [`,`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -33,7 +34,7 @@ describe(`[auto generated] Validates comma parsing (mostly covered elsewhere)`, const code = [`f(,)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -65,7 +66,7 @@ describe(`[auto generated] Validates comma parsing (mostly covered elsewhere)`, const code = [`p,`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/comments.spec.ts b/libs/tests/tokenizer/src/lib/comments.spec.ts index 19337e02b..8eec3b891 100644 --- a/libs/tests/tokenizer/src/lib/comments.spec.ts +++ b/libs/tests/tokenizer/src/lib/comments.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates comment parsing`, () => { const code = [` ; something()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -33,7 +34,7 @@ describe(`[auto generated] Validates comment parsing`, () => { const code = [`a = b() ; something()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -83,7 +84,7 @@ describe(`[auto generated] Validates comment parsing`, () => { const code = [` ; TODO: something()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -103,7 +104,7 @@ describe(`[auto generated] Validates comment parsing`, () => { const code = [`a = b() ; TODO: something()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -153,7 +154,7 @@ describe(`[auto generated] Validates comment parsing`, () => { const code = [`a = $ ; TODO: something()`, ` b()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/control.1.spec.ts b/libs/tests/tokenizer/src/lib/control.1.spec.ts index b086f527c..573219d1f 100644 --- a/libs/tests/tokenizer/src/lib/control.1.spec.ts +++ b/libs/tests/tokenizer/src/lib/control.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -20,7 +21,7 @@ describe(`[auto generated] Validates control statement parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -136,7 +137,7 @@ describe(`[auto generated] Validates control statement parsing`, () => { const code = [`if !true then break`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -186,7 +187,7 @@ describe(`[auto generated] Validates control statement parsing`, () => { const code = [`if !true then continue`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -236,7 +237,7 @@ describe(`[auto generated] Validates control statement parsing`, () => { const code = [`for i=0,99 do begin`, ` continue`, ` break`, `endfor`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -334,7 +335,7 @@ describe(`[auto generated] Validates control statement parsing`, () => { const code = [`for i=0,99 do begin`, ` jump:`, `endfor`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -433,7 +434,7 @@ describe(`[auto generated] Validates control statement parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -639,7 +640,7 @@ describe(`[auto generated] Validates control statement parsing`, () => { const code = [`if not wild then goto, done else printf, outunit`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -755,7 +756,7 @@ describe(`[auto generated] Validates control statement parsing`, () => { const code = [`GOTO, do_six & END`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/executive-command.spec.ts b/libs/tests/tokenizer/src/lib/executive-command.spec.ts index 093d84264..a68351f6a 100644 --- a/libs/tests/tokenizer/src/lib/executive-command.spec.ts +++ b/libs/tests/tokenizer/src/lib/executive-command.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates executive command parsing`, () => { const code = [`.compile`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -33,7 +34,7 @@ describe(`[auto generated] Validates executive command parsing`, () => { const code = [`.run myfile.pro`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -53,7 +54,7 @@ describe(`[auto generated] Validates executive command parsing`, () => { const code = [` .run myfile.pro`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -73,7 +74,7 @@ describe(`[auto generated] Validates executive command parsing`, () => { const code = [`obj.method`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -105,7 +106,7 @@ describe(`[auto generated] Validates executive command parsing`, () => { const code = [`!null = obj.method`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/include.spec.ts b/libs/tests/tokenizer/src/lib/include.spec.ts index bd54e6563..cf8970311 100644 --- a/libs/tests/tokenizer/src/lib/include.spec.ts +++ b/libs/tests/tokenizer/src/lib/include.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates include statements, but not correct locatio const code = [`@includeme`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -33,7 +34,7 @@ describe(`[auto generated] Validates include statements, but not correct locatio const code = [`a = myfunc(@bad)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -83,7 +84,7 @@ describe(`[auto generated] Validates include statements, but not correct locatio const code = [`for i=0,99 do @very_wrong`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -163,7 +164,7 @@ describe(`[auto generated] Validates include statements, but not correct locatio const code = [`a = @include_wrong + @way_bad`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -219,7 +220,7 @@ describe(`[auto generated] Validates include statements, but not correct locatio const code = [`@include.pro ; comment`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/lambda.spec.ts b/libs/tests/tokenizer/src/lib/lambda.spec.ts index e1021c54a..4f36b21a6 100644 --- a/libs/tests/tokenizer/src/lib/lambda.spec.ts +++ b/libs/tests/tokenizer/src/lib/lambda.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates lambda functions parsed as special token`, const code = [`a = lambda(x:x+2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/line-modifiers.spec.ts b/libs/tests/tokenizer/src/lib/line-modifiers.spec.ts index f824301e0..d81eaf58a 100644 --- a/libs/tests/tokenizer/src/lib/line-modifiers.spec.ts +++ b/libs/tests/tokenizer/src/lib/line-modifiers.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates line modifier (separators)`, () => { const code = [`a = a + b() & proc & a.procMethod,1 & $`, `a={struct}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -209,7 +210,7 @@ describe(`[auto generated] Validates line modifier (separators)`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -534,7 +535,7 @@ describe(`[auto generated] Validates line modifier (separators)`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -776,7 +777,7 @@ describe(`[auto generated] Validates line modifier (separators)`, () => { const code = [`scale=scale, $, $`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/logic.case.1.spec.ts b/libs/tests/tokenizer/src/lib/logic.case.1.spec.ts index a301bbd70..3ee0295d2 100644 --- a/libs/tests/tokenizer/src/lib/logic.case.1.spec.ts +++ b/libs/tests/tokenizer/src/lib/logic.case.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -24,7 +25,7 @@ describe(`[auto generated] Validates case statement`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -268,7 +269,7 @@ describe(`[auto generated] Validates case statement`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/logic.if-then-else.1.spec.ts b/libs/tests/tokenizer/src/lib/logic.if-then-else.1.spec.ts index 68e5030f1..c43737551 100644 --- a/libs/tests/tokenizer/src/lib/logic.if-then-else.1.spec.ts +++ b/libs/tests/tokenizer/src/lib/logic.if-then-else.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates for if-then-else parsing [1]`, () => { const code = [`if !true then print, 'yes'`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -81,7 +82,7 @@ describe(`[auto generated] Validates for if-then-else parsing [1]`, () => { const code = [`if ~doIt then print, 'yes' else a = yellow()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -203,7 +204,7 @@ describe(`[auto generated] Validates for if-then-else parsing [1]`, () => { const code = [`if !true $`, ` then print, 'yes'`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -283,7 +284,7 @@ describe(`[auto generated] Validates for if-then-else parsing [1]`, () => { const code = [`if !true $`, ` then print $`, ` , 'yes'`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -375,7 +376,7 @@ describe(`[auto generated] Validates for if-then-else parsing [1]`, () => { const code = [`if !true then print, 'yes' $`, ` else print, 'no'`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -491,7 +492,7 @@ describe(`[auto generated] Validates for if-then-else parsing [1]`, () => { const code = [`if !true then print, 'yes' $`, ` else $`, ` print, 'no'`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -619,7 +620,7 @@ describe(`[auto generated] Validates for if-then-else parsing [1]`, () => { const code = [`if ~(myFunc(_a17$) * 2) then if !false then print, 'yes'`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/logic.if-then-else.2.spec.ts b/libs/tests/tokenizer/src/lib/logic.if-then-else.2.spec.ts index 535c7ca54..60cd88686 100644 --- a/libs/tests/tokenizer/src/lib/logic.if-then-else.2.spec.ts +++ b/libs/tests/tokenizer/src/lib/logic.if-then-else.2.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -17,7 +18,7 @@ describe(`[auto generated] Validates for if-then-else parsing [2]`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/logic.if-then-else.3.spec.ts b/libs/tests/tokenizer/src/lib/logic.if-then-else.3.spec.ts index 34771d7f8..f48970b20 100644 --- a/libs/tests/tokenizer/src/lib/logic.if-then-else.3.spec.ts +++ b/libs/tests/tokenizer/src/lib/logic.if-then-else.3.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates for if-then-else parsing [3]`, () => { const code = [`if i ne 0 then a[0, m1-i] = y ;Symmetrical`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -159,7 +160,7 @@ describe(`[auto generated] Validates for if-then-else parsing [3]`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -392,7 +393,7 @@ describe(`[auto generated] Validates for if-then-else parsing [3]`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -667,7 +668,7 @@ describe(`[auto generated] Validates for if-then-else parsing [3]`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -837,7 +838,7 @@ describe(`[auto generated] Validates for if-then-else parsing [3]`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/logic.switch.1.spec.ts b/libs/tests/tokenizer/src/lib/logic.switch.1.spec.ts index 8e49a043d..34c0bdc4d 100644 --- a/libs/tests/tokenizer/src/lib/logic.switch.1.spec.ts +++ b/libs/tests/tokenizer/src/lib/logic.switch.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -24,7 +25,7 @@ describe(`[auto generated] Validates switch statement`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -268,7 +269,7 @@ describe(`[auto generated] Validates switch statement`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/logic.ternary.1.spec.ts b/libs/tests/tokenizer/src/lib/logic.ternary.1.spec.ts index 108b8adc1..00bac30dd 100644 --- a/libs/tests/tokenizer/src/lib/logic.ternary.1.spec.ts +++ b/libs/tests/tokenizer/src/lib/logic.ternary.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates for ternary statement parsing`, () => { const code = [`a = something ? 5 : 6`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -87,7 +88,7 @@ describe(`[auto generated] Validates for ternary statement parsing`, () => { const code = [`a = !true ? (!false ? 7 : 8) : 6`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -209,7 +210,7 @@ describe(`[auto generated] Validates for ternary statement parsing`, () => { const code = [`mypro, something ? ~something ? 7 : 8 : 6, 2`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -343,7 +344,7 @@ describe(`[auto generated] Validates for ternary statement parsing`, () => { const code = [`a = myfunc(something ? otherfunc() : !awesomesauce) + 3`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -453,7 +454,7 @@ describe(`[auto generated] Validates for ternary statement parsing`, () => { const code = [`a = 5*something ? 5- 4 : 6^3`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -581,7 +582,7 @@ describe(`[auto generated] Validates for ternary statement parsing`, () => { const code = [`a = _myvar $`, ` ? 'jello' : "jelly"`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -672,7 +673,7 @@ describe(`[auto generated] Validates for ternary statement parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -818,7 +819,7 @@ describe(`[auto generated] Validates for ternary statement parsing`, () => { const code = [`_17 = arr[!true ? 0 : -3: -1]`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/loops.for.spec.ts b/libs/tests/tokenizer/src/lib/loops.for.spec.ts index 00f19a6d0..4e00ea09f 100644 --- a/libs/tests/tokenizer/src/lib/loops.for.spec.ts +++ b/libs/tests/tokenizer/src/lib/loops.for.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates for loop parsing`, () => { const code = [`for i=0, 99 do print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -111,7 +112,7 @@ describe(`[auto generated] Validates for loop parsing`, () => { const code = [`for i=0, 99, 2 do !null = myFunc(i)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -233,7 +234,7 @@ describe(`[auto generated] Validates for loop parsing`, () => { const code = [`for i=0, jj do $`, ` print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -343,7 +344,7 @@ describe(`[auto generated] Validates for loop parsing`, () => { const code = [`for i=0, 99 do begin`, ` !null = myFunc(i)`, `endfor`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -465,7 +466,7 @@ describe(`[auto generated] Validates for loop parsing`, () => { const code = [`for i=0, 99 do for j=0, 99 do print, i + j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/loops.foreach.spec.ts b/libs/tests/tokenizer/src/lib/loops.foreach.spec.ts index f1c728da2..6d4356210 100644 --- a/libs/tests/tokenizer/src/lib/loops.foreach.spec.ts +++ b/libs/tests/tokenizer/src/lib/loops.foreach.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates foreach loop parsing`, () => { const code = [`foreach val, arr do print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -93,7 +94,7 @@ describe(`[auto generated] Validates foreach loop parsing`, () => { const code = [`foreach val, arr, idx do !null = myFunc(i)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -197,7 +198,7 @@ describe(`[auto generated] Validates foreach loop parsing`, () => { const code = [`foreach val, arr do $`, ` print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -293,7 +294,7 @@ describe(`[auto generated] Validates foreach loop parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -397,7 +398,7 @@ describe(`[auto generated] Validates foreach loop parsing`, () => { const code = [`foreach val, arr do foreach val2, val do print, val2`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/loops.repeat.spec.ts b/libs/tests/tokenizer/src/lib/loops.repeat.spec.ts index e1d2fc052..7c6d4867e 100644 --- a/libs/tests/tokenizer/src/lib/loops.repeat.spec.ts +++ b/libs/tests/tokenizer/src/lib/loops.repeat.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates repeat loop parsing`, () => { const code = [`REPEAT A = A * 2 UNTIL A GT B`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -117,7 +118,7 @@ describe(`[auto generated] Validates repeat loop parsing`, () => { const code = [`REPEAT PRINT, A UNTIL A GT B`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -203,7 +204,7 @@ describe(`[auto generated] Validates repeat loop parsing`, () => { const code = [`REPEAT A = $`, ` A * 2 UNTIL $`, ` A GT B`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -331,7 +332,7 @@ describe(`[auto generated] Validates repeat loop parsing`, () => { const code = [`REPEAT BEGIN`, ` A = A * 2`, `ENDREP UNTIL A GT B`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -447,7 +448,7 @@ describe(`[auto generated] Validates repeat loop parsing`, () => { const code = [`repeat if !true then break until !true`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/loops.while.spec.ts b/libs/tests/tokenizer/src/lib/loops.while.spec.ts index 730f4817b..40706d2ba 100644 --- a/libs/tests/tokenizer/src/lib/loops.while.spec.ts +++ b/libs/tests/tokenizer/src/lib/loops.while.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates while loop parsing`, () => { const code = [`while !true do print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -81,7 +82,7 @@ describe(`[auto generated] Validates while loop parsing`, () => { const code = [`while !true do while !false do print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -179,7 +180,7 @@ describe(`[auto generated] Validates while loop parsing`, () => { const code = [`while !true do $`, ` print, $`, ` i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -271,7 +272,7 @@ describe(`[auto generated] Validates while loop parsing`, () => { const code = [`while (a eq 5) do begin`, ` !null = myFunc(i)`, `endwhile`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/methods.functions.spec.ts b/libs/tests/tokenizer/src/lib/methods.functions.spec.ts index 1e2c0a0fb..99f32091a 100644 --- a/libs/tests/tokenizer/src/lib/methods.functions.spec.ts +++ b/libs/tests/tokenizer/src/lib/methods.functions.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates function method parsing`, () => { const code = [`!NULL = a.myFunc(1)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -69,7 +70,7 @@ describe(`[auto generated] Validates function method parsing`, () => { const code = [`!NULL = a->myFunc(1)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -125,7 +126,7 @@ describe(`[auto generated] Validates function method parsing`, () => { const code = [`a.super::myfunc(1)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -163,7 +164,7 @@ describe(`[auto generated] Validates function method parsing`, () => { const code = [`a->super::myfunc(a)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -201,7 +202,7 @@ describe(`[auto generated] Validates function method parsing`, () => { const code = [`!NULL = a.myFunc( $`, ` 1)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -269,7 +270,7 @@ describe(`[auto generated] Validates function method parsing`, () => { const code = [`a.b()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/methods.procedures.spec.ts b/libs/tests/tokenizer/src/lib/methods.procedures.spec.ts index 480b89f1d..ca8a37def 100644 --- a/libs/tests/tokenizer/src/lib/methods.procedures.spec.ts +++ b/libs/tests/tokenizer/src/lib/methods.procedures.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates procedure method parsing`, () => { const code = [`a.myProc, 1`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -57,7 +58,7 @@ describe(`[auto generated] Validates procedure method parsing`, () => { const code = [`a->myProc, a`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -101,7 +102,7 @@ describe(`[auto generated] Validates procedure method parsing`, () => { const code = [`a.super::myProc, 1`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -145,7 +146,7 @@ describe(`[auto generated] Validates procedure method parsing`, () => { const code = [`a->super::myProc, a`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -189,7 +190,7 @@ describe(`[auto generated] Validates procedure method parsing`, () => { const code = [`a.myProc, $`, ` 1`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -249,7 +250,7 @@ describe(`[auto generated] Validates procedure method parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -485,7 +486,7 @@ describe(`[auto generated] Validates procedure method parsing`, () => { const code = [`a.b`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/number-string.complex-i.spec.ts b/libs/tests/tokenizer/src/lib/number-string.complex-i.spec.ts index ddc7d2a31..6cae5b49a 100644 --- a/libs/tests/tokenizer/src/lib/number-string.complex-i.spec.ts +++ b/libs/tests/tokenizer/src/lib/number-string.complex-i.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"36i + "45i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -51,7 +52,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"36bi + "45ulli`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -89,7 +90,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'101010'bi`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -109,7 +110,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'10101'xi`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -129,7 +130,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'10101'oi`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -149,7 +150,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"101010"bi`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -169,7 +170,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"10101"xi`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -189,7 +190,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"10101"oi`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -209,7 +210,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'7FFF'XSi`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -229,7 +230,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'8FFF'XSi`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/number-string.complex-j.spec.ts b/libs/tests/tokenizer/src/lib/number-string.complex-j.spec.ts index b7e458bd6..8f789fdca 100644 --- a/libs/tests/tokenizer/src/lib/number-string.complex-j.spec.ts +++ b/libs/tests/tokenizer/src/lib/number-string.complex-j.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"36j + "45j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -51,7 +52,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"36bj + "45ullj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -89,7 +90,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'101010'bj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -109,7 +110,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'10101'xj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -129,7 +130,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'10101'oj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -149,7 +150,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"101010"bj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -169,7 +170,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"10101"xj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -189,7 +190,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"10101"oj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -209,7 +210,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'7FFF'XSj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -229,7 +230,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'8FFF'XSj`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/number-string.spec.ts b/libs/tests/tokenizer/src/lib/number-string.spec.ts index ac48c161c..784603b07 100644 --- a/libs/tests/tokenizer/src/lib/number-string.spec.ts +++ b/libs/tests/tokenizer/src/lib/number-string.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"36 + "45`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -51,7 +52,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"36b + "45ull`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -89,7 +90,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'101010'b`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -109,7 +110,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'10101'x`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -129,7 +130,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'10101'o`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -149,7 +150,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"101010"b`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -169,7 +170,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"10101"x`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -189,7 +190,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`"10101"o`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -209,7 +210,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'7FFF'XS`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -229,7 +230,7 @@ describe(`[auto generated] Validates special cases for number string parsing`, ( const code = [`'8FFF'XS`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/numbers.spec.ts b/libs/tests/tokenizer/src/lib/numbers.spec.ts index 30e8c6e9f..ec9666e63 100644 --- a/libs/tests/tokenizer/src/lib/numbers.spec.ts +++ b/libs/tests/tokenizer/src/lib/numbers.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = -1e34`, `a = -1e34i`, `a = -1e34j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -135,7 +136,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = 1e-34`, `a = 1e-34i`, `a = 1e-34j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -221,7 +222,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = 1e-`, `a = 1e-i`, `a = 1e-j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -311,7 +312,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -509,7 +510,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -707,7 +708,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -901,7 +902,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = -1d34`, `a = -1d34i`, `a = -1d34j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1023,7 +1024,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = 1d-34`, `a = 1d-34i`, `a = 1d-34j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1109,7 +1110,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = 1d-`, `a = 1d-i`, `a = 1d-j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1195,7 +1196,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = 1i`, `a = 1j`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1257,7 +1258,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a.`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1283,7 +1284,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = b.`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1327,7 +1328,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = .`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1365,7 +1366,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`a = .1e+12`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1403,7 +1404,7 @@ describe(`[auto generated] Validates special cases for number parsing`, () => { const code = [`.`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/operators.compound.spec.ts b/libs/tests/tokenizer/src/lib/operators.compound.spec.ts index 1fcff226b..3ce7bb45f 100644 --- a/libs/tests/tokenizer/src/lib/operators.compound.spec.ts +++ b/libs/tests/tokenizer/src/lib/operators.compound.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates compound operator parsing`, () => { const code = [`z *= $`, ` 5`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -63,7 +64,7 @@ describe(`[auto generated] Validates compound operator parsing`, () => { const code = [`a || b || c`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -119,7 +120,7 @@ describe(`[auto generated] Validates compound operator parsing`, () => { const code = [`a or b or c`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -175,7 +176,7 @@ describe(`[auto generated] Validates compound operator parsing`, () => { const code = [`a && b && c`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -231,7 +232,7 @@ describe(`[auto generated] Validates compound operator parsing`, () => { const code = [`a and b and c`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/operators.spec.ts b/libs/tests/tokenizer/src/lib/operators.spec.ts index fe0531620..1e1e11911 100644 --- a/libs/tests/tokenizer/src/lib/operators.spec.ts +++ b/libs/tests/tokenizer/src/lib/operators.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`{1+2}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -45,7 +46,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`[1+2]`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -95,7 +96,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`(1+2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -145,7 +146,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`1+2,3+4`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -213,7 +214,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`if 1+2 then a = 3+4 else a = 4^3`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -371,7 +372,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`for i=0, 99-1 do print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -487,7 +488,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`zach + $ `, `awesome`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -542,7 +543,7 @@ describe(`[auto generated] Validates operator parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -826,7 +827,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`*(*pState).pTitle->SetProperty, color=[255, 255, 255]`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -966,7 +967,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`*pTitle.SetProperty, color=[255, 255, 255]`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1076,7 +1077,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`*pTitle.SetProperty(color=[255, 255, 255])`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1180,7 +1181,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`*pTitle->SetProperty(color=[255, 255, 255])`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1284,7 +1285,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`++a`, `a++`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1334,7 +1335,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`--a`, `a+--`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -1396,7 +1397,7 @@ describe(`[auto generated] Validates operator parsing`, () => { const code = [`a = b++ + 5`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/parentheses.spec.ts b/libs/tests/tokenizer/src/lib/parentheses.spec.ts index 0bf08f666..31f7b6347 100644 --- a/libs/tests/tokenizer/src/lib/parentheses.spec.ts +++ b/libs/tests/tokenizer/src/lib/parentheses.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates parentheses parsing`, () => { const code = [`(1 + 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -63,7 +64,7 @@ describe(`[auto generated] Validates parentheses parsing`, () => { const code = [`myFunc(1 + 2) * (1 + 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -161,7 +162,7 @@ describe(`[auto generated] Validates parentheses parsing`, () => { const code = [`(1 + $ `, ` 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/prompts.spec.ts b/libs/tests/tokenizer/src/lib/prompts.spec.ts index f7b0db41b..c146ccaa8 100644 --- a/libs/tests/tokenizer/src/lib/prompts.spec.ts +++ b/libs/tests/tokenizer/src/lib/prompts.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates prompt parsing`, () => { const code = [`IDL> print, 42`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -57,7 +58,7 @@ describe(`[auto generated] Validates prompt parsing`, () => { const code = [`ENVI> print, 17`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/properties.spec.ts b/libs/tests/tokenizer/src/lib/properties.spec.ts index f11044729..9d6b1f9ab 100644 --- a/libs/tests/tokenizer/src/lib/properties.spec.ts +++ b/libs/tests/tokenizer/src/lib/properties.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates property parsing`, () => { const code = [`a.thing = 5`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -57,7 +58,7 @@ describe(`[auto generated] Validates property parsing`, () => { const code = [`b = a.thing`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -101,7 +102,7 @@ describe(`[auto generated] Validates property parsing`, () => { const code = [`b = $`, ` a.thing`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -157,7 +158,7 @@ describe(`[auto generated] Validates property parsing`, () => { const code = [`b = a.thing1.thing2`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -207,7 +208,7 @@ describe(`[auto generated] Validates property parsing`, () => { const code = [`myPro, a.thing, b.thing`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -269,7 +270,7 @@ describe(`[auto generated] Validates property parsing`, () => { const code = [`(*state).bottomSelection = lMarkers.Count()-1`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -367,7 +368,7 @@ describe(`[auto generated] Validates property parsing`, () => { const code = [`a = b.(i)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/python.spec.ts b/libs/tests/tokenizer/src/lib/python.spec.ts index fb243895e..e7984ab66 100644 --- a/libs/tests/tokenizer/src/lib/python.spec.ts +++ b/libs/tests/tokenizer/src/lib/python.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates Python code parsing`, () => { const code = [`>>>import numpy as np`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/quotes.edge-cases.spec.ts b/libs/tests/tokenizer/src/lib/quotes.edge-cases.spec.ts index 8a91df593..48d26307b 100644 --- a/libs/tests/tokenizer/src/lib/quotes.edge-cases.spec.ts +++ b/libs/tests/tokenizer/src/lib/quotes.edge-cases.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates edge case quote parsing`, () => { const code = [`a = "5"`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/quotes.escaped.spec.ts b/libs/tests/tokenizer/src/lib/quotes.escaped.spec.ts index 79b44937a..898bfbe33 100644 --- a/libs/tests/tokenizer/src/lib/quotes.escaped.spec.ts +++ b/libs/tests/tokenizer/src/lib/quotes.escaped.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates escaped quote parsing`, () => { const code = [`'Resolve_Routine, ''%s'', Is_Function=%d'`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -45,7 +46,7 @@ describe(`[auto generated] Validates escaped quote parsing`, () => { const code = [`"Resolve_Routine, ""%s"", Is_Function=%d"`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -79,7 +80,7 @@ describe(`[auto generated] Validates escaped quote parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -131,7 +132,7 @@ describe(`[auto generated] Validates escaped quote parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/quotes.spec.ts b/libs/tests/tokenizer/src/lib/quotes.spec.ts index db4b8beb2..ac3a8de89 100644 --- a/libs/tests/tokenizer/src/lib/quotes.spec.ts +++ b/libs/tests/tokenizer/src/lib/quotes.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`'myFunc(1 + 2)'`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -33,7 +34,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`"myFunc(1 + 2)"`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -53,7 +54,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`'string`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -73,7 +74,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`"string`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -95,7 +96,7 @@ describe(`[auto generated] Validates quote parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -183,7 +184,7 @@ describe(`[auto generated] Validates quote parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -269,7 +270,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`if "bad-quote"then "bad-quote"else`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -331,7 +332,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`case "bad-quote"of`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -363,7 +364,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`for "bad-quote"do`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -407,7 +408,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`repeat 'bad-quote'until`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -451,7 +452,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`if 'bad-quote'then 'bad-quote'else`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -513,7 +514,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`case 'bad-quote'of`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -545,7 +546,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`for 'bad-quote'do`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -589,7 +590,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`repeat 'bad-quote'until`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -633,7 +634,7 @@ describe(`[auto generated] Validates quote parsing`, () => { const code = [`arr = ["0.00000000"]`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/routines.definitions.1.spec.ts b/libs/tests/tokenizer/src/lib/routines.definitions.1.spec.ts index 3b6ec3b7d..c0f043a42 100644 --- a/libs/tests/tokenizer/src/lib/routines.definitions.1.spec.ts +++ b/libs/tests/tokenizer/src/lib/routines.definitions.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -19,7 +20,7 @@ describe(`[auto generated] Validates routine parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -177,7 +178,7 @@ describe(`[auto generated] Validates routine parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -336,7 +337,7 @@ describe(`[auto generated] Validates routine parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -502,7 +503,7 @@ describe(`[auto generated] Validates routine parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -662,7 +663,7 @@ describe(`[auto generated] Validates routine parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/routines.definitions.2.spec.ts b/libs/tests/tokenizer/src/lib/routines.definitions.2.spec.ts index f757375f1..24a63a83f 100644 --- a/libs/tests/tokenizer/src/lib/routines.definitions.2.spec.ts +++ b/libs/tests/tokenizer/src/lib/routines.definitions.2.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates routine parsing`, () => { const code = [`PRO EndMagic, Unit, Id`, ` PRINTF, Unit`, `END`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -99,7 +100,7 @@ describe(`[auto generated] Validates routine parsing`, () => { const code = [`pro !sosobad,`, `END`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -143,7 +144,7 @@ describe(`[auto generated] Validates routine parsing`, () => { const code = [`pro !sosobad::method,`, `END`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -187,7 +188,7 @@ describe(`[auto generated] Validates routine parsing`, () => { const code = [`FUNCTION VarName, Ptr & RETURN,'' & END`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/routines.functions.spec.ts b/libs/tests/tokenizer/src/lib/routines.functions.spec.ts index b1e84008f..743a11c8e 100644 --- a/libs/tests/tokenizer/src/lib/routines.functions.spec.ts +++ b/libs/tests/tokenizer/src/lib/routines.functions.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates function parsing`, () => { const code = [`myFunc(1 + 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -63,7 +64,7 @@ describe(`[auto generated] Validates function parsing`, () => { const code = [`myFunc(myFunc2(_y7$) + 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -125,7 +126,7 @@ describe(`[auto generated] Validates function parsing`, () => { const code = [`myFunc(1 + 2) * (1 + 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -223,7 +224,7 @@ describe(`[auto generated] Validates function parsing`, () => { const code = [`myFunc(1 $`, ` + 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -285,7 +286,7 @@ describe(`[auto generated] Validates function parsing`, () => { const code = [`a()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/routines.keywords.spec.ts b/libs/tests/tokenizer/src/lib/routines.keywords.spec.ts index c54e56355..f184b4bec 100644 --- a/libs/tests/tokenizer/src/lib/routines.keywords.spec.ts +++ b/libs/tests/tokenizer/src/lib/routines.keywords.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates keyword parsing`, () => { const code = [`myfunc(a = 5)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -63,7 +64,7 @@ describe(`[auto generated] Validates keyword parsing`, () => { const code = [`_otherfunc(a = 5, _b=42)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -143,7 +144,7 @@ describe(`[auto generated] Validates keyword parsing`, () => { const code = [`myfunc(a = 5, $`, ` _b=42)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -235,7 +236,7 @@ describe(`[auto generated] Validates keyword parsing`, () => { const code = [`myfunc(a $`, ` = 5)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -297,7 +298,7 @@ describe(`[auto generated] Validates keyword parsing`, () => { const code = [`_y = _superFunction(a = 5)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/routines.procedures.spec.ts b/libs/tests/tokenizer/src/lib/routines.procedures.spec.ts index a663d6171..7f25c0679 100644 --- a/libs/tests/tokenizer/src/lib/routines.procedures.spec.ts +++ b/libs/tests/tokenizer/src/lib/routines.procedures.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates procedure parsing`, () => { const code = [`myPro, 1 + 2`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -69,7 +70,7 @@ describe(`[auto generated] Validates procedure parsing`, () => { const code = [`myPro, arg1, arg2`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -119,7 +120,7 @@ describe(`[auto generated] Validates procedure parsing`, () => { const code = [`myPro, $`, ` arg1, arg2`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -181,7 +182,7 @@ describe(`[auto generated] Validates procedure parsing`, () => { const code = [`for i=0, 2*5-jello(1) do print, i`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -327,7 +328,7 @@ describe(`[auto generated] Validates procedure parsing`, () => { const code = [`a`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/routines.spacing.spec.ts b/libs/tests/tokenizer/src/lib/routines.spacing.spec.ts index e0c10870c..73452d901 100644 --- a/libs/tests/tokenizer/src/lib/routines.spacing.spec.ts +++ b/libs/tests/tokenizer/src/lib/routines.spacing.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates routine spacing`, () => { const code = [`myFunc (1 + 2)`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -63,7 +64,7 @@ describe(`[auto generated] Validates routine spacing`, () => { const code = [`mypro ,`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -95,7 +96,7 @@ describe(`[auto generated] Validates routine spacing`, () => { const code = [`a . method ()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -127,7 +128,7 @@ describe(`[auto generated] Validates routine spacing`, () => { const code = [`a -> method ()`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -159,7 +160,7 @@ describe(`[auto generated] Validates routine spacing`, () => { const code = [`a . method `]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -191,7 +192,7 @@ describe(`[auto generated] Validates routine spacing`, () => { const code = [`a . method , `]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -229,7 +230,7 @@ describe(`[auto generated] Validates routine spacing`, () => { const code = [`a -> method `]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -261,7 +262,7 @@ describe(`[auto generated] Validates routine spacing`, () => { const code = [`a -> method , `]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/string-literal.escape.spec.ts b/libs/tests/tokenizer/src/lib/string-literal.escape.spec.ts index 00ae203aa..9dc3cfa8a 100644 --- a/libs/tests/tokenizer/src/lib/string-literal.escape.spec.ts +++ b/libs/tests/tokenizer/src/lib/string-literal.escape.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -27,7 +28,7 @@ describe(`[auto generated] Verify string literal escape characters`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/string-literal.multiline.spec.ts b/libs/tests/tokenizer/src/lib/string-literal.multiline.spec.ts index 16340d192..873f502e8 100644 --- a/libs/tests/tokenizer/src/lib/string-literal.multiline.spec.ts +++ b/libs/tests/tokenizer/src/lib/string-literal.multiline.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -21,7 +22,7 @@ describe(`[auto generated] Verify string literal processing with multi-line stat ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/string-literal.spec.ts b/libs/tests/tokenizer/src/lib/string-literal.spec.ts index 7fc42783b..30aa13030 100644 --- a/libs/tests/tokenizer/src/lib/string-literal.spec.ts +++ b/libs/tests/tokenizer/src/lib/string-literal.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Verify string literal processing`, () => { const code = [`a = \`my string with \${expression + 5}\``]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -99,7 +100,7 @@ describe(`[auto generated] Verify string literal processing`, () => { const code = [`a = \`my string without substitution\``]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -149,7 +150,7 @@ describe(`[auto generated] Verify string literal processing`, () => { const code = [`a = \`start \${ \`nested\` } else\``]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -237,7 +238,7 @@ describe(`[auto generated] Verify string literal processing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -515,7 +516,7 @@ describe(`[auto generated] Verify string literal processing`, () => { const code = [`a = \`something \\\` included \``]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -577,7 +578,7 @@ describe(`[auto generated] Verify string literal processing`, () => { const code = [`a = \` first \``]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -627,7 +628,7 @@ describe(`[auto generated] Verify string literal processing`, () => { const code = [`a = \`\${1.234,"%10.3f"}\``]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/structures.1.spec.ts b/libs/tests/tokenizer/src/lib/structures.1.spec.ts index 69460cac5..634c0fec5 100644 --- a/libs/tests/tokenizer/src/lib/structures.1.spec.ts +++ b/libs/tests/tokenizer/src/lib/structures.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`_z5$ = {thing}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -69,7 +70,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`_17$ = { $`, ` thing}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -137,7 +138,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`_17$ = {thing:z}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -199,7 +200,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`_17$ = { $`, ` thing:z}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -273,7 +274,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`_z5$ = {thing1:{thing2:z}}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -359,7 +360,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`_z5$ = {thing, inherits _jklol}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -427,7 +428,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`a17 = {_th1g, abc:def, b:5, c:f()}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/structures.2.spec.ts b/libs/tests/tokenizer/src/lib/structures.2.spec.ts index 48ece0fc5..737a4f8b4 100644 --- a/libs/tests/tokenizer/src/lib/structures.2.spec.ts +++ b/libs/tests/tokenizer/src/lib/structures.2.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -15,7 +16,7 @@ describe(`[auto generated] Validates structure parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -149,7 +150,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`_17$ = { $ ; something`, ` thing: {name, some:value}}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -274,7 +275,7 @@ describe(`[auto generated] Validates structure parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -438,7 +439,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`a = {!exciting}`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -494,7 +495,7 @@ describe(`[auto generated] Validates structure parsing`, () => { const code = [`void = {mlLabelingTool_GraphicOverlay $`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -547,7 +548,7 @@ describe(`[auto generated] Validates structure parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/unexpected.1.spec.ts b/libs/tests/tokenizer/src/lib/unexpected.1.spec.ts index 611aaf853..f36d24f90 100644 --- a/libs/tests/tokenizer/src/lib/unexpected.1.spec.ts +++ b/libs/tests/tokenizer/src/lib/unexpected.1.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -26,7 +27,7 @@ describe(`[auto generated] Validates unexpected closer parsing`, () => { ]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/tests/tokenizer/src/lib/unknown.spec.ts b/libs/tests/tokenizer/src/lib/unknown.spec.ts index b05a79600..2ddbc0b4f 100644 --- a/libs/tests/tokenizer/src/lib/unknown.spec.ts +++ b/libs/tests/tokenizer/src/lib/unknown.spec.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { IBaseToken, StripIDs, @@ -13,7 +14,7 @@ describe(`[auto generated] Validates unknown token parsing`, () => { const code = [`sEvent.component-> $`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -57,7 +58,7 @@ describe(`[auto generated] Validates unknown token parsing`, () => { const code = [`a = $ bad bad`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ @@ -101,7 +102,7 @@ describe(`[auto generated] Validates unknown token parsing`, () => { const code = [`scale=scale, $, $`]; // extract tokens - const tokenized = Tokenizer(code); + const tokenized = Tokenizer(code, new CancellationToken()); // define expected tokens const expected: IBaseToken[] = [ diff --git a/libs/vscode/notebooks/src/lib/controller/idl-notebook-controller.class.ts b/libs/vscode/notebooks/src/lib/controller/idl-notebook-controller.class.ts index f4f72964b..78543e126 100644 --- a/libs/vscode/notebooks/src/lib/controller/idl-notebook-controller.class.ts +++ b/libs/vscode/notebooks/src/lib/controller/idl-notebook-controller.class.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { CleanIDLOutput, IDL, @@ -645,7 +646,7 @@ export class IDLNotebookController { /** * Parse code and see if we have a main level program */ - const parsed = Parser(strings); + const parsed = Parser(strings, new CancellationToken()); // check for main level program if (parsed.tree[parsed.tree.length - 1]?.name === TOKEN_NAMES.MAIN_LEVEL) { diff --git a/libs/vscode/server/src/lib/events/custom-events/on-add-docs.ts b/libs/vscode/server/src/lib/events/custom-events/on-add-docs.ts index a299c91eb..5a198ca6a 100644 --- a/libs/vscode/server/src/lib/events/custom-events/on-add-docs.ts +++ b/libs/vscode/server/src/lib/events/custom-events/on-add-docs.ts @@ -1,4 +1,5 @@ import { Assembler } from '@idl/assembler'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { IDL_LSP_LOG } from '@idl/logger'; import { IDL_TRANSLATION } from '@idl/translation'; import { IAddDocsMessagePayload } from '@idl/vscode/events/messages'; @@ -43,7 +44,7 @@ export const ON_ADD_DOCS = async (event: IAddDocsMessagePayload) => { ); // format - const formatted = Assembler(tokens, { + const formatted = Assembler(tokens, new CancellationToken(), { autoDoc: true, styleAndFormat: false, autoFix: false, diff --git a/libs/vscode/server/src/lib/events/documents/on-document-formatting.ts b/libs/vscode/server/src/lib/events/documents/on-document-formatting.ts index 5103f942c..50bbb8416 100644 --- a/libs/vscode/server/src/lib/events/documents/on-document-formatting.ts +++ b/libs/vscode/server/src/lib/events/documents/on-document-formatting.ts @@ -4,6 +4,7 @@ import { FormatterType, IAssemblerOptions, } from '@idl/assembling/config'; +import { CancellationToken } from '@idl/cancellation-tokens'; import { ParsedTask } from '@idl/data-types/tasks'; import { IDL_LSP_LOG } from '@idl/logger'; import { LoadTask } from '@idl/schemas/tasks'; @@ -109,7 +110,7 @@ export const ON_DOCUMENT_FORMATTING = async ( const tokens = await IDL_INDEX.getParsedProCode(info.fsPath, info.code); // format - formatted = Assembler(tokens, config); + formatted = Assembler(tokens, new CancellationToken(), config); // remove info.fsPath from memory cache IDL_INDEX.tokensByFile.remove(info.fsPath); diff --git a/libs/vscode/server/src/lib/events/notebooks/on-did-change-notebook.ts b/libs/vscode/server/src/lib/events/notebooks/on-did-change-notebook.ts index 4ee5ed2d1..bf5bb9730 100644 --- a/libs/vscode/server/src/lib/events/notebooks/on-did-change-notebook.ts +++ b/libs/vscode/server/src/lib/events/notebooks/on-did-change-notebook.ts @@ -48,7 +48,7 @@ export const ON_DID_CHANGE_NOTEBOOK = async ( const fsPath = GetFSPath(notebook.uri); // index file - await IDL_INDEX.getParsedNotebook(fsPath, idlNotebook); + await IDL_INDEX.parseAndTrackNotebook(fsPath, idlNotebook); // send problems SendNotebookProblems(notebook); diff --git a/libs/vscode/server/src/lib/events/notebooks/on-did-open-notebook.ts b/libs/vscode/server/src/lib/events/notebooks/on-did-open-notebook.ts index a3634975e..722e53384 100644 --- a/libs/vscode/server/src/lib/events/notebooks/on-did-open-notebook.ts +++ b/libs/vscode/server/src/lib/events/notebooks/on-did-open-notebook.ts @@ -41,7 +41,7 @@ export const ON_DID_OPEN_NOTEBOOK = async (notebook: NotebookDocument) => { const fsPath = GetFSPath(notebook.uri); // index file - await IDL_INDEX.getParsedNotebook(fsPath, idlNotebook); + await IDL_INDEX.parseAndTrackNotebook(fsPath, idlNotebook); // send problems SendNotebookProblems(notebook); diff --git a/libs/workers/parsing/src/lib/lsp-worker-thread-pool.interface.ts b/libs/workers/parsing/src/lib/lsp-worker-thread-pool.interface.ts index cfc01601f..cc4a7326e 100644 --- a/libs/workers/parsing/src/lib/lsp-worker-thread-pool.interface.ts +++ b/libs/workers/parsing/src/lib/lsp-worker-thread-pool.interface.ts @@ -6,7 +6,10 @@ import { PayloadFromLSPWorker, PayloadToLSPWorker, } from './lsp-worker-thread.payloads.interface'; -import { ILSPWorkerWorkerIO } from './lsp-workerio.interface'; +import { + ILSPWorkerWorkerIO, + ILSPWorkerWorkerIOPostAndReceiveMessageResult, +} from './lsp-workerio.interface'; /** * Strictly typed interface for sending and receiving messages to a pool of @@ -24,7 +27,7 @@ export interface ILSPWorkerThreadPool<_Message extends LSPWorkerThreadMessage> type: T, payload: PayloadToLSPWorker, options?: IPostMessageOptions - ): Promise>; + ): ILSPWorkerWorkerIOPostAndReceiveMessageResult; /** * Sends message to all worker threads diff --git a/libs/workers/parsing/src/lib/lsp-worker-thread.payloads.interface.ts b/libs/workers/parsing/src/lib/lsp-worker-thread.payloads.interface.ts index fee3fbf9f..c376b4670 100644 --- a/libs/workers/parsing/src/lib/lsp-worker-thread.payloads.interface.ts +++ b/libs/workers/parsing/src/lib/lsp-worker-thread.payloads.interface.ts @@ -1,3 +1,4 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { PayloadFromWorkerBaseMessage, WorkerIOBaseMessage, @@ -146,5 +147,6 @@ export type PayloadFromLSPWorker = * Callback handler for LSP worker thread */ export type LSPMessageHandler = ( - payload: PayloadToLSPWorker + payload: PayloadToLSPWorker, + token: CancellationToken ) => Promise>; diff --git a/libs/workers/parsing/src/lib/lsp-workerio.interface.ts b/libs/workers/parsing/src/lib/lsp-workerio.interface.ts index da7113db4..d7a83373b 100644 --- a/libs/workers/parsing/src/lib/lsp-workerio.interface.ts +++ b/libs/workers/parsing/src/lib/lsp-workerio.interface.ts @@ -1,4 +1,4 @@ -import { IWorkerIO } from '@idl/workers/workerio'; +import { IPostAndReceiveMessageResult, IWorkerIO } from '@idl/workers/workerio'; import { Subject } from 'rxjs'; import { LSPWorkerThreadMessage } from './lsp-worker-thread.messages.interface'; @@ -7,6 +7,18 @@ import { PayloadToLSPWorker, } from './lsp-worker-thread.payloads.interface'; +/** + * Response when we are waiting for a message + */ +export interface ILSPWorkerWorkerIOPostAndReceiveMessageResult< + _Message extends LSPWorkerThreadMessage +> extends IPostAndReceiveMessageResult<_Message> { + /** + * Response from the server + */ + response: Promise>; +} + /** * Interface for our WorkerIO so that we have something we can * correctly type which we do by creating an extension of this @@ -31,7 +43,7 @@ export interface ILSPWorkerWorkerIO<_Message extends LSPWorkerThreadMessage> type: T, payload: PayloadToLSPWorker, timeout?: number - ): Promise>; + ): ILSPWorkerWorkerIOPostAndReceiveMessageResult; /** * Subscribe to all messages with the same ID from any worker diff --git a/libs/workers/workerio/src/index.ts b/libs/workers/workerio/src/index.ts index 475261345..231f0d474 100644 --- a/libs/workers/workerio/src/index.ts +++ b/libs/workers/workerio/src/index.ts @@ -3,6 +3,7 @@ export * from './lib/messages/workerio.payloads.interface'; export * from './lib/workerio.class'; export * from './lib/workerio.class.interface'; export * from './lib/workerio.interface'; +export * from './lib/workerio-cancellation-token.class'; export * from './lib/workerio-client.class'; export * from './lib/workerio-client.class.interface'; export * from './lib/workerio-pool.class'; diff --git a/libs/workers/workerio/src/lib/messages/workerio.messages.interface.ts b/libs/workers/workerio/src/lib/messages/workerio.messages.interface.ts index 858b374f8..3ab7dfb16 100644 --- a/libs/workers/workerio/src/lib/messages/workerio.messages.interface.ts +++ b/libs/workers/workerio/src/lib/messages/workerio.messages.interface.ts @@ -1,3 +1,8 @@ +/** + * Cancel/interrupt pending work + */ +export type CancelMessage = 'cancel'; + /** * Error running process */ @@ -16,4 +21,8 @@ export type UnhandledError = 'unhandled-error'; /** * Base messages that are always supported with workerio */ -export type WorkerIOBaseMessage = ErrorMessage | LogMessage | UnhandledError; +export type WorkerIOBaseMessage = + | CancelMessage + | ErrorMessage + | LogMessage + | UnhandledError; diff --git a/libs/workers/workerio/src/lib/messages/workerio.payloads.interface.ts b/libs/workers/workerio/src/lib/messages/workerio.payloads.interface.ts index cdecefa4b..c1558a029 100644 --- a/libs/workers/workerio/src/lib/messages/workerio.payloads.interface.ts +++ b/libs/workers/workerio/src/lib/messages/workerio.payloads.interface.ts @@ -1,4 +1,5 @@ import { + CancelMessage, ErrorMessage, WorkerIOBaseMessage, } from './workerio.messages.interface'; @@ -6,7 +7,8 @@ import { /** * What is the response from our worker thread */ -export type PayloadToWorkerBaseMessage<_Message = WorkerIOBaseMessage> = any; +export type PayloadToWorkerBaseMessage<_Message = WorkerIOBaseMessage> = + _Message extends CancelMessage ? { messageId: string } : any; /** * What is the response from our worker thread diff --git a/libs/workers/workerio/src/lib/workerio-cancellation-token.class.ts b/libs/workers/workerio/src/lib/workerio-cancellation-token.class.ts new file mode 100644 index 000000000..d4b25bd4f --- /dev/null +++ b/libs/workers/workerio/src/lib/workerio-cancellation-token.class.ts @@ -0,0 +1,47 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; + +import { WorkerIOBaseMessage } from './messages/workerio.messages.interface'; +import { WorkerIO } from './workerio.class'; + +/** + * Cancellation token for workers that manages sending a cancellation + * message to a worker if cancelled + */ +export class WorkerIOCancellationToken extends CancellationToken { + /** + * The worker to send a cancellation message to + */ + private io: WorkerIO; + + /** + * The ID of the worker to send a message to + */ + private workerId: string; + + /** + * The ID of the message that we cancel + */ + private messageId: string; + + constructor(io: WorkerIO, workerId: string, messageId: string) { + super(); + + // set properties + this.io = io; + this.workerId = workerId; + this.messageId = messageId; + } + + /** + * Cancels our token + */ + cancel() { + super.cancel(); + + // send message + this.io.postMessage(this.workerId, 'cancel', { messageId: this.messageId }); + + // delete references to clean up + delete this.io; + } +} diff --git a/libs/workers/workerio/src/lib/workerio-client.class.interface.ts b/libs/workers/workerio/src/lib/workerio-client.class.interface.ts index 95cfc331b..068a4fb19 100644 --- a/libs/workers/workerio/src/lib/workerio-client.class.interface.ts +++ b/libs/workers/workerio/src/lib/workerio-client.class.interface.ts @@ -1,3 +1,5 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; + import { ErrorMessage, UnhandledError, @@ -15,7 +17,7 @@ export interface IWorkerIOClient<_Message extends string> { */ on( message: T, - promiseGenerator: (arg: any) => Promise + promiseGenerator: (arg: any, cancel: CancellationToken) => Promise ): void; /** diff --git a/libs/workers/workerio/src/lib/workerio-client.class.ts b/libs/workers/workerio/src/lib/workerio-client.class.ts index 3a5155677..c8144b063 100644 --- a/libs/workers/workerio/src/lib/workerio-client.class.ts +++ b/libs/workers/workerio/src/lib/workerio-client.class.ts @@ -1,13 +1,18 @@ +import { CancellationToken } from '@idl/cancellation-tokens'; import { ObjectifyError } from '@idl/logger'; import { SimplePromiseQueue } from '@idl/shared'; import { MessagePort } from 'worker_threads'; import { + CancelMessage, ErrorMessage, LogMessage, UnhandledError, } from './messages/workerio.messages.interface'; -import { PayloadFromWorkerBaseMessage } from './messages/workerio.payloads.interface'; +import { + PayloadFromWorkerBaseMessage, + PayloadToWorkerBaseMessage, +} from './messages/workerio.payloads.interface'; import { IMessageFromWorker, ISentMessageToWorker } from './workerio.interface'; import { IWorkerIOClient } from './workerio-client.class.interface'; @@ -23,9 +28,12 @@ export class WorkerIOClient<_Message extends string> /** Events that we listen to */ private events: { - [P in _Message]?: (payload: any) => Promise; + [P in _Message]?: (payload: any, cancel: CancellationToken) => Promise; } = {}; + /** Track cancellation tokens for requests */ + private cancels: { [key: string]: CancellationToken } = {}; + /** Promise queue to throttle number of things we do at once */ private queue: SimplePromiseQueue; @@ -60,7 +68,10 @@ export class WorkerIOClient<_Message extends string> } /** Subscribe to messages from our parent thread */ - on(message: _Message, promiseGenerator: (arg: any) => Promise) { + on( + message: _Message, + promiseGenerator: (arg: any, cancel: CancellationToken) => Promise + ) { this.events[message] = promiseGenerator; } @@ -93,6 +104,15 @@ export class WorkerIOClient<_Message extends string> this._postMessage(outgoing as IMessageFromWorker<_Message>); } + /** + * Cancel message by ID + */ + cancel(messageId: string) { + if (messageId in this.cancels) { + this.cancels[messageId].cancel(); + } + } + /** * Send message to parent process */ @@ -122,12 +142,29 @@ export class WorkerIOClient<_Message extends string> * Actual message handler, separate scope than even callback */ private async _handleMessage(message: ISentMessageToWorker<_Message>) { + // check for cancellation which we handle immediately + if (message.type === 'cancel') { + // indicate that our process should be cancelled + this.cancel( + (message.payload as PayloadToWorkerBaseMessage).messageId + ); + + // return and dont process below + return; + } + await this.queue.add(async () => { if (message.type in this.events) { + // create a new cancellation token + const cancel = new CancellationToken(); + + // track by our message ID + this.cancels[message._id] = cancel; + // handle errors which makes the worker threads, assuming everything goes through here, invincible! try { // do something based on our message - const res = await this.events[message.type](message.payload); + const res = await this.events[message.type](message.payload, cancel); // check if we need to respond, otherwise we will be silent if (!message.noResponse) { @@ -146,6 +183,9 @@ export class WorkerIOClient<_Message extends string> message._id ); } + + // clean up cancellation + delete this.cancels[message._id]; } }); } diff --git a/libs/workers/workerio/src/lib/workerio-pool.class.interface.ts b/libs/workers/workerio/src/lib/workerio-pool.class.interface.ts index 2446116ac..7d4516df7 100644 --- a/libs/workers/workerio/src/lib/workerio-pool.class.interface.ts +++ b/libs/workers/workerio/src/lib/workerio-pool.class.interface.ts @@ -1,12 +1,12 @@ import { LogManager } from '@idl/logger'; import { Subject } from 'rxjs'; -import { - PayloadFromWorkerBaseMessage, - PayloadToWorkerBaseMessage, -} from './messages/workerio.payloads.interface'; +import { PayloadToWorkerBaseMessage } from './messages/workerio.payloads.interface'; import { IWorkerIO } from './workerio.class.interface'; -import { IMessageFromWorker } from './workerio.interface'; +import { + IMessageFromWorker, + IPostAndReceiveMessageResult, +} from './workerio.interface'; import { IPostMessageOptions } from './workerio-pool.interface'; /** @@ -30,7 +30,7 @@ export interface IWorkerIOPool<_Message extends string> { type: T, payload: PayloadToWorkerBaseMessage, options?: IPostMessageOptions - ): Promise>; + ): IPostAndReceiveMessageResult; /** * Sends message to all worker threads diff --git a/libs/workers/workerio/src/lib/workerio-pool.class.ts b/libs/workers/workerio/src/lib/workerio-pool.class.ts index efc4da7c8..f0a9275fa 100644 --- a/libs/workers/workerio/src/lib/workerio-pool.class.ts +++ b/libs/workers/workerio/src/lib/workerio-pool.class.ts @@ -2,12 +2,12 @@ import { LogManager } from '@idl/logger'; import { nanoid } from 'nanoid'; import { Worker } from 'worker_threads'; -import { - PayloadFromWorkerBaseMessage, - PayloadToWorkerBaseMessage, -} from './messages/workerio.payloads.interface'; +import { PayloadToWorkerBaseMessage } from './messages/workerio.payloads.interface'; import { WorkerIO } from './workerio.class'; -import { IMessageToWorker } from './workerio.interface'; +import { + IMessageToWorker, + IPostAndReceiveMessageResult, +} from './workerio.interface'; import { IWorkerIOPool } from './workerio-pool.class.interface'; import { IPostMessageOptions } from './workerio-pool.interface'; @@ -75,7 +75,7 @@ export class WorkerIOPool<_Message extends string> type: T, payload: PayloadToWorkerBaseMessage, options?: IPostMessageOptions - ): Promise> { + ): IPostAndReceiveMessageResult { // get the next ID and shift our array let next = this.ids.shift() as string; this.ids.push(next); diff --git a/libs/workers/workerio/src/lib/workerio.class.interface.ts b/libs/workers/workerio/src/lib/workerio.class.interface.ts index 7f8d02ad3..a1d1c02ef 100644 --- a/libs/workers/workerio/src/lib/workerio.class.interface.ts +++ b/libs/workers/workerio/src/lib/workerio.class.interface.ts @@ -5,6 +5,7 @@ import { PayloadFromWorkerBaseMessage, PayloadToWorkerBaseMessage, } from './messages/workerio.payloads.interface'; +import { IPostAndReceiveMessageResult } from './workerio.interface'; /** * Interface for our WorkerIO so that we have something we can @@ -32,7 +33,7 @@ export interface IWorkerIO<_Message extends string> { type: T, payload: PayloadToWorkerBaseMessage, timeout?: number - ): Promise>; + ): IPostAndReceiveMessageResult; /** * Subscribe to all messages with the same ID from any worker diff --git a/libs/workers/workerio/src/lib/workerio.class.ts b/libs/workers/workerio/src/lib/workerio.class.ts index a422767d1..e4adab1b8 100644 --- a/libs/workers/workerio/src/lib/workerio.class.ts +++ b/libs/workers/workerio/src/lib/workerio.class.ts @@ -12,7 +12,9 @@ import { IMessageFromWorker, IMessagePromise, IMessageToWorker, + IPostAndReceiveMessageResult, } from './workerio.interface'; +import { WorkerIOCancellationToken } from './workerio-cancellation-token.class'; /** * Object class that manages sending and receiving messages from worker threads in a @@ -21,14 +23,17 @@ import { export class WorkerIO<_Message extends string> implements IWorkerIO<_Message> { /** Worker threads by worker ID */ workers: { [key: string]: Worker } = {}; + /** Promises for messages by ID (i.e. waiting for a response) */ private messages: { [key: string]: IMessagePromise } = {}; + /** Message subscriptions we have created. Keep references to clean up */ private subscriptions: { [key: string]: { [key: string]: Subject>; }; } = {}; + /** Message subscriptions we have created. Keep references to clean up */ private globalSubscriptions: { [key: string]: Subject>; @@ -313,7 +318,7 @@ export class WorkerIO<_Message extends string> implements IWorkerIO<_Message> { type: T, payload: PayloadToWorkerBaseMessage, timeout?: number - ): Promise> { + ): IPostAndReceiveMessageResult { // make sure we have an ID to send a message to if (!(workerId in this.workers)) { throw new Error( @@ -329,21 +334,28 @@ export class WorkerIO<_Message extends string> implements IWorkerIO<_Message> { // make a unique ID for tracking the return message const idReturn = nanoid(); + // create cancellation token + const cancel = new WorkerIOCancellationToken(this, workerId, idReturn); + // make our new promise - return new Promise>((resolve, reject) => { - // save our callback information - this.messages[idReturn] = { - resolve: resolve, - reject: reject, - timeout: - timeout !== undefined - ? setTimeout(() => reject('Request exceeded timeout'), timeout) - : undefined, - }; - - // send our message - this._postMessage(workerId, msg, idReturn); - }); + const response = new Promise>( + (resolve, reject) => { + // save our callback information + this.messages[idReturn] = { + resolve: resolve, + reject: reject, + timeout: + timeout !== undefined + ? setTimeout(() => reject('Request exceeded timeout'), timeout) + : undefined, + }; + + // send our message + this._postMessage(workerId, msg, idReturn); + } + ); + + return { token: cancel, response }; } /** diff --git a/libs/workers/workerio/src/lib/workerio.interface.ts b/libs/workers/workerio/src/lib/workerio.interface.ts index a46576488..1b8586a69 100644 --- a/libs/workers/workerio/src/lib/workerio.interface.ts +++ b/libs/workers/workerio/src/lib/workerio.interface.ts @@ -1,3 +1,6 @@ +import { PayloadFromWorkerBaseMessage } from './messages/workerio.payloads.interface'; +import { WorkerIOCancellationToken } from './workerio-cancellation-token.class'; + export interface IMessagePromise { resolve: (data: any) => void; reject: (data: any) => void; @@ -56,3 +59,17 @@ export interface IMessageFromWorker<_Message extends string> */ payload: any; } + +/** + * Response when we are waiting for a message + */ +export interface IPostAndReceiveMessageResult<_Message extends string> { + /** + * Cancellation token + */ + token: WorkerIOCancellationToken; + /** + * Response from the server + */ + response: Promise>; +} diff --git a/package-lock.json b/package-lock.json index e6e338a31..596a4421d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "idl-for-vscode", - "version": "3.2.1", + "version": "3.2.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "idl-for-vscode", - "version": "3.2.1", + "version": "3.2.2", "license": "MIT", "dependencies": { "@analytics-debugger/ga4mp": "^0.0.8", diff --git a/package.json b/package.json index d94adb7a1..e653a34f3 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "idl-for-vscode", "displayName": "%packageJSON.displayName%", "description": "%packageJSON.description%", - "version": "3.2.1", + "version": "3.2.2", "publisher": "idl", "license": "MIT", "encryption": "yes", diff --git a/tsconfig.base.json b/tsconfig.base.json index 6d6441adb..999bef91e 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -28,6 +28,7 @@ "@idl/assembling/tree-handlers": [ "libs/assembling/tree-handlers/src/index.ts" ], + "@idl/cancellation-tokens": ["libs/cancellation-tokens/src/index.ts"], "@idl/data-types/core": ["libs/data-types/core/src/index.ts"], "@idl/data-types/tasks": ["libs/data-types/tasks/src/index.ts"], "@idl/debug-types": ["libs/debug-types/src/index.ts"],