Merge branch 'develop' into main

.vscode/settings.json

@@ -15,6 +15,7 @@
     "bendrep",
     "bendswitch",
     "bendwhile",
+    "bubbleanimation",
     "defint",
     "Deref",
     "Doesnt",
@@ -40,6 +41,7 @@
     "idlwv",
     "imagefromuri",
     "INCREMENTERS",
+    "lineanimation",
     "minilog",
     "mlog",
     "nbands",

CHANGELOG.md

@@ -42,6 +42,28 @@ Here are some of the features that notebooks bring:
 
   - After each cell is executed we issue a `retall` command to make sure that we are at the top-level and not stopped in a weird state
 
+## 3.2.1 August 2023
+
+Notebook key behavior change: If you are running one or more cells, and there is an error from IDL for any cell being executed, all pending cells are cleared and not executed. This makes sure that, if later cells depend on earlier ones, that you don't have cascading failures.
+
+Resolve a problem with the language server where you can get into an infinite loop trying to resolve include statements in IDL when they end up including one another
+
+Add a potential fix for always-increasing loops when doing change detection for parsed files
+
+Fix an issue where, if you have the same folder or sub folder on IDL's path and in an open workspace, then you would get duplicate problems being reported. We now get the unique files from all folders on startup at once to resolve this.
+
+Fixed a bug where notebook cells would sometimes process in the wrong worker thread, causing inconsistencies with things like semantic tokens.
+
+Fixed a bug where semantic tokens (highlighting static class references) was wrong when tokens were built out of order from top down and left to right
+
+Added a new problem code that detects when the IDL include statement creates a circular include pattern
+
+Fixed a bug with semantic tokens in notebooks where the text would be highlighted as a semantic token in cells that don't have any semantic tokens
+
+When you click into the outputs from an IDL notebook that is being rendered by the IDL Extension, a blue outline appears over what you have selected to match VSCode's behavior
+
+Added logic when retrieving outputs from IDL to be more backwards compatible. As it was, you needed at least IDL 8.8.3, but it now supports pre-8.8.3 at your own risk.
+
 ## 3.2.0 August 2023
 
 When the language server does not use a full parse, extract structure definitions. Before, this was a logic gap (and made the code faster), but they should be correctly resolved now with minimal performance hits.

apps/client-e2e/src/tests/interactions/notebooks-interact-right.ts

@@ -83,13 +83,13 @@ export const NotebooksInteractRight: RunnerFunction = async (init) => {
     },
   };
 
-  // verify semantic tokens
+  // verify semantic tokens (have none in NB, so get none)
   expect(
     await init.client.client.sendRequest(
       'textDocument/semanticTokens/full',
       tokenParams
     )
-  ).toBeTruthy();
+  ).toBeNull();
 
   // short pause
   await Sleep(250);

apps/client-e2e/src/tests/interactions/pro-code-interacts-right.ts

@@ -76,13 +76,13 @@ export const ProCodeInteractRight: RunnerFunction = async (init) => {
     },
   };
 
-  // verify semantic tokens
+  // verify semantic tokens (have none in code, so get none)
   expect(
     await init.client.client.sendRequest(
       'textDocument/semanticTokens/full',
       tokenParams
     )
-  ).toBeTruthy();
+  ).toBeNull();
 
   // short pause
   await Sleep(250);

apps/client-e2e/src/tests/notebooks/_notebook-runner.ts

@@ -4,8 +4,10 @@ import { Runner } from '../runner.class';
 import { NotebookFormats_1_0_0 } from './notebook-formats-1.0.0';
 import { NotebookFormats_2_0_0 } from './notebook-formats-2.0.0';
 import { NotebookProblemsTrackRight } from './notebook-problems-track-right';
-import { RunNotebookRestart } from './notebook-restart';
+import { RunNotebookReset } from './notebook-reset';
 import { RunNotebookStop } from './notebook-stop';
+import { RunENVIMessageListenerTestNotebook } from './run-envi-message-listener-test-notebook';
+import { RunProblemNotebooks } from './run-problem-notebooks';
 import { RunTestENVIMapNotebook } from './run-test-envi-map-notebook';
 import { RunTestENVINotebook } from './run-test-envi-notebook';
 import { RunTestNotebook } from './run-test-notebook';
@@ -39,11 +41,17 @@ NOTEBOOK_RUNNER.addTest({
 });
 
 NOTEBOOK_RUNNER.addTest({
-  name: 'Run notebook that tests everything',
+  name: 'Run notebook that tests successes',
   fn: RunTestNotebook,
   critical: true,
 });
 
+NOTEBOOK_RUNNER.addTest({
+  name: 'Run notebooks that test problems are handled right',
+  fn: RunProblemNotebooks,
+  critical: true,
+});
+
 NOTEBOOK_RUNNER.addTest({
   name: 'Save output and reload',
   fn: SaveAndClearNotebook,
@@ -61,10 +69,16 @@ NOTEBOOK_RUNNER.addTest({
   fn: RunTestENVIMapNotebook,
 });
 
+// can get ENVI progress messages
+NOTEBOOK_RUNNER.addTest({
+  name: 'Notebooks can display progress messages from ENVI',
+  fn: RunENVIMessageListenerTestNotebook,
+});
+
 // reset goes first
 NOTEBOOK_RUNNER.addTest({
   name: 'Reset does the right thing',
-  fn: RunNotebookRestart,
+  fn: RunNotebookReset,
   critical: true,
 });
 

apps/client-e2e/src/tests/notebooks/helpers/compare-cells.interface.ts

@@ -13,7 +13,7 @@ export interface ICompareCellOutputs {
   /** Cell we are comparing against */
   idx: number;
   /** did we succeed */
-  success: boolean;
+  success: boolean | undefined;
   /** All of the cell mimetypes */
   mimeTypes: MimeTypes[];
 }

apps/client-e2e/src/tests/notebooks/helpers/compare-cells.ts

@@ -37,8 +37,10 @@ export function CompareCellOutputs(
     expect(nbCell).not.toBeUndefined();
 
     // validate success if we have it
-    if (nbCell.executionSummary.success !== undefined) {
-      expect(nbCell.executionSummary.success).toEqual(expected.success);
+    if (expected.success) {
+      expect(nbCell.executionSummary?.success).toBeTruthy();
+    } else {
+      expect(nbCell.executionSummary?.success).toBeFalsy();
     }
 
     // get output mime types

apps/client-e2e/src/tests/notebooks/helpers/run-notebook-and-compare-cells.ts

@@ -0,0 +1,68 @@
+import { NOTEBOOK_FOLDER } from '@idl/notebooks/shared';
+import { Sleep } from '@idl/shared';
+import { IDLNotebookController } from '@idl/vscode/notebooks';
+import { OpenNotebookInVSCode, VSCODE_COMMANDS } from '@idl/vscode/shared';
+import expect from 'expect';
+import { existsSync, rmSync } from 'fs';
+import * as vscode from 'vscode';
+
+import { CompareCellOutputs } from './compare-cells';
+import { ICompareCellOutputs } from './compare-cells.interface';
+
+/**
+ * helper function to:
+ *
+ * 1. Open notebook
+ * 2. Clear existing outputs
+ * 3. Run notebook
+ * 4. Compare expected outputs to actual outputs
+ * 5. Clear outputs
+ * 6. Close
+ */
+export async function RunNotebookAndCompareCells(
+  file: string,
+  cells: ICompareCellOutputs[],
+  controller: IDLNotebookController,
+  clear = true
+) {
+  // nuke .idl folder if it exists
+  if (existsSync(NOTEBOOK_FOLDER)) {
+    rmSync(NOTEBOOK_FOLDER, { recursive: true, force: true });
+  }
+
+  /**
+   * Open the notebook
+   */
+  const nb = await OpenNotebookInVSCode(file);
+
+  // clear any existing outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_CLEAR_OUTPUTS);
+
+  // save to disk
+  await nb.save();
+
+  // run all cells
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_RUN_ALL);
+
+  // make sure launched
+  expect(controller.isStarted()).toBeTruthy();
+
+  // short pause
+  await Sleep(100);
+
+  // compare cells
+  CompareCellOutputs(nb, cells);
+
+  if (clear) {
+    // clear outputs
+    await vscode.commands.executeCommand(
+      VSCODE_COMMANDS.NOTEBOOK_CLEAR_OUTPUTS
+    );
+  }
+
+  // save again
+  await nb.save();
+
+  // clear any existing outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.CLOSE_EDITOR);
+}

apps/client-e2e/src/tests/notebooks/notebook-restart.ts → apps/client-e2e/src/tests/notebooks/notebook-reset.ts

@@ -22,7 +22,7 @@ export const CELL_OUTPUT: ICompareCellOutputs[] = [
  * Function that verifies that we can do basic debugging of IDL sessions
  * and launch a new debugging session.
  */
-export const RunNotebookRestart: RunnerFunction = async (init) => {
+export const RunNotebookReset: RunnerFunction = async (init) => {
   /**
    * Get the file we are going to open
    */
@@ -56,12 +56,12 @@ export const RunNotebookRestart: RunnerFunction = async (init) => {
   // make sure stopped
   expect(init.notebooks.controller.isStarted()).toBeTruthy();
 
-  // clear outputs
-  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_CLEAR_OUTPUTS);
-
   // compare state
   CompareCellOutputs(nb, CELL_OUTPUT);
 
+  // clear outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_CLEAR_OUTPUTS);
+
   // clear any existing outputs
   await vscode.commands.executeCommand(VSCODE_COMMANDS.CLOSE_EDITOR);
 };

apps/client-e2e/src/tests/notebooks/notebook-stop.ts

@@ -56,9 +56,6 @@ export const RunNotebookStop: RunnerFunction = async (init) => {
   // make sure stopped
   expect(init.notebooks.controller.isStarted()).toBeFalsy();
 
-  // clear outputs
-  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_CLEAR_OUTPUTS);
-
   // compare state
   CompareCellOutputs(nb, CELL_OUTPUT);
 

apps/client-e2e/src/tests/notebooks/run-envi-message-listener-test-notebook.ts

@@ -0,0 +1,61 @@
+import { GetExtensionPath } from '@idl/shared';
+
+import { RunnerFunction } from '../runner.interface';
+import { ICompareCellOutputs } from './helpers/compare-cells.interface';
+import { RunNotebookAndCompareCells } from './helpers/run-notebook-and-compare-cells';
+
+/**
+ * Types of outputs from cells that we expect to have
+ */
+export const CELL_OUTPUT: ICompareCellOutputs[] = [
+  {
+    idx: 0,
+    success: true,
+    mimeTypes: [],
+  },
+  {
+    idx: 1,
+    success: true,
+    mimeTypes: [],
+  },
+  {
+    idx: 2,
+    success: true,
+    mimeTypes: ['text/plain'],
+  },
+  {
+    idx: 3,
+    success: true,
+    mimeTypes: [],
+  },
+  {
+    idx: 4,
+    success: true,
+    mimeTypes: [],
+  },
+  {
+    idx: 5,
+    success: true,
+    mimeTypes: [],
+  },
+  {
+    idx: 6,
+    success: true,
+    mimeTypes: ['text/plain'],
+  },
+];
+
+/**
+ * Function that runs basic tests for ENVI message listeners
+ */
+export const RunENVIMessageListenerTestNotebook: RunnerFunction = async (
+  init
+) => {
+  await RunNotebookAndCompareCells(
+    GetExtensionPath(
+      'idl/test/client-e2e/notebooks/envi-message-listener.idlnb'
+    ),
+    CELL_OUTPUT,
+    init.notebooks.controller
+  );
+};