Merge branch 'develop' into main

.gitignore

@@ -19,6 +19,8 @@
 
 # test helper file that we auto-generate
 parse-test/
+isolate*.log
+perf.txt
 
 # IDE - VSCode
 .vscode/*

.vscodeignore

@@ -51,13 +51,15 @@ tmp/**
 angular.json
 compile_speed_test.pro
 decorate-angular-cli.js
+isolate*.log
 jest.config.ts
 jest.preset.js
 jest.preset.ts
 migrations.json
 nx.json
 NX.md
 package-lock.json
+perf.txt
 problems.json
 tsconfig.base.json
 webpack.base.config.js

CHANGELOG.md

@@ -4,6 +4,52 @@ All notable changes to the "idl" extension will be documented in this file.
 
 For much more detail on incremental work for large features, see our [developer notes](./extension/docs/developer/dev-notes/README.md).
 
+## 3.1.1 Unreleased
+
+Preview release of IDL Notebooks! This is a first pass at adding notebook support for IDL which is independent from Jupyter.
+
+> Notebooks are a preview feature and, based on early adopter feedback, will likely change
+
+Here are some of the features that notebooks bring:
+
+- Notebook files should end with the extension ".idlnb" which are managed and rendered by the IDL extension.
+
+  - At this time, notebooks only save markdown and code cells. We do not save/restore outputs from cells, but this will be coming in the future.
+
+  - Notebooks do not embed graphics as a part of the notebook. However, this will likely be coming in the future.
+
+  - Notebooks do not embed widgets.
+
+- Notebooks support highlighting, problem reporting, hover help, auto-complete, and go-to-definition
+
+- Basic ability to run cells.
+
+  - This does **NOT** include debugging and interactively stepping through code at this point in time
+
+  - Because we do not have debugging, after each cell is executed we issue a `retall` command to IDL to make sure that we are at the top-level
+
+- Only support for IDL code (no Python or other languages at this time)
+
+## 3.1.0 July 2023
+
+Fixed an issue where garbage collection was not turning on and caused out-of-memory errors which led to language server crashes
+
+Added a cache to reduce memory usage for worker threads (large workspaces with 300+ files should use about 50% less RAM). Coincidentally this also improved performance as well.
+
+On startup, a new log statement prints to show the state of garbage collection: `idl-lsp info Garbage collection enabled: true`
+
+Improved the on-enter commands that automatically continue comment blocks as you type within them. They were close, but not quite there are some rules conflicted with one another so they didn't work right.
+
+Fixed a major performance issue when doing a quick parse of PRO code. For almost 8000 files on a developer machine, we went from 13 seconds down to 3 with a parse rate of 650k lines/second!
+
+For quick parsing, we now also extract docs for your code to give a better hover help and auto-complete user experience with a low impact to performance.
+
+Fixed a bug where the IDL icon was pointing to the wrong file for light themes
+
+Added a new button to the IDL sidebar which allows you to easily specify the location of IDL without needing to rummage through the command palette
+
+Fixed some import bugs if you have an older version of node.js on your path where "performance" was undefined.
+
 ## 3.0.6 - July 2023
 
 Improved message when language server crashes and a button that opens documentation for workarounds for the memory problem

apps/README.md

@@ -28,10 +28,12 @@ Here's what we have so far
 
 - **parsing-worker**: Node.js worker thread to index IDL code in the background.
 
+- **performance**: Node.js app meant to be a place where we can test the speed of parsing without needing to open VSCode (also, cross platform)
+
 - **server**: The VS Code language server implementation for IDL.
 
 - **test-tokenizer**: An app used for testing the tokenizer and creating our automated tests.
 
 - **tmlang-maker**: Live-reloading application to convert our YAML language config file to plist for VSCode to digest.
 
-- **vscode-e2e-runner**: Basic application that runs our VSCode integration tests
+- **vscode-e2e-runner**: Basic application that runs our VSCode integration tests which are located in `client-e2e`

apps/client-e2e/src/tests/notebooks/_notebook-runner.ts

@@ -0,0 +1,46 @@
+import { Logger } from '@idl/logger';
+
+import { Runner } from '../runner.class';
+import { RunNotebookRestart } from './notebook-restart';
+import { RunNotebookStop } from './notebook-stop';
+import { RunTestNotebook } from './run-test-notebook';
+import { SaveAndClearNotebook } from './save-and-clear-output';
+
+/*
+ * Logger to be used for tests related to debugging
+ */
+export const NOTEBOOK_TEST_LOGGER = new Logger(
+  'tests-notebook',
+  false,
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  () => {}
+);
+
+/**
+ * Test runner for debugging
+ */
+export const NOTEBOOK_RUNNER = new Runner(NOTEBOOK_TEST_LOGGER);
+
+NOTEBOOK_RUNNER.addTest({
+  name: 'Run notebook that tests everything',
+  fn: RunTestNotebook,
+  critical: true,
+});
+
+NOTEBOOK_RUNNER.addTest({
+  name: 'Save output and reload',
+  fn: SaveAndClearNotebook,
+  critical: true,
+});
+
+NOTEBOOK_RUNNER.addTest({
+  name: 'Stop does the right thing',
+  fn: RunNotebookStop,
+  critical: true,
+});
+
+NOTEBOOK_RUNNER.addTest({
+  name: 'Reset does the right thing',
+  fn: RunNotebookRestart,
+  critical: true,
+});

apps/client-e2e/src/tests/notebooks/helpers/compare-cells.interface.ts

@@ -0,0 +1,13 @@
+type MimeTypes = 'text/html' | 'text/plain' | 'image/png';
+
+/**
+ * Output for cells
+ */
+export interface ICompareCells {
+  /** Cell we are comparing against */
+  idx: number;
+  /** did we succeed */
+  success: boolean;
+  /** Retrieve all of the cell mimetypes */
+  mimeTypes: MimeTypes[];
+}

apps/client-e2e/src/tests/notebooks/helpers/compare-cells.ts

@@ -0,0 +1,50 @@
+import expect from 'expect';
+import * as vscode from 'vscode';
+
+import { ICompareCells } from './compare-cells.interface';
+
+/**
+ * Compares cells from a notebook against what we expect them to be
+ */
+export function CompareCells(
+  nb: vscode.NotebookDocument,
+  cellOutput: ICompareCells[]
+) {
+  /**
+   * get notebook cells
+   */
+  const cells = nb.getCells();
+
+  // validate outputs from all known cells
+  for (let i = 0; i < cellOutput.length; i++) {
+    /**
+     * Get expected cell output
+     */
+    const expected = cellOutput[i];
+
+    // debug log
+    console.log(`  Comparing known output cell "${expected.idx}"`);
+
+    /**
+     * get notebook cell
+     */
+    const nbCell = cells[expected.idx];
+
+    // make sure it exists
+    expect(nbCell).not.toBeUndefined();
+
+    // validate success if we have it
+    if (nbCell.executionSummary.success !== undefined) {
+      expect(nbCell.executionSummary.success).toEqual(expected.success);
+    }
+
+    // get output mime types
+    let mimes: string[] = [];
+    for (let j = 0; j < nbCell.outputs.length; j++) {
+      mimes = mimes.concat(nbCell.outputs[j].items.map((item) => item.mime));
+    }
+
+    // validate cell mime types
+    expect(mimes).toEqual(expected.mimeTypes);
+  }
+}

apps/client-e2e/src/tests/notebooks/helpers/verify-empty.ts

@@ -0,0 +1,35 @@
+import expect from 'expect';
+import * as vscode from 'vscode';
+
+/**
+ * Verifies that a notebook is empty
+ */
+export function VerifyEmpty(nb: vscode.NotebookDocument) {
+  /**
+   * get notebook cells
+   */
+  const cells = nb.getCells();
+
+  // track number of output cells
+  let nOut = 0;
+
+  // validate outputs from all known cells
+  for (let i = 0; i < cells.length; i++) {
+    /**
+     * get notebook cell
+     */
+    const nbCell = cells[i];
+
+    // get output mime types
+    let mimes: string[] = [];
+    for (let j = 0; j < nbCell.outputs.length; j++) {
+      mimes = mimes.concat(nbCell.outputs[j].items.map((item) => item.mime));
+    }
+
+    // track total output
+    nOut += mimes.length;
+  }
+
+  // validate cell mime types
+  expect(nOut).toEqual(0);
+}

apps/client-e2e/src/tests/notebooks/notebook-restart.ts

@@ -0,0 +1,65 @@
+import { GetExtensionPath, IDL_COMMANDS, Sleep } from '@idl/shared';
+import { OpenNotebookInVSCode, VSCODE_COMMANDS } from '@idl/vscode/shared';
+import expect from 'expect';
+import * as vscode from 'vscode';
+
+import { RunnerFunction } from '../runner.interface';
+import { CompareCells } from './helpers/compare-cells';
+import { ICompareCells } from './helpers/compare-cells.interface';
+
+/**
+ * Types of outputs from cells that we expect to have
+ */
+export const CELL_OUTPUT: ICompareCells[] = [
+  {
+    idx: 0,
+    success: false,
+    mimeTypes: [],
+  },
+];
+
+/**
+ * Function that verifies that we can do basic debugging of IDL sessions
+ * and launch a new debugging session.
+ */
+export const RunNotebookRestart: RunnerFunction = async (init) => {
+  /**
+   * Get the file we are going to open
+   */
+  const file = GetExtensionPath('idl/test/client-e2e/stop-notebook.idlnb');
+
+  /**
+   * Open the notebook
+   */
+  const nb = await OpenNotebookInVSCode(file);
+
+  // trigger a run to start IDL
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_RUN_ALL);
+
+  // make sure launched
+  expect(init.notebooks.controller.isLaunched()).toBeTruthy();
+
+  // trigger a run
+  vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_RUN_ALL);
+
+  // short pause
+  await Sleep(100);
+
+  // stop execution
+  await vscode.commands.executeCommand(IDL_COMMANDS.NOTEBOOKS.RESET);
+
+  // short pause
+  await Sleep(100);
+
+  // make sure stopped
+  expect(init.notebooks.controller.isLaunched()).toBeTruthy();
+
+  // clear outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_CLEAR_OUTPUTS);
+
+  // compare state
+  CompareCells(nb, CELL_OUTPUT);
+
+  // clear any existing outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.CLOSE_EDITOR);
+};

apps/client-e2e/src/tests/notebooks/notebook-stop.ts

@@ -0,0 +1,65 @@
+import { GetExtensionPath, IDL_COMMANDS, Sleep } from '@idl/shared';
+import { OpenNotebookInVSCode, VSCODE_COMMANDS } from '@idl/vscode/shared';
+import expect from 'expect';
+import * as vscode from 'vscode';
+
+import { RunnerFunction } from '../runner.interface';
+import { CompareCells } from './helpers/compare-cells';
+import { ICompareCells } from './helpers/compare-cells.interface';
+
+/**
+ * Types of outputs from cells that we expect to have
+ */
+export const CELL_OUTPUT: ICompareCells[] = [
+  {
+    idx: 0,
+    success: false,
+    mimeTypes: [],
+  },
+];
+
+/**
+ * Function that verifies that we can do basic debugging of IDL sessions
+ * and launch a new debugging session.
+ */
+export const RunNotebookStop: RunnerFunction = async (init) => {
+  /**
+   * Get the file we are going to open
+   */
+  const file = GetExtensionPath('idl/test/client-e2e/stop-notebook.idlnb');
+
+  /**
+   * Open the notebook
+   */
+  const nb = await OpenNotebookInVSCode(file);
+
+  // trigger a run to start IDL
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_RUN_ALL);
+
+  // make sure launched
+  expect(init.notebooks.controller.isLaunched()).toBeTruthy();
+
+  // trigger a run
+  vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_RUN_ALL);
+
+  // short pause
+  await Sleep(100);
+
+  // stop execution
+  await vscode.commands.executeCommand(IDL_COMMANDS.NOTEBOOKS.STOP);
+
+  // short pause
+  await Sleep(100);
+
+  // make sure stopped
+  expect(init.notebooks.controller.isLaunched()).toBeFalsy();
+
+  // clear outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.NOTEBOOK_CLEAR_OUTPUTS);
+
+  // compare state
+  CompareCells(nb, CELL_OUTPUT);
+
+  // clear any existing outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.CLOSE_EDITOR);
+};