Manually merge in develop for 3.2.0

.prettierignore

@@ -6,6 +6,7 @@
 /.vscode-test
 /dist
 /coverage
+/idl/test/client-e2e
 routines.json
 global.json
 extension/language/syntaxes/src/*

.vscode/settings.json

@@ -23,11 +23,22 @@
     "ENVIGLT",
     "enviraster",
     "ENVIRPC",
+    "EPSG",
     "FFIDL",
     "fflate",
     "Fuzzysort",
+    "geojsofromnuri",
+    "IDLJSON",
     "IDLNB",
+    "idlnotebook",
+    "idlnotebookimage_animationfrompngs",
+    "idlnotebookimage_animationfromuris",
+    "idlnotebookimage_fromuri",
+    "idlnotebookimage_png",
+    "idlnotebookmap",
+    "idlnotebookplot",
     "idlwv",
+    "imagefromuri",
     "INCREMENTERS",
     "minilog",
     "mlog",
@@ -44,6 +55,8 @@
     "Recurser",
     "Recursor",
     "RETALL",
+    "shapefile",
+    "shapefiles",
     "SPATIALREF",
     "stepover",
     "strictarr",
@@ -56,7 +69,13 @@
     "Uncaptured",
     "vsix",
     "workerio",
-    "wschanges"
+    "wschanges",
+    "xmax",
+    "xmin",
+    "xsize",
+    "ymax",
+    "ymin",
+    "ysize"
   ],
   "cSpell.enableFiletypes": ["!json"],
   "[yaml]": {
@@ -66,6 +85,6 @@
     "editor.formatOnSave": false
   },
   "[html]": {
-    "editor.formatOnSave": false
+    "editor.formatOnSave": true
   }
 }

CHANGELOG.md

@@ -4,6 +4,116 @@ All notable changes to the "idl" extension will be documented in this file.
 
 For much more detail on incremental work for large features, see our [developer notes](./extension/docs/developer/dev-notes/README.md).
 
+## Unreleased
+
+Preview release of IDL Notebooks! This is a first pass at adding notebook support for IDL which is independent from Jupyter.
+
+> Notebooks are a preview feature and, based on early adopter feedback, will likely change
+
+Here are some of the features that notebooks bring:
+
+- Notebook files should end with the extension ".idlnb" which are managed and rendered by the IDL extension.
+
+  - Notebooks support highlighting, problem reporting, hover help, auto-complete, go-to-definition, formatting, and semantic token highlighting.
+
+    > Pro tip: See the FORMATTING.md doc for information regarding how to format notebooks on save.
+
+  - Basic notebook functionality for saving (including outputs and images) and restoring all works as expected
+
+  - Notebooks embed images. If you use function or object graphics, all windows will be embedded.
+
+    - At this time, there may be some graphics that shouldn't be grabbed (like when you run ENVI processing with the ENVI UI open)
+
+  - Notebooks do not embed widgets.
+
+  - Notebooks only support Markdown and IDL cell types
+
+- When running cells, notebooks automatically start an IDL process. Notebooks provide two custom buttons for managing IDL: Reset and Stop.
+
+  - Reset will stop and restart IDL so that it is fresh (this way you can interrupt cell execution)
+
+  - Stop will stop the IDL process and interrupt cell execution. A new IDL session won't be launched until you run a new cell.
+
+- Ability to run cells:
+
+  - Cells are executed as-is and don't support debugging or interactive processes.
+
+  - See the "Hello World" notebook for details on how cell execution works and how you can write code
+
+  - After each cell is executed we issue a `retall` command to make sure that we are at the top-level and not stopped in a weird state
+
+## 3.2.0 August 2023
+
+When the language server does not use a full parse, extract structure definitions. Before, this was a logic gap (and made the code faster), but they should be correctly resolved now with minimal performance hits.
+
+- For context, using non-full parse can give a 4x improvement in performance (parse rate with 6 workers goes from 90k lines/second to 425k lines/second).
+
+Change execution path for notebooks so cells are compiled in the same folder that a notebook lives in. This makes it much easier to load datasets that are next to the notebook (for data scientists) and mimics the behavior of PRO code. This means that, if you use "routine_dir()" or "routine_filepath()" from a notebook, it will resolve to the path that you expect.
+
+For images, added a button that allows you to save the graphic to disk. All files saved to disk are in PNG format and come from the embedded images within the notebook file format. However, this makes it easy to generate graphics and export them from notebooks!
+
+Added auto-complete for structure names when there are no properties or only the beginning of a name has been typed
+
+Fixed a bug where task files and idl.json files were being processed as PRO files and reporting crazy errors
+
+Re-worked the notebook file format to be human readable (as JSON). **Do not edit the files by hand as you risk breaking your notebooks**. This new format:
+
+- Is pretty-printed JSON which can easily be read and is easier for git GUI applications to manage
+
+- Has a reduced size when we embed graphics
+
+- Uses a schema for complex output types (i.e. images, animations) with the pipes in place to have custom renderers or applets embedded in notebooks
+
+- Normalizes line endings on save which makes sure notebooks are the same on Windows vs Linux/Mac
+
+- Allows some transparency into the notebook format with what gets stored
+
+- All outputs from IDL are stored as a JSON string. This is to reduce file size and improve parsing speed (future concept is embedding plots/graphs which requires data that pretty-printed JSON would make unreasonably large)
+
+For all notebooks, we add extra catches when attempting to restore notebook outputs saved in the file. This means that we can update the way outputs are stored/processed in the future without breaking notebooks completely. You would just need to re-run cells and regenerate outputs in the format that we expect it to be.
+
+Fix an un handled case for auto-doc with structures where we didn't add spaces after an empty structure definition (even though that is invalid for IDL).
+
+Fixed a bug with notebooks where cells weren't cleaned up properly and you would get fake duplicate routines reported as a problem. Added tests to catch this in the future.
+
+Fixed a bug where docs header regular expressions were being over-zealous and grabbing more than they should.
+
+Fixed some bugs with styles of embedded graphics not appearing quite like they should when the notebooks are small.
+
+Fixed a bug where error resolving completion would show a message about hover help
+
+Fixed a bug where auto-complete would fail when you had fatal syntax errors in code and for/foreach loops
+
+Fixed a bug where auto-complete in notebooks wouldn't get the code from the cell for the right auto-complete user experience
+
+Further increase node.js timeout for edge cases where it wouldn't respond/start fast enough
+
+Changed the formatting behavior for structures when they have line continuations.
+
+- Before:
+
+  ```idl
+  !null = $
+    {MyStruct, $
+    _foo: 5}
+
+  !null = $
+    { $
+    _foo: 5}
+  ```
+
+- After:
+
+  ```idl
+  !null = $
+    {MyStruct, $
+      _foo: 5}
+
+  !null = $
+    { $
+      _foo: 5}
+  ```
+
 ## 3.1.4 August 2023
 
 For routine documentation, add button "Open Examples in Notebook" Which opens the routine, the description, and likely code examples as runnable notebook cells

apps/README.md

@@ -24,8 +24,6 @@ Here's what we have so far
 
 - **package-json**: Creates the content for the contribution points in the main `package.json` file. Built with typescript and does some basic checks for translations and auxiliary files to make sure we aren't missing anything.
 
-- **parsing-global-index**: Not used, but placeholder to have tokens and problems tracked.manages separate from the main thread. A little over-engineering for this early in the game, but it is there for future use.
-
 - **parsing-worker**: Node.js worker thread to index IDL code in the background.
 
 - **performance**: Node.js app meant to be a place where we can test the speed of parsing without needing to open VSCode (also, cross platform)

apps/client-e2e/src/tests/interactions/_interactions-runner.ts

@@ -2,7 +2,13 @@ import { Logger } from '@idl/logger';
 
 import { Runner } from '../runner.class';
 import { AddDocs } from './add-docs';
+import { IDLJSONInteractRight } from './idl-json-interact-right';
 import { IndexIDLFolderRightAndOpenEditClose } from './index-idl-folder-right-and-open-edit-close';
+import { NotebookCompletionBasic } from './notebooks-completion-basic';
+import { NotebooksInteractRight } from './notebooks-interact-right';
+import { NotebooksNoDuplicates } from './notebooks-no-duplicates';
+import { ProCodeInteractRight } from './pro-code-interacts-right';
+import { TasksInteractRight } from './tasks-interact-right';
 
 /*
  * Logger to be used for tests related to debugging
@@ -28,3 +34,34 @@ INTERACTIONS_RUNNER.addTest({
   name: 'Index IDL folder right on startup and properly sync problems on open, edit, close',
   fn: IndexIDLFolderRightAndOpenEditClose,
 });
+
+INTERACTIONS_RUNNER.addTest({
+  name: 'Task interaction does the right thing',
+  fn: TasksInteractRight,
+});
+
+INTERACTIONS_RUNNER.addTest({
+  name: 'idl.json interaction does the right thing',
+  fn: IDLJSONInteractRight,
+});
+
+INTERACTIONS_RUNNER.addTest({
+  name: 'PRO code interacts right',
+  fn: ProCodeInteractRight,
+});
+
+INTERACTIONS_RUNNER.addTest({
+  name: 'Notebooks interact right',
+  fn: NotebooksInteractRight,
+});
+
+INTERACTIONS_RUNNER.addTest({
+  name: 'Notebook problems track right',
+  fn: NotebooksNoDuplicates,
+});
+
+INTERACTIONS_RUNNER.addTest({
+  name: 'Verify notebook auto-complete passes basic tests',
+  fn: NotebookCompletionBasic,
+  critical: true,
+});

apps/client-e2e/src/tests/interactions/idl-json-interact-right.ts

@@ -0,0 +1,85 @@
+import { GetExtensionPath, Sleep } from '@idl/shared';
+import {
+  OpenFileInVSCode,
+  ReplaceDocumentContent,
+  VSCODE_COMMANDS,
+} from '@idl/vscode/shared';
+import expect from 'expect';
+import { readFileSync } from 'fs';
+import * as vscode from 'vscode';
+import {
+  SemanticTokensParams,
+  TextDocumentPositionParams,
+} from 'vscode-languageserver';
+
+import { RunnerFunction } from '../runner.interface';
+
+/**
+ * Verifies idl.json does the right thing with changes and doesnt trigger
+ * PRO code parsing in the language server
+ */
+export const IDLJSONInteractRight: RunnerFunction = async (init) => {
+  const doc = await OpenFileInVSCode(
+    GetExtensionPath('idl/test/client-e2e/idl.json')
+  );
+
+  // short pause to make sure we open and parse
+  await Sleep(250);
+
+  /**
+   * Get number of original diagnostics
+   */
+  const nOrig = vscode.languages.getDiagnostics(doc.uri).length;
+
+  // replace the content in our document
+  await ReplaceDocumentContent(
+    doc,
+    readFileSync(
+      GetExtensionPath('idl/test/client-e2e/idl-changes.json'),
+      'utf-8'
+    )
+  );
+
+  // short pause
+  await Sleep(250);
+
+  // verify problems
+  expect(vscode.languages.getDiagnostics(doc.uri).length).toEqual(nOrig);
+
+  /**
+   * Event params for LSP user interaction
+   */
+  const params: TextDocumentPositionParams = {
+    textDocument: {
+      uri: doc.uri.toString(),
+    },
+    position: {
+      line: 0,
+      character: 0,
+    },
+  };
+
+  /**
+   * Event params for LSP semantic token request
+   */
+  const tokenParams: SemanticTokensParams = {
+    textDocument: {
+      uri: doc.uri.toString(),
+    },
+  };
+
+  // send all requests that might trigger parsing
+  await init.client.client.sendRequest('textDocument/hover', params);
+  await init.client.client.sendRequest('textDocument/completion', params);
+  await init.client.client.sendRequest('textDocument/definition', params);
+  await init.client.client.sendRequest(
+    'textDocument/semanticTokens/full',
+    tokenParams
+  );
+
+  // short pause
+  await Sleep(250);
+
+  // clear any existing outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.CLOSE_EDITOR);
+};

apps/client-e2e/src/tests/interactions/notebooks-completion-basic.ts

@@ -0,0 +1,73 @@
+import { GetExtensionPath, Sleep } from '@idl/shared';
+import { OpenNotebookInVSCode, VSCODE_COMMANDS } from '@idl/vscode/shared';
+import expect from 'expect';
+import * as vscode from 'vscode';
+import {
+  CompletionItem as LanguageServerCompletionItem,
+  TextDocumentPositionParams,
+} from 'vscode-languageserver';
+
+import { RunnerFunction } from '../runner.interface';
+
+/**
+ * Verifies working with notebooks does the right thing with changes and doesnt trigger
+ * PRO code parsing in the language server
+ */
+export const NotebookCompletionBasic: RunnerFunction = async (init) => {
+  const doc = await OpenNotebookInVSCode(
+    GetExtensionPath('idl/test/client-e2e/notebooks/auto-complete-test.idlnb')
+  );
+
+  // short pause
+  await Sleep(250);
+
+  /**
+   * Event params for LSP user interaction
+   */
+  const completionParams: TextDocumentPositionParams = {
+    textDocument: {
+      uri: doc.cellAt(0).document.uri.toString(),
+    },
+    position: {
+      line: 0,
+      character: 7,
+    },
+  };
+
+  // verify definition has return
+  const completion1: LanguageServerCompletionItem[] =
+    await init.client.client.sendRequest(
+      'textDocument/completion',
+      completionParams
+    );
+
+  // verify definition has return
+  expect(completion1).not.toBeUndefined();
+  expect(completion1.length).toEqual(0);
+
+  /**
+   * Event params for LSP user interaction
+   */
+  const completionParams2: TextDocumentPositionParams = {
+    textDocument: {
+      uri: doc.cellAt(1).document.uri.toString(),
+    },
+    position: {
+      line: 1,
+      character: 0,
+    },
+  };
+
+  const completion2: LanguageServerCompletionItem[] =
+    await init.client.client.sendRequest(
+      'textDocument/completion',
+      completionParams2
+    );
+
+  // verify definition has return
+  expect(completion2).not.toBeUndefined();
+  expect(completion2.length).not.toEqual(0);
+
+  // clear any existing outputs
+  await vscode.commands.executeCommand(VSCODE_COMMANDS.CLOSE_EDITOR);
+};