Merge branch 'develop' into main

.gitignore

@@ -4,6 +4,8 @@
 /dist
 /tmp
 /out-tsc
+.angular
+.nx
 
 # dependencies
 /node_modules
@@ -55,9 +57,4 @@ idl.extension.host.log
 compliance/3rdpartylicenses_condensed.md
 compliance/3rdpartylicenses_encryption.csv
 compliance/3rdpartylicenses.csv
-compliance/compliance.json
-
-# IDL notebooks
-idl/routines/notebooks/**
-
-.angular
+compliance/compliance.json

.prettierignore

@@ -16,3 +16,5 @@ compliance
 *settings.json
 apps/test-tokenizer/src/test-maker/cache/cache.json
 .angular
+
+/.nx/cache

.vscode/settings.json

@@ -18,6 +18,7 @@
     "bubbleanimation",
     "defint",
     "Deref",
+    "didnt",
     "Doesnt",
     "dont",
     "ENVI",

.vscodeignore

@@ -1,6 +1,7 @@
 .angular/**
 .github/**
 .idl/idl.json
+.nx/**
 .vscode/**
 .vscode-test/**
 apps/**
@@ -19,6 +20,7 @@ dist/apps/client/assets/.gitkeep
 dist/apps/client-e2e/**
 dist/apps/idl-webview/3rdpartylicenses.txt
 dist/apps/i18n/**
+dist/apps/notebook/components/3rdpartylicenses.txt
 dist/apps/package-json/**
 dist/apps/parsing-worker/3rdpartylicenses.txt
 dist/apps/parsing-worker/assets/.gitkeep

CHANGELOG.md

@@ -4,11 +4,39 @@ All notable changes to the "idl" extension will be documented in this file.
 
 For much more detail on incremental work for large features, see our [developer notes](./extension/docs/developer/dev-notes/README.md).
 
-## Unreleased
+## Preview Features
 
-Preview release of IDL Notebooks! This is a first pass at adding notebook support for IDL which is independent from Jupyter.
+This section of the CHANGELOG documents features that have been added to the extension, but are still in an experimental phase. Feel free to try them out and provide feedback via discussions or issues on our GitHub page.
 
-> Notebooks are a preview feature and, based on early adopter feedback, will likely change
+Code style revamp! We reworked how routines, routine methods, properties, and structure names get formatted. This includes:
+
+- Support for camel case ("camelCase") and pascal case ("PascalCase") styling. Mileage may vary here, based on the routine names, so please let us know if this doesn't look quite right or do what you expect. Some routines that start with "IDL" or "ENVI" might surprise you with the case conversion.
+
+- New style setting called "routineMethods" so that you can indicate the styling for methods apart from normal functions or procedures
+
+- New style setting called "structureNames" so that you can have structure names have consistent formatting.
+
+- We also now format the structure names in inheritance statements.
+
+- We also use structure formatting preferences when auto-completing structure names
+
+- Routine formatting now gives you the benefit-of-the-doubt when formatting routines and routine methods. In the past, if we encountered an unknown routine, we would not change the appearance. Now, even if we don't know the class method or routine, we apply styling. Milage may vary here based on the style you use when we can't get the source information.
+
+- New defaults:
+
+  - Properties: camelCase
+
+  - Routines: match definition
+
+  - Routine methods: camelCase
+
+  - Structure names: PascalCase
+
+- When generating ENVI and IDL tasks, using our new case libraries, we attempt to make a pretty display name from parameter names. For example converting the keyword "my_keyword" to "My Keyword". This applied to task and parameter display names.
+
+## 4.0.0 November 2023
+
+The official release of IDL Notebooks! This is a first pass at adding notebook support for IDL (which is independent from Jupyter). We are hoping to have early adopters try it out and provide feedback on how notebooks behave. Read below to learn more or find an example IDL and ENVI Notebook directly within VSCode in the IDL sidebar.
 
 Here are some of the features that notebooks bring:
 
@@ -42,6 +70,52 @@ Here are some of the features that notebooks bring:
 
   - After each cell is executed we issue a `retall` command to make sure that we are at the top-level and not stopped in a weird state
 
+Updated the IDL and ENVI icons throughout the extension and on our github pages. This is the same icon that will be used as part of the next official release of ENVI and IDL.
+
+We now use colored icons for file icons and added some new file associations for our ENVI within the editor. If you have a theme that doesn't look great with our icon colors, let us know!
+
+With the new icons, updated our custom icon theme to include the new ENVI logo for key ENVI file extensions.
+
+Added the ability to convert your IDL notebooks to PRO code! This exciting features uses intimate knowledge of IDL to break down the code in your cells and put it back together as a single PRO file.
+
+- Any routines and non-main level program code is put together first and follows the order of the cells. Then, any main-level programs within cells or single lines of code are added as one, large main level program at the end.
+
+- Milage may vary based on how you wrote your notebook, but it is very easy to get out your routine definitions
+
+- Use GitHub to let us know if this should behave differently!
+
+Expanded sidebar for notebooks with buttons for:
+
+- Creating a new notebook
+
+- Formatting the IDL Code in your notebook cells (shortcut for VSCode native command)
+
+- Ability to open an example IDL Notebook
+
+- Ability to open example ENVI Notebook
+
+- Converting notebooks to PRO code
+
+There's also a command added that will reset your example notebooks to what we had originally. it is called "IDL: Reset IDL and ENVI Example Notebooks'.
+
+When automatically opening ENVI files, the text displayed adds a note about how you can disable the behavior by looking at the extension documentation.
+
+Optimize the generation of the code outline and semantic tokens (static class references) to calculate both at the same time we are parsing a file or cell of code. This reduces extra CPU usage from retrieving complex information from our cache.
+
+When notebooks start, we now perform an extra check to make sure we have the routines needed to function correctly. This is mainly for developers and edge-cases where you weren't provided any feedback on the startup process and wouldn't know there was a problem.
+
+Resolved an issue where you couldn't view the output from a notebook cell in VSCode. We provided a default VSCode configuration which fixes this problem and added a new doc to the extension to capture known issues and how to solve them.
+
+Fixed an issue where statements being sent to IDL were not executing correctly for implied print
+
+When generating ENVI and IDL tasks, we try to make a nice display name for task and parameter names
+
+Added auto-complete for "inherits" statements in structure definitions
+
+Fixed an issue where we incorrectly identified main level programs in parsed PRO code that were only comments
+
+When we parse documentation for routines, we now accept docs within routine definitions. If you have a comment block immediately following parameters, then that takes priority over comment blocks above.
+
 ## 3.2.4 October 2023
 
 Fixed an issue where improper versions of node.js were used for the language server startup. This caused failures in the language server starting which would prevent any of the feature goodness of the language server from being accessible.

README.md

@@ -36,22 +36,42 @@
   </video>
 </div>
 
-**3.0 is here!**
+**4.0.0 is here!**
 
-This release offers many exciting and new features, such as syntax-error checking, linting, debugging, and profiling. With all these new features, there also may be some undocumented software enhancements (i.e. bugs). For these please use [GitHub](https://github.com/interactive-data-language/vscode-idl) to file bug/feature requests. There is also a quick link for submitting bugs for the extension in the IDL View.
+This release headlines our newest feature: IDL Notebooks! The IDL Notebook format is standalone and only requires this extension to function (no extra dependencies on Python or Jupyter).
 
-See below and the [CHANGELOG.md](./CHANGELOG.md) for full details.
+Within the extension, you'll find a new sidebar entry to help create a new notebook, open IDL and ENVI Notebook examples, and convert your IDL Notebooks to a PRO file as well. If you have questions, feel free to reach out to us on GitHub with any feedback or issues you encounter.
+
+In addition to this, we have revamped our tools for styling/formatting your code to include other case styles like pascal and camel case.
+
+That's not all that is included in this release, so check out the [CHANGELOG.md](./CHANGELOG.md) for full details.
 
 > If you are looking to contribute, or get your development environment setup, see [CONTRIBUTING.md](./CONTRIBUTING.md).
 
 ---
 
-This extension adds syntax highlighting, code snippets, debugging, problem reporting (undefined variables and syntax errors, 100+ in total!), and much more for the Interactive Data Language (IDL) from NV5 Geospatial Software (formerly Research Systems, Inc.).
+This extension adds syntax highlighting, code snippets, debugging, problem reporting (undefined variables and syntax errors, 100+ in total!), IDL Notebooks, and much more for the Interactive Data Language (IDL) from NV5 Geospatial Software (formerly Research Systems, Inc.).
 
 For a full list of changes, see [CHANGELOG.md](./CHANGELOG.md) for lots of details.
 
+## IDL Requirement
+
+Using the "debug" sessions of IDL requires that you have a version of IDL with lists, hashes, and orderedhashes (and the ability to parse/serialize JSON).
+
+The extension is tested against:
+
+- IDL 8.9
+
+- IDL 9.0
+
+Anything else is use-at-your-own-risk and may not work.
+
+For notebooks, we only test against the above versions, and the ENVI Notebook API does require ENVI in order to function.
+
 ## Features
 
+- IDL Notebooks for a new, modern, and ad-hoc way to develop IDL code! Check out the examples to learn about how they work (accessible through the sidebar of the extension).
+
 - Before compiling any code, more than 100 syntax errors/problems/hints are detected and reported to users.
 
 - Syntax errors you encounter while writing code offer information/details on why they are problems to help you learn the in's and out's of IDL better.
@@ -128,14 +148,6 @@ For a full list of changes, see [CHANGELOG.md](./CHANGELOG.md) for lots of detai
 
   - You may occasionally see some text that is printed to the console when IDL is running, this is a known bug that happens for a few rare cases and can be ignored.
 
-- Using the "debug" sessions of IDL requires that you have a version of IDL with lists, hashes, and orderedhashes (and the ability to parse/serialize JSON). The extension has been tested against:
-
-  - IDL 8.8
-
-  - IDL 8.9
-
-  Anything else is use-at-your-own-risk and may not work.
-
 ## Release Notes
 
 See [CHANGELOG](CHANGELOG.md).

apps/client-e2e/project.json

@@ -43,7 +43,7 @@
       }
     },
     "lint": {
-      "executor": "@nx/linter:eslint",
+      "executor": "@nx/eslint:lint",
       "outputs": ["{options.outputFile}"],
       "options": {
         "lintFilePatterns": ["apps/client-e2e/**/*.ts"]

apps/client-e2e/src/main.ts

@@ -1,5 +1,8 @@
+import { FindIDL } from '@idl/idl';
 import { EXTENSION_FULL_NAME, GetExtensionPath } from '@idl/shared';
 import { Sleep } from '@idl/test-helpers';
+import { GetWorkspaceConfig, IIDLWorkspaceConfig } from '@idl/vscode/config';
+import { IDL_EXTENSION_CONFIG_KEYS } from '@idl/vscode/extension-config';
 import { IInitializeType } from '@idl/vscode/initialize-types';
 import { OpenFileInVSCode, VSCODE_COMMANDS } from '@idl/vscode/shared';
 import expect from 'expect';
@@ -34,11 +37,23 @@ export async function run(): Promise<void> {
 
   // run our tests
   try {
+    /**
+     * Manually specify IDL folder
+     */
+    const idlDir = FindIDL();
+
+    // validate we know where it is
+    if (!idlDir) {
+      throw new Error('Unable to find IDL, cannot run tests');
+    }
+
+    // alert user which IDL we are using
+    console.log(` `);
+    console.log(`Test are using this IDL: "${idlDir}"`);
+
     // get extension
     const ext = vscode.extensions.getExtension(EXTENSION_FULL_NAME);
 
-    console.log(ext);
-
     // activate extension
     ACTIVATION_RESULT = await ext.activate();
 
@@ -89,6 +104,16 @@ export async function run(): Promise<void> {
     // close editor
     await vscode.commands.executeCommand(VSCODE_COMMANDS.CLOSE_EDITOR);
 
+    // get the current workspace config
+    const config = GetWorkspaceConfig();
+
+    // set latest IDL folder
+    (config as IIDLWorkspaceConfig).update(
+      IDL_EXTENSION_CONFIG_KEYS.IDLDirectory,
+      idlDir,
+      true
+    );
+
     // check if we allow debug logs
     if (DEBUG_LOGS) {
       ACTIVATION_RESULT.client.logger.setDebug(DEBUG_LOGS);

apps/client-e2e/src/tests/debugging/_debugging-runner.ts

@@ -6,6 +6,7 @@ import { Compile } from './compile';
 import { Continue } from './continue';
 import { Edit } from './edit';
 import { Exit } from './exit';
+import { ImpliedPrint } from './implied-print';
 import { RigorousAlwaysReturn } from './rigorous-always-return';
 import { StartDebugging } from './start';
 import { VariableReplacement } from './variable-replacement';
@@ -53,6 +54,12 @@ DEBUGGING_RUNNER.addTest({
   critical: true,
 });
 
+DEBUGGING_RUNNER.addTest({
+  name: 'Verify implied print works right',
+  fn: ImpliedPrint,
+  critical: true,
+});
+
 DEBUGGING_RUNNER.addTest({
   name: 'Manual exit closes cleanly',
   fn: Exit,

apps/client-e2e/src/tests/debugging/implied-print.ts

@@ -0,0 +1,30 @@
+import { CleanIDLOutput } from '@idl/idl';
+import { IDL_COMMANDS } from '@idl/shared';
+import expect from 'expect';
+import * as vscode from 'vscode';
+
+import { RunnerFunction } from '../runner.interface';
+
+/**
+ * Verifies that simple statements that should be executed as implied print are
+ * indeed interpreted like that
+ */
+export const ImpliedPrint: RunnerFunction = async (init) => {
+  /**
+   * Start IDL
+   */
+  const started = await vscode.commands.executeCommand(
+    IDL_COMMANDS.DEBUG.START
+  );
+
+  // verify we started
+  expect(started).toBeTruthy();
+
+  // compile file
+  const res = CleanIDLOutput(
+    await init.debug.adapter.evaluate(`!version`, { echo: true })
+  ).toLowerCase();
+
+  // make sure we compile
+  expect(res.includes('attempt to call undefined procedure')).toBeFalsy();
+};

apps/client-e2e/src/tests/interactions/notebooks-interact-right.ts

@@ -89,7 +89,7 @@ export const NotebooksInteractRight: RunnerFunction = async (init) => {
       'textDocument/semanticTokens/full',
       tokenParams
     )
-  ).toBeNull();
+  ).toEqual({ data: [] });
 
   // short pause
   await Sleep(250);

apps/client-e2e/src/tests/interactions/pro-code-interacts-right.ts

@@ -82,7 +82,7 @@ export const ProCodeInteractRight: RunnerFunction = async (init) => {
       'textDocument/semanticTokens/full',
       tokenParams
     )
-  ).toBeFalsy();
+  ).toEqual({ data: [] });
 
   // short pause
   await Sleep(250);