Merge pull request #66 from carocad/antlr-js

Support for Browser environments

.gitignore

@@ -16,7 +16,8 @@ pom.xml.asc
 /node_modules/
 /out/
 /src/java/
-/src/javascript
 /figwheel_server.log
 package*.json
 /.eastwood
+pom.xml
+/src/clojure/parcera/antlr/js/

.travis.yml

@@ -8,9 +8,13 @@ branches:
 
 language: generic
 
+addons:
+  chrome: stable
+
 jobs:
   include:
-    - stage: Tests
+    - stage: tests
+      name: Java tests
       script:
         - curl -O https://www.antlr.org/download/antlr-4.7.1-complete.jar
         # generate java
@@ -19,17 +23,28 @@ jobs:
         - lein do clean, compile, check, eastwood
         - lein trampoline test
         - lein trampoline test :benchmark
-        # todo - re-enable js
-        # generate javascript - todo
-        #- java -jar antlr-4.7.1-complete.jar -Xexact-output-dir -o src/javascript/parcera/antlr -package parcera.antlr -Dlanguage=JavaScript -no-listener -no-visitor src/Clojure.g4
-        #- nvm install 10.10 && nvm use 10.10 && lein trampoline cljsbuild test
+    # still test stage
+    - name: JavaScript tests
+      script:
+        - curl -O https://www.antlr.org/download/antlr-4.7.1-complete.jar
+        # generate javascript
+        - java -jar antlr-4.7.1-complete.jar -Xexact-output-dir -o src/clojure/parcera/antlr/js -package parcera.antlr -Dlanguage=JavaScript -no-listener -no-visitor src/Clojure.g4
+        # pre-bundle javascript
+        - npm init --yes && npm add webpack webpack-cli antlr4
+        - npx webpack --config ./resources/webpack.config.js
+        - lein test-runner
 
     - stage: Release
       if: tag IS present
       script:
         - curl -O https://www.antlr.org/download/antlr-4.7.1-complete.jar
         # generate java
         - java -jar antlr-4.7.1-complete.jar -Xexact-output-dir -o src/java/parcera/antlr -package parcera.antlr -Dlanguage=Java -no-listener -no-visitor src/Clojure.g4
+        # generate javascript
+        - java -jar antlr-4.7.1-complete.jar -Xexact-output-dir -o src/clojure/parcera/antlr/js -package parcera.antlr -Dlanguage=JavaScript -no-listener -no-visitor src/Clojure.g4
+        # pre-bundle javascript
+        - npm init --yes && npm add webpack webpack-cli antlr4
+        - npx webpack --config ./resources/webpack.config.js
         - lein do javac, compile, check
       deploy:
         - provider: script

README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://travis-ci.com/carocad/parcera.svg?branch=master)](https://travis-ci.com/carocad/parcera)
 [![Clojars Project](https://img.shields.io/clojars/v/carocad/parcera.svg)](https://clojars.org/carocad/parcera)
 
-Grammar-based Clojure parser.
+Grammar-based Clojure(script) parser.
 
 Parcera can safely read any Clojure file without any code evaluation.
 
@@ -12,8 +12,15 @@ parsing engine and focuses on the grammar definition instead.
 
 ## usage
 
-Add `[org.antlr/antlr4-runtime "4.7.1"]` to your dependencies in addition to parcera. Parcera assumes that
-this dependency will be in the classpath to avoid collisions.
+- Java
+
+Add `[org.antlr/antlr4-runtime "4.7.1"]` to your dependencies in addition to parcera. 
+This is to avoid adding an unnecessary dependency for the JavaScript users.
+
+- Javascript
+
+All necessary files are delivered with parcera. However, currently only Browser support
+has been tested. 
 
 ```clojure
 (ns example.core
@@ -32,11 +39,11 @@ this dependency will be in the classpath to avoid collisions.
   (:symbol "parcera.core")
   (:whitespace " ")
   (:list
-   (:simple_keyword ":require")
+   (:keyword ":require")
    (:whitespace " ")
-   (:vector (:symbol "clojure.data") (:whitespace " ") (:simple_keyword ":as") (:whitespace " ") (:symbol "data"))
+   (:vector (:symbol "clojure.data") (:whitespace " ") (:keyword ":as") (:whitespace " ") (:symbol "data"))
    (:whitespace " ")
-   (:vector (:symbol "clojure.string") (:whitespace " ") (:simple_keyword ":as") (:whitespace " ") (:symbol "str")))))
+   (:vector (:symbol "clojure.string") (:whitespace " ") (:keyword ":as") (:whitespace " ") (:symbol "str")))))
 
 ;; get meta data from the parsed code
 (meta (second (parcera/ast (str :hello))))

dev.cljs.edn

@@ -0,0 +1,14 @@
+;; https://figwheel.org/docs/testing.html
+^{:auto-testing true
+  :watch-dirs   ["src/clojure" "test"]}
+{:main          parcera.core
+ :npm-deps      {"antlr4" "^4.7.2"}
+ :install-deps  true
+ :infer-externs true}
+ ;:target :nodejs
+;{:file        "src/javascript/parcera/antlr/ClojureLexer.js"
+; :provides    ["antlr.clojure.lexer"]
+; :module-type :commonjs}
+;{:file        "src/javascript/parcera/antlr/Clojure.Parser.js"
+; :provides    ["antlr.clojure.parser"]
+; :module-type :commonjs}]}

project.clj

@@ -1,5 +1,5 @@
-(defproject carocad/parcera "0.10.2"
-  :description "Grammar-based Clojure parser"
+(defproject carocad/parcera "0.11.0"
+  :description "Grammar-based Clojure(script) parser"
   :url "https://github.com/carocad/parcera"
   :license {:name "LGPLv3"
             :url  "https://github.com/carocad/parcera/blob/master/LICENSE.md"}
@@ -8,14 +8,22 @@
   :javac-options ["-target" "1.8" "-source" "1.8"]
   :dependencies [[org.clojure/clojure "1.10.1"]]
   :profiles {:dev      {:dependencies   [[criterium/criterium "0.4.5"] ;; benchmark
-                                         [org.clojure/test.check "0.10.0"]] ;; generative testing
-                        :plugins        [[jonase/eastwood "0.3.5"] ;; linter
-                                         ;; java reloader
-                                         [lein-virgil "0.1.9"]]
+                                         [org.clojure/test.check "0.10.0"] ;; generative testing
+                                         ;; https://github.com/bhauman/figwheel-main/issues/161
+                                         [com.bhauman/figwheel-main "0.2.0"]] ;; cljs repl
+                        :plugins        [[jonase/eastwood "0.3.5"]] ;; linter
+                        ;; java reloader
+                        ;[lein-virgil "0.1.9"]]
                         :resource-paths ["target"]
                         :clean-targets  ^{:protect false} ["target"]}
 
-             :provided {:dependencies [[org.antlr/antlr4-runtime "4.7.1"]]}}
+             :provided {:dependencies [[org.clojure/clojurescript "1.10.597"]
+                                       [org.antlr/antlr4-runtime "4.7.1"]]}}
+
+  :aliases {"test-runner" ["trampoline" "run"
+                           "-m" "figwheel.main"
+                           "-co" "tests.cljs.edn"
+                           "-m" "parcera.test-runner"]}
 
   :test-selectors {:default     (fn [m] (not (some #{:benchmark} (keys m))))
                    :benchmark   :benchmark}

resources/clojureReader.js

@@ -0,0 +1,22 @@
+/**
+ * Utility module to load all antlr related functionality; including
+ * the generated one.
+ *
+ * This file will then be passed through Webpack to create a single
+ * bundle
+ */
+const antlr4 = require('antlr4')
+const {ClojureLexer} = require('../src/clojure/parcera/antlr/js/ClojureLexer')
+const {ClojureParser} = require('../src/clojure/parcera/antlr/js/ClojureParser')
+
+const reader = {
+    charStreams: (input) => new antlr4.CharStreams.fromString(input),
+    lexer: (chars) => new ClojureLexer(chars),
+    tokens: (lexer) => new antlr4.CommonTokenStream(lexer),
+    parser: (tokens) => new ClojureParser(tokens),
+}
+
+module.exports = reader
+// global.ClojureReader = reader
+
+// console.log(`DONE 💫`)

resources/webpack.config.js

@@ -0,0 +1,19 @@
+const path = require('path')
+
+module.exports = {
+  mode: "production",
+  entry: path.resolve(__dirname, './clojureReader.js'),
+  output: {
+    filename: 'clojure.reader.bundle.js',
+    path: path.resolve(__dirname, '../src/clojure/parcera/antlr/js'),
+    // the url to the output directory resolved relative to the HTML page
+    library: "ClojureReader",
+    // the name of the exported library
+    libraryTarget: "commonjs2",
+  },
+  target: 'web',
+  node: {
+    module: "empty",
+    net: "empty",
+    fs: "empty" }
+}

src/Clojure.g4

@@ -33,11 +33,9 @@ vector: '[' input* ']';
 
 map: '{' input* '}';
 
-literal: keyword | string | number | character | symbol;
+literal: keyword | macro_keyword | string | number | character | symbol;
 
-keyword: simple_keyword | macro_keyword;
-
-simple_keyword: SIMPLE_KEYWORD;
+keyword: KEYWORD;
 
 macro_keyword: MACRO_KEYWORD;
 
@@ -69,10 +67,10 @@ metadata: ((metadata_entry | deprecated_metadata_entry) whitespace?)+
           | set
           | namespaced_map
           | tag
-          | function
+          | fn
           );
 
-metadata_entry: '^' ( map | symbol | string | keyword );
+metadata_entry: '^' ( map | symbol | string | keyword | macro_keyword );
 
 /**
  * According to https://github.com/clojure/clojure-site/blob/7493bdb10222719923519bfd6d2699a26677ee82/content/guides/weird_characters.adoc#-and----metadata
@@ -81,7 +79,7 @@ metadata_entry: '^' ( map | symbol | string | keyword );
  * In order to support roundtrip of parser rules it is required to exactly identify the
  * character used which would not be possible with something like '#'? '^'
  */
-deprecated_metadata_entry: '#^' ( map | symbol | string | keyword );
+deprecated_metadata_entry: '#^' ( map | symbol | string | keyword | macro_keyword );
 
 backtick: '`' whitespace? form;
 
@@ -93,7 +91,7 @@ unquote_splicing: '~@' whitespace? form;
 
 deref: '@' whitespace? form;
 
-dispatch: ( function
+dispatch: ( fn
           | regex
           | set
           | conditional
@@ -105,13 +103,13 @@ dispatch: ( function
           | eval
           );
 
-function: '#' list; // no whitespace allowed
+fn: '#' list; // no whitespace allowed
 
 regex: '#' STRING;
 
 set: '#{' input* '}'; // no whitespace allowed
 
-namespaced_map: '#' (keyword | auto_resolve)
+namespaced_map: '#' (keyword | macro_keyword | auto_resolve)
                     whitespace?
                     map;
 
@@ -160,7 +158,7 @@ MACRO_KEYWORD: '::' KEYWORD_HEAD KEYWORD_BODY*;
  * technically that is NOT a valid keyword. However in order to maintain
  * backwards compatibility the Clojure team didnt remove it from LispReader
  */
-SIMPLE_KEYWORD: ':' ((KEYWORD_HEAD KEYWORD_BODY*) | '/');
+KEYWORD: ':' ((KEYWORD_HEAD KEYWORD_BODY*) | '/');
 
 /**
  * a symbol must start with a valid character and can be followed

src/clojure/deps.cljs

@@ -0,0 +1,10 @@
+; NOTE: I feel this more as a hack than as an actual solution.}
+; Ideally I should just let ClojureScript import the lexer and parser
+; from a browser perspective. However it seems that the Closure compiler
+; doesnt like conditional imports so it throws saying that 'fs' is not
+; defined which is clearly a node.js dependency.
+; On the other hand, if I pass the module as node.js target then the Closure
+; compiler corrupts the source code which makes it unusable :(
+{:foreign-libs [{:file        "parcera/antlr/js/clojure.reader.bundle.js"
+                 :provides    ["antlr.clojure.reader"]
+                 :module-type :commonjs}]}