Merge pull request #96 from returntocorp/matthew/ci_scan_complete

Add type for `/complete` data
semgrep · Jul 25, 2023 · 407f582 · 407f582
2 parents ef5767d + ba48b56
commit 407f582
Show file tree

Hide file tree

Showing 6 changed files with 3,252 additions and 460 deletions.
diff --git a/semgrep_output_v1.atd b/semgrep_output_v1.atd
@@ -824,13 +824,49 @@ type found_dependency <ocaml attr="deriving show"> = {
   ?children: dependency_child list option;
 }
 
+
+(* json names are to maintain backwards compatibility with the python enum it is replacing *)
+type sca_parser_name <ocaml attr="deriving show"> = [
+  | Gemfile_lock <json name="gemfile_lock">
+  | Go_mod <json name="go_mod">
+  | Go_sum <json name="go_sum">
+  | Gradle_lockfile <json name="gradle_lockfile">
+  | Gradle_build <json name="gradle_build">
+  | Jsondoc <json name="jsondoc">
+  | Pipfile <json name="pipfile">
+  | Pnpm_lock <json name="pnpm_lock">
+  | Poetry_lock <json name="poetry_lock">
+  | Pyproject_toml <json name="pyproject_toml">
+  | Requirements <json name="requirements">
+  | Yarn_1 <json name="yarn_1">
+  | Yarn_2 <json name="yarn_2">
+  | Pomtree <json name="pomtree">
+  | Cargo_parser <json name="cargo">
+  | Composer_lock <json name="composer_lock">
+]
+
+
+
+type dependency_parser_error <ocaml attr="deriving show"> = {
+  path: string;
+  parser: sca_parser_name;
+  reason: string;
+  (* Not using `position` because this type must be backwards compatible with the python class it is replacing *)
+  ?line: int option;
+  ?col: int option;
+  ?text: string option;
+}
+
+
 (*****************************************************************************)
 (* Semgrep CLI findings output for the App *)
 (*****************************************************************************)
 (* TODO: see semgrep/app/scans.py
  * there is also api_scans_meta, api_scans_ignores, and api_scans_complete
  * EXPERIMENTAL
 *)
+
+(* Sent by the CLI to /findings_and_ignores aka /results *)
 type ci_scan_results <ocaml attr="deriving show"> = {
   (* TODO: ?version: version option; *)
    findings: finding list;
@@ -845,6 +881,48 @@ type ci_scan_results <ocaml attr="deriving show"> = {
 
 
 
+type parsing_stats <ocaml attr="deriving show"> = {
+  targets_parsed: int;
+  num_targets: int;
+  bytes_parsed: int;
+  num_bytes: int;
+}
+
+type ci_scan_complete_stats <ocaml attr="deriving show"> = {
+  findings: int;
+  errors: cli_error list;
+  total_time: float;
+  unsupported_exts: (string * int) list
+    <json repr="object">
+    <python repr="dict">
+    <ts repr="map">;
+  lockfile_scan_info: (string * int) list
+    <json repr="object">
+    <python repr="dict">
+    <ts repr="map">;
+  parse_rate: (string * parsing_stats) list
+    <json repr="object">
+    <python repr="dict">
+    <ts repr="map">;
+  (* This is EngineType from python, which is different from engine_kind used in this file *)
+  ?engine_requested: string option;
+}
+
+type ci_scan_dependencies <ocaml attr="deriving show"> =  (string * found_dependency list) list
+    <json repr="object">
+    <python repr="dict">
+    <ts repr="map">
+
+(* Seny by the CLI to /complete *)
+type ci_scan_complete_response <ocaml attr="deriving show"> = {
+  exit_code: int;
+  stats: ci_scan_complete_stats;
+  ?dependencies: ci_scan_dependencies option;
+  ?dependency_parser_errors: dependency_parser_error list option;
+  ?task_id: string option;
+}
+
+
 type finding_hashes <ocaml attr="deriving show"> = {
   start_line_hash: string;
   end_line_hash: string;

diff --git a/semgrep_output_v1.jsonschema b/semgrep_output_v1.jsonschema
@@ -733,6 +733,38 @@
         }
       }
     },
+    "sca_parser_name": {
+      "oneOf": [
+        { "const": "gemfile_lock" },
+        { "const": "go_mod" },
+        { "const": "go_sum" },
+        { "const": "gradle_lockfile" },
+        { "const": "gradle_build" },
+        { "const": "jsondoc" },
+        { "const": "pipfile" },
+        { "const": "pnpm_lock" },
+        { "const": "poetry_lock" },
+        { "const": "pyproject_toml" },
+        { "const": "requirements" },
+        { "const": "yarn_1" },
+        { "const": "yarn_2" },
+        { "const": "pomtree" },
+        { "const": "cargo" },
+        { "const": "composer_lock" }
+      ]
+    },
+    "dependency_parser_error": {
+      "type": "object",
+      "required": [ "path", "parser", "reason" ],
+      "properties": {
+        "path": { "type": "string" },
+        "parser": { "$ref": "#/definitions/sca_parser_name" },
+        "reason": { "type": "string" },
+        "line": { "type": "integer" },
+        "col": { "type": "integer" },
+        "text": { "type": "string" }
+      }
+    },
     "ci_scan_results": {
       "type": "object",
       "required": [
@@ -755,6 +787,67 @@
         "contributions": { "$ref": "#/definitions/contributions" }
       }
     },
+    "parsing_stats": {
+      "type": "object",
+      "required": [
+        "targets_parsed", "num_targets", "bytes_parsed", "num_bytes"
+      ],
+      "properties": {
+        "targets_parsed": { "type": "integer" },
+        "num_targets": { "type": "integer" },
+        "bytes_parsed": { "type": "integer" },
+        "num_bytes": { "type": "integer" }
+      }
+    },
+    "ci_scan_complete_stats": {
+      "type": "object",
+      "required": [
+        "findings", "errors", "total_time", "unsupported_exts",
+        "lockfile_scan_info", "parse_rate"
+      ],
+      "properties": {
+        "findings": { "type": "integer" },
+        "errors": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/cli_error" }
+        },
+        "total_time": { "type": "number" },
+        "unsupported_exts": {
+          "type": "object",
+          "additionalProperties": { "type": "integer" }
+        },
+        "lockfile_scan_info": {
+          "type": "object",
+          "additionalProperties": { "type": "integer" }
+        },
+        "parse_rate": {
+          "type": "object",
+          "additionalProperties": { "$ref": "#/definitions/parsing_stats" }
+        },
+        "engine_requested": { "type": "string" }
+      }
+    },
+    "ci_scan_dependencies": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "array",
+        "items": { "$ref": "#/definitions/found_dependency" }
+      }
+    },
+    "ci_scan_complete_response": {
+      "type": "object",
+      "required": [ "exit_code", "stats" ],
+      "properties": {
+        "exit_code": { "type": "integer" },
+        "stats": { "$ref": "#/definitions/ci_scan_complete_stats" },
+        "dependencies": { "$ref": "#/definitions/ci_scan_dependencies" },
+        "dependency_parser_errors": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/dependency_parser_error" }
+        },
+        "task_id": { "type": "string" }
+      }
+    },
     "finding_hashes": {
       "type": "object",
       "required": [