Added a VQL modifer to the sigma evaluator (#3164) (#3165)

This allows a sigma rule to call arbitrary VQL

artifacts/definitions/Server/Import/UpdatedBuiltin.yaml

@@ -0,0 +1,37 @@
+name: Server.Import.UpdatedBuiltin
+description: |
+  This artifact allows importing updated versions of some common built
+  in artifacts. If you do not want to wait for the next full release
+  you can use this artifact to import a more recent version of some
+  select artifacts which might include later feature.
+
+  NOTE: There is no guarantees that the updated artifact will work on
+  an older version. Make sure to test properly.
+
+type: SERVER
+
+required_permissions:
+- SERVER_ADMIN
+
+parameters:
+  - name: PackageName
+    type: choices
+    default: Windows.KapeFiles.Targets
+    choices:
+      - Windows.KapeFiles.Targets
+      - Generic.Forensic.SQLiteHunter
+
+  - name: Prefix
+    description: Add artifacts with this prefix
+    default: Updated.
+
+sources:
+  - query: |
+      LET URLlookup = dict(
+        `Windows.KapeFiles.Targets`="https://raw.githubusercontent.com/Velocidex/velociraptor/master/artifacts/definitions/Windows/KapeFiles/Targets.yaml",
+        `Generic.Forensic.SQLiteHunter`="https://raw.githubusercontent.com/Velocidex/SQLiteHunter/main/output/SQLiteHunter.yaml"
+      )
+
+      SELECT artifact_set(definition=Content, prefix="Updated.") AS Artifact
+      FROM http_client(url=get(item=URLlookup, field=PackageName))
+      WHERE Response = 200

vql/sigma/evaluator/evaluate.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/Velocidex/ordereddict"
 	"github.com/bradleyjkemp/sigma-go"
 	"www.velocidex.com/golang/vfilter"
 	"www.velocidex.com/golang/vfilter/types"
@@ -48,18 +49,29 @@ func (self *VQLRuleEvaluator) evaluateAggregationExpression(
 
 func (self *VQLRuleEvaluator) Match(ctx context.Context,
 	scope types.Scope, event *Event) (Result, error) {
+	subscope := scope.Copy().AppendVars(
+		ordereddict.NewDict().
+			Set("Event", event).
+			Set("Rule", self.Rule))
+	defer subscope.Close()
+
 	result := Result{
 		Match:            false,
 		SearchResults:    map[string]bool{},
 		ConditionResults: make([]bool, len(self.Detection.Conditions)),
 	}
 
+	// TODO: This needs to be done lazily so conditions do not need to
+	// be evaluated needlessly.
 	for identifier, search := range self.Detection.Searches {
 		var err error
-		result.SearchResults[identifier], err = self.evaluateSearch(ctx, scope, search, event)
+
+		eval_result, err := self.evaluateSearch(ctx, subscope, search, event)
 		if err != nil {
 			return Result{}, fmt.Errorf("error evaluating search %s: %w", identifier, err)
 		}
+
+		result.SearchResults[identifier] = eval_result
 	}
 
 	for conditionIndex, condition := range self.Detection.Conditions {

vql/sigma/evaluator/evaluate_search.go

@@ -141,7 +141,9 @@ eventMatcher:
 			if err != nil {
 				return false, err
 			}
-			if !self.matcherMatchesValues(matcherValues, comparator, allValuesMustMatch, values) {
+			if !self.matcherMatchesValues(
+				ctx, scope,
+				matcherValues, comparator, allValuesMustMatch, values) {
 				// this field didn't match so the overall matcher
 				// doesn't match, try the next EventMatcher
 				continue eventMatcher
@@ -195,14 +197,16 @@ func (self *VQLRuleEvaluator) GetFieldValuesFromEvent(
 }
 
 func (self *VQLRuleEvaluator) matcherMatchesValues(
+	ctx context.Context, scope types.Scope,
 	matcherValues []string, comparator modifiers.ComparatorFunc, allValuesMustMatch bool, actualValues []interface{}) bool {
 	matched := allValuesMustMatch
 	for _, expectedValue := range matcherValues {
 		valueMatchedEvent := false
 		// There are multiple possible event fields that each expected
 		// value needs to be compared against
 		for _, actualValue := range actualValues {
-			comparatorMatched, err := comparator(actualValue, expectedValue)
+			comparatorMatched, err := comparator(
+				ctx, scope, actualValue, expectedValue)
 			if err != nil {
 				// todo
 			}

vql/sigma/evaluator/modifiers/modifiers.go

@@ -1,6 +1,7 @@
 package modifiers
 
 import (
+	"context"
 	"encoding/base64"
 	"fmt"
 	"net"
@@ -9,6 +10,7 @@ import (
 	"strings"
 
 	"gopkg.in/yaml.v3"
+	"www.velocidex.com/golang/vfilter/types"
 )
 
 func GetComparator(modifiers ...string) (ComparatorFunc, error) {
@@ -54,7 +56,9 @@ func getComparator(comparators map[string]Comparator, modifiers ...string) (Comp
 		comparator = baseComparator{}
 	}
 
-	return func(actual, expected any) (bool, error) {
+	return func(
+		ctx context.Context, scope types.Scope,
+		actual, expected any) (bool, error) {
 		var err error
 		for _, modifier := range eventValueModifiers {
 			actual, err = modifier.Modify(actual)
@@ -69,17 +73,23 @@ func getComparator(comparators map[string]Comparator, modifiers ...string) (Comp
 			}
 		}
 
-		return comparator.Matches(actual, expected)
+		return comparator.Matches(
+			ctx, scope, actual, expected)
 	}, nil
 }
 
-// Comparator defines how the comparison between actual and expected field values is performed (the default is exact string equality).
-// For example, the `cidr` modifier uses a check based on the *net.IPNet Contains function
+// Comparator defines how the comparison between actual and expected
+// field values is performed (the default is exact string equality).
+// For example, the `cidr` modifier uses a check based on the
+// *net.IPNet Contains function
 type Comparator interface {
-	Matches(actual any, expected any) (bool, error)
+	Matches(ctx context.Context, scope types.Scope,
+		actual any, expected any) (bool, error)
 }
 
-type ComparatorFunc func(actual, expected any) (bool, error)
+type ComparatorFunc func(
+	ctx context.Context, scope types.Scope,
+	actual, expected any) (bool, error)
 
 // ValueModifier modifies the expected value before it is passed to the comparator.
 // For example, the `base64` modifier converts the expected value to base64.
@@ -97,6 +107,7 @@ var Comparators = map[string]Comparator{
 	"gte":        gte{},
 	"lt":         lt{},
 	"lte":        lte{},
+	"vql":        vql{},
 }
 
 var ComparatorsCaseSensitive = map[string]Comparator{
@@ -109,6 +120,7 @@ var ComparatorsCaseSensitive = map[string]Comparator{
 	"gte":        gte{},
 	"lt":         lt{},
 	"lte":        lte{},
+	"vql":        vql{},
 }
 
 var ValueModifiers = map[string]ValueModifier{
@@ -120,7 +132,9 @@ var EventValueModifiers = map[string]ValueModifier{}
 
 type baseComparator struct{}
 
-func (baseComparator) Matches(actual, expected any) (bool, error) {
+func (baseComparator) Matches(
+	ctx context.Context, scope types.Scope,
+	actual, expected any) (bool, error) {
 	switch {
 	case actual == nil && expected == "null":
 		// special case: "null" should match the case where a field isn't present (and so actual is nil)
@@ -133,40 +147,52 @@ func (baseComparator) Matches(actual, expected any) (bool, error) {
 
 type contains struct{}
 
-func (contains) Matches(actual, expected any) (bool, error) {
+func (contains) Matches(
+	ctx context.Context, scope types.Scope,
+	actual, expected any) (bool, error) {
 	// The Sigma spec defines that by default comparisons are case-insensitive
 	return strings.Contains(strings.ToLower(coerceString(actual)), strings.ToLower(coerceString(expected))), nil
 }
 
 type endswith struct{}
 
-func (endswith) Matches(actual, expected any) (bool, error) {
+func (endswith) Matches(
+	ctx context.Context, scope types.Scope,
+	actual, expected any) (bool, error) {
 	// The Sigma spec defines that by default comparisons are case-insensitive
 	return strings.HasSuffix(strings.ToLower(coerceString(actual)), strings.ToLower(coerceString(expected))), nil
 }
 
 type startswith struct{}
 
-func (startswith) Matches(actual, expected any) (bool, error) {
+func (startswith) Matches(
+	ctx context.Context, scope types.Scope,
+	actual, expected any) (bool, error) {
 	// The Sigma spec defines that by default comparisons are case-insensitive
 	return strings.HasPrefix(strings.ToLower(coerceString(actual)), strings.ToLower(coerceString(expected))), nil
 }
 
 type containsCS struct{}
 
-func (containsCS) Matches(actual, expected any) (bool, error) {
+func (containsCS) Matches(
+	ctx context.Context, scope types.Scope,
+	actual, expected any) (bool, error) {
 	return strings.Contains(coerceString(actual), coerceString(expected)), nil
 }
 
 type endswithCS struct{}
 
-func (endswithCS) Matches(actual, expected any) (bool, error) {
+func (endswithCS) Matches(
+	ctx context.Context, scope types.Scope,
+	actual, expected any) (bool, error) {
 	return strings.HasSuffix(coerceString(actual), coerceString(expected)), nil
 }
 
 type startswithCS struct{}
 
-func (startswithCS) Matches(actual, expected any) (bool, error) {
+func (startswithCS) Matches(
+	ctx context.Context, scope types.Scope,
+	actual, expected any) (bool, error) {
 	return strings.HasPrefix(coerceString(actual), coerceString(expected)), nil
 }
 
@@ -178,7 +204,9 @@ func (b64) Modify(value any) (any, error) {
 
 type re struct{}
 
-func (re) Matches(actual any, expected any) (bool, error) {
+func (re) Matches(
+	ctx context.Context, scope types.Scope,
+	actual any, expected any) (bool, error) {
 	re, err := regexp.Compile("(?i)" + coerceString(expected))
 	if err != nil {
 		return false, err
@@ -189,7 +217,9 @@ func (re) Matches(actual any, expected any) (bool, error) {
 
 type cidr struct{}
 
-func (cidr) Matches(actual any, expected any) (bool, error) {
+func (cidr) Matches(
+	ctx context.Context, scope types.Scope,
+	actual any, expected any) (bool, error) {
 	_, cidr, err := net.ParseCIDR(coerceString(expected))
 	if err != nil {
 		return false, err
@@ -201,28 +231,36 @@ func (cidr) Matches(actual any, expected any) (bool, error) {
 
 type gt struct{}
 
-func (gt) Matches(actual any, expected any) (bool, error) {
+func (gt) Matches(
+	ctx context.Context, scope types.Scope,
+	actual any, expected any) (bool, error) {
 	gt, _, _, _, err := compareNumeric(actual, expected)
 	return gt, err
 }
 
 type gte struct{}
 
-func (gte) Matches(actual any, expected any) (bool, error) {
+func (gte) Matches(
+	ctx context.Context, scope types.Scope,
+	actual any, expected any) (bool, error) {
 	_, gte, _, _, err := compareNumeric(actual, expected)
 	return gte, err
 }
 
 type lt struct{}
 
-func (lt) Matches(actual any, expected any) (bool, error) {
+func (lt) Matches(
+	ctx context.Context, scope types.Scope,
+	actual any, expected any) (bool, error) {
 	_, _, lt, _, err := compareNumeric(actual, expected)
 	return lt, err
 }
 
 type lte struct{}
 
-func (lte) Matches(actual any, expected any) (bool, error) {
+func (lte) Matches(
+	ctx context.Context, scope types.Scope,
+	actual any, expected any) (bool, error) {
 	_, _, _, lte, err := compareNumeric(actual, expected)
 	return lte, err
 }

vql/sigma/evaluator/modifiers/modifiers_test.go

@@ -1,9 +1,12 @@
 package modifiers
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 	"testing"
+
+	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 )
 
 func Test_compareNumeric(t *testing.T) {
@@ -49,6 +52,9 @@ func Test_compareNumeric(t *testing.T) {
 func BenchmarkContains(b *testing.B) {
 	needle := "abcdefg"
 
+	ctx := context.Background()
+	scope := vql_subsystem.MakeScope()
+
 	var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 	haystack := make([]rune, 1_000_000)
 	for i := range haystack {
@@ -57,7 +63,8 @@ func BenchmarkContains(b *testing.B) {
 	haystackString := string(haystack)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		_, err := contains{}.Matches(string(haystackString), needle)
+		_, err := contains{}.Matches(
+			ctx, scope, string(haystackString), needle)
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -67,6 +74,9 @@ func BenchmarkContains(b *testing.B) {
 func BenchmarkContainsCS(b *testing.B) {
 	needle := "abcdefg"
 
+	ctx := context.Background()
+	scope := vql_subsystem.MakeScope()
+
 	var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 	haystack := make([]rune, 1_000_000)
 	for i := range haystack {
@@ -75,7 +85,8 @@ func BenchmarkContainsCS(b *testing.B) {
 	haystackString := string(haystack)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		_, err := containsCS{}.Matches(string(haystackString), needle)
+		_, err := containsCS{}.Matches(
+			ctx, scope, string(haystackString), needle)
 		if err != nil {
 			b.Fatal(err)
 		}