Add Dir method to DotFS value; Add Migrate to DotDB

providers/db.go

@@ -4,8 +4,14 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"io/fs"
 	"log/slog"
+	"regexp"
+	"slices"
+	"strconv"
 	"time"
+
+	"github.com/google/uuid"
 )
 
 // DotDB is used to create a dot field value that can query a SQL database. When
@@ -149,3 +155,103 @@ func (c *DotDB) rollback() error {
 	}
 	return nil
 }
+
+// Migrate applies the migration files in the specified directory that match the
+// given regular expression in ascending order based on their ID. Each migration
+// file is run in its own transaction. Migrations must have id greater than 0.
+//
+// Parameters:
+//   - dir: The directory containing the migration files.
+//   - matchExpr: The regular expression used to match the migration file names. The expression must have
+//     at least one subexpression to capture the migration ID of the file.
+//   - currentId: The current migration ID. Only migration files with an ID greater than this value will be applied.
+//   - includeIds: A list of specific migration IDs to apply. If provided other ids are skipped.
+//
+// Returns the number of migrations applied, and returns an error on these conditions:
+//   - Rexex expr does not have a submatch
+//   - Fails to list files in the dir
+//   - The submatch expression is not convertible to a uint64
+//   - The migration file could not be read
+//   - The migration statement failed during execution
+//
+// The function uses a unique UUID to identify each migration run and logs
+// various events at DEBUG level during the migration process.
+func (c *DotDB) Migrate(dir Dir, matchExpr string, currentId int64, includeIds ...int64) (int, error) {
+	type M struct {
+		id   int64
+		file fs.DirEntry
+	}
+	re, err := regexp.Compile(matchExpr)
+	if err != nil {
+		return 0, err
+	}
+	if re.NumSubexp() == 0 {
+		return 0, fmt.Errorf("regex expr must have at least one subexpression to match the migration id")
+	}
+	idx := 1
+	if nidx := re.SubexpIndex("id"); nidx != -1 {
+		idx = nidx
+	}
+	files, err := dir.List(".")
+	if err != nil {
+		return 0, fmt.Errorf("failed to list files in dir")
+	}
+	slices.Sort(includeIds)
+	UUID := uuid.NewString()
+	log := c.log.WithGroup("migration").With(slog.String("uuid", UUID))
+	log.Debug("performing migration", slog.Int("migration_files", len(files)), slog.Int64("current_id", currentId), slog.String("matchExpr", matchExpr), slog.Int("submatch_idx", idx))
+	matches := []M{}
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		m := re.FindStringSubmatch(file.Name())
+		if m == nil {
+			log.Debug("file did not match expr", slog.String("name", file.Name()))
+			continue
+		}
+		id, err := strconv.ParseInt(m[idx], 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("submatch not convertible to int. uuid: %s, file: '%s', submatch: '%s'", UUID, file.Name(), m[idx])
+		}
+		if id <= currentId {
+			log.Debug("skipping file before current id", slog.String("name", file.Name()), slog.Int64("id", id))
+			continue
+		}
+		if _, found := slices.BinarySearch(includeIds, id); len(includeIds) > 0 && !found {
+			log.Debug("skipping file not listed in includeIds", slog.String("name", file.Name()), slog.Int64("id", id))
+			continue
+		}
+		matches = append(matches, M{id, file})
+		log.Debug("added file", slog.String("name", file.Name()), slog.Int64("id", id))
+	}
+	slices.SortFunc(matches, func(a, b M) int {
+		if a.id < b.id {
+			return -1
+		} else if a.id > b.id {
+			return 1
+		} else {
+			return 0
+		}
+	})
+	err = c.commit()
+	if err != nil {
+		return 0, fmt.Errorf("failed to commit ambient tx. uuid: %s, error: %w", UUID, err)
+	}
+	for i, file := range matches {
+		statement, err := dir.Read(file.file.Name())
+		if err != nil {
+			return i, fmt.Errorf("failed to read migration file. uuid: %s, file: '%s', id: %d, error: %w", UUID, file.file.Name(), file.id, err)
+		}
+		_, err = c.Exec(statement)
+		if err != nil {
+			return i, fmt.Errorf("failed to execute migration statement. uuid: %s, file: '%s', id: %d, error: %w", UUID, file.file.Name(), file.id, err)
+		}
+		err = c.commit()
+		if err != nil {
+			return i, fmt.Errorf("failed to commit migration tx. uuid: %s, file: '%s', id: %d, error: %w", UUID, file.file.Name(), file.id, err)
+		}
+		log.Debug("applied migration file", slog.String("name", file.file.Name()), slog.Int64("id", file.id))
+	}
+	return len(matches), nil
+}

providers/fs.go

@@ -6,79 +6,79 @@ import (
 	"io"
 	"io/fs"
 	"log/slog"
-	"net/http"
 	"path"
 	"sync"
 )
 
-// DotFS is used to create an xtemplate dot field value that can access files in
-// a local directory, or any [fs.FS].
-//
-// All public methods on DotFS are
-type DotFS struct {
+type dotFS struct {
 	fs     fs.FS
 	log    *slog.Logger
-	w      http.ResponseWriter
-	r      *http.Request
 	opened map[fs.File]struct{}
 }
 
+// Dir
+type Dir struct {
+	dot  *dotFS
+	path string
+}
+
 var bufPool = sync.Pool{
 	New: func() any {
 		return new(bytes.Buffer)
 	},
 }
 
-// List reads and returns a slice of names from the given directory
-// relative to the FS root.
-func (c *DotFS) List(name string) ([]string, error) {
-	entries, err := fs.ReadDir(c.fs, path.Clean(name))
-	if err != nil {
-		return nil, err
-	}
-
-	names := make([]string, 0, len(entries))
-	for _, dirEntry := range entries {
-		names = append(names, dirEntry.Name())
+// Dir returns a
+func (d Dir) Dir(name string) (Dir, error) {
+	name = path.Clean(name)
+	if st, err := d.Stat(name); err != nil {
+		return Dir{}, err
+	} else if !st.IsDir() {
+		return Dir{}, fmt.Errorf("not a directory: %s", name)
 	}
+	return Dir{dot: d.dot, path: path.Join(d.path, name)}, nil
+}
 
-	return names, nil
+// List reads and returns a slice of names from the given directory relative to
+// the FS root.
+func (d Dir) List(name string) ([]fs.DirEntry, error) {
+	return fs.ReadDir(d.dot.fs, path.Join(d.path, path.Clean(name)))
 }
 
 // Exists returns true if filename can be opened successfully.
-func (c *DotFS) Exists(filename string) (bool, error) {
-	file, err := c.fs.Open(filename)
+func (d Dir) Exists(name string) bool {
+	name = path.Join(d.path, path.Clean(name))
+	file, err := d.Open(name)
 	if err == nil {
 		file.Close()
-		return true, nil
+		return true
 	}
-	return false, nil
+	return false
 }
 
 // Stat returns Stat of a filename.
 //
 // Note: if you intend to read the file, afterwards, calling .Open instead may
 // be more efficient.
-func (c *DotFS) Stat(filename string) (fs.FileInfo, error) {
-	filename = path.Clean(filename)
-	file, err := c.fs.Open(filename)
+func (d Dir) Stat(name string) (fs.FileInfo, error) {
+	name = path.Join(d.path, path.Clean(name))
+	file, err := d.dot.fs.Open(name)
 	if err != nil {
 		return nil, err
 	}
 	defer file.Close()
-
 	return file.Stat()
 }
 
-// Read returns the contents of a filename relative to the FS root as a
-// string.
-func (c *DotFS) Read(filename string) (string, error) {
+// Read returns the contents of a filename relative to the FS root as a string.
+func (d Dir) Read(name string) (string, error) {
+	name = path.Join(d.path, path.Clean(name))
+
 	buf := bufPool.Get().(*bytes.Buffer)
 	buf.Reset()
 	defer bufPool.Put(buf)
 
-	filename = path.Clean(filename)
-	file, err := c.fs.Open(filename)
+	file, err := d.dot.fs.Open(name)
 	if err != nil {
 		return "", err
 	}
@@ -93,16 +93,16 @@ func (c *DotFS) Read(filename string) (string, error) {
 }
 
 // Open opens the file
-func (c *DotFS) Open(path_ string) (fs.File, error) {
-	path_ = path.Clean(path_)
+func (d Dir) Open(name string) (fs.File, error) {
+	name = path.Join(d.path, path.Clean(name))
 
-	file, err := c.fs.Open(path_)
+	file, err := d.dot.fs.Open(name)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open file at path '%s': %w", path_, err)
+		return nil, fmt.Errorf("failed to open file at path '%s': %w", name, err)
 	}
 
-	c.log.Debug("opened file", slog.String("path", path_))
-	c.opened[file] = struct{}{}
+	d.dot.log.Debug("opened file", slog.String("path", name))
+	d.dot.opened[file] = struct{}{}
 
-	return file, nil
+	return d.dot.fs.Open(name)
 }

providers/fsprovider.go

@@ -81,14 +81,14 @@ func (p *DotFSProvider) Value(r xtemplate.Request) (any, error) {
 		if _, err := newfs.(interface {
 			Stat(string) (fs.FileInfo, error)
 		}).Stat("."); err != nil {
-			return &DotFS{}, fmt.Errorf("failed to stat fs current directory '%s': %w", p.Path, err)
+			return Dir{}, fmt.Errorf("failed to stat fs current directory '%s': %w", p.Path, err)
 		}
 		p.FS = newfs
 	}
-	return &DotFS{p.FS, xtemplate.GetLogger(r.R.Context()), r.W, r.R, make(map[fs.File]struct{})}, nil
+	return Dir{dot: &dotFS{p.FS, xtemplate.GetLogger(r.R.Context()), make(map[fs.File]struct{})}, path: "."}, nil
 }
 func (p *DotFSProvider) Cleanup(a any, err error) error {
-	v := a.(*DotFS)
+	v := a.(Dir).dot
 	errs := []error{}
 	for file := range v.opened {
 		if err := file.Close(); err != nil {

test/config.json

@@ -5,6 +5,11 @@
             "name": "FSW",
             "path": "./dataw"
         },
+        {
+            "type": "fs",
+            "name": "Migrations",
+            "path": "./migrations"
+        },
         {
             "type": "kv",
             "name": "KV",

test/migrations/schema.1.sql

@@ -0,0 +1,3 @@
+CREATE TABLE test(data);
+
+PRAGMA user_version = 1;

test/migrations/schema.10.sql

@@ -0,0 +1,3 @@
+DROP INDEX idx_test;
+
+PRAGMA user_version = 10;

test/migrations/schema.2.sql

@@ -0,0 +1,3 @@
+CREATE INDEX idx_test ON test(data);
+
+PRAGMA user_version = 2;

test/templates/db/index.html

@@ -0,0 +1,5 @@
+<!DOCTYPE html>
+
+<p>{{.DB.Migrate .Migrations `^schema\.(\d+)\.sql$` (.DB.QueryVal `PRAGMA user_version;`)}} migrations applied.</p>
+
+<p>Current user_version: {{.DB.QueryVal `PRAGMA user_version`}}</p>

test/templates/fs/{filepath...}.html → test/templates/fs/browse/{filepath...}.html

@@ -1,6 +1,6 @@
 <!DOCTYPE html>
 {{$path := .Req.PathValue "filepath"}}
-{{if ne $path ""}}<p><a href="/fs/{{dir $path}}">Go up</a></p>{{else}}{{$path = "."}}{{end}}
+{{if ne $path ""}}<p><a href="/fs/browse/{{dir $path}}">Go up</a></p>{{else}}{{$path = "."}}{{end}}
 {{$result := try .FS "Stat" $path}}
 {{if not $result.OK}}
 <p>Path <code>{{$path}}</code>&nbsp;doesn't exist</p>
@@ -13,7 +13,7 @@
     {{range .FS.List $path}}
     {{$lpath := list $path . | join "/"}}
     {{$lstat := try $.FS "Stat" $lpath}}
-    <li><a href="/fs/{{$path}}/{{.}}">{{.}}</a> {{if $lstat.OK}}{{$lstat.Value.Mode}} {{printf "%+v" $lstat.Value.Sys}}{{else}}{{$lstat.Error}}{{end}}</li>
+    <li><a href="/fs/browse/{{$path}}/{{.}}">{{.}}</a> {{if $lstat.OK}}{{$lstat.Value.Mode}} {{printf "%+v" $lstat.Value.Sys}}{{else}}{{$lstat.Error}}{{end}}</li>
     {{end}}
 </ul>
 {{else}}

test/tests/db.hurl

@@ -0,0 +1,14 @@
+GET http://localhost:8080/db
+
+HTTP 200
+[Asserts]
+body contains "3 migrations applied"
+body contains "Current user_version: 10"
+
+
+GET http://localhost:8080/db
+
+HTTP 200
+[Asserts]
+body contains "0 migrations applied"
+body contains "Current user_version: 10"

test/tests/fs.hurl

@@ -1,11 +1,19 @@
 # reading files from fs
-GET http://localhost:8080/fs/
+GET http://localhost:8080/fs/browse/
 
 HTTP 200
 [Asserts]
 body contains "<!doctype html>"
 body contains "listing"
 
+# reading files from fs
+GET http://localhost:8080/fs/browse/subdir/
+
+HTTP 200
+[Asserts]
+body contains "<!doctype html>"
+body contains "world.txt"
+
 # serve content
 GET http://localhost:8080/fs/serve