Add an abstraction layer 'FileSystem' to implement a VFS

This just shuffles some APIs around, no real changes other than adding
some indirection.

index/index.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/hound-search/hound/codesearch/index"
 	"github.com/hound-search/hound/codesearch/regexp"
+	"github.com/hound-search/hound/vcs"
 )
 
 const (
@@ -256,9 +257,9 @@ func (n *Index) Search(pat string, opt *SearchOptions) (*SearchResponse, error)
 	}, nil
 }
 
-func isTextFile(filename string) (bool, error) {
+func isTextFile(filename string, fs vcs.FileSystem) (bool, error) {
 	buf := make([]byte, filePeekSize)
-	r, err := os.Open(filename)
+	r, err := fs.Open(filename)
 	if err != nil {
 		return false, err
 	}
@@ -307,13 +308,8 @@ func validUTF8IgnoringPartialTrailingRune(p []byte) bool {
 	return true
 }
 
-func addFileToIndex(ix *index.IndexWriter, dst, src, path string) (string, error) {
-	rel, err := filepath.Rel(src, path)
-	if err != nil {
-		return "", err
-	}
-
-	r, err := os.Open(path)
+func addFileToIndex(ix *index.IndexWriter, dst string, fs vcs.FileSystem, rel string) (string, error) {
+	r, err := fs.Open(rel)
 	if err != nil {
 		return "", err
 	}
@@ -332,12 +328,7 @@ func addFileToIndex(ix *index.IndexWriter, dst, src, path string) (string, error
 	return ix.Add(rel, io.TeeReader(r, g)), nil
 }
 
-func addDirToIndex(dst, src, path string) error {
-	rel, err := filepath.Rel(src, path)
-	if err != nil {
-		return err
-	}
-
+func addDirToIndex(dst string, fs vcs.FileSystem, rel string) error {
 	if rel == "." {
 		return nil
 	}
@@ -366,7 +357,7 @@ func containsString(haystack []string, needle string) bool {
 	return false
 }
 
-func indexAllFiles(opt *IndexOptions, dst, src string) error {
+func indexAllFiles(opt *IndexOptions, dst string, fs vcs.FileSystem) error {
 	ix := index.Create(filepath.Join(dst, "tri"))
 	defer ix.Close()
 
@@ -379,19 +370,8 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error {
 	}
 	defer fileHandle.Close()
 
-	// Resolve the symbolic link
-	if fi, err := os.Stat(src); err == nil && fi.Mode()|os.ModeSymlink != 0 {
-		if s, err := os.Readlink(src); err == nil {
-			src = s
-		}
-	}
-
-	if err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { //nolint
+	if err := fs.Walk(func(rel string, info os.FileInfo, err error) error { //nolint
 		name := info.Name()
-		rel, err := filepath.Rel(src, path) //nolint
-		if err != nil {
-			return err
-		}
 
 		// Is this file considered "special", this means it's not even a part
 		// of the source repository (like .git or .svn).
@@ -415,7 +395,7 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error {
 		}
 
 		if info.IsDir() {
-			return addDirToIndex(dst, src, path)
+			return addDirToIndex(dst, fs, rel)
 		}
 
 		if info.Mode()&os.ModeType != 0 {
@@ -426,7 +406,7 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error {
 			return nil
 		}
 
-		txt, err := isTextFile(path)
+		txt, err := isTextFile(rel, fs)
 		if err != nil {
 			return err
 		}
@@ -439,7 +419,7 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error {
 			return nil
 		}
 
-		reasonForExclusion, err := addFileToIndex(ix, dst, src, path)
+		reasonForExclusion, err := addFileToIndex(ix, dst, fs, rel)
 		if err != nil {
 			return err
 		}
@@ -485,7 +465,7 @@ func Read(dir string) (*IndexRef, error) {
 	return m, nil
 }
 
-func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) {
+func Build(opt *IndexOptions, dst string, fs vcs.FileSystem, url, rev string) (*IndexRef, error) {
 	if _, err := os.Stat(dst); err != nil {
 		if err := os.MkdirAll(dst, os.ModePerm); err != nil {
 			return nil, err
@@ -496,7 +476,7 @@ func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) {
 		return nil, err
 	}
 
-	if err := indexAllFiles(opt, dst, src); err != nil {
+	if err := indexAllFiles(opt, dst, fs); err != nil {
 		return nil, err
 	}
 

index/index_test.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"testing"
+
+	"github.com/hound-search/hound/vcs"
 )
 
 const (
@@ -26,9 +28,14 @@ func buildIndex(url, rev string) (*IndexRef, error) {
 		return nil, err
 	}
 
+	dirFs, err := vcs.NewDirFilesystem(thisDir())
+	if err != nil {
+		return nil, err
+	}
+
 	var opt IndexOptions
 
-	return Build(&opt, dir, thisDir(), url, rev)
+	return Build(&opt, dir, dirFs, url, rev)
 }
 
 func TestSearch(t *testing.T) {

searcher/searcher.go

@@ -232,13 +232,13 @@ func findExistingRefs(dbpath string) (*foundRefs, error) {
 // one will be built.
 func buildAndOpenIndex(
 	opt *index.IndexOptions,
-	dbpath,
-	vcsDir,
+	dbpath string,
+	fs vcs.FileSystem,
 	idxDir,
 	url,
 	rev string) (*index.Index, error) {
 	if _, err := os.Stat(idxDir); err != nil {
-		r, err := index.Build(opt, idxDir, vcsDir, url, rev)
+		r, err := index.Build(opt, idxDir, fs, url, rev)
 		if err != nil {
 			return nil, err
 		}
@@ -366,11 +366,17 @@ func updateAndReindex(
 		return rev, false
 	}
 
+	fs, err := wd.FileSystem(vcsDir)
+	if err != nil {
+		log.Printf("vcs fs error: %s", err)
+		return rev, false
+	}
+
 	log.Printf("Rebuilding %s for %s", name, newRev)
 	idx, err := buildAndOpenIndex(
 		opt,
 		dbpath,
-		vcsDir,
+		fs,
 		nextIndexDir(dbpath),
 		repo.Url,
 		newRev)
@@ -434,10 +440,15 @@ func newSearcher(
 		refs.claim(ref)
 	}
 
+	fs, err := wd.FileSystem(vcsDir)
+	if err != nil {
+		return nil, err
+	}
+
 	idx, err := buildAndOpenIndex(
 		opt,
 		dbpath,
-		vcsDir,
+		fs,
 		idxDir,
 		repo.Url,
 		rev)

vcs/bzr.go

@@ -81,3 +81,7 @@ func (g *BzrDriver) SpecialFiles() []string {
 func (g *BzrDriver) AutoGeneratedFiles(dir string) []string {
 	return []string{}
 }
+
+func (g *BzrDriver) FileSystem(dir string) (FileSystem, error) {
+	return NewDirFilesystem(dir)
+}

vcs/dir_fs.go

@@ -0,0 +1,44 @@
+package vcs
+
+import (
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+type dirFilesystem struct {
+	dir string
+}
+
+func NewDirFilesystem(dir string) (FileSystem, error) {
+	// Resolve the symbolic link
+	if fi, err := os.Stat(dir); err == nil && fi.Mode()|os.ModeSymlink != 0 {
+		if s, err := os.Readlink(dir); err == nil {
+			dir = s
+		}
+	}
+
+	return &dirFilesystem{dir: dir}, nil
+}
+
+func (dir *dirFilesystem) Open(name string) (io.ReadCloser, error) {
+	if strings.HasPrefix(name, "/") {
+		return nil, fmt.Errorf("Expected relative path, got absolute: %s", name)
+	}
+	return os.Open(path.Join(dir.dir, name))
+}
+
+func (dir *dirFilesystem) Walk(fn filepath.WalkFunc) error {
+	return filepath.Walk(dir.dir, func(path string, info fs.FileInfo, err error) error {
+		rel, err := filepath.Rel(dir.dir, path)
+		if err != nil {
+			return err
+		}
+
+		return fn(rel, info, err)
+	})
+}

vcs/git.go

@@ -227,3 +227,7 @@ func (d *headBranchDetector) detectRef(dir string) string {
 
 	return matches[1]
 }
+
+func (g *GitDriver) FileSystem(dir string) (FileSystem, error) {
+	return NewDirFilesystem(dir)
+}

vcs/hg.go

@@ -83,3 +83,7 @@ func (g *MercurialDriver) SpecialFiles() []string {
 func (g *MercurialDriver) AutoGeneratedFiles(dir string) []string {
 	return []string{}
 }
+
+func (g *MercurialDriver) FileSystem(dir string) (FileSystem, error) {
+	return NewDirFilesystem(dir)
+}

vcs/local.go

@@ -105,3 +105,7 @@ func (g *LocalDriver) SpecialFiles() []string {
 func (g *LocalDriver) AutoGeneratedFiles(dir string) []string {
 	return []string{}
 }
+
+func (g *LocalDriver) FileSystem(dir string) (FileSystem, error) {
+	return NewDirFilesystem(dir)
+}

vcs/svn.go

@@ -104,3 +104,7 @@ func (g *SVNDriver) SpecialFiles() []string {
 func (g *SVNDriver) AutoGeneratedFiles(dir string) []string {
 	return []string{}
 }
+
+func (g *SVNDriver) FileSystem(dir string) (FileSystem, error) {
+	return NewDirFilesystem(dir)
+}

vcs/vcs.go

@@ -2,15 +2,25 @@ package vcs
 
 import (
 	"fmt"
+	"io"
 	"log"
 	"os"
+	"path/filepath"
 )
 
 // A collection that maps vcs names to their underlying
 // factory. A factory allows the vcs to have unserialized
 // json config passed in to be parsed.
 var drivers = make(map[string]func(c []byte) (Driver, error))
 
+// A filesystem API abstraction to allow accessing vcs objects directly
+// instead of duplicating them on disk
+type FileSystem interface {
+	Open(name string) (io.ReadCloser, error)
+
+	Walk(fn filepath.WalkFunc) error
+}
+
 // A "plugin" for each vcs that supports the very limited set of vcs
 // operations that hound needs.
 type Driver interface {
@@ -29,6 +39,8 @@ type Driver interface {
 
 	// Return a list of filenames that are marked as auto-generated.
 	AutoGeneratedFiles(dir string) []string
+
+	FileSystem(dir string) (FileSystem, error)
 }
 
 // An API to interact with a vcs working directory. This is