Implement core: walker, chunker, embedder, index, CLI

1 files changed, 109 insertions, 0 deletions

diff --git a/internal/walker/walker.go b/internal/walker/walker.go
new file mode 100644
index 0000000..0ac470d
--- /dev/null
+++ b/internal/walker/walker.go
@@ -0,0 +1,109 @@
+package walker
+
+import (
+        "os"
+        "path/filepath"
+        "strings"
+
+        ignore "github.com/sabhiram/go-gitignore"
+)
+
+// DefaultIgnore patterns applied to all walks
+var DefaultIgnore = []string{
+        "vendor/",
+        "node_modules/",
+        ".git/",
+        ".codevec/",
+}
+
+// Walker walks a directory tree finding files to index
+type Walker struct {
+        root       string
+        extensions []string // e.g., [".go"]
+        gitignore  *ignore.GitIgnore
+}
+
+// New creates a walker for the given root directory
+func New(root string, extensions []string) (*Walker, error) {
+        root, err := filepath.Abs(root)
+        if err != nil {
+                return nil, err
+        }
+
+        w := &Walker{
+                root:       root,
+                extensions: extensions,
+        }
+
+        // Load .gitignore if present
+        gitignorePath := filepath.Join(root, ".gitignore")
+        if _, err := os.Stat(gitignorePath); err == nil {
+                gi, err := ignore.CompileIgnoreFile(gitignorePath)
+                if err == nil {
+                        w.gitignore = gi
+                }
+        }
+
+        return w, nil
+}
+
+// Walk returns all matching files in the directory tree
+func (w *Walker) Walk() ([]string, error) {
+        var files []string
+
+        err := filepath.WalkDir(w.root, func(path string, d os.DirEntry, err error) error {
+                if err != nil {
+                        return err
+                }
+
+                // Get path relative to root for ignore matching
+                relPath, err := filepath.Rel(w.root, path)
+                if err != nil {
+                        return err
+                }
+
+                // Skip default ignored directories
+                if d.IsDir() {
+                        for _, pattern := range DefaultIgnore {
+                                if strings.HasPrefix(relPath+"/", pattern) || relPath+"/" == pattern {
+                                        return filepath.SkipDir
+                                }
+                        }
+                }
+
+                // Skip if matched by .gitignore
+                if w.gitignore != nil && w.gitignore.MatchesPath(relPath) {
+                        if d.IsDir() {
+                                return filepath.SkipDir
+                        }
+                        return nil
+                }
+
+                // Skip directories and non-matching extensions
+                if d.IsDir() {
+                        return nil
+                }
+
+                if !w.matchesExtension(path) {
+                        return nil
+                }
+
+                files = append(files, path)
+                return nil
+        })
+
+        return files, err
+}
+
+func (w *Walker) matchesExtension(path string) bool {
+        if len(w.extensions) == 0 {
+                return true
+        }
+        ext := filepath.Ext(path)
+        for _, e := range w.extensions {
+                if ext == e {
+                        return true
+                }
+        }
+        return false
+}