feat(api)!: give the embedding engine an explicit store and Close
The embedding engine now backs its graph with the SQLite store instead of an in-memory graph, so it holds a database handle and a write-ahead-log checkpointer that have to be released. New returns an error because opening the store can fail, and the new Close is required: it closes the store and removes the temp directory the zero-config constructor creates. WithStorePath opts into a store that outlives the process, so an embedder can index once and query the same graph on later runs.
This commit is contained in:
@@ -116,6 +116,17 @@ internal/
|
||||
pkg/gortex/ Public API for embedding
|
||||
```
|
||||
|
||||
## Public API (`pkg/gortex`)
|
||||
|
||||
`New` returns `(*Engine, error)` — it opens a SQLite graph store, which can
|
||||
fail. Pass `WithStorePath` to keep the store at a path of your choosing and
|
||||
reuse the index on the next run; without it the store lives in a temp
|
||||
directory.
|
||||
|
||||
Every `Engine` must be closed. `Close` checkpoints the write-ahead log, closes
|
||||
the database handle, and removes the temp directory when `New` created one —
|
||||
skipping it leaks a file handle and a background goroutine.
|
||||
|
||||
## Questions?
|
||||
|
||||
Open an issue or start a discussion. We're happy to help!
|
||||
|
||||
+89
-10
@@ -2,10 +2,15 @@
|
||||
package gortex
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zzet/gortex/internal/config"
|
||||
"github.com/zzet/gortex/internal/graph"
|
||||
"github.com/zzet/gortex/internal/graph/store_sqlite"
|
||||
"github.com/zzet/gortex/internal/indexer"
|
||||
"github.com/zzet/gortex/internal/parser"
|
||||
"github.com/zzet/gortex/internal/parser/languages"
|
||||
@@ -13,40 +18,114 @@ import (
|
||||
)
|
||||
|
||||
// Engine is the public entry point for the Gortex code intelligence engine.
|
||||
//
|
||||
// An Engine owns a SQLite graph store — an open database handle plus the
|
||||
// background bookkeeping that keeps its write-ahead log in check — so every
|
||||
// Engine must be closed when the caller is done with it. See Close.
|
||||
type Engine struct {
|
||||
graph *graph.Graph
|
||||
store *store_sqlite.Store
|
||||
indexer *indexer.Indexer
|
||||
query *query.Engine
|
||||
|
||||
// tmpDir is non-empty when New created the store in a temp directory it
|
||||
// owns; Close removes it.
|
||||
tmpDir string
|
||||
}
|
||||
|
||||
// Option configures an Engine.
|
||||
type Option func(*config.IndexConfig)
|
||||
type Option func(*settings)
|
||||
|
||||
// settings collects everything the options can influence before the Engine
|
||||
// and its store are constructed.
|
||||
type settings struct {
|
||||
index config.IndexConfig
|
||||
storePath string
|
||||
}
|
||||
|
||||
// WithWorkers sets the number of parallel parsing workers.
|
||||
func WithWorkers(n int) Option {
|
||||
return func(c *config.IndexConfig) { c.Workers = n }
|
||||
return func(s *settings) { s.index.Workers = n }
|
||||
}
|
||||
|
||||
// WithExclude adds exclude patterns.
|
||||
func WithExclude(patterns ...string) Option {
|
||||
return func(c *config.IndexConfig) { c.Exclude = append(c.Exclude, patterns...) }
|
||||
return func(s *settings) { s.index.Exclude = append(s.index.Exclude, patterns...) }
|
||||
}
|
||||
|
||||
// WithStorePath puts the graph store at path, creating the file and any
|
||||
// missing parent directories. The store survives Close, so a later Engine
|
||||
// opened on the same path starts from the graph the previous run indexed.
|
||||
//
|
||||
// Without this option New keeps the store in a temp directory that Close
|
||||
// deletes, which suits one-shot analysis but throws the index away.
|
||||
func WithStorePath(path string) Option {
|
||||
return func(s *settings) { s.storePath = path }
|
||||
}
|
||||
|
||||
// New creates a new Gortex Engine with the given options.
|
||||
func New(opts ...Option) *Engine {
|
||||
//
|
||||
// The caller owns the returned Engine's store and must call Close on it.
|
||||
func New(opts ...Option) (*Engine, error) {
|
||||
cfg := config.Default()
|
||||
set := &settings{index: cfg.Index}
|
||||
for _, o := range opts {
|
||||
o(&cfg.Index)
|
||||
o(set)
|
||||
}
|
||||
|
||||
path := set.storePath
|
||||
tmpDir := ""
|
||||
if path == "" {
|
||||
dir, err := os.MkdirTemp("", "gortex-engine-*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create temporary graph store directory: %w", err)
|
||||
}
|
||||
tmpDir = dir
|
||||
path = filepath.Join(dir, "graph.sqlite")
|
||||
} else if dir := filepath.Dir(path); dir != "" && dir != "." {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create graph store directory %s: %w", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
st, err := store_sqlite.Open(path)
|
||||
if err != nil {
|
||||
if tmpDir != "" {
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
}
|
||||
return nil, fmt.Errorf("open graph store %s: %w", path, err)
|
||||
}
|
||||
|
||||
g := graph.New()
|
||||
reg := parser.NewRegistry()
|
||||
languages.RegisterAll(reg)
|
||||
|
||||
idx := indexer.New(g, reg, cfg.Index, zap.NewNop())
|
||||
eng := query.NewEngine(g)
|
||||
return &Engine{
|
||||
store: st,
|
||||
indexer: indexer.New(st, reg, set.index, zap.NewNop()),
|
||||
query: query.NewEngine(st),
|
||||
tmpDir: tmpDir,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return &Engine{graph: g, indexer: idx, query: eng}
|
||||
// Close releases the graph store: it checkpoints the write-ahead log and
|
||||
// closes the database handle, and removes the temp directory when New created
|
||||
// one. Every Engine must be closed exactly once; calling it a second time is a
|
||||
// no-op. After Close the Engine's query and index methods must not be used.
|
||||
func (e *Engine) Close() error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
if e.store != nil {
|
||||
err = e.store.Close()
|
||||
e.store = nil
|
||||
}
|
||||
if e.tmpDir != "" {
|
||||
if rmErr := os.RemoveAll(e.tmpDir); rmErr != nil && err == nil {
|
||||
err = rmErr
|
||||
}
|
||||
e.tmpDir = ""
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// IndexResult is the result of an indexing operation.
|
||||
|
||||
+47
-2
@@ -9,15 +9,24 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEngine_IndexAndQuery(t *testing.T) {
|
||||
func writeSample(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "main.go"), []byte(`package main
|
||||
|
||||
func Hello() {}
|
||||
func World() { Hello() }
|
||||
`), 0o644))
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestEngine_IndexAndQuery(t *testing.T) {
|
||||
dir := writeSample(t)
|
||||
|
||||
eng, err := New(WithWorkers(1))
|
||||
require.NoError(t, err)
|
||||
defer func() { require.NoError(t, eng.Close()) }()
|
||||
|
||||
eng := New(WithWorkers(1))
|
||||
result, err := eng.Index(dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -34,3 +43,39 @@ func World() { Hello() }
|
||||
assert.Greater(t, stats.TotalNodes, 0)
|
||||
assert.Equal(t, stats.TotalNodes, result.NodeCount)
|
||||
}
|
||||
|
||||
func TestEngine_CloseRemovesTemporaryStore(t *testing.T) {
|
||||
eng, err := New(WithWorkers(1))
|
||||
require.NoError(t, err)
|
||||
tmpDir := eng.tmpDir
|
||||
require.NotEmpty(t, tmpDir)
|
||||
|
||||
require.NoError(t, eng.Close())
|
||||
_, statErr := os.Stat(tmpDir)
|
||||
assert.True(t, os.IsNotExist(statErr), "temp store directory should be gone after Close")
|
||||
|
||||
// Close is idempotent.
|
||||
require.NoError(t, eng.Close())
|
||||
}
|
||||
|
||||
func TestEngine_WithStorePathPersistsAcrossRuns(t *testing.T) {
|
||||
dir := writeSample(t)
|
||||
storePath := filepath.Join(t.TempDir(), "nested", "graph.sqlite")
|
||||
|
||||
eng, err := New(WithWorkers(1), WithStorePath(storePath))
|
||||
require.NoError(t, err)
|
||||
_, err = eng.Index(dir)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, eng.Close())
|
||||
|
||||
_, statErr := os.Stat(storePath)
|
||||
require.NoError(t, statErr, "store file should survive Close")
|
||||
|
||||
reopened, err := New(WithWorkers(1), WithStorePath(storePath))
|
||||
require.NoError(t, err)
|
||||
defer func() { require.NoError(t, reopened.Close()) }()
|
||||
|
||||
nodes := reopened.FindSymbols("Hello")
|
||||
require.Len(t, nodes, 1)
|
||||
assert.Equal(t, "Hello", nodes[0].Name)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user