Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.

// Package indexstore provides the deliberately small Pebble surface used by
// Package common provides the deliberately small Pebble surface used by
// immutable collection index artifacts.
package indexstore
package common

import (
"bytes"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright 2026-present the xvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
package indexstore
package common

import (
"errors"
Expand Down
20 changes: 10 additions & 10 deletions internal/db/index/column/fts_column/fts_pebble.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
"math"
"os"

"github.com/gorse-io/xvec/internal/indexstore"
"github.com/gorse-io/xvec/internal/db/common"
)

const (
Expand Down Expand Up @@ -51,7 +51,7 @@ func (d *FTSTermDictionary) Save(ctx context.Context, path string) error {
if err := requireEmptyFTSDirectory(path); err != nil {
return err
}
store, err := indexstore.Open(path, indexstore.Options{})
store, err := common.Open(path, common.Options{})
if err != nil {
return fmt.Errorf("core: create FTS Pebble store: %w", err)
}
Expand Down Expand Up @@ -118,7 +118,7 @@ func OpenFTSTermDictionary(ctx context.Context, path string) (*FTSTermDictionary
if err := ctx.Err(); err != nil {
return nil, err
}
store, err := indexstore.Open(path, indexstore.Options{ReadOnly: true})
store, err := common.Open(path, common.Options{ReadOnly: true})
if err != nil {
return nil, ftsDictionaryCorruption("open Pebble store", err)
}
Expand Down Expand Up @@ -277,12 +277,12 @@ func requireEmptyFTSDirectory(path string) error {
}

type ftsStoreWriter struct {
store *indexstore.Store
batch *indexstore.Batch
store *common.Store
batch *common.Batch
size int
}

func newFTSStoreWriter(store *indexstore.Store) *ftsStoreWriter {
func newFTSStoreWriter(store *common.Store) *ftsStoreWriter {
return &ftsStoreWriter{store: store, batch: store.NewBatch()}
}

Expand Down Expand Up @@ -335,7 +335,7 @@ func writeFTSDocumentLengths(ctx context.Context, writer *ftsStoreWriter, length
return nil
}

func readFTSDocumentLengths(ctx context.Context, store *indexstore.Store) ([]uint32, error) {
func readFTSDocumentLengths(ctx context.Context, store *common.Store) ([]uint32, error) {
iterator, err := store.NewPrefixIterator([]byte{'d'})
if err != nil {
return nil, ftsDictionaryCorruption("open document-length iterator", err)
Expand Down Expand Up @@ -385,7 +385,7 @@ func writeFTSPostingChunks(ctx context.Context, writer *ftsStoreWriter, ordinal
return nil
}

func readFTSPostingChunks(ctx context.Context, store *indexstore.Store, ordinal uint32) ([]byte, error) {
func readFTSPostingChunks(ctx context.Context, store *common.Store, ordinal uint32) ([]byte, error) {
prefix := make([]byte, 5)
prefix[0] = 'p'
binary.BigEndian.PutUint32(prefix[1:], ordinal)
Expand Down Expand Up @@ -418,7 +418,7 @@ func readFTSPostingChunks(ctx context.Context, store *indexstore.Store, ordinal
return data, nil
}

func validateFTSPostingKeys(ctx context.Context, store *indexstore.Store, termCount uint32) error {
func validateFTSPostingKeys(ctx context.Context, store *common.Store, termCount uint32) error {
iterator, err := store.NewPrefixIterator([]byte{'p'})
if err != nil {
return ftsDictionaryCorruption("open posting-key iterator", err)
Expand Down Expand Up @@ -475,7 +475,7 @@ func ftsDictionaryCorruption(message string, err error) error {
}

func inspectFTSStoreKeys(path string) ([][]byte, error) {
store, err := indexstore.Open(path, indexstore.Options{ReadOnly: true})
store, err := common.Open(path, common.Options{ReadOnly: true})
if err != nil {
return nil, err
}
Expand Down
20 changes: 10 additions & 10 deletions internal/db/index/column/fts_column/fts_pebble_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import (
"path/filepath"
"testing"

"github.com/gorse-io/xvec/internal/db/common"
"github.com/gorse-io/xvec/internal/db/index/column/fts_column/tokenizer"
"github.com/gorse-io/xvec/internal/indexstore"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -60,7 +60,7 @@ func TestValidateFTSPostingKeysHonorsCancellation(t *testing.T) {
dictionary := buildFTSTestDictionary(t, [][]tokenizer.Token{{{Text: "alpha", Position: 0}}})
path := filepath.Join(t.TempDir(), "cancel.pebble")
require.NoError(t, dictionary.Save(context.Background(), path))
store, err := indexstore.Open(path, indexstore.Options{ReadOnly: true})
store, err := common.Open(path, common.Options{ReadOnly: true})
require.NoError(t, err)
defer func() { require.NoError(t, store.Close()) }()

Expand Down Expand Up @@ -95,29 +95,29 @@ func TestFTSTermDictionaryPebbleRejectsInvalidInputsAndCorruption(t *testing.T)
badMaximumTF.maximumTF = []uint32{0}
require.ErrorIs(t, badMaximumTF.Save(context.Background(), filepath.Join(t.TempDir(), "tf.pebble")), ErrInvalidFTSDictionary)

mutations := map[string]func(*indexstore.Store) error{
"missing format": func(store *indexstore.Store) error { return store.Delete(ftsFormatKey) },
"short stats": func(store *indexstore.Store) error { return store.Set(ftsStatsKey, []byte{1}) },
"too many documents": func(store *indexstore.Store) error {
mutations := map[string]func(*common.Store) error{
"missing format": func(store *common.Store) error { return store.Delete(ftsFormatKey) },
"short stats": func(store *common.Store) error { return store.Set(ftsStatsKey, []byte{1}) },
"too many documents": func(store *common.Store) error {
stats := make([]byte, 16)
binary.LittleEndian.PutUint64(stats, uint64(math.MaxUint32)+1)
return store.Set(ftsStatsKey, stats)
},
"missing lengths": func(store *indexstore.Store) error {
"missing lengths": func(store *common.Store) error {
return store.Delete([]byte{'d', 0, 0, 0, 0})
},
"invalid term": func(store *indexstore.Store) error {
"invalid term": func(store *common.Store) error {
return store.Set([]byte{'t', 0, 0, 0, 0}, []byte{1})
},
"orphan posting": func(store *indexstore.Store) error {
"orphan posting": func(store *common.Store) error {
return store.Set([]byte{'p', 0, 0, 0, 1, 0, 0, 0, 0}, []byte{1})
},
}
for name, mutate := range mutations {
t.Run(name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "corrupt.pebble")
require.NoError(t, dictionary.Save(context.Background(), path))
store, err := indexstore.Open(path, indexstore.Options{})
store, err := common.Open(path, common.Options{})
require.NoError(t, err)
require.NoError(t, mutate(store))
require.NoError(t, store.Close())
Expand Down
16 changes: 8 additions & 8 deletions internal/db/sqlengine/inverted_pebble.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
"unicode/utf8"

"github.com/gorse-io/xvec/internal/ailego/container"
"github.com/gorse-io/xvec/internal/indexstore"
"github.com/gorse-io/xvec/internal/db/common"
)

const (
Expand Down Expand Up @@ -71,7 +71,7 @@ func (i *InvertedIndex) Save(ctx context.Context, path string) error {
if err != nil {
return fmt.Errorf("sql: marshal inverted field: %w", err)
}
store, err := indexstore.Open(path, indexstore.Options{})
store, err := common.Open(path, common.Options{})
if err != nil {
return fmt.Errorf("sql: create inverted index store: %w", err)
}
Expand Down Expand Up @@ -145,7 +145,7 @@ func OpenInvertedIndex(ctx context.Context, path string) (*InvertedIndex, error)
if err := ctx.Err(); err != nil {
return nil, err
}
store, err := indexstore.Open(path, indexstore.Options{ReadOnly: true})
store, err := common.Open(path, common.Options{ReadOnly: true})
if err != nil {
return nil, invertedCorruption("open Pebble store", err)
}
Expand Down Expand Up @@ -290,12 +290,12 @@ func requireEmptyInvertedDirectory(path string) error {
}

type invertedStoreWriter struct {
store *indexstore.Store
batch *indexstore.Batch
store *common.Store
batch *common.Batch
size int
}

func newInvertedStoreWriter(store *indexstore.Store) *invertedStoreWriter {
func newInvertedStoreWriter(store *common.Store) *invertedStoreWriter {
return &invertedStoreWriter{store: store, batch: store.NewBatch()}
}

Expand Down Expand Up @@ -356,7 +356,7 @@ func writeInvertedBitmap(ctx context.Context, writer *invertedStoreWriter, prefi
return nil
}

func readInvertedBitmap(ctx context.Context, store *indexstore.Store, prefix []byte) (*container.Bitmap, error) {
func readInvertedBitmap(ctx context.Context, store *common.Store, prefix []byte) (*container.Bitmap, error) {
iterator, err := store.NewPrefixIterator(prefix)
if err != nil {
return nil, invertedCorruption("open bitmap iterator", err)
Expand Down Expand Up @@ -537,7 +537,7 @@ func ensureInvertedJSONEOF(decoder *json.Decoder) error {
}

func inspectInvertedStoreKeys(path string) ([][]byte, error) {
store, err := indexstore.Open(path, indexstore.Options{ReadOnly: true})
store, err := common.Open(path, common.Options{ReadOnly: true})
if err != nil {
return nil, err
}
Expand Down
16 changes: 8 additions & 8 deletions internal/db/sqlengine/inverted_pebble_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"path/filepath"
"testing"

"github.com/gorse-io/xvec/internal/indexstore"
"github.com/gorse-io/xvec/internal/db/common"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -125,16 +125,16 @@ func TestInvertedIndexPebbleRejectsInvalidInputsAndCorruption(t *testing.T) {
require.NoError(t, err)
require.Error(t, unsealed.Save(context.Background(), filepath.Join(t.TempDir(), "unsealed.pebble")))

mutations := map[string]func(*indexstore.Store) error{
"missing format": func(store *indexstore.Store) error { return store.Delete(invertedFormatKey) },
"invalid field": func(store *indexstore.Store) error { return store.Set(invertedFieldKey, []byte("{} {}")) },
"missing rows": func(store *indexstore.Store) error {
mutations := map[string]func(*common.Store) error{
"missing format": func(store *common.Store) error { return store.Delete(invertedFormatKey) },
"invalid field": func(store *common.Store) error { return store.Set(invertedFieldKey, []byte("{} {}")) },
"missing rows": func(store *common.Store) error {
return store.Delete([]byte{'b', 0, 0, 0, 0, 0})
},
"invalid term": func(store *indexstore.Store) error {
"invalid term": func(store *common.Store) error {
return store.Set([]byte{'t', 0, 0, 0, 0}, []byte{byte(ValueInt64)})
},
"invalid length key": func(store *indexstore.Store) error {
"invalid length key": func(store *common.Store) error {
key := make([]byte, 9)
key[0] = 'l'
binary.BigEndian.PutUint32(key[1:5], 1)
Expand All @@ -145,7 +145,7 @@ func TestInvertedIndexPebbleRejectsInvalidInputsAndCorruption(t *testing.T) {
t.Run(name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "corrupt.pebble")
require.NoError(t, index.Save(context.Background(), path))
store, err := indexstore.Open(path, indexstore.Options{})
store, err := common.Open(path, common.Options{})
require.NoError(t, err)
require.NoError(t, mutate(store))
require.NoError(t, store.Close())
Expand Down
Loading