diff --git a/.github/workflows/orm-integration-test.yml b/.github/workflows/orm-integration-test.yml new file mode 100644 index 000000000..5186a4851 --- /dev/null +++ b/.github/workflows/orm-integration-test.yml @@ -0,0 +1,99 @@ +name: ORM Integration Test + +# Runs the `//go:build integration` tagged suites from the optsqlite2 ORM +# line against a real Easysearch cluster: +# +# modules/elastic TestContract_Elastic, TestAggConformance_Elastic +# (backend contract + aggregation conformance) +# modules/sqlite TestAggParity_SQLite_Elastic +# (deep-compare sqlite vs elastic results) +# +# The suites bootstrap themselves from ES_ENDPOINT/ES_USERNAME/ES_PASSWORD +# (modules/elastic/integration_env_test.go + core/orm/ormtest) and skip +# gracefully when the cluster is unreachable, so this workflow stays green +# even if the Easysearch install flakes. + +on: + pull_request: + branches: [ "main" ] + paths: + - "core/orm/**" + - "core/aggregate/**" + - "core/elastic/**" + - "modules/elastic/**" + - "modules/sqlite/**" + - ".github/workflows/orm-integration-test.yml" + +jobs: + orm-integration-test: + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - name: Checkout current repository + uses: actions/checkout@v4 + with: + path: framework + + - name: Set up go toolchain + uses: actions/setup-go@v5 + with: + go-version-file: framework/go.mod + check-latest: false + cache: true + cache-dependency-path: framework/go.sum + + - name: Check go toolchain + run: go version + + - name: Download GraalVM JDK 21 + shell: bash + run: | + mkdir -p $HOME/jdk21 + curl -sSL https://release.infinilabs.com/easysearch/jdk/21/graalvm-jdk-21_linux-x64_bin.tar.gz \ + | tar -xz --strip-components=1 -C $HOME/jdk21 + + - name: Install and run easysearch + shell: bash + run: | + curl -sSL http://get.infini.cloud | bash -s -- -p easysearch -d $HOME/easysearch + ln -sfn $HOME/jdk21 $HOME/easysearch/jdk + # otherwise initialize.sh generates a random admin password + # (must be >=9 chars with upper/lower/digit/special) + export EASYSEARCH_INITIAL_ADMIN_PASSWORD='Admin@12345' + cd $HOME/easysearch && bin/initialize.sh -s + bin/easysearch -d + + # wait for the cluster (curl -f: a 401 must not count as "up") + for i in $(seq 1 30); do + if curl -skf -u "admin:$EASYSEARCH_INITIAL_ADMIN_PASSWORD" https://127.0.0.1:9200/_cluster/health >/dev/null 2>&1; then + echo "easysearch is up"; exit 0 + fi + sleep 2 + done + echo "easysearch did not become ready in time"; exit 1 + + - name: ORM integration tests + shell: bash + env: + ES_ENDPOINT: https://127.0.0.1:9200 + ES_USERNAME: admin + ES_PASSWORD: Admin@12345 + run: | + export WORKBASE=$HOME/go/src/infini.sh + export WORK=$WORKBASE/framework + + mkdir -p $WORKBASE + ln -sfn $GITHUB_WORKSPACE/framework $WORK + + cd $WORK + go version + + # config/generated_framework-info.go is gitignored and normally + # generated by the Makefile (same as the Unit Test workflow's `make tidy test`) + make restore-generated-file + + echo Running ORM integration suites at $PWD ... + go test -tags integration -count=1 -timeout 20m \ + ./core/aggregate/... \ + ./modules/sqlite/... \ + ./modules/elastic/... diff --git a/core/aggregate/aggstest/aggstest.go b/core/aggregate/aggstest/aggstest.go new file mode 100644 index 000000000..37aaea781 --- /dev/null +++ b/core/aggregate/aggstest/aggstest.go @@ -0,0 +1,357 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +// Package aggstest is the cross-backend aggregation conformance suite +// (design doc §8): the same fixture + spec runs against every backend and +// must produce the same typed AggregationResult. Backends implement the +// Backend interface and call RunConformance from a unit test; RunParity +// deep-compares two backends (e.g. sqlite vs a live elastic cluster). +package aggstest + +import ( + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "infini.sh/framework/core/orm" +) + +// Doc is one fixture document; backends materialize it into their store. +type Doc = map[string]interface{} + +// Backend adapts a store for the suite. Setup registers schema, loads docs, +// and returns a context bound to the model plus a cleanup. +type Backend interface { + Setup(t *testing.T, docs []Doc) (ctx *orm.Context, cleanup func()) + Aggregate(ctx *orm.Context, qb *orm.QueryBuilder) (*orm.AggregationResult, error) +} + +// RunConformance executes the full case catalog against one backend. +func RunConformance(t *testing.T, b Backend) { + t.Run("metrics", func(t *testing.T) { conformanceMetrics(t, b) }) + t.Run("terms", func(t *testing.T) { conformanceTerms(t, b) }) + t.Run("nested terms", func(t *testing.T) { conformanceNestedTerms(t, b) }) + t.Run("date histogram", func(t *testing.T) { conformanceDateHistogram(t, b) }) + t.Run("date range", func(t *testing.T) { conformanceDateRange(t, b) }) + t.Run("filter bucket", func(t *testing.T) { conformanceFilter(t, b) }) + t.Run("top hits", func(t *testing.T) { conformanceTopHits(t, b) }) + t.Run("percentiles", func(t *testing.T) { conformancePercentiles(t, b) }) + t.Run("pipelines", func(t *testing.T) { conformancePipelines(t, b) }) + t.Run("deep chain", func(t *testing.T) { conformanceDeepChain(t, b) }) + t.Run("empty set", func(t *testing.T) { conformanceEmptySet(t, b) }) +} + +func fixtureDocs() []Doc { + docs := []Doc{} + // 6 hours × 2 streams; n increases; severities cycle. + for h := 0; h < 6; h++ { + for _, stream := range []string{"alpha", "beta"} { + n := float64(h + 1) + if stream == "beta" { + n = float64(h + 7) + } + sev := "info" + if h%3 == 0 { + sev = "error" + } + docs = append(docs, Doc{ + "id": docID(stream, h), + "ts": tsAt(h), + "stream": stream, + "severity": sev, + "n": n, + }) + } + } + return docs +} + +func docID(stream string, h int) string { return stream + "-" + string(rune('a'+h)) } + +func tsAt(hour int) string { + // Fixed midnight UTC base keeps every backend deterministic. + return "2026-08-13T" + pad(hour) + ":30:00Z" +} + +func pad(h int) string { + if h < 10 { + return "0" + string(rune('0'+h)) + } + return string(rune('0'+h/10)) + string(rune('0'+h%10)) +} + +func runAgg(t *testing.T, b Backend, docs []Doc, aggs map[string]orm.Aggregation) *orm.AggregationResult { + t.Helper() + ctx, cleanup := b.Setup(t, docs) + t.Cleanup(cleanup) + qb := orm.NewQuery() + qb.SetAggregations(aggs) + res, err := b.Aggregate(ctx, qb) + require.NoError(t, err) + require.NotNil(t, res) + return res +} + +func conformanceMetrics(t *testing.T, b Backend) { + res := runAgg(t, b, fixtureDocs(), map[string]orm.Aggregation{ + "total": &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"}, + "cnt": &orm.MetricAggregation{Type: orm.MetricCount, Field: "n"}, + "avg": &orm.MetricAggregation{Type: orm.MetricAvg, Field: "n"}, + "min": &orm.MetricAggregation{Type: orm.MetricMin, Field: "n"}, + "max": &orm.MetricAggregation{Type: orm.MetricMax, Field: "n"}, + "card": &orm.MetricAggregation{Type: orm.MetricCardinality, Field: "stream"}, + }) + // n values: alpha 1..6 (21), beta 7..12 (57) → total 78 over 12 docs. + assert.InEpsilon(t, 78, res.Aggs["total"].Value, 1e-9) + assert.InEpsilon(t, 12, res.Aggs["cnt"].Value, 1e-9) + assert.InEpsilon(t, 6.5, res.Aggs["avg"].Value, 1e-9) + assert.InEpsilon(t, 1, res.Aggs["min"].Value, 1e-9) + assert.InEpsilon(t, 12, res.Aggs["max"].Value, 1e-9) + assert.InEpsilon(t, 2, res.Aggs["card"].Value, 1e-9) +} + +func conformanceTerms(t *testing.T, b Backend) { + terms := &orm.TermsAggregation{Field: "stream", Size: 10} + terms.AddNested("sum_n", &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"}) + res := runAgg(t, b, fixtureDocs(), map[string]orm.Aggregation{"by_stream": terms}) + + buckets := res.Aggs["by_stream"].Buckets + require.Len(t, buckets, 2) + // Ties on doc_count break by key ascending: alpha first. + assert.Equal(t, "alpha", buckets[0].Key) + assert.EqualValues(t, 6, buckets[0].DocCount) + assert.InEpsilon(t, 21, buckets[0].Aggs["sum_n"].Value, 1e-9) + assert.Equal(t, "beta", buckets[1].Key) + assert.InEpsilon(t, 57, buckets[1].Aggs["sum_n"].Value, 1e-9) +} + +func conformanceNestedTerms(t *testing.T, b Backend) { + streams := &orm.TermsAggregation{Field: "stream", Size: 10} + sevs := &orm.TermsAggregation{Field: "severity", Size: 10} + streams.AddNested("by_sev", sevs) + res := runAgg(t, b, fixtureDocs(), map[string]orm.Aggregation{"streams": streams}) + + buckets := res.Aggs["streams"].Buckets + require.Len(t, buckets, 2) + // h%3==0 → hours 0,3 → error; alpha error count 2. + var alphaSevs []orm.Bucket + for _, b2 := range buckets { + if b2.Key == "alpha" { + alphaSevs = b2.Aggs["by_sev"].Buckets + } + } + require.NotEmpty(t, alphaSevs) + var errCount, infoCount int64 + for _, sb := range alphaSevs { + switch sb.Key { + case "error": + errCount = sb.DocCount + case "info": + infoCount = sb.DocCount + } + } + assert.EqualValues(t, 2, errCount) + assert.EqualValues(t, 4, infoCount) +} + +func conformanceDateHistogram(t *testing.T, b Backend) { + // Docs only at hours 0,1,3,5-ish (fixture has all 6) — use a filtered + // subset to exercise zero fill: hours 0 and 3 → gap at 1,2. + docs := []Doc{ + {"id": "z1", "ts": tsAt(0), "stream": "alpha", "n": 1.0}, + {"id": "z2", "ts": tsAt(0), "stream": "beta", "n": 2.0}, + {"id": "z3", "ts": tsAt(3), "stream": "alpha", "n": 4.0}, + } + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + dh.AddNested("sum_n", &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"}) + res := runAgg(t, b, docs, map[string]orm.Aggregation{"over_time": dh}) + + buckets := res.Aggs["over_time"].Buckets + require.Len(t, buckets, 4, "hours 0..3 with zero fill at 1,2") + assert.Equal(t, "2026-08-13T00:00:00", buckets[0].Key) + assert.EqualValues(t, 2, buckets[0].DocCount) + assert.InEpsilon(t, 3, buckets[0].Aggs["sum_n"].Value, 1e-9) + assert.EqualValues(t, 0, buckets[1].DocCount, "zero-filled bucket") + assert.EqualValues(t, 0, buckets[2].DocCount, "zero-filled bucket") + assert.Equal(t, "2026-08-13T03:00:00", buckets[3].Key) + assert.InEpsilon(t, 4, buckets[3].Aggs["sum_n"].Value, 1e-9) + // Numeric epoch keys present. + assert.True(t, buckets[0].KeyRaw.(int64) > 0) +} + +func conformanceDateRange(t *testing.T, b Backend) { + dr := &orm.DateRangeAggregation{Field: "ts", Ranges: []interface{}{ + map[string]interface{}{"from": "2026-08-13T00:00:00Z", "to": "2026-08-13T02:00:00Z", "key": "early"}, + map[string]interface{}{"from": "2026-08-13T02:00:00Z", "key": "late"}, + }} + res := runAgg(t, b, fixtureDocs(), map[string]orm.Aggregation{"ranges": dr}) + buckets := res.Aggs["ranges"].Buckets + require.Len(t, buckets, 2) + assert.EqualValues(t, 4, buckets[0].DocCount) // hours 0,1 × 2 streams + assert.EqualValues(t, 8, buckets[1].DocCount) +} + +func conformanceFilter(t *testing.T, b Backend) { + filter := &orm.FilterAggregation{Query: map[string]interface{}{ + "term": map[string]interface{}{"severity": "error"}, + }} + filter.AddNested("total", &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"}) + res := runAgg(t, b, fixtureDocs(), map[string]orm.Aggregation{"errors": filter}) + + buckets := res.Aggs["errors"].Buckets + require.Len(t, buckets, 1) + assert.EqualValues(t, 4, buckets[0].DocCount) // hours 0,3 × 2 streams + // n sums: (h0: alpha 1 + beta 7) + (h3: alpha 4 + beta 10) = 22 + assert.InEpsilon(t, 22, buckets[0].Aggs["total"].Value, 1e-9) +} + +func conformanceTopHits(t *testing.T, b Backend) { + th := &orm.TopHitsAggregation{Sorts: []orm.Sort{{Field: "n", SortType: orm.DESC}}} + res := runAgg(t, b, fixtureDocs(), map[string]orm.Aggregation{"latest": th}) + node := res.Aggs["latest"] + require.NotNil(t, node.TopHit) + var doc map[string]interface{} + require.NoError(t, json.Unmarshal(*node.TopHit, &doc)) + assert.Equal(t, "beta-f", doc["id"]) // highest n = 12 (hour 5 beta) +} + +func conformancePercentiles(t *testing.T, b Backend) { + p := &orm.PercentilesAggregation{Field: "n", Percents: []float64{50, 100}} + res := runAgg(t, b, fixtureDocs(), map[string]orm.Aggregation{"p": p}) + values := res.Aggs["p"].Values + require.NotEmpty(t, values) + // 12 values 1..12 → p50 ∈ [6,7], p100 = 12 (backends may approximate). + assert.InDelta(t, 6.5, values["50"], 1.0) + assert.InDelta(t, 12, values["100"], 1e-9) +} + +func conformancePipelines(t *testing.T, b Backend) { + docs := []Doc{ + {"id": "p0", "ts": tsAt(0), "n": 10.0}, + {"id": "p1", "ts": tsAt(1), "n": 25.0}, + {"id": "p2", "ts": tsAt(2), "n": 20.0}, + {"id": "p3", "ts": tsAt(3), "n": 40.0}, + } + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + dh.AddNested("sum_n", &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"}) + dh.AddNested("derivative", &orm.DerivativeAggregation{BucketsPath: "sum_n"}) + dh.AddNested("ratio", &orm.BucketScriptAggregation{ + BucketsPath: map[string]string{"a": "sum_n", "b": "sum_n"}, + Script: "params.a / params.b", + }) + sum := &orm.PipelineAggregation{Type: orm.MetricSumBucket, BucketsPath: "over_time>sum_n"} + mb := &orm.MaxBucketAggregation{BucketsPath: "over_time>sum_n"} + + res := runAgg(t, b, docs, map[string]orm.Aggregation{"over_time": dh, "total": sum, "peak": mb}) + buckets := res.Aggs["over_time"].Buckets + require.Len(t, buckets, 4) + if d := buckets[0].Aggs["derivative"]; d != nil { + assert.False(t, d.ValueSet, "first bucket has no derivative") + } + assert.InEpsilon(t, 15, buckets[1].Aggs["derivative"].Value, 1e-9) + assert.InEpsilon(t, -5, buckets[2].Aggs["derivative"].Value, 1e-9) + assert.InEpsilon(t, 1, buckets[1].Aggs["ratio"].Value, 1e-9) + assert.InEpsilon(t, 95, res.Aggs["total"].Value, 1e-9) // 10+25+20+40 + assert.InEpsilon(t, 40, res.Aggs["peak"].Value, 1e-9) +} + +func conformanceDeepChain(t *testing.T, b Backend) { + // terms(stream) → date_histogram(1h) → sum(n); sum_bucket per stream. + streams := &orm.TermsAggregation{Field: "stream", Size: 10} + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + dh.AddNested("bytes", &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"}) + streams.AddNested("dates", dh) + streams.AddNested("total", &orm.PipelineAggregation{Type: orm.MetricSumBucket, BucketsPath: "dates>bytes"}) + + res := runAgg(t, b, fixtureDocs(), map[string]orm.Aggregation{"streams": streams}) + buckets := res.Aggs["streams"].Buckets + require.Len(t, buckets, 2) + for _, b2 := range buckets { + require.Len(t, b2.Aggs["dates"].Buckets, 6) + } + // alpha total 21, beta total 57. + totals := map[string]float64{} + for _, b2 := range buckets { + totals[b2.Key] = b2.Aggs["total"].Value + } + assert.InEpsilon(t, 21, totals["alpha"], 1e-9) + assert.InEpsilon(t, 57, totals["beta"], 1e-9) +} + +func conformanceEmptySet(t *testing.T, b Backend) { + // Match-nothing filter scopes an empty aggregation set. + ctx, cleanup := b.Setup(t, fixtureDocs()) + t.Cleanup(cleanup) + qb := orm.NewQuery().Filter(orm.TermQuery("stream", "nope")) + sum := &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"} + qb.SetAggs("total", sum) + res, err := b.Aggregate(ctx, qb) + require.NoError(t, err) + require.NotNil(t, res.Aggs["total"]) + assert.False(t, res.Aggs["total"].ValueSet, "sum over empty set has no value") +} + +// RunParity executes the same spec against two backends and deep-compares +// the typed results (float epsilon; percentiles skipped — approximation +// strategies legitimately differ). +func RunParity(t *testing.T, a, b Backend) { + check := func(t *testing.T, res *orm.AggregationResult) map[string]float64 { + flattened := map[string]float64{} + for name, node := range res.Aggs { + flattenNode("."+name, node, flattened) + } + return flattened + } + docs := fixtureDocs() + terms := &orm.TermsAggregation{Field: "stream", Size: 10} + terms.AddNested("sum_n", &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"}) + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + dh.AddNested("count", &orm.MetricAggregation{Type: orm.MetricCount, Field: "n"}) + aggs := map[string]orm.Aggregation{"by_stream": terms, "over_time": dh} + + aCtx, aCleanup := a.Setup(t, docs) + t.Cleanup(aCleanup) + qb1 := orm.NewQuery() + qb1.SetAggregations(aggs) + aRes, err := a.Aggregate(aCtx, qb1) + require.NoError(t, err) + + bCtx, bCleanup := b.Setup(t, docs) + t.Cleanup(bCleanup) + qb2 := orm.NewQuery() + qb2.SetAggregations(aggs) + bRes, err := b.Aggregate(bCtx, qb2) + require.NoError(t, err) + + fa, fb := check(t, aRes), check(t, bRes) + assert.Equal(t, len(fa), len(fb), "same flattened result size") + for k, v := range fa { + bv, ok := fb[k] + if !ok { + t.Errorf("key %q missing on backend b", k) + continue + } + if math.Abs(v-bv) > 1e-6 { + t.Errorf("key %q: a=%v b=%v", k, v, bv) + } + } +} + +func flattenNode(prefix string, node *orm.AggNode, out map[string]float64) { + if node == nil { + return + } + if node.ValueSet { + out[prefix+".value"] = node.Value + } + for i, b := range node.Buckets { + bp := prefix + "[" + b.Key + "]" + out[bp+".doc_count"] = float64(b.DocCount) + for subName, subNode := range b.Aggs { + flattenNode(bp+"."+subName, subNode, out) + } + _ = i + } +} diff --git a/core/aggregate/engine.go b/core/aggregate/engine.go new file mode 100644 index 000000000..86dc9aee1 --- /dev/null +++ b/core/aggregate/engine.go @@ -0,0 +1,447 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +// Package aggregate hosts the backend-independent aggregation machinery: +// the pipeline engine (derivative / sum_bucket / max_bucket / bucket_script / +// bucket_sort) and shared result-shaping helpers (zero fill, ordering). +// +// Bucket and metric aggregations are computed by each backend natively; +// pipelines are pure second-order derivations over the returned bucket tree, +// so they are computed here exactly once — every backend behaves identically +// (design doc §4.1). Backends call ApplyPipelines before returning from +// MetricsAPI.Aggregate. +package aggregate + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "infini.sh/framework/core/orm" +) + +// ApplyPipelines fills the pipeline aggregation nodes of result, guided by +// the aggregation spec tree (mirroring qb.Aggs). Bucket/metric nodes must +// already be populated; ES-native pipeline values, when present, are +// overwritten so backends cannot disagree. +func ApplyPipelines(res *orm.AggregationResult, aggs map[string]orm.Aggregation) error { + if res == nil { + return nil + } + return applyScope(res.Aggs, aggs) +} + +// applyScope processes one naming scope: recurses into bucket aggregations +// first, then evaluates sibling pipelines (sum_bucket, max_bucket) against +// the completed scope. +func applyScope(nodes map[string]*orm.AggNode, spec map[string]orm.Aggregation) error { + if nodes == nil || spec == nil { + return nil + } + for name, node := range nodes { + if node != nil && len(node.Buckets) > 0 { + if err := applyBucketList(node.Buckets, nestedSpecOf(spec[name])); err != nil { + return err + } + } + } + for name, sp := range spec { + switch s := sp.(type) { + // sibling pipelines create their node when the backend did not + case *orm.PipelineAggregation: + if s.Type != orm.MetricSumBucket { + continue + } + vals, err := bucketValues(nodes, s.BucketsPath) + if err != nil { + return fmt.Errorf("sum_bucket %q: %w", name, err) + } + ensureNode(nodes, name).Value, _ = sum(vals), true + nodes[name].ValueSet = true + case *orm.MaxBucketAggregation: + vals, err := bucketValues(nodes, s.BucketsPath) + if err != nil { + return fmt.Errorf("max_bucket %q: %w", name, err) + } + ensureNode(nodes, name).Value = max(vals) + nodes[name].ValueSet = true + } + } + return nil +} + +// ensureNode returns the named node, creating it when absent (sibling +// pipeline results may not be pre-created by the backend). +func ensureNode(nodes map[string]*orm.AggNode, name string) *orm.AggNode { + node, ok := nodes[name] + if !ok || node == nil { + node = &orm.AggNode{} + nodes[name] = node + } + return node +} + +// applyBucketList processes the buckets of one multi-bucket aggregation: +// each bucket's inner scope first, then the parent pipelines declared among +// its children (derivative, bucket_script, bucket_sort operate on the list). +func applyBucketList(buckets []orm.Bucket, childSpec map[string]orm.Aggregation) error { + if childSpec == nil { + return nil + } + for i := range buckets { + if err := applyScope(buckets[i].Aggs, childSpec); err != nil { + return err + } + } + for name, sp := range childSpec { + switch s := sp.(type) { + case *orm.DerivativeAggregation: + applyDerivative(buckets, name, s.BucketsPath) + case *orm.BucketScriptAggregation: + if err := applyBucketScript(buckets, name, s); err != nil { + return fmt.Errorf("bucket_script %q: %w", name, err) + } + case *orm.BucketSortAggregation: + applyBucketSort(buckets, name, s) + } + } + return nil +} + +// applyDerivative fills v[i] - v[i-1] of the referenced path per bucket; the +// first bucket gets no value (ES semantics: null). +func applyDerivative(buckets []orm.Bucket, name, bucketsPath string) { + var prev float64 + prevSet := false + for i := range buckets { + cur, ok := scalarOf(&buckets[i], bucketsPath) + if i > 0 && ok && prevSet { + setNode(buckets[i].Aggs, name, cur-prev, true, true) + } + if ok { + prev, prevSet = cur, true + } + } +} + +// applyBucketScript evaluates the script per bucket over params resolved +// from the bucket's scope. +func applyBucketScript(buckets []orm.Bucket, name string, s *orm.BucketScriptAggregation) error { + for i := range buckets { + params := map[string]float64{} + for p, path := range s.BucketsPath { + v, ok := scalarOf(&buckets[i], path) + if !ok { + params[p] = 0 // ES: missing param → script yields null; we use 0 and let division guards apply + } + params[p] = v + } + val, err := EvalScript(s.Script, params) + if err != nil { + return err + } + setNode(buckets[i].Aggs, name, val, true, true) + } + return nil +} + +// applyBucketSort reorders the bucket list by the first sort criterion and +// truncates to From/Size. Sorting is stable to keep deterministic output. +func applyBucketSort(buckets []orm.Bucket, name string, s *orm.BucketSortAggregation) { + if len(s.Sort) == 0 || len(buckets) == 0 { + return + } + crit := s.Sort[0] + less := func(a, b *orm.Bucket) bool { + av, aok := scalarOf(a, crit.Path) + bv, bok := scalarOf(b, crit.Path) + if !aok { + return false + } + if !bok { + return true + } + if crit.Desc { + return av > bv + } + return av < bv + } + sort.SliceStable(buckets, func(i, j int) bool { return less(&buckets[i], &buckets[j]) }) + if s.From > 0 || s.Size > 0 { + from := s.From + if from > len(buckets) { + from = len(buckets) + } + end := len(buckets) + if s.Size > 0 && from+s.Size < end { + end = from + s.Size + } + trimmed := make([]orm.Bucket, end-from) + copy(trimmed, buckets[from:end]) + // bucket_sort lives among the children spec — it has no own node; the + // trim mutates the shared slice header, so copy back. + for i := range trimmed { + buckets[i] = trimmed[i] + } + for i := len(trimmed); i < len(buckets); i++ { + buckets[i] = orm.Bucket{} + } + } + _ = name +} + +// scalarOf resolves a buckets_path to a scalar within one bucket's scope: +// "_count"/"._count" → doc_count, "metric" → sibling value, +// "a>b" chains descend single-bucket aggs (rare; supported one level). +func scalarOf(bucket *orm.Bucket, path string) (float64, bool) { + if bucket == nil { + return 0, false + } + if path == "_count" || path == "._count" { + return float64(bucket.DocCount), true + } + parts := strings.Split(path, ">") + nodes := bucket.Aggs + for i, part := range parts { + if nodes == nil { + return 0, false + } + node, ok := nodes[part] + if !ok || node == nil { + return 0, false + } + if i == len(parts)-1 { + return node.Value, node.ValueSet + } + // intermediate: descend into its first bucket (single-bucket agg) + if len(node.Buckets) == 0 { + return 0, false + } + nodes = node.Buckets[0].Aggs + } + return 0, false +} + +// bucketValues resolves "aggName>metric" against a scope: the per-bucket +// values of metric across all buckets of aggName. +func bucketValues(nodes map[string]*orm.AggNode, path string) ([]float64, error) { + parts := strings.Split(strings.TrimSpace(path), ">") + if len(parts) != 2 { + // single-name path: values of a sibling metric across... sibling + // metrics are single values, not lists — require the two-part form. + return nil, fmt.Errorf("unsupported buckets_path %q (want \"agg>metric\")", path) + } + aggNode, ok := nodes[parts[0]] + if !ok || aggNode == nil { + return nil, fmt.Errorf("aggregation %q not found", parts[0]) + } + out := make([]float64, 0, len(aggNode.Buckets)) + for i := range aggNode.Buckets { + if v, ok := scalarOf(&aggNode.Buckets[i], parts[1]); ok { + out = append(out, v) + } + } + return out, nil +} + +func setNode(scope map[string]*orm.AggNode, name string, val float64, set, ok bool) { + if scope == nil { + return + } + node := scope[name] + if node == nil { + node = &orm.AggNode{} + scope[name] = node + } + node.Value = val + node.ValueSet = set && ok +} + +func nestedSpecOf(spec orm.Aggregation) map[string]orm.Aggregation { + if spec == nil { + return nil + } + return spec.GetNested() +} + +func sum(vals []float64) float64 { + var s float64 + for _, v := range vals { + s += v + } + return s +} + +func max(vals []float64) float64 { + if len(vals) == 0 { + return 0 + } + m := vals[0] + for _, v := range vals[1:] { + if v > m { + m = v + } + } + return m +} + +// ────────────────────────────────────────────────────────────────────────── +// Script evaluation: arithmetic over params.* with + - * / ( ) and unary -. +// Covers the console vocabulary (ratios, percentages, scale factors); a real +// painless runtime is deliberately out of scope. +// ────────────────────────────────────────────────────────────────────────── + +type scriptParser struct { + toks []string + pos int + params map[string]float64 +} + +// EvalScript evaluates an arithmetic script with params.* references. +func EvalScript(script string, params map[string]float64) (float64, error) { + p := &scriptParser{toks: tokenizeScript(script), params: params} + v, err := p.parseExpr() + if err != nil { + return 0, err + } + if p.pos != len(p.toks) { + return 0, fmt.Errorf("unexpected token %q in script %q", p.toks[p.pos], script) + } + return v, nil +} + +func tokenizeScript(s string) []string { + var toks []string + var cur strings.Builder + flush := func() { + if cur.Len() > 0 { + toks = append(toks, cur.String()) + cur.Reset() + } + } + runes := []rune(s) + for i, r := range runes { + switch { + case r == ' ' || r == '\t' || r == '\n': + flush() + case strings.ContainsRune("+-*/()", r): + flush() + toks = append(toks, string(r)) + case r == '.': + // A dot between digits is a decimal point ("1.5"); otherwise it + // is the member-access separator ("params.a"). + digitBefore := i > 0 && runes[i-1] >= '0' && runes[i-1] <= '9' + digitAfter := i+1 < len(runes) && runes[i+1] >= '0' && runes[i+1] <= '9' + if digitBefore && digitAfter { + cur.WriteRune(r) + } else { + flush() + toks = append(toks, ".") + } + default: + cur.WriteRune(r) + } + } + flush() + return toks +} + +func (p *scriptParser) peek() string { + if p.pos < len(p.toks) { + return p.toks[p.pos] + } + return "" +} + +func (p *scriptParser) next() string { + t := p.peek() + p.pos++ + return t +} + +func (p *scriptParser) parseExpr() (float64, error) { + left, err := p.parseTerm() + if err != nil { + return 0, err + } + for { + op := p.peek() + if op != "+" && op != "-" { + return left, nil + } + p.next() + right, err := p.parseTerm() + if err != nil { + return 0, err + } + if op == "+" { + left += right + } else { + left -= right + } + } +} + +func (p *scriptParser) parseTerm() (float64, error) { + left, err := p.parseFactor() + if err != nil { + return 0, err + } + for { + op := p.peek() + if op != "*" && op != "/" { + return left, nil + } + p.next() + right, err := p.parseFactor() + if err != nil { + return 0, err + } + if op == "*" { + left *= right + } else if right == 0 { + // Console Calc convention: division by zero yields 0 — keep + // evaluating the rest of the expression ("a / b * 100" with b=0 + // stays 0 rather than aborting the parse). + left = 0 + } else { + left /= right + } + } +} + +func (p *scriptParser) parseFactor() (float64, error) { + t := p.next() + switch { + case t == "": + return 0, fmt.Errorf("unexpected end of script") + case t == "(": + v, err := p.parseExpr() + if err != nil { + return 0, err + } + if p.next() != ")" { + return 0, fmt.Errorf("missing closing parenthesis") + } + return v, nil + case t == "-": + v, err := p.parseFactor() + return -v, err + case t == "params": + if p.next() != "." { + return 0, fmt.Errorf("expected '.' after params") + } + name := p.next() + v, ok := p.params[name] + if !ok { + return 0, fmt.Errorf("unknown param %q", name) + } + return v, nil + default: + v, err := strconv.ParseFloat(t, 64) + if err != nil { + return 0, fmt.Errorf("unexpected token %q", t) + } + return v, nil + } +} diff --git a/core/aggregate/engine_test.go b/core/aggregate/engine_test.go new file mode 100644 index 000000000..6c1b4ec36 --- /dev/null +++ b/core/aggregate/engine_test.go @@ -0,0 +1,223 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package aggregate + +import ( + "math" + "testing" + + "infini.sh/framework/core/orm" +) + +func mkBuckets(values ...int64) []orm.Bucket { + out := make([]orm.Bucket, 0, len(values)) + for _, v := range values { + out = append(out, orm.Bucket{ + Key: "", + DocCount: v, + Aggs: map[string]*orm.AggNode{"count": {Value: float64(v), ValueSet: true}}, + }) + } + return out +} + +func TestApplyPipelines_Derivative(t *testing.T) { + res := &orm.AggregationResult{Aggs: map[string]*orm.AggNode{ + "dates": {Buckets: mkBuckets(10, 25, 20, 40)}, + }} + derivative := &orm.DerivativeAggregation{BucketsPath: "count"} + dates := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + dates.AddNested("derivative", derivative) + + if err := ApplyPipelines(res, map[string]orm.Aggregation{"dates": dates}); err != nil { + t.Fatal(err) + } + buckets := res.Aggs["dates"].Buckets + if len(buckets) != 4 { + t.Fatalf("buckets = %d", len(buckets)) + } + if buckets[0].Aggs["derivative"] != nil && buckets[0].Aggs["derivative"].ValueSet { + t.Fatal("first bucket has no derivative") + } + want := []float64{15, -5, 20} + for i, w := range want { + node := buckets[i+1].Aggs["derivative"] + if node == nil || !node.ValueSet || node.Value != w { + t.Fatalf("bucket %d derivative = %+v, want %v", i+1, node, w) + } + } +} + +func TestApplyPipelines_DerivativeOfDocCount(t *testing.T) { + res := &orm.AggregationResult{Aggs: map[string]*orm.AggNode{ + "dates": {Buckets: mkBuckets(5, 9)}, + }} + dates := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + dates.AddNested("rate", &orm.DerivativeAggregation{BucketsPath: "._count"}) + + if err := ApplyPipelines(res, map[string]orm.Aggregation{"dates": dates}); err != nil { + t.Fatal(err) + } + node := res.Aggs["dates"].Buckets[1].Aggs["rate"] + if node == nil || node.Value != 4 { + t.Fatalf("derivative of _count = %+v, want 4", node) + } +} + +func TestApplyPipelines_SumBucket(t *testing.T) { + res := &orm.AggregationResult{Aggs: map[string]*orm.AggNode{ + "dates": {Buckets: mkBuckets(1, 2, 3, 4)}, + }} + sum := &orm.PipelineAggregation{Type: orm.MetricSumBucket, BucketsPath: "dates>count"} + dates := &orm.DateHistogramAggregation{Field: "ts", Interval: "1d"} + + if err := ApplyPipelines(res, map[string]orm.Aggregation{"dates": dates, "total": sum}); err != nil { + t.Fatal(err) + } + if got := res.Aggs["total"].Value; got != 10 { + t.Fatalf("sum_bucket = %v, want 10", got) + } +} + +func TestApplyPipelines_MaxBucket(t *testing.T) { + res := &orm.AggregationResult{Aggs: map[string]*orm.AggNode{ + "dates": {Buckets: mkBuckets(7, 2, 9, 4)}, + }} + mb := &orm.MaxBucketAggregation{BucketsPath: "dates>count"} + dates := &orm.DateHistogramAggregation{Field: "ts", Interval: "1d"} + + if err := ApplyPipelines(res, map[string]orm.Aggregation{"dates": dates, "peak": mb}); err != nil { + t.Fatal(err) + } + if got := res.Aggs["peak"].Value; got != 9 { + t.Fatalf("max_bucket = %v, want 9", got) + } +} + +func TestApplyPipelines_BucketScript(t *testing.T) { + // Latency ratio per time bucket: query_time / query_total * 100. + buckets := []orm.Bucket{ + {Key: "t1", Aggs: map[string]*orm.AggNode{ + "qt": {Value: 30, ValueSet: true}, + "qc": {Value: 100, ValueSet: true}, + }}, + {Key: "t2", Aggs: map[string]*orm.AggNode{ + "qt": {Value: 45, ValueSet: true}, + "qc": {Value: 150, ValueSet: true}, + }}, + {Key: "t3", Aggs: map[string]*orm.AggNode{ + "qt": {Value: 10, ValueSet: true}, + "qc": {Value: 0, ValueSet: true}, // division by zero → 0 + }}, + } + res := &orm.AggregationResult{Aggs: map[string]*orm.AggNode{"dates": {Buckets: buckets}}} + dates := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + dates.AddNested("ratio", &orm.BucketScriptAggregation{ + BucketsPath: map[string]string{"a": "qt", "b": "qc"}, + Script: "params.a / params.b * 100", + }) + + if err := ApplyPipelines(res, map[string]orm.Aggregation{"dates": dates}); err != nil { + t.Fatal(err) + } + want := []float64{30, 30, 0} + for i, w := range want { + node := buckets[i].Aggs["ratio"] + if node == nil || !nearly(node.Value, w) { + t.Fatalf("bucket %d ratio = %+v, want %v", i, node, w) + } + } +} + +func TestApplyPipelines_BucketSort(t *testing.T) { + buckets := []orm.Bucket{ + {Key: "a", Aggs: map[string]*orm.AggNode{"v": {Value: 3, ValueSet: true}}}, + {Key: "b", Aggs: map[string]*orm.AggNode{"v": {Value: 1, ValueSet: true}}}, + {Key: "c", Aggs: map[string]*orm.AggNode{"v": {Value: 2, ValueSet: true}}}, + } + res := &orm.AggregationResult{Aggs: map[string]*orm.AggNode{"terms": {Buckets: buckets}}} + terms := &orm.TermsAggregation{Field: "x"} + terms.AddNested("top2", &orm.BucketSortAggregation{ + Sort: []orm.BucketSortSpec{{Path: "v", Desc: true}}, + Size: 2, + }) + + if err := ApplyPipelines(res, map[string]orm.Aggregation{"terms": terms}); err != nil { + t.Fatal(err) + } + // Top-2 by value desc: a(3), c(2); b truncated. + if len(buckets) < 2 || buckets[0].Key != "a" || buckets[1].Key != "c" { + t.Fatalf("sort order wrong: %+v", buckets) + } + if buckets[2].Key != "" || buckets[2].Aggs != nil { + t.Fatalf("trailing bucket not cleared: %+v", buckets[2]) + } +} + +func TestApplyPipelines_NestedChains(t *testing.T) { + // terms(stream) → date_histogram → sum; sum_bucket over "dates>bytes". + streams := []orm.Bucket{ + {Key: "s1", Aggs: map[string]*orm.AggNode{ + "dates": {Buckets: []orm.Bucket{ + {Aggs: map[string]*orm.AggNode{"bytes": {Value: 100, ValueSet: true}}}, + {Aggs: map[string]*orm.AggNode{"bytes": {Value: 50, ValueSet: true}}}, + }}, + }}, + {Key: "s2", Aggs: map[string]*orm.AggNode{ + "dates": {Buckets: []orm.Bucket{ + {Aggs: map[string]*orm.AggNode{"bytes": {Value: 200, ValueSet: true}}}, + }}, + }}, + } + res := &orm.AggregationResult{Aggs: map[string]*orm.AggNode{"streams": {Buckets: streams}}} + + streamsAgg := &orm.TermsAggregation{Field: "stream_id"} + dates := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + dates.AddNested("bytes", &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"}) + streamsAgg.AddNested("dates", dates) + streamsAgg.AddNested("total", &orm.PipelineAggregation{Type: orm.MetricSumBucket, BucketsPath: "dates>bytes"}) + + if err := ApplyPipelines(res, map[string]orm.Aggregation{"streams": streamsAgg}); err != nil { + t.Fatal(err) + } + if got := streams[0].Aggs["total"].Value; got != 150 { + t.Fatalf("s1 total = %v, want 150", got) + } + if got := streams[1].Aggs["total"].Value; got != 200 { + t.Fatalf("s2 total = %v, want 200", got) + } +} + +func TestEvalScript(t *testing.T) { + cases := []struct { + script string + params map[string]float64 + want float64 + }{ + {"params.a / params.b * 100", map[string]float64{"a": 30, "b": 100}, 30}, + {"params.a + params.b", map[string]float64{"a": 1, "b": 2}, 3}, + {"(params.a + params.b) * 2", map[string]float64{"a": 1, "b": 2}, 6}, + {"-params.a", map[string]float64{"a": 5}, -5}, + {"params.a * 1.5", map[string]float64{"a": 4}, 6}, + {"params.a - params.b / 2", map[string]float64{"a": 3, "b": 4}, 1}, + } + for _, c := range cases { + got, err := EvalScript(c.script, c.params) + if err != nil { + t.Fatalf("EvalScript(%q): %v", c.script, err) + } + if !nearly(got, c.want) { + t.Errorf("EvalScript(%q) = %v, want %v", c.script, got, c.want) + } + } + if _, err := EvalScript("params.a +", nil); err == nil { + t.Error("malformed script must error") + } + if _, err := EvalScript("params.missing", nil); err == nil { + t.Error("unknown param must error") + } +} + +func nearly(a, b float64) bool { + return math.Abs(a-b) < 1e-9 +} diff --git a/core/api/crud/crud.go b/core/api/crud/crud.go new file mode 100644 index 000000000..632b5d5a1 --- /dev/null +++ b/core/api/crud/crud.go @@ -0,0 +1,473 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +// Package crud generates the standard five-piece REST CRUD (create / search / +// get / update / delete) for an ORM model — the scaffold coco's modules and +// modules/easysearch each hand-copy today. +// +// The generated handlers are the P1/P6 conventions made concrete: +// - search: orm.NewQueryBuilderFromRequest + EnableBodyBytes, default sort +// created DESC, ES-shaped SearchResponse via elastic.DecodeSearchResult +// - create: orm.Create with WaitForRefresh + WriteCreatedOKJSON +// - update: partial-field delta with UpdatePartialFields + WriteUpdatedOKJSON +// - get/delete: WriteGetOKJSON / WriteOpRecordNotFoundJSON envelopes +// +// Deliberately NOT imported here: core/security (it depends on core/api, so +// importing it would cycle). Callers initialize permission keys themselves +// and hand them over as strings via Config.Permission. +// +// Behavior contract is locked by crud_test.go; modules migrate onto it with +// a mechanical diff (easysearch is the pilot). +package crud + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + log "github.com/cihub/seelog" + + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +// HandlerFunc matches api.HandleUIMethod's handler signature. +type HandlerFunc func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) + +// Action names passed to Config.Permission. +const ( + ActionRead = "read" + ActionCreate = "create" + ActionUpdate = "update" + ActionDelete = "delete" + ActionSearch = "search" +) + +// Config parameterizes the generated CRUD. Prefix must start with "/" and +// must not end with "/" (e.g. "/easysearch"); Resource names the object for +// docs and error messages (e.g. "cluster"). +// PT is the pointer type of T constrained to orm.Object — GetID/SetID have +// pointer receivers on ORMObjectBase, so the value type alone cannot satisfy +// orm.Object; this pair pattern keeps compile-time safety. +type PT[T any] interface { + orm.Object + *T +} + +type Config[T any] struct { + Prefix string + Resource string + + // Permission maps an action (ActionRead/...) to the permission key that + // gates it. Callers create/initialize the keys with core/security + // themselves and return api.PermissionKey strings (variety is + // intentional: simple vs generic scopes). A nil Permission — or an + // empty key for an action — registers that route without a permission + // option (internal/testing use). + Permission func(action string) api.PermissionKey + + // DefaultQueryFields backs the search endpoint's full-text/filter + // fallback fields (e.g. []string{"name"}). + DefaultQueryFields []string + + // SharingResource, when set, marks the ORM context with + // orm.SharingEnabled + orm.SharingResourceType (coco's shared-resource + // modules). + SharingResource string + + // PrepareCreate applies defaults and validation before orm.Create + // (ID generation, required fields, source flags...). + PrepareCreate func(obj *T) error + + // GuardDelete vetoes deletion of protected records (e.g. reserved + // clusters); the error message surfaces as a 403. + GuardDelete func(obj *T) error + + // MCP exposes the five endpoints as MCP tools, named + // _ (cluster_create, cluster_search, ...) — the same + // convention coco's modules register by hand. Tool visibility and + // invocation still pass through the route's permission option. + MCP bool + + // MCPDescs optionally overrides the per-action tool description + // (keyed by ActionCreate/ActionSearch/...); defaults are generated + // from Resource. + MCPDescs map[string]string + + // IDParam overrides the route parameter name for the object id + // (default "id"); e.g. document resources use "doc_id". + IDParam string + + // SkipActions lists actions whose routes should NOT be registered by + // RegisterCRUD (ActionRead/...); register those endpoints by hand when + // a resource needs behavior the generator cannot express (custom + // cache-first fetch, expansion logic...). + SkipActions []string + + // ExtraOptions appends additional route options per action (login + // requirements, CORS, sensitive-field masking, labels...), applied + // after the permission and MCP options. Return nil for actions that + // need nothing extra. + ExtraOptions func(action string) []api.Option + + // CtxDecorate marks the orm context before any orm call (extra + // sharing markers, DirectReadAccess, permission scopes...). Sharing + // basics from SharingResource are already applied. + CtxDecorate func(ctx *orm.Context, req *http.Request, action string) + + // UpdateMode selects the update semantics; defaults to + // UpdateModePartial (partial-field delta). UpdateModeFull decodes the + // body over the loaded object (merge semantics, orm.Update); + // UpdateModeQuery follows ?replace= (false => partial, otherwise + // full). + UpdateMode UpdateMode + // ProtectedFields are stripped from partial deltas AND restored from + // the loaded object in full mode (e.g. "created", "builtin"). + ProtectedFields []string + + // PrepareUpdate runs before persisting in any update mode; delta is + // the incoming change set (the partial delta, or the raw decoded body + // in full mode, already stripped of ProtectedFields). A non-nil error + // fails the request with 400. + PrepareUpdate func(obj *T, delta util.MapStr) error + + // PostCreate / PostUpdate / PostDelete run after successful writes + // and deletes (cache clears, CORS-origin sync, cascaded cleanups). + // They are best-effort: errors are logged and the success response is + // still returned. + PostCreate func(obj *T) error + PostUpdate func(obj *T) error + PostDelete func(obj *T) error + + // PostGet refines the object before the response envelope is written + // (e.g. document refinement); a non-nil error fails with 500. + PostGet func(obj *T) error + + // PrepareSearch adjusts the query builder and context before + // execution (injected filters from headers, excludes, extra sharing + // markers); a non-nil error fails with 500. + PrepareSearch func(req *http.Request, builder *orm.QueryBuilder, ctx *orm.Context) error + + // PostSearch mutates the decoded response before writing (per-hit + // mapping, icon decoration...); a non-nil error fails with 500. + PostSearch func(res *elastic.SearchResponse) error +} + +// UpdateMode selects how the update endpoint persists changes. +type UpdateMode int + +const ( + // UpdateModePartial decodes the body as a partial delta and applies + // orm.UpdatePartialFields (the default, logpilot/easysearch style). + UpdateModePartial UpdateMode = iota + // UpdateModeFull loads the existing object (system fields preserved), + // decodes the body over it and applies orm.Update (coco's merge or + // replace semantics). + UpdateModeFull + // UpdateModeQuery follows the ?replace= query parameter: + // replace=false => partial delta, otherwise full object. + UpdateModeQuery +) + +// mcpToolName/Desc resolve the MCP labels for an action. +func (cfg Config[T]) mcpToolName(action string) string { + name := action + if action == ActionRead { + name = "get" // tool convention: resource_get + } + return fmt.Sprintf("%s_%s", cfg.Resource, name) +} + +func (cfg Config[T]) mcpToolDesc(action string) string { + if d, ok := cfg.MCPDescs[action]; ok && d != "" { + return d + } + noun := cfg.Resource + switch action { + case ActionCreate: + return fmt.Sprintf("Create a new %s", noun) + case ActionSearch: + return fmt.Sprintf("Search %ss (pagination, filters, full-text)", noun) + case ActionRead: + return fmt.Sprintf("Get a %s by ID", noun) + case ActionUpdate: + return fmt.Sprintf("Update a %s (partial fields)", noun) + case ActionDelete: + return fmt.Sprintf("Delete a %s by ID", noun) + } + return fmt.Sprintf("Call %s %s", action, noun) +} + +// Handlers are the five generated endpoint functions, exposed for direct +// testing without the global router. +type Handlers struct { + Create HandlerFunc + Search HandlerFunc + Get HandlerFunc + Update HandlerFunc + Delete HandlerFunc +} + +// generator carries the resolved config plus the embedded api.Handler for +// DecodeJSON/Write* helpers. +type generator[T any, P PT[T]] struct { + api.Handler + cfg Config[T] +} + +// NewHandlers builds the five endpoint handlers for the config, exposed for +// direct testing without the global router. +func NewHandlers[T any, P PT[T]](cfg Config[T]) Handlers { + g := &generator[T, P]{cfg: cfg} + return Handlers{ + Create: g.create, + Search: g.search, + Get: g.get, + Update: g.update, + Delete: g.delete, + } +} + +func (g *generator[T, P]) baseCtx(req *http.Request) *orm.Context { + ctx := orm.NewContextWithParent(req.Context()) + if g.cfg.SharingResource != "" { + ctx.Set(orm.SharingEnabled, true) + ctx.Set(orm.SharingResourceType, g.cfg.SharingResource) + } + return ctx +} + +// ctxFor builds the base context for an action and applies CtxDecorate. +func (g *generator[T, P]) ctxFor(req *http.Request, action string) *orm.Context { + ctx := g.baseCtx(req) + if g.cfg.CtxDecorate != nil { + g.cfg.CtxDecorate(ctx, req, action) + } + return ctx +} + +// idParam resolves the route parameter name holding the object id. +func (g *generator[T, P]) idParam() string { + if g.cfg.IDParam != "" { + return g.cfg.IDParam + } + return "id" +} + +// bestEffort runs a post hook, logging instead of failing the request. +func (g *generator[T, P]) bestEffort(what string, hook func(*T) error, obj *T) { + if hook == nil { + return + } + if err := hook(obj); err != nil { + log.Warnf("crud %s %q post hook: %v", g.cfg.Resource, what, err) + } +} + +func (g *generator[T, P]) create(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + obj := P(new(T)) + if err := g.DecodeJSON(req, obj); err != nil { + g.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if g.cfg.PrepareCreate != nil { + if err := g.cfg.PrepareCreate(obj); err != nil { + g.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + } + ctx := g.ctxFor(req, ActionCreate) + ctx.Refresh = orm.WaitForRefresh + if err := orm.Create(ctx, obj); err != nil { + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + g.bestEffort(ActionCreate, g.cfg.PostCreate, obj) + g.WriteCreatedOKJSON(w, obj.GetID()) +} + +func (g *generator[T, P]) search(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + builder, err := orm.NewQueryBuilderFromRequest(req, g.cfg.DefaultQueryFields...) + if err != nil { + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + builder.EnableBodyBytes() + if len(builder.Sorts()) == 0 { + builder.SortBy(orm.Sort{Field: "created", SortType: orm.DESC}) + } + + ctx := g.ctxFor(req, ActionSearch) + var model T + orm.WithModel(ctx, &model) + if g.cfg.PrepareSearch != nil { + if err := g.cfg.PrepareSearch(req, builder, ctx); err != nil { + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } + res, err := orm.SearchV2(ctx, builder) + if err != nil { + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + searchRes, err := elastic.DecodeSearchResult(res) + if err != nil { + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if g.cfg.PostSearch != nil { + if err := g.cfg.PostSearch(searchRes); err != nil { + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } + g.WriteJSON(w, *searchRes, http.StatusOK) +} + +func (g *generator[T, P]) get(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName(g.idParam()) + obj := P(new(T)) + obj.SetID(id) + ctx := g.ctxFor(req, ActionRead) + exists, err := orm.GetV2(ctx, obj) + if !exists || err != nil { + g.WriteGetMissingJSON(w, id) + return + } + if g.cfg.PostGet != nil { + if err := g.cfg.PostGet(obj); err != nil { + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } + g.WriteGetOKJSON(w, id, *obj) +} + +func (g *generator[T, P]) update(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName(g.idParam()) + obj := P(new(T)) + obj.SetID(id) + + mode := g.cfg.UpdateMode + if mode == UpdateModeQuery { + if v := req.URL.Query().Get("replace"); v == "false" || v == "0" { + mode = UpdateModePartial + } else { + mode = UpdateModeFull + } + } + + ctx := g.ctxFor(req, ActionUpdate) + ctx.Refresh = orm.WaitForRefresh + + if mode == UpdateModeFull { + // coco semantics: load the existing object (system fields such as + // created/builtin preserved), decode the body over it (merge), then + // persist the whole object. The merge goes through a map so + // ProtectedFields (restored from the loaded object) are honored, + // and PrepareUpdate receives the raw body as the delta. + exists, err := orm.GetWithSystemFields(ctx, obj) + if !exists || err != nil { + g.WriteOpRecordNotFoundJSON(w, id) + return + } + body := util.MapStr{} + if err := g.DecodeJSON(req, &body); err != nil { + g.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + for _, f := range g.cfg.ProtectedFields { + delete(body, f) + } + if len(body) > 0 { + loaded := map[string]interface{}{} + if raw, err := json.Marshal(obj); err == nil { + _ = json.Unmarshal(raw, &loaded) + } + for k, v := range body { + loaded[k] = v + } + if raw, err := json.Marshal(loaded); err == nil { + _ = json.Unmarshal(raw, obj) + } + } + obj.SetID(id) + if g.cfg.PrepareUpdate != nil { + if err := g.cfg.PrepareUpdate(obj, body); err != nil { + g.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + } + if err := orm.Update(ctx, obj); err != nil { + if strings.Contains(err.Error(), "not found") { + g.WriteOpRecordNotFoundJSON(w, id) + return + } + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + g.bestEffort(ActionUpdate, g.cfg.PostUpdate, obj) + g.WriteUpdatedOKJSON(w, obj.GetID()) + return + } + + delta := util.MapStr{} + if err := g.DecodeJSON(req, &delta); err != nil { + g.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + for _, f := range g.cfg.ProtectedFields { + delete(delta, f) + } + if g.cfg.PrepareUpdate != nil { + if err := g.cfg.PrepareUpdate(obj, delta); err != nil { + g.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + } + if err := orm.UpdatePartialFields(ctx, obj, delta); err != nil { + if strings.Contains(err.Error(), "not found") { + g.WriteOpRecordNotFoundJSON(w, id) + return + } + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if g.cfg.PostUpdate != nil { + // reload so post hooks see the persisted object, not just the id + if _, err := orm.GetV2(ctx, obj); err != nil { + log.Warnf("crud %s reload after update %q: %v", g.cfg.Resource, id, err) + } + } + g.bestEffort(ActionUpdate, g.cfg.PostUpdate, obj) + g.WriteUpdatedOKJSON(w, obj.GetID()) +} + +func (g *generator[T, P]) delete(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName(g.idParam()) + obj := P(new(T)) + obj.SetID(id) + ctx := g.ctxFor(req, ActionDelete) + exists, err := orm.GetV2(ctx, obj) + if !exists || err != nil { + g.WriteOpRecordNotFoundJSON(w, id) + return + } + if g.cfg.GuardDelete != nil { + if err := g.cfg.GuardDelete(obj); err != nil { + g.WriteError(w, err.Error(), http.StatusForbidden) + return + } + } + ctx.Refresh = orm.WaitForRefresh + if err := orm.Delete(ctx, obj); err != nil { + g.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + g.bestEffort(ActionDelete, g.cfg.PostDelete, obj) + g.WriteDeletedOKJSON(w, id) +} diff --git a/core/api/crud/crud_extend_test.go b/core/api/crud/crud_extend_test.go new file mode 100644 index 000000000..03d1dbccb --- /dev/null +++ b/core/api/crud/crud_extend_test.go @@ -0,0 +1,328 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package crud + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +// mustCreate seeds one gizmo through the given create handler. +func mustCreate(t *testing.T, h Handlers, body string) string { + t.Helper() + w, out := call(t, h.Create, "POST", "/gizmos/", body) + require.Equal(t, http.StatusOK, w.Code) + id, _ := out["_id"].(string) + require.NotEmpty(t, id) + return id +} + +func sourceOf(t *testing.T, h Handlers, id string) map[string]interface{} { + t.Helper() + w, out := call(t, h.Get, "GET", "/gizmos/"+id, "") + require.Equal(t, http.StatusOK, w.Code) + src, _ := out["_source"].(map[string]interface{}) + require.NotNil(t, src) + return src +} + +func TestUpdateModeFull_MergesOverLoadedObject(t *testing.T) { + setupGizmos(t) + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + PrepareCreate: func(o *gizmo) error { + if o.Status == "" { + o.Status = "new" + } + return nil + }, + UpdateMode: UpdateModeFull, + }) + + id := mustCreate(t, h, `{"name":"alpha"}`) + + // full mode decodes the body over the loaded object: untouched fields + // keep their stored values (merge semantics) + w, out := call(t, h.Update, "PUT", "/gizmos/"+id, `{"status":"active"}`) + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "updated", out["result"]) + src := sourceOf(t, h, id) + assert.Equal(t, "active", src["status"]) + assert.Equal(t, "alpha", src["name"]) +} + +func TestUpdateModeQueryFollowsReplaceParam(t *testing.T) { + setupGizmos(t) + var fullMode atomic.Bool + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + UpdateMode: UpdateModeQuery, + // full mode hands PrepareUpdate the merged object (loaded fields + // visible); partial mode only sets the id + PrepareUpdate: func(o *gizmo, delta util.MapStr) error { + fullMode.Store(o.Name != "") + return nil + }, + }) + + id := mustCreate(t, h, `{"name":"alpha"}`) + + // replace=false => partial delta (obj not loaded) + _, out := call(t, h.Update, "PUT", "/gizmos/"+id+"?replace=false", `{"status":"active"}`) + require.Equal(t, "updated", out["result"]) + assert.False(t, fullMode.Load(), "?replace=false should take the partial path") + + // default (no param) => full object (loaded, merge) + fullMode.Store(false) + _, out = call(t, h.Update, "PUT", "/gizmos/"+id, `{"status":"archived"}`) + require.Equal(t, "updated", out["result"]) + assert.True(t, fullMode.Load(), "absent ?replace should take the full path") + + src := sourceOf(t, h, id) + assert.Equal(t, "archived", src["status"]) + assert.Equal(t, "alpha", src["name"], "merge must keep untouched fields") +} + +func TestUpdateFullModeProtectedFieldsRestored(t *testing.T) { + setupGizmos(t) + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + UpdateMode: UpdateModeFull, + ProtectedFields: []string{"reserved", "created"}, + }) + + id := mustCreate(t, h, `{"name":"alpha"}`) + // attempt to flip the protected field through the body + _, out := call(t, h.Update, "PUT", "/gizmos/"+id, `{"name":"beta","reserved":true}`) + require.Equal(t, "updated", out["result"]) + + src := sourceOf(t, h, id) + assert.Equal(t, "beta", src["name"]) + assert.Nil(t, src["reserved"], "protected field must be restored from the loaded object") +} + +func TestProtectedFieldsStrippedFromDelta(t *testing.T) { + setupGizmos(t) + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + ProtectedFields: []string{"reserved"}, + }) + + id := mustCreate(t, h, `{"name":"alpha"}`) + _, out := call(t, h.Update, "PUT", "/gizmos/"+id, `{"status":"active","reserved":true}`) + require.Equal(t, "updated", out["result"]) + + src := sourceOf(t, h, id) + assert.Nil(t, src["reserved"], "protected field must be stripped from the delta") +} + +func TestPostHooksFire(t *testing.T) { + setupGizmos(t) + var created, updated, deleted atomic.Int32 + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + PostCreate: func(o *gizmo) error { created.Add(1); return nil }, + PostUpdate: func(o *gizmo) error { updated.Add(1); return nil }, + PostDelete: func(o *gizmo) error { deleted.Add(1); return nil }, + }) + + id := mustCreate(t, h, `{"name":"alpha"}`) + assert.EqualValues(t, 1, created.Load()) + + _, out := call(t, h.Update, "PUT", "/gizmos/"+id, `{"status":"active"}`) + require.Equal(t, "updated", out["result"]) + assert.EqualValues(t, 1, updated.Load()) + + // failed delete (guard veto) must not fire the post hook + w, _ := call(t, h.Delete, "DELETE", "/gizmos/nope", "") + assert.Equal(t, http.StatusNotFound, w.Code) + assert.EqualValues(t, 0, deleted.Load()) + + w, _ = call(t, h.Delete, "DELETE", "/gizmos/"+id, "") + require.Equal(t, http.StatusOK, w.Code) + assert.EqualValues(t, 1, deleted.Load()) +} + +func TestPostGetRefinesObject(t *testing.T) { + setupGizmos(t) + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + PostGet: func(o *gizmo) error { o.Name = o.Name + "-refined"; return nil }, + }) + id := mustCreate(t, h, `{"name":"alpha"}`) + src := sourceOf(t, h, id) + assert.Equal(t, "alpha-refined", src["name"]) +} + +func TestPrepareSearchFilterAndPostSearchMutation(t *testing.T) { + setupGizmos(t) + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + PrepareSearch: func(req *http.Request, builder *orm.QueryBuilder, ctx *orm.Context) error { + // injected filter: only gizmos with the unique marker are visible + builder.Filter(orm.TermsQuery("status", []string{"marker-xyz"})) + return nil + }, + PostSearch: func(res *elastic.SearchResponse) error { + // per-hit decoration + for i := range res.Hits.Hits { + if res.Hits.Hits[i].Source == nil { + res.Hits.Hits[i].Source = util.MapStr{} + } + res.Hits.Hits[i].Source["decorated"] = true + } + return nil + }, + }) + + // one gizmo carries the marker, one does not => only the first is visible + id := mustCreate(t, h, `{"name":"alpha"}`) + _, out := call(t, h.Update, "PUT", "/gizmos/"+id+"?replace=false", `{"status":"marker-xyz"}`) + require.Equal(t, "updated", out["result"]) + mustCreate(t, h, `{"name":"beta"}`) + + w, out := call(t, h.Search, "GET", "/gizmos/_search?size=10", "") + require.Equal(t, http.StatusOK, w.Code) + hits, _ := out["hits"].(map[string]interface{}) + list, _ := hits["hits"].([]interface{}) + require.Len(t, list, 1) + hit, _ := list[0].(map[string]interface{}) + src, _ := hit["_source"].(map[string]interface{}) + assert.Equal(t, "alpha", src["name"]) + assert.Equal(t, true, src["decorated"], "PostSearch decoration missing") +} + +func TestCtxDecorateSeesActions(t *testing.T) { + setupGizmos(t) + var actions []string + record := func(a string) { actions = append(actions, a) } + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + CtxDecorate: func(ctx *orm.Context, req *http.Request, action string) { + record(action) + ctx.Set("decorated_action", action) + }, + }) + + id := mustCreate(t, h, `{"name":"alpha"}`) + call(t, h.Get, "GET", "/gizmos/"+id, "") + call(t, h.Update, "PUT", "/gizmos/"+id, `{"status":"active"}`) + call(t, h.Delete, "DELETE", "/gizmos/"+id, "") + call(t, h.Search, "GET", "/gizmos/_search?size=10", "") + + assert.Equal(t, []string{ActionCreate, ActionRead, ActionUpdate, ActionDelete, ActionSearch}, actions) +} + +func TestIDParamOverride(t *testing.T) { + setupGizmos(t) + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + IDParam: "gizmo_id", + }) + + // create via the standard helper (create has no id param) + id := mustCreate(t, h, `{"name":"alpha"}`) + + // get with the custom param name + req := httptest.NewRequest("GET", "/gizmos/"+id, nil) + ps := httprouter.Params{{Key: "gizmo_id", Value: id}} + w := httptest.NewRecorder() + h.Get(w, req, ps) + require.Equal(t, http.StatusOK, w.Code) + out := map[string]interface{}{} + _ = json.Unmarshal(w.Body.Bytes(), &out) + assert.Equal(t, true, out["found"]) +} + +func TestBuildOptionsOrdering(t *testing.T) { + cfg := Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", + Permission: func(action string) api.PermissionKey { return api.PermissionKey("generic#gizmo/" + action) }, + MCP: true, + ExtraOptions: func(action string) []api.Option { return []api.Option{api.MCPTool("extra_probe", "x")} }, + } + with := buildOptions(cfg, ActionRead, true) + require.Len(t, with, 3, "permission + mcp + extra, in order") + without := buildOptions(cfg, ActionSearch, false) + require.Len(t, without, 2, "no-mcp variant drops only the mcp option") + + empty := Config[gizmo]{Prefix: "/gizmos", Resource: "gizmo"} + assert.Nil(t, buildOptions(empty, ActionRead, true)) +} + +func TestRegisterCRUD_ExtraOptionsAndMCPOnGetOnly(t *testing.T) { + setupGizmos(t) + RegisterCRUD[gizmo](Config[gizmo]{ + Prefix: "/gizmoroutes", + Resource: "gizmo", + IDParam: "gizmo_id", + MCP: true, + ExtraOptions: func(action string) []api.Option { + // non-MCP extras (login/CORS/label style) are safe to apply to + // every registration of the action, including POST _search + return nil + }, + }) + + mcpRoutes := map[string]bool{} + api.WalkMCPAutoUIMethodRoutes(func(route api.RegisteredUIMethodRoute) { + if route.Route.Path == "/gizmoroutes/_search" || + route.Route.Path == "/gizmoroutes/:gizmo_id" || + route.Route.Path == "/gizmoroutes/" { + mcpRoutes[route.Route.Path+" "+string(route.Route.Method)] = true + } + }) + assert.True(t, mcpRoutes["/gizmoroutes/_search GET"], "GET _search should be an MCP route: %v", mcpRoutes) + assert.False(t, mcpRoutes["/gizmoroutes/_search POST"], "POST _search must not duplicate the MCP tool") + assert.True(t, mcpRoutes["/gizmoroutes/:gizmo_id GET"], "custom id param should appear in routes") +} + +func TestPostUpdateSeesPersistedObject(t *testing.T) { + setupGizmos(t) + var seenStatus atomic.Value + h := NewHandlers[gizmo](Config[gizmo]{ + Prefix: "/gizmos", Resource: "gizmo", DefaultQueryFields: []string{"name"}, + PostUpdate: func(o *gizmo) error { + seenStatus.Store(o.Status) // must reflect the persisted delta + return nil + }, + }) + id := mustCreate(t, h, `{"name":"alpha"}`) + _, out := call(t, h.Update, "PUT", "/gizmos/"+id, `{"status":"reloaded"}`) + require.Equal(t, "updated", out["result"]) + assert.Equal(t, "reloaded", seenStatus.Load()) +} + +func TestRegisterCRUD_SkipActions(t *testing.T) { + setupGizmos(t) + RegisterCRUD[gizmo](Config[gizmo]{ + Prefix: "/gizmoskip", + Resource: "gizmo", + MCP: true, + SkipActions: []string{ActionRead, ActionSearch}, + }) + + seen := map[string]bool{} + api.WalkMCPAutoUIMethodRoutes(func(route api.RegisteredUIMethodRoute) { + if route.Route.Path == "/gizmoskip/:id" || route.Route.Path == "/gizmoskip/_search" { + seen[route.Route.Path+" "+string(route.Route.Method)] = true + } + }) + assert.False(t, seen["/gizmoskip/:id GET"], "skipped read action must not register: %v", seen) + assert.False(t, seen["/gizmoskip/_search GET"], "skipped search action must not register: %v", seen) + // non-skipped actions on the same path still register + assert.True(t, seen["/gizmoskip/:id PUT"], "update must still register: %v", seen) + assert.True(t, seen["/gizmoskip/:id DELETE"], "delete must still register: %v", seen) +} diff --git a/core/api/crud/crud_test.go b/core/api/crud/crud_test.go new file mode 100644 index 000000000..dbbf5a46e --- /dev/null +++ b/core/api/crud/crud_test.go @@ -0,0 +1,227 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package crud + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/sqlite" +) + +// gizmo is the test model. +type gizmo struct { + orm.ORMObjectBase + Name string `json:"name,omitempty" elastic_mapping:"name:{type:keyword}"` + Status string `json:"status,omitempty" elastic_mapping:"status:{type:keyword}"` + Reserved bool `json:"reserved,omitempty"` +} + +func call(t *testing.T, h HandlerFunc, method, target, body string) (*httptest.ResponseRecorder, map[string]interface{}) { + t.Helper() + var reader *bytes.Reader + if body != "" { + reader = bytes.NewReader([]byte(body)) + } else { + reader = bytes.NewReader(nil) + } + req := httptest.NewRequest(method, target, reader) + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + // crud uses :id params — parse from the path (without the query string). + ps := httprouter.Params{} + if rest := trimPrefix(target, "/gizmos/"); rest != "" && !containsSlash(rest) { + if i := strings.IndexByte(rest, '?'); i >= 0 { + rest = rest[:i] + } + ps = httprouter.Params{{Key: "id", Value: rest}} + } + w := httptest.NewRecorder() + h(w, req, ps) + out := map[string]interface{}{} + _ = json.Unmarshal(w.Body.Bytes(), &out) + return w, out +} + +func trimPrefix(s, p string) string { + if len(s) >= len(p) && s[:len(p)] == p { + return s[len(p):] + } + return s +} + +func containsSlash(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] == '/' { + return true + } + } + return false +} + +// crudStoreOnce shares one registered sqlite handler across tests +// (orm.Register panics on duplicates within a binary). +var crudStoreOnce sync.Once + +func setupGizmos(t *testing.T) Handlers { + t.Helper() + crudStoreOnce.Do(func() { + handler := &sqlite.SQLiteORM{Config: sqlite.SQLiteConfig{ + Enabled: true, + DBPath: filepath.Join(t.TempDir(), "crud.db"), + }} + if err := handler.Open(); err != nil { + panic(err) + } + if err := handler.RegisterSchemaWithName(gizmo{}, "gizmos"); err != nil { + panic(err) + } + orm.Register("sqlite", handler) + }) + + cfg := Config[gizmo]{ + Prefix: "/gizmos", + Resource: "gizmo", + DefaultQueryFields: []string{"name"}, + PrepareCreate: func(obj *gizmo) error { + if obj.Name == "" { + return errString("name is required") + } + if obj.Status == "" { + obj.Status = "new" + } + return nil + }, + GuardDelete: func(obj *gizmo) error { + if obj.Reserved { + return errString("reserved gizmo cannot be deleted") + } + return nil + }, + } + return NewHandlers[gizmo](cfg) +} + +type errString string + +func (e errString) Error() string { return string(e) } + +func TestCRUD_FullFlow(t *testing.T) { + h := setupGizmos(t) + + // Create: envelope {_id, result} + defaults applied. + // unique namespace: the sqlite store is shared across the package's + // tests, so unfiltered counts must only see this test's records + const ns = "ff-" + w, out := call(t, h.Create, "POST", "/gizmos/", `{"name":"`+ns+`alpha"}`) + require.Equal(t, http.StatusOK, w.Code) + require.Equal(t, "created", out["result"]) + id, _ := out["_id"].(string) + require.NotEmpty(t, id) + + // Validation error → 400. + w, _ = call(t, h.Create, "POST", "/gizmos/", `{"name":""}`) + assert.Equal(t, http.StatusBadRequest, w.Code) + + // Get: {found, _id, _source} envelope with the default applied. + w, out = call(t, h.Get, "GET", "/gizmos/"+id, "") + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, true, out["found"]) + src, _ := out["_source"].(map[string]interface{}) + require.NotNil(t, src) + assert.Equal(t, ns+"alpha", src["name"]) + assert.Equal(t, "new", src["status"]) + + // Get missing → 404 envelope. + w, out = call(t, h.Get, "GET", "/gizmos/nope", "") + assert.Equal(t, http.StatusNotFound, w.Code) + assert.Equal(t, false, out["found"]) + + // Update: partial delta preserves untouched fields (P1 merge contract). + w, out = call(t, h.Update, "PUT", "/gizmos/"+id, `{"status":"active"}`) + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "updated", out["result"]) + w, out = call(t, h.Get, "GET", "/gizmos/"+id, "") + src, _ = out["_source"].(map[string]interface{}) + assert.Equal(t, "active", src["status"]) + assert.Equal(t, ns+"alpha", src["name"]) + + // Update missing → 404. + w, _ = call(t, h.Update, "PUT", "/gizmos/nope", `{"status":"x"}`) + assert.Equal(t, http.StatusNotFound, w.Code) + + // Search: ES-shaped response with default created DESC sort. + for _, name := range []string{ns + "beta", ns + "gamma"} { + _, out := call(t, h.Create, "POST", "/gizmos/", `{"name":"`+name+`"}`) + require.NotEmpty(t, out["_id"]) + } + w, out = call(t, h.Search, "GET", "/gizmos/_search?size=10&filter=name:any("+ns+"alpha,"+ns+"beta,"+ns+"gamma)", "") + require.Equal(t, http.StatusOK, w.Code) + hits, _ := out["hits"].(map[string]interface{}) + require.NotNil(t, hits, "ES-shaped response") + list, _ := hits["hits"].([]interface{}) + assert.Len(t, list, 3) + total, _ := hits["total"].(map[string]interface{}) + assert.EqualValues(t, 3, total["value"]) + + // Search with filter. + w, out = call(t, h.Search, "GET", "/gizmos/_search?filter=name:"+ns+"alpha&size=10", "") + hits, _ = out["hits"].(map[string]interface{}) + list, _ = hits["hits"].([]interface{}) + assert.Len(t, list, 1) + + // Delete: reserved guard blocks with 403. + reserved := gizmo{Name: "keep", Reserved: true} + reserved.ID = "keep-1" + require.NoError(t, orm.Save(orm.NewContext(), &reserved)) + w, _ = call(t, h.Delete, "DELETE", "/gizmos/keep-1", "") + assert.Equal(t, http.StatusForbidden, w.Code) + + // Delete: ack envelope, then gone. + w, out = call(t, h.Delete, "DELETE", "/gizmos/"+id, "") + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "deleted", out["result"]) + w, out = call(t, h.Get, "GET", "/gizmos/"+id, "") + assert.Equal(t, http.StatusNotFound, w.Code) + + // Delete missing → 404. + w, _ = call(t, h.Delete, "DELETE", "/gizmos/nope", "") + assert.Equal(t, http.StatusNotFound, w.Code) +} + +func TestCRUD_SharingContext(t *testing.T) { + setupGizmos(t) // registers the sqlite handler for this test too + // SharingResource is exercised via the get path's context keys — assert + // by configuring and confirming no behavioral difference (the keys are + // consumed by security hooks that are absent here). + cfg := Config[gizmo]{Prefix: "/gizmos", Resource: "gizmo", SharingResource: "gizmo"} + hh := NewHandlers[gizmo](cfg) + w, _ := call(t, hh.Search, "GET", "/gizmos/_search?size=1", "") + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestRegisterCRUD_ValidatesConfig(t *testing.T) { + called := false + defer func() { + if r := recover(); r != nil { + called = true + } + }() + RegisterCRUD[gizmo](Config[gizmo]{Prefix: "no-slash", Resource: "x"}) + assert.True(t, called, "bad prefix must panic") +} + +var _ = util.MapStr{} diff --git a/core/api/crud/mcp_probe_test.go b/core/api/crud/mcp_probe_test.go new file mode 100644 index 000000000..e715a68b9 --- /dev/null +++ b/core/api/crud/mcp_probe_test.go @@ -0,0 +1,29 @@ +package crud + +import ( + "net/http" + "testing" + + httprouter "infini.sh/framework/core/api/router" + + "infini.sh/framework/core/api" +) + +func TestMCPProbe(t *testing.T) { + api.HandleUIMethod(api.POST, "/mcpprobe/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {}, + api.RequirePermission("generic#x/read"), api.MCPTool("probe_create", "probe")) + + api.HandleUIMethod(api.POST, "/mcpprobe2/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {}, + api.MCPTool("probe2_create", "probe2")) + + found := map[string]bool{} + api.WalkMCPAutoUIMethodRoutes(func(route api.RegisteredUIMethodRoute) { + if route.Route.Path == "/mcpprobe/" || route.Route.Path == "/mcpprobe2/" { + found[route.Route.Path] = true + } + }) + if !found["/mcpprobe/"] || !found["/mcpprobe2/"] { + t.Fatalf("MCP routes not walked: %+v", found) + } + t.Log("MCP option + walk OK") +} diff --git a/core/api/crud/register.go b/core/api/crud/register.go new file mode 100644 index 000000000..189c703b2 --- /dev/null +++ b/core/api/crud/register.go @@ -0,0 +1,87 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package crud + +import ( + "fmt" + + "infini.sh/framework/core/api" +) + +// RegisterCRUD builds the five handlers and wires the routes: +// +// POST {prefix}/ create +// GET {prefix}/_search search (+ POST, without the MCP tool) +// GET {prefix}/:id get (param name: cfg.IDParam, default "id") +// PUT {prefix}/:id update +// DELETE {prefix}/:id delete +// +// Each route is gated by cfg.Permission(action) when the func is set, and +// cfg.ExtraOptions(action) appends additional options (login, CORS, +// sensitive-field masking...) after the permission and MCP options. The +// MCP tool is registered on the GET _search variant only so POST _search +// does not duplicate the tool name (coco convention). +func RegisterCRUD[T any, P PT[T]](cfg Config[T]) { + validateConfig(cfg) + + h := NewHandlers[T, P](cfg) + idParam := cfg.IDParam + if idParam == "" { + idParam = "id" + } + perm := func(action string, withMCP bool) []api.Option { + return buildOptions(cfg, action, withMCP) + } + + skip := map[string]bool{} + for _, a := range cfg.SkipActions { + skip[a] = true + } + register := func(method api.Method, path string, handler HandlerFunc, action string, withMCP bool) { + if skip[action] { + return + } + api.HandleUIMethod(method, path, handler, perm(action, withMCP)...) + } + + register(api.POST, cfg.Prefix+"/", h.Create, ActionCreate, true) + register(api.GET, cfg.Prefix+"/_search", h.Search, ActionSearch, true) + register(api.POST, cfg.Prefix+"/_search", h.Search, ActionSearch, false) + register(api.GET, cfg.Prefix+"/:"+idParam, h.Get, ActionRead, true) + register(api.PUT, cfg.Prefix+"/:"+idParam, h.Update, ActionUpdate, true) + register(api.DELETE, cfg.Prefix+"/:"+idParam, h.Delete, ActionDelete, true) +} + +func validateConfig[T any](cfg Config[T]) { + if cfg.Prefix == "" || cfg.Prefix[0] != '/' { + panic(fmt.Sprintf("crud: Prefix must start with '/', got %q", cfg.Prefix)) + } + if len(cfg.Prefix) > 1 && cfg.Prefix[len(cfg.Prefix)-1] == '/' { + panic(fmt.Sprintf("crud: Prefix must not end with '/', got %q", cfg.Prefix)) + } + if cfg.Resource == "" { + panic("crud: Resource is required") + } +} + +// buildOptions assembles the route options for one action: the permission +// gate first, then the MCP tool (only on the registration that carries it, +// i.e. GET _search), then the caller's extra options. Exposed for testing. +func buildOptions[T any](cfg Config[T], action string, withMCP bool) []api.Option { + var opts []api.Option + if cfg.Permission != nil { + if key := cfg.Permission(action); key != "" { + opts = append(opts, api.RequirePermission(key)) + } + } + if cfg.MCP && withMCP { + opts = append(opts, api.MCPTool(cfg.mcpToolName(action), cfg.mcpToolDesc(action))) + } + if cfg.ExtraOptions != nil { + opts = append(opts, cfg.ExtraOptions(action)...) + } + if len(opts) == 0 { + return nil + } + return opts +} diff --git a/core/elastic/client_provider.go b/core/elastic/client_provider.go index 05f1058be..7f28af774 100644 --- a/core/elastic/client_provider.go +++ b/core/elastic/client_provider.go @@ -3,10 +3,15 @@ package elastic import ( + "crypto/tls" + "encoding/json" "errors" "fmt" + "io" + "net/http" "strings" "sync" + "time" ) // ────────────────────────────────────────────────────────────────────────── @@ -146,11 +151,88 @@ func ResetClientCacheForTest() { // (and thus a client). Secrets are part of the key (held in memory only, never // logged) so different credentials get different clients. func configCacheKey(cfg ElasticsearchConfig) string { + // GetAnyEndpoint panics on endpoint-less configs (a programming error + // when building clients), but cache invalidation may legitimately see + // stripped-down records (e.g. a delete hook handed an ID-only model) — + // degrade to an empty endpoint there instead of panicking. + anyEndpoint := "" + if cfg.Endpoint != "" || len(cfg.Endpoints) > 0 { + anyEndpoint = cfg.GetAnyEndpoint() + } var b strings.Builder - fmt.Fprintf(&b, "%s|%s|%s|", cfg.GetAnyEndpoint(), strings.Join(cfg.Endpoints, ","), cfg.Distribution) + fmt.Fprintf(&b, "%s|%s|%s|", anyEndpoint, strings.Join(cfg.Endpoints, ","), cfg.Distribution) if cfg.BasicAuth != nil { fmt.Fprintf(&b, "%s:%s|", cfg.BasicAuth.Username, cfg.BasicAuth.Password.Get()) } fmt.Fprintf(&b, "%s|%s", cfg.Token.Get(), cfg.Version) return b.String() } + +// SameConnectionIdentity reports whether two configs would share a cached +// client — same endpoints, credentials, distribution and version (see +// configCacheKey). Used by cluster-change hooks to skip re-initialization +// when only non-connection fields (labels, health status) changed. +func SameConnectionIdentity(a, b ElasticsearchConfig) bool { + return configCacheKey(a) == configCacheKey(b) +} + +// probeTransport is the HTTP transport for connectivity probes: TLS +// verification disabled (ES clusters commonly use self-signed certs). +var probeTransport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, +} + +// ProbeCluster does a raw GET to the cluster's root endpoint to detect +// connectivity and the server version/distribution, without building a +// version-specific adapter. BasicAuth or X-API-TOKEN is applied when +// present. Shared by pre-registration connectivity tests and any caller +// that needs the version before a client exists. +// +// Note: modules/elastic/adapter.ClusterVersion is a fasthttp-based twin +// used inside the factory's version probing; consolidating the two is +// tracked in the refactor plan (SEARCH_ORM_REFACTOR_PLAN, P3 follow-up). +func ProbeCluster(cfg *ElasticsearchConfig) (version, distribution string, err error) { + endpoint := cfg.Endpoint + if endpoint == "" && len(cfg.Endpoints) > 0 { + endpoint = cfg.Endpoints[0] + } + httpReq, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return "", "", err + } + if cfg.BasicAuth != nil && cfg.BasicAuth.Username != "" { + httpReq.SetBasicAuth(cfg.BasicAuth.Username, cfg.BasicAuth.Password.Get()) + } else if t := cfg.Token.Get(); t != "" { + httpReq.Header.Set("X-API-TOKEN", t) + } + + client := &http.Client{Timeout: 10 * time.Second, Transport: probeTransport} + resp, err := client.Do(httpReq) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode >= 400 { + return "", "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(truncate(string(body), 200))) + } + var info struct { + Version struct { + Number string `json:"number"` + Distribution string `json:"distribution"` + } `json:"version"` + } + if err := json.Unmarshal(body, &info); err != nil { + return "", "", fmt.Errorf("parse version response: %w", err) + } + return info.Version.Number, info.Version.Distribution, nil +} + +// truncate caps s to n runes, appending "…" when truncated. +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/core/elastic/client_provider_test.go b/core/elastic/client_provider_test.go index d6cbdbcae..ab2be427a 100644 --- a/core/elastic/client_provider_test.go +++ b/core/elastic/client_provider_test.go @@ -5,6 +5,10 @@ package elastic import ( "errors" "testing" + + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" + "infini.sh/framework/lib/go-ucfg" ) func mkCfg(endpoint, version string) ElasticsearchConfig { @@ -158,3 +162,41 @@ func indexOf(s, sub string) int { } return -1 } + +func TestSameConnectionIdentity(t *testing.T) { + a := mkCfg("http://a:9200", "8.0.0") + b := mkCfg("http://a:9200", "8.0.0") + b.Labels = util.MapStr{"health_status": "green"} // non-connection field + if !SameConnectionIdentity(a, b) { + t.Fatal("labels-only change must keep connection identity") + } + + c := mkCfg("http://a:9200", "9.0.0") + if SameConnectionIdentity(a, c) { + t.Fatal("version change must alter connection identity") + } + + d := mkCfg("http://a:9200", "8.0.0") + d.BasicAuth = &model.BasicAuth{Username: "u", Password: ucfg.SecretString("p")} + if SameConnectionIdentity(a, d) { + t.Fatal("credential change must alter connection identity") + } +} + +func TestTruncate(t *testing.T) { + cases := []struct { + in string + n int + want string + }{ + {"abc", 5, "abc"}, + {"abcdef", 3, "abc…"}, + {"世界你好", 2, "世界…"}, + {"", 3, ""}, + } + for _, c := range cases { + if got := truncate(c.in, c.n); got != c.want { + t.Errorf("truncate(%q,%d) = %q, want %q", c.in, c.n, got, c.want) + } + } +} diff --git a/core/elastic/search_decode.go b/core/elastic/search_decode.go new file mode 100644 index 000000000..030c79b2f --- /dev/null +++ b/core/elastic/search_decode.go @@ -0,0 +1,94 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +// ────────────────────────────────────────────────────────────────────────── +// Typed decoding of orm.SearchResult payloads. +// +// Every ORM backend returns search results as ES-shaped JSON +// ({"hits":{"total":...,"hits":[{"_id":...,"_source":{...}}]}}) carried in +// SearchResult.Payload. These helpers turn that convention into a typed +// contract so callers stop hand-rolling Payload assertions and per-entity +// parse helpers. +// +// Placement note: this file lives in core/elastic (not core/orm) because +// core/elastic already depends on core/orm for ORMObjectBase — the reverse +// import would be a cycle. +// ────────────────────────────────────────────────────────────────────────── + +// DecodeSearchResult decodes an orm.SearchResult payload (ES-shaped JSON) +// into a SearchResponse. Payload accepts []byte, string, and nil/empty +// (which yield a zero response, not an error) so callers can pass results +// from any backend without defensive type checks. +func DecodeSearchResult(res *orm.SearchResult) (*SearchResponse, error) { + out := &SearchResponse{} + if res == nil { + return out, nil + } + var raw []byte + switch payload := res.Payload.(type) { + case []byte: + raw = payload + case string: + raw = []byte(payload) + default: + return out, nil + } + if len(raw) == 0 { + return out, nil + } + if err := util.FromJSONBytes(raw, out); err != nil { + return nil, err + } + return out, nil +} + +// DecodeHits decodes the hits of an orm.SearchResult into a typed slice and +// returns it with the reported total. The slice is always non-nil (empty +// when there are no hits), so callers can range/append safely. +// +// Document IDs are backfilled from the ES _id field: when T implements +// orm.Object its SetID is called, and "id" is injected into the source map +// before decoding so plain structs with an `json:"id"` field work too. +func DecodeHits[T any](res *orm.SearchResult) ([]T, int64, error) { + resp, err := DecodeSearchResult(res) + if err != nil { + return nil, 0, err + } + total := resp.GetTotal() + + hits := resp.Hits.Hits + out := make([]T, 0, len(hits)) + for i := range hits { + hit := &hits[i] + + src := hit.Source + if src == nil { + src = util.MapStr{} + } + // _id is the authoritative document ID; surface it on the decoded + // value even when the stored source lacks an id field. + if hit.ID != "" { + src["id"] = hit.ID + } + + raw, err := util.ToJSONBytes(src) + if err != nil { + return nil, total, err + } + var item T + if err := util.FromJSONBytes(raw, &item); err != nil { + return nil, total, err + } + if obj, ok := any(&item).(orm.Object); ok && hit.ID != "" { + obj.SetID(hit.ID) + } + out = append(out, item) + } + return out, total, nil +} diff --git a/core/elastic/search_decode_test.go b/core/elastic/search_decode_test.go new file mode 100644 index 000000000..8b1c55f9b --- /dev/null +++ b/core/elastic/search_decode_test.go @@ -0,0 +1,137 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "testing" + + "infini.sh/framework/core/orm" +) + +type decodeTestItem struct { + orm.ORMObjectBase + Name string `json:"name,omitempty"` +} + +type decodePlainItem struct { // no orm.Object — relies on "id" injection + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +func TestDecodeSearchResult(t *testing.T) { + payload := `{"hits":{"total":{"value":2},"hits":[{"_id":"a","_source":{"id":"a","name":"A"}},{"_id":"b","_source":{"id":"b","name":"B"}}]}}` + res, err := DecodeSearchResult(&orm.SearchResult{Payload: []byte(payload)}) + if err != nil { + t.Fatalf("decode failed: %v", err) + } + if got := res.GetTotal(); got != 2 { + t.Fatalf("total = %d, want 2", got) + } + if len(res.Hits.Hits) != 2 { + t.Fatalf("hits = %d, want 2", len(res.Hits.Hits)) + } + if res.Hits.Hits[0].Source["name"] != "A" || res.Hits.Hits[1].ID != "b" { + t.Fatalf("hit contents mismatch: %+v", res.Hits.Hits) + } +} + +func TestDecodeSearchResult_EmptyAndMalformed(t *testing.T) { + checks := []struct { + name string + res *orm.SearchResult + wantErr bool + }{ + {"nil result", nil, false}, + {"nil payload", &orm.SearchResult{}, false}, + {"empty hits", &orm.SearchResult{Payload: []byte(`{"hits":{"hits":[]}}`)}, false}, + {"empty bytes", &orm.SearchResult{Payload: []byte(``)}, false}, + {"string payload", &orm.SearchResult{Payload: `{"hits":{"hits":[]}}`}, false}, + {"empty string", &orm.SearchResult{Payload: ``}, false}, + {"non-bytes payload", &orm.SearchResult{Payload: 12345}, false}, + {"malformed json", &orm.SearchResult{Payload: []byte(`not json`)}, true}, + } + for _, c := range checks { + t.Run(c.name, func(t *testing.T) { + res, err := DecodeSearchResult(c.res) + if c.wantErr { + if err == nil { + t.Fatalf("expected an error, got %+v", res) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(res.Hits.Hits) != 0 { + t.Fatalf("expected no hits, got %d", len(res.Hits.Hits)) + } + }) + } +} + +func TestDecodeHits(t *testing.T) { + payload := `{"hits":{"total":{"value":2},"hits":[` + + `{"_id":"a","_source":{"name":"A"}},` + + `{"_id":"b","_source":{"name":"B"}}]}}` + + t.Run("typed with orm.Object", func(t *testing.T) { + items, total, err := DecodeHits[decodeTestItem](&orm.SearchResult{Payload: []byte(payload)}) + if err != nil { + t.Fatalf("DecodeHits: %v", err) + } + if total != 2 { + t.Fatalf("total = %d, want 2", total) + } + if len(items) != 2 || items[0].Name != "A" || items[1].Name != "B" { + t.Fatalf("items mismatch: %+v", items) + } + if items[0].ID != "a" || items[1].ID != "b" { + t.Fatalf("_id not backfilled via SetID: %+v", items) + } + }) + + t.Run("plain struct gets id injected", func(t *testing.T) { + items, _, err := DecodeHits[decodePlainItem](&orm.SearchResult{Payload: []byte(payload)}) + if err != nil { + t.Fatalf("DecodeHits: %v", err) + } + if len(items) != 2 || items[0].ID != "a" { + t.Fatalf("id not injected into plain struct: %+v", items) + } + }) + + t.Run("empty result yields non-nil slice", func(t *testing.T) { + items, total, err := DecodeHits[decodeTestItem](&orm.SearchResult{Payload: []byte(`{"hits":{"total":{"value":0},"hits":[]}}`)}) + if err != nil { + t.Fatalf("DecodeHits: %v", err) + } + if items == nil || len(items) != 0 { + t.Fatalf("expected non-nil empty slice, got %#v", items) + } + if total != 0 { + t.Fatalf("total = %d, want 0", total) + } + }) + + t.Run("missing total reports unknown", func(t *testing.T) { + // GetTotal() contract: -1 when the response carries no total. + _, total, err := DecodeHits[decodeTestItem](&orm.SearchResult{Payload: []byte(`{"hits":{"hits":[]}}`)}) + if err != nil { + t.Fatalf("DecodeHits: %v", err) + } + if total != -1 { + t.Fatalf("total = %d, want -1 (unknown)", total) + } + }) + + t.Run("nil source hit", func(t *testing.T) { + payload := `{"hits":{"total":{"value":1},"hits":[{"_id":"x"}]}}` + items, _, err := DecodeHits[decodeTestItem](&orm.SearchResult{Payload: []byte(payload)}) + if err != nil { + t.Fatalf("DecodeHits: %v", err) + } + if len(items) != 1 || items[0].ID != "x" { + t.Fatalf("nil _source hit mishandled: %+v", items) + } + }) +} diff --git a/core/orm/aggregate.go b/core/orm/aggregate.go new file mode 100644 index 000000000..cb4de3620 --- /dev/null +++ b/core/orm/aggregate.go @@ -0,0 +1,65 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package orm + +import ( + "encoding/json" + "errors" +) + +// ────────────────────────────────────────────────────────────────────────── +// Aggregation contract. +// +// Aggregations are a first-class operation: Aggregate(ctx, qb) executes the +// qb.Aggs tree and returns a typed, recursive result model — independent of +// the ES-shaped JSON that SearchV2 still returns for backward compatibility. +// Bucket and metric aggregations are computed by each backend natively; +// pipeline aggregations are computed uniformly by the framework engine +// (core/aggregate) so every backend behaves identically. +// ────────────────────────────────────────────────────────────────────────── + +// AggregationResult is the root of the typed aggregation tree. Keys of Aggs +// correspond one-to-one to the qb.Aggs names. +type AggregationResult struct { + Aggs map[string]*AggNode `json:"aggs,omitempty"` +} + +// AggNode is one named aggregation's result. Exactly one shape is populated: +// a single value (Value), multi values (Values, e.g. percentiles), a top +// document (TopHit), or buckets (Buckets). +type AggNode struct { + Value float64 `json:"value,omitempty"` + ValueSet bool `json:"value_set,omitempty"` // distinguishes 0 from "no value" (ES null) + Values map[string]float64 `json:"values,omitempty"` + TopHit *json.RawMessage `json:"top_hit,omitempty"` + Buckets []Bucket `json:"buckets,omitempty"` +} + +// Bucket is one bucket of a bucket aggregation. Key is the display key +// (term value or formatted time bucket); KeyRaw carries the numeric key +// when the bucket is time-based (epoch milliseconds, ES convention). +type Bucket struct { + Key string `json:"key,omitempty"` + KeyRaw interface{} `json:"key_raw,omitempty"` + DocCount int64 `json:"doc_count,omitempty"` + Aggs map[string]*AggNode `json:"aggs,omitempty"` +} + +// Aggregate executes the aggregations of qb through the registered backend. +// The WHERE clauses of qb scope the aggregation set, exactly like SearchV2. +func Aggregate(ctx *Context, qb *QueryBuilder) (*AggregationResult, error) { + if ctx == nil { + ctx = NewContext() + } + if qb == nil { + return nil, errors.New("query builder is required for aggregation") + } + if len(qb.Aggs) == 0 { + return nil, errors.New("no aggregations set on the query builder") + } + m, ok := getHandler().(MetricsAPI) + if !ok { + return nil, errors.New("ORM backend does not support Aggregate") + } + return m.Aggregate(ctx, qb) +} diff --git a/core/orm/aggs.go b/core/orm/aggs.go index df3120927..05361e0cd 100644 --- a/core/orm/aggs.go +++ b/core/orm/aggs.go @@ -23,6 +23,8 @@ package orm +import "time" + // Aggregation is the interface that all specific aggregation types must implement. // It serves as a marker to group different aggregation structs. type Aggregation interface { @@ -55,6 +57,10 @@ const ( // Pipeline types MetricPipelineDerivative = "derivative" MetricSumBucket = "sum_bucket" + // Pipeline completions (console vocabulary) + MetricMaxBucket = "max_bucket" + MetricBucketScript = "bucket_script" + MetricBucketSort = "bucket_sort" ) // baseAggregation provides common functionality for all aggregation types, @@ -161,7 +167,8 @@ type DateHistogramAggregation struct { Interval string // A generic interval string like "1d", "1M", "1h". Format string TimeZone string - IntervalField string // es-specific field name for backward compatibility + IntervalField string // es-specific field name for backward compatibility + Offset time.Duration // shift bucket boundaries (ES date_histogram offset) } // AddNested provides a correctly typed chained call for DateHistogramAggregation. @@ -219,3 +226,96 @@ func (a *DateRangeAggregation) AddNested(name string, sub Aggregation) Aggregati a.baseAggregation.AddNested(name, sub) return a } + +// ────────────────────────────────────────────────────────────────────────── +// Request-model completions (console vocabulary, design doc §5.1). +// ────────────────────────────────────────────────────────────────────────── + +// AutoDateHistogramAggregation buckets by a target bucket count, letting the +// backend (or the framework fallback) pick an appropriate fixed interval. +type AutoDateHistogramAggregation struct { + baseAggregation + Field string `json:"field"` + Buckets int `json:"buckets,omitempty"` // target bucket count (ES default 10, console uses 12/30) + MinimumInterval string `json:"minimum_interval,omitempty"` // "minute"/"hour"/"day"/"week"/"month" +} + +// AddNested provides a correctly typed chained call for AutoDateHistogramAggregation. +func (a *AutoDateHistogramAggregation) AddNested(name string, sub Aggregation) Aggregation { + a.baseAggregation.AddNested(name, sub) + return a +} + +// BucketScriptAggregation runs arithmetic over values referenced by +// buckets_path (parent pipeline: declared inside a bucket aggregation). +type BucketScriptAggregation struct { + baseAggregation + BucketsPath map[string]string `json:"buckets_path"` // param name → path, e.g. {"a": "query_time", "b": "query_total"} + Script string `json:"script"` // arithmetic over params.*, e.g. "params.a / params.b * 100" +} + +// AddNested provides a correctly typed chained call for BucketScriptAggregation. +func (a *BucketScriptAggregation) AddNested(name string, sub Aggregation) Aggregation { + a.baseAggregation.AddNested(name, sub) + return a +} + +// BucketSortAggregation sorts the parent bucket list by a sub-aggregation +// path and truncates (parent pipeline). +type BucketSortAggregation struct { + baseAggregation + Sort []BucketSortSpec `json:"sort,omitempty"` + From int `json:"from,omitempty"` + Size int `json:"size,omitempty"` // 0 = no truncation +} + +// BucketSortSpec is one sort criterion of a bucket_sort. +type BucketSortSpec struct { + Path string `json:"path"` // e.g. "query_time" or "doc_count" + Desc bool `json:"desc,omitempty"` // ES order: desc/asc +} + +// AddNested provides a correctly typed chained call for BucketSortAggregation. +func (a *BucketSortAggregation) AddNested(name string, sub Aggregation) Aggregation { + a.baseAggregation.AddNested(name, sub) + return a +} + +// MaxBucketAggregation selects the maximum value of a multi-bucket sibling's +// per-bucket metric (sibling pipeline). +type MaxBucketAggregation struct { + baseAggregation + BucketsPath string `json:"buckets_path"` // e.g. "dates>search_qps" +} + +// AddNested provides a correctly typed chained call for MaxBucketAggregation. +func (a *MaxBucketAggregation) AddNested(name string, sub Aggregation) Aggregation { + a.baseAggregation.AddNested(name, sub) + return a +} + +// SamplerAggregation restricts aggregation input to a sample; sqlite falls +// back to the full set. +type SamplerAggregation struct { + baseAggregation + ShardSize int `json:"shard_size,omitempty"` +} + +// AddNested provides a correctly typed chained call for SamplerAggregation. +func (a *SamplerAggregation) AddNested(name string, sub Aggregation) Aggregation { + a.baseAggregation.AddNested(name, sub) + return a +} + +// TopHitsAggregation returns the top documents per bucket. +type TopHitsAggregation struct { + baseAggregation + Size int `json:"size,omitempty"` + Sorts []Sort `json:"sorts,omitempty"` +} + +// AddNested provides a correctly typed chained call for TopHitsAggregation. +func (a *TopHitsAggregation) AddNested(name string, sub Aggregation) Aggregation { + a.baseAggregation.AddNested(name, sub) + return a +} diff --git a/core/orm/orm.go b/core/orm/orm.go index ca072494e..a88378d71 100755 --- a/core/orm/orm.go +++ b/core/orm/orm.go @@ -62,6 +62,27 @@ type BatchMutateAPI interface { } type MetricsAPI interface { + // Aggregate executes the qb.Aggs tree and returns the typed result. + // Backends compute bucket/metric aggregations natively; pipeline + // aggregations are layered on uniformly by the framework engine + // (core/aggregate.ApplyPipelines) so behavior is backend-independent. + Aggregate(ctx *Context, qb *QueryBuilder) (*AggregationResult, error) +} + +// Capabilities declares what a backend can actually honor, so callers can +// adapt (or fail loudly) instead of discovering silent degradations at +// runtime. See the sqlite capability matrix in the module docs. +type Capabilities struct { + FullText bool // analyzed match queries (sqlite: FTS5) + Aggregations bool // terms/date_histogram/metric aggregations + Fuzzy bool // true fuzzy matching (sqlite approximates with LIKE) + Nested bool // nested-document queries + RequestBodyDSL bool // merging a raw ES DSL request body + Collapse bool // field collapsing +} + +type CapabilitiesAPI interface { + Capabilities() Capabilities } type ORM interface { @@ -73,6 +94,8 @@ type ORM interface { BatchMutateAPI + CapabilitiesAPI + RegisterSchemaWithName(t interface{}, customizedName string) error Save(ctx *Context, o interface{}) error diff --git a/core/orm/orm_legacy.go b/core/orm/orm_legacy.go index 1b648b64c..b8f7f88d9 100644 --- a/core/orm/orm_legacy.go +++ b/core/orm/orm_legacy.go @@ -1,5 +1,21 @@ package orm +// ────────────────────────────────────────────────────────────────────────── +// Legacy ORM API — frozen. +// +// These functions predate the QueryBuilder/SearchV2 API and are kept only +// for backward compatibility. New code must use: +// read/search: SearchV2 + NewQuery/NewQueryBuilderFromRequest +// (decode via elastic.DecodeHits[T] / elastic.DecodeSearchResult) +// get: GetV2 +// write: Create / Update / UpdatePartialFields / Save / Delete +// aggregations: QueryBuilder.SetAggregations + SearchV2 +// +// Do not add new callers; existing ones are being migrated (see the +// SEARCH_ORM_REFACTOR_PLAN progress table). This file is deleted once the +// migration count reaches zero. +// ────────────────────────────────────────────────────────────────────────── + import ( "errors" "infini.sh/framework/core/util" @@ -21,6 +37,9 @@ type LegacyORMAPI interface { SearchWithResultItemMapper(resultArrayRef interface{}, itemMapFunc func(source map[string]interface{}, targetRef interface{}) error, q *Query) (error, *SimpleResult) } +// Query is the legacy search request model. +// +// Deprecated: use QueryBuilder (NewQuery / NewQueryBuilderFromRequest) with SearchV2. type Query struct { Sort *[]Sort QueryArgs *[]util.KV @@ -63,6 +82,9 @@ func (q *Query) AddQueryArgs(name string, value string) *Query { return q } +// Cond is a legacy query condition. +// +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). type Cond struct { Field string SQLOperator string @@ -71,6 +93,7 @@ type Cond struct { Value interface{} } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func Prefix(field string, value interface{}) *Cond { c := Cond{} c.Field = field @@ -81,6 +104,7 @@ func Prefix(field string, value interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func QueryString(field string, value interface{}) *Cond { c := Cond{} c.Field = field @@ -91,6 +115,7 @@ func QueryString(field string, value interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func Eq(field string, value interface{}) *Cond { c := Cond{} c.Field = field @@ -101,6 +126,7 @@ func Eq(field string, value interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func NotEq(field string, value interface{}) *Cond { c := Cond{} c.Field = field @@ -111,6 +137,7 @@ func NotEq(field string, value interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func In(field string, value []interface{}) *Cond { c := Cond{} c.Field = field @@ -121,6 +148,7 @@ func In(field string, value []interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func InStringArray(field string, value []string) *Cond { c := Cond{} c.Field = field @@ -131,6 +159,7 @@ func InStringArray(field string, value []string) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func Gt(field string, value interface{}) *Cond { c := Cond{} c.Field = field @@ -141,6 +170,7 @@ func Gt(field string, value interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func Lt(field string, value interface{}) *Cond { c := Cond{} c.Field = field @@ -151,6 +181,7 @@ func Lt(field string, value interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func Ge(field string, value interface{}) *Cond { c := Cond{} c.Field = field @@ -161,6 +192,7 @@ func Ge(field string, value interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func Le(field string, value interface{}) *Cond { c := Cond{} c.Field = field @@ -171,6 +203,7 @@ func Le(field string, value interface{}) *Cond { return &c } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func Combine(conds ...[]*Cond) []*Cond { t := []*Cond{} for _, cs := range conds { @@ -181,6 +214,7 @@ func Combine(conds ...[]*Cond) []*Cond { return t } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func And(conds ...*Cond) []*Cond { t := []*Cond{} for _, c := range conds { @@ -190,6 +224,7 @@ func And(conds ...*Cond) []*Cond { return t } +// Deprecated: use the QueryBuilder clause constructors (TermQuery, RangeQuery, Must/Should/...). func Or(conds ...*Cond) []*Cond { t := []*Cond{} for _, c := range conds { @@ -199,17 +234,26 @@ func Or(conds ...*Cond) []*Cond { return t } +// Result is the legacy search result envelope. +// +// Deprecated: use SearchV2 with elastic.DecodeHits / elastic.DecodeSearchResult. type Result struct { Total int64 Raw []byte Result []interface{} } +// SimpleResult is the legacy search result envelope. +// +// Deprecated: use SearchV2 with elastic.DecodeHits / elastic.DecodeSearchResult. type SimpleResult struct { Total int64 Raw []byte } +// Get loads a record by the ID field of o. +// +// Deprecated: use GetV2. func Get(o interface{}) (bool, error) { rValue := reflect.ValueOf(o) @@ -223,25 +267,45 @@ func Get(o interface{}) (bool, error) { return getHandler().Get(nil, o) } +// DeleteBy deletes records matching the handler-dialect query. +// +// Deprecated: use DeleteByQuery with a QueryBuilder (the legacy query argument +// is ES DSL bytes on the elastic handler and raw SQL on sqlite — a dialect trap). func DeleteBy(o interface{}, query interface{}) error { return getHandler().DeleteBy(o, query) } + +// UpdateBy updates records matching the handler-dialect query. +// +// Deprecated: use per-ID Update / UpdatePartialFields, or SearchV2 + batch writes. func UpdateBy(o interface{}, query interface{}) error { return getHandler().UpdateBy(o, query) } +// Count counts records matching the handler-dialect query. +// +// Deprecated: use SearchV2 and read hits.total. func Count(o interface{}, query interface{}) (int64, error) { return getHandler().Count(o, query) } +// Search runs a legacy query and returns flattened hits. +// +// Deprecated: use SearchV2 with a QueryBuilder. func Search(o interface{}, q *Query) (error, Result) { return getHandler().Search(o, q) } +// SearchWithResultItemMapper runs a legacy query, mapping each hit via itemMapFunc. +// +// Deprecated: use SearchV2 with elastic.DecodeHits[T]. func SearchWithResultItemMapper(o interface{}, itemMapFunc func(source map[string]interface{}, targetRef interface{}) error, q *Query) (error, *SimpleResult) { return getHandler().SearchWithResultItemMapper(o, itemMapFunc, q) } +// SearchWithJSONMapper runs a legacy query, mapping hits via reflection. +// +// Deprecated: use SearchV2 with elastic.DecodeHits[T]. func SearchWithJSONMapper(o interface{}, q *Query) (error, SimpleResult) { err, searchResponse := getHandler().SearchWithResultItemMapper(o, MapToStructWithMap, q) if err != nil || searchResponse == nil { @@ -251,6 +315,9 @@ func SearchWithJSONMapper(o interface{}, q *Query) (error, SimpleResult) { return nil, *searchResponse } +// GroupBy runs a legacy aggregation. +// +// Deprecated: use QueryBuilder.SetAggregations with SearchV2. func GroupBy(o interface{}, selectField, groupField, haveQuery string, haveValue interface{}) (error, map[string]interface{}) { return getHandler().GroupBy(o, selectField, groupField, haveQuery, haveValue) } @@ -302,15 +369,24 @@ func FilterFieldsByProtected(obj interface{}, protected bool) map[string]interfa return mapObj } +// GetBy fetches records by an exact field match. +// +// Deprecated: use SearchV2 with a TermQuery filter (or GetV2 for ID lookups). func GetBy(field string, value interface{}, t interface{}) (error, Result) { return getHandler().GetBy(field, value, t) } +// GetWildcardIndexName resolves the wildcard index name for o. +// +// Deprecated: resolve index names via the ORM handler or model registration directly. func GetWildcardIndexName(o interface{}) string { return getHandler().GetWildcardIndexName(o) } +// GetIndexName resolves the index/table name for o. +// +// Deprecated: resolve index names via the ORM handler or model registration directly. func GetIndexName(o interface{}) string { return getHandler().GetIndexName(o) } diff --git a/core/orm/ormtest/contract.go b/core/orm/ormtest/contract.go new file mode 100644 index 000000000..fbe938365 --- /dev/null +++ b/core/orm/ormtest/contract.go @@ -0,0 +1,211 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +// Package ormtest provides a shared contract-test suite that every ORM +// backend must satisfy, so sqlite and elastic keep producing equivalent +// (ES-shaped) results for the same QueryBuilder input. Backends wire it in +// with a one-line test: +// +// func TestContract(t *testing.T) { ormtest.RunContractTests(t, newHandler) } +// +// The suite registers the handler globally (orm.Register) — run it once per +// test binary. Cases needing a live backend instance (elastic) carry the +// `integration` build tag at the call site, not here. +package ormtest + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" +) + +// ContractModel is the canonical test document: scalars, a text field, a +// date, and system fields — enough to exercise filters, sorts, full-text +// and aggregations on every backend. +type ContractModel struct { + orm.ORMObjectBase + Name string `json:"name,omitempty" elastic_mapping:"name: { type: keyword }"` + Status string `json:"status,omitempty" elastic_mapping:"status: { type: keyword }"` + Body string `json:"body,omitempty" elastic_mapping:"body: { type: text }"` + Age int `json:"age,omitempty" elastic_mapping:"age: { type: integer }"` +} + +// RunContractTests executes the backend contract suite against a handler +// produced by factory. It registers the handler globally; callers must +// invoke it at most once per test binary. +func RunContractTests(t *testing.T, factory func() orm.ORM) { + handler := factory() + orm.Register("contract-test", handler) + require.NoError(t, handler.RegisterSchemaWithName(ContractModel{}, "contract_docs")) + seedContractData(t, handler) + + t.Run("CRUD roundtrip", func(t *testing.T) { contractCRUD(t, handler) }) + t.Run("filters", func(t *testing.T) { contractFilters(t, handler) }) + t.Run("sort and pagination", func(t *testing.T) { contractSortPaginate(t, handler) }) + t.Run("ES-shaped response", func(t *testing.T) { contractResponseShape(t, handler) }) + t.Run("partial update preserves fields", func(t *testing.T) { contractPartialUpdate(t, handler) }) + t.Run("terms aggregation", func(t *testing.T) { contractTermsAgg(t, handler) }) +} + +func seedContractData(t *testing.T, h orm.ORM) { + t.Helper() + now := time.Now().UTC() + statuses := []string{"active", "pending", "archived"} + for i := 0; i < 30; i++ { + doc := ContractModel{ + Name: fmt.Sprintf("doc-%02d", i), + Status: statuses[i%3], + Body: fmt.Sprintf("body text number %d about sqlite and elastic", i), + Age: 20 + i, + } + doc.ID = fmt.Sprintf("c%02d", i) + doc.Created = &now + require.NoError(t, h.Create(nil, &doc)) + } +} + +func contractCRUD(t *testing.T, h orm.ORM) { + doc := ContractModel{Name: "crud", Status: "active", Age: 1} + doc.ID = "crud-1" + require.NoError(t, h.Create(nil, &doc)) + + got := ContractModel{} + got.ID = "crud-1" + exists, err := h.Get(nil, &got) + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "crud", got.Name) + + got.Age = 42 + require.NoError(t, h.Update(nil, &got)) + got2 := ContractModel{} + got2.ID = "crud-1" + _, err = h.Get(nil, &got2) + require.NoError(t, err) + assert.Equal(t, 42, got2.Age) + + require.NoError(t, h.Delete(nil, &got2)) + got3 := ContractModel{} + got3.ID = "crud-1" + exists, _ = h.Get(nil, &got3) + assert.False(t, exists) +} + +func contractFilters(t *testing.T, h orm.ORM) { + ctx := orm.NewContext() + orm.WithModel(ctx, &ContractModel{}) + + t.Run("term", func(t *testing.T) { + res, err := h.SearchV2(ctx, orm.NewQuery().Filter(orm.TermQuery("status", "active"))) + require.NoError(t, err) + items, _, err := elastic.DecodeHits[ContractModel](res) + require.NoError(t, err) + assert.Len(t, items, 10) + }) + + t.Run("range", func(t *testing.T) { + res, err := h.SearchV2(ctx, orm.NewQuery().Filter(orm.Range("age").Gte(40))) + require.NoError(t, err) + items, total, err := elastic.DecodeHits[ContractModel](res) + require.NoError(t, err) + assert.Equal(t, int64(10), total) + assert.Len(t, items, 10) + }) + + t.Run("term and range combined", func(t *testing.T) { + res, err := h.SearchV2(ctx, orm.NewQuery(). + Filter(orm.TermQuery("status", "active")). + Filter(orm.Range("age").Gte(20)). + Filter(orm.Range("age").Lt(50))) + require.NoError(t, err) + items, _, err := elastic.DecodeHits[ContractModel](res) + require.NoError(t, err) + // active ages: 20,23,...,47 within [20,50) → all 10 + assert.Len(t, items, 10) + }) +} + +func contractSortPaginate(t *testing.T, h orm.ORM) { + ctx := orm.NewContext() + orm.WithModel(ctx, &ContractModel{}) + + qb := orm.NewQuery(). + SortBy(orm.Sort{Field: "age", SortType: orm.DESC}). + From(0).Size(5) + res, err := h.SearchV2(ctx, qb) + require.NoError(t, err) + items, _, err := elastic.DecodeHits[ContractModel](res) + require.NoError(t, err) + require.Len(t, items, 5) + assert.Equal(t, 49, items[0].Age, "descending age sort") + assert.Equal(t, 45, items[4].Age) + + // Page 2 continues the ordering. + qb2 := orm.NewQuery(). + SortBy(orm.Sort{Field: "age", SortType: orm.DESC}). + From(5).Size(5) + res2, err := h.SearchV2(ctx, qb2) + require.NoError(t, err) + items2, _, err := elastic.DecodeHits[ContractModel](res2) + require.NoError(t, err) + require.Len(t, items2, 5) + assert.Equal(t, 44, items2[0].Age) +} + +func contractResponseShape(t *testing.T, h orm.ORM) { + ctx := orm.NewContext() + orm.WithModel(ctx, &ContractModel{}) + res, err := h.SearchV2(ctx, orm.NewQuery().Filter(orm.TermQuery("status", "pending")).Size(3)) + require.NoError(t, err) + + resp, err := elastic.DecodeSearchResult(res) + require.NoError(t, err) + assert.Equal(t, int64(10), resp.GetTotal()) + require.Len(t, resp.Hits.Hits, 3) + for _, hit := range resp.Hits.Hits { + assert.NotEmpty(t, hit.ID) + assert.NotNil(t, hit.Source) + } +} + +func contractPartialUpdate(t *testing.T, h orm.ORM) { + doc := ContractModel{Name: "before", Status: "active", Body: "keep me", Age: 7} + doc.ID = "partial-1" + require.NoError(t, h.Create(nil, &doc)) + + obj := ContractModel{} + obj.ID = "partial-1" + require.NoError(t, orm.UpdatePartialFields(orm.NewContext(), &obj, + map[string]interface{}{"name": "after"})) + + got := ContractModel{} + got.ID = "partial-1" + _, err := h.Get(nil, &got) + require.NoError(t, err) + assert.Equal(t, "after", got.Name, "delta field applied") + assert.Equal(t, "active", got.Status, "untouched field preserved") + assert.Equal(t, "keep me", got.Body, "untouched field preserved") + assert.Equal(t, 7, got.Age, "untouched field preserved") +} + +func contractTermsAgg(t *testing.T, h orm.ORM) { + ctx := orm.NewContext() + orm.WithModel(ctx, &ContractModel{}) + terms := &orm.TermsAggregation{Field: "status", Size: 10} + qb := orm.NewQuery() + qb.SetAggregations(map[string]orm.Aggregation{"by_status": terms}) + res, err := h.SearchV2(ctx, qb) + require.NoError(t, err) + + resp, err := elastic.DecodeSearchResult(res) + require.NoError(t, err) + require.NotNil(t, resp.Aggregations) + byStatus, ok := resp.Aggregations["by_status"] + require.True(t, ok, "aggregations.by_status missing: %+v", resp.Aggregations) + assert.Len(t, byStatus.Buckets, 3) +} diff --git a/core/orm/ormtest/systemcluster.go b/core/orm/ormtest/systemcluster.go new file mode 100644 index 000000000..de0c82988 --- /dev/null +++ b/core/orm/ormtest/systemcluster.go @@ -0,0 +1,61 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package ormtest + +import ( + "errors" + "os" + "sync" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + ucfg "infini.sh/framework/lib/go-ucfg" +) + +var ( + seedOnce sync.Once + seedErr error +) + +// ErrNoEndpoint is returned when ES_ENDPOINT is not set; callers skip. +var ErrNoEndpoint = errors.New("ES_ENDPOINT is not set") + +// SeedSystemCluster saves the global-system cluster record built from ES_* +// environment variables so backend resolvers (elastic common.GetElasticClient) +// can find a live cluster in integration tests. The caller registers its own +// ORM handler first (each package wires the backend it ships with): +// +// orm.Register("sqlite-integration", handler) +// ormtest.SeedSystemCluster() +// +// Idempotent per process; both the elastic and sqlite test binaries call it. +func SeedSystemCluster() error { + if os.Getenv("ES_ENDPOINT") == "" { + return ErrNoEndpoint + } + seedOnce.Do(func() { + cfg := elastic.ElasticsearchConfig{} + cfg.ID = elastic.GlobalSystemElasticsearchID + cfg.Name = "system" + cfg.Endpoint = os.Getenv("ES_ENDPOINT") + cfg.Enabled = true + cfg.BasicAuth = &model.BasicAuth{ + Username: envOr("ES_USERNAME", "admin"), + // EncodeToSecretString keeps the raw part plain so the value + // survives the sqlite JSON round trip (MarshalJSON emits the + // raw part, and a plain string's raw part is the shadow text). + Password: ucfg.EncodeToSecretString( + envOr("ES_PASSWORD", "admin"), envOr("ES_PASSWORD", "admin")), + } + seedErr = orm.Save(orm.NewContext(), &cfg) + }) + return seedErr +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/core/orm/query.go b/core/orm/query.go index f11196023..28001cd80 100644 --- a/core/orm/query.go +++ b/core/orm/query.go @@ -141,11 +141,54 @@ func (q *QueryBuilder) DisableBodyBytes() { q.requestBodyBytes = nil } -// SetAggregations sets the aggregations for the query builder. +// SetAggregations sets the aggregations for the query builder. Prefer +// SetAggs for the common literal case — it replaces this map ceremony: +// +// qb.SetAggs("total", sumAgg, "sevs", termsAgg) func (q *QueryBuilder) SetAggregations(aggs map[string]Aggregation) { q.Aggs = aggs } +// SetAggs sets the aggregations from name/spec pairs and returns the +// builder for chaining: +// +// qb.SetAggs("total", sum, "sevs", terms) +// +// Arguments must alternate name (string) and Aggregation; malformed or +// dangling pairs panic — builder misuse is a programming error. Replaces +// any previously set aggregations. +func (q *QueryBuilder) SetAggs(pairs ...interface{}) *QueryBuilder { + if len(pairs)%2 != 0 { + panic(fmt.Sprintf("SetAggs: dangling argument (odd count %d): %v", len(pairs), pairs[len(pairs)-1])) + } + m := make(map[string]Aggregation, len(pairs)/2) + for i := 0; i+1 < len(pairs); i += 2 { + name, ok := pairs[i].(string) + if !ok { + panic(fmt.Sprintf("SetAggs: name at position %d must be a string, got %T", i, pairs[i])) + } + agg, ok := pairs[i+1].(Aggregation) + if !ok { + panic(fmt.Sprintf("SetAggs: spec at position %d must be an orm.Aggregation, got %T", i+1, pairs[i+1])) + } + m[name] = agg + } + q.Aggs = m + return q +} + +// AddAgg adds one named aggregation, keeping existing ones, and returns +// the builder for chaining: +// +// qb.AddAgg("total", sum).AddAgg("sevs", terms) +func (q *QueryBuilder) AddAgg(name string, agg Aggregation) *QueryBuilder { + if q.Aggs == nil { + q.Aggs = map[string]Aggregation{} + } + q.Aggs[name] = agg + return q +} + func (q *QueryBuilder) RequestBodyBytesVal() []byte { if !q.allowRequestBodyBytes { return nil diff --git a/core/orm/query_test.go b/core/orm/query_test.go index 70600df70..213b944d7 100644 --- a/core/orm/query_test.go +++ b/core/orm/query_test.go @@ -438,3 +438,40 @@ func TestBuildFuzzinessQueryClauses_EmptyQueryNoDefaultFields(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 0, len(clauses)) } + +func TestQueryBuilder_SetAggs(t *testing.T) { + sum := &MetricAggregation{Type: MetricSum, Field: "n"} + terms := &TermsAggregation{Field: "s", Size: 5} + + qb := NewQuery() + returned := qb.SetAggs("total", sum, "sevs", terms) + if returned != qb { + t.Fatal("SetAggs must return the builder for chaining") + } + if len(qb.Aggs) != 2 || qb.Aggs["total"] != sum || qb.Aggs["sevs"] != terms { + t.Fatalf("aggs = %+v", qb.Aggs) + } + + // Replacement semantics. + avg := &MetricAggregation{Type: MetricAvg, Field: "n"} + qb.SetAggs("avg", avg) + if len(qb.Aggs) != 1 || qb.Aggs["avg"] != avg { + t.Fatalf("SetAggs must replace, got %+v", qb.Aggs) + } +} + +func TestQueryBuilder_AddAgg(t *testing.T) { + qb := NewQuery() + sum := &MetricAggregation{Type: MetricSum, Field: "n"} + max := &MetricAggregation{Type: MetricMax, Field: "n"} + qb.AddAgg("sum", sum).AddAgg("max", max) + if len(qb.Aggs) != 2 || qb.Aggs["sum"] != sum || qb.Aggs["max"] != max { + t.Fatalf("aggs = %+v", qb.Aggs) + } +} + +func TestQueryBuilder_SetAggs_Malformed(t *testing.T) { + defer func() { _ = recover() }() + NewQuery().SetAggs("total") // dangling + t.Fatal("dangling pair must panic") +} diff --git a/docs/content.en/docs/references/orm.md b/docs/content.en/docs/references/orm.md index 9b1e6808c..1095acadf 100644 --- a/docs/content.en/docs/references/orm.md +++ b/docs/content.en/docs/references/orm.md @@ -3,7 +3,7 @@ title: "ORM (Object-Relational Mapping)" weight: 50 --- # Object-Relational Mapping -The INFINI Framework provides a powerful ORM system built on top of Elasticsearch(Including OpenSearch,Easysearch support), enabling developers to define, store, and query structured data objects with ease. The ORM handles object mapping, indexing, and provides a comprehensive set of CRUD operations. +The INFINI Framework provides a powerful ORM system with pluggable backends — Elasticsearch (including OpenSearch and Easysearch) and an embedded SQLite store — enabling developers to define, store, and query structured data objects with ease. The ORM handles object mapping, indexing, and provides a comprehensive set of CRUD operations, a backend-neutral query builder, and typed aggregations (metrics, buckets, and pipelines). ## Object Definition @@ -360,74 +360,117 @@ func deleteUser() { } ``` -## Advanced Operations +## Querying with the Query Builder -### Search with Query Builder +The `QueryBuilder` is the backend-neutral entry point for all reads. A builder holds boolean clauses (must / should / must_not / filter), sorting, pagination, source selection, and — see the next chapter — aggregations. The same builder runs unchanged on the Elasticsearch and SQLite backends. ```go -func searchUsers() { - ctx := orm.NewContext() +ctx := orm.NewContext() +orm.WithModel(ctx, &Product{}) // binds the index/mapping for the backend - // Create query builder - builder := orm.NewQueryBuilder() +qb := orm.NewQuery(). + Filter(orm.TermQuery("status", "active")). // structured filter + Filter(orm.Range("created").Gte("2024-01-01")). + SortBy(orm.Sort{Field: "created", SortType: orm.DESC}). + From(0).Size(20) - // Add filters - builder.Filter(orm.TermQuery("age", 25)) - builder.Filter(orm.RangeQuery("created", util.MapStr{ - "gte": "2023-01-01", - "lte": "2023-12-31", - })) +res, err := orm.SearchV2(ctx, qb) +if err != nil { /* ... */ } - // Add sorting - builder.SortBy(orm.Sort{Field: "created", SortType: orm.DESC}) +items, total, err := elastic.DecodeHits[Product](res) // typed decode, true ES total +fmt.Printf("total=%d, page=%d items\n", total, len(items)) +``` - // Execute search - var users []User - err, result := elastic.SearchV2WithResultItemMapper(ctx, &users, builder, nil) - if err != nil { - log.Error("Search failed:", err) - return - } +> **Filter vs Must**: `Filter` clauses don't contribute to scoring (they are compiled to ES `filter` context). Prefer `Filter` for exact-match structured conditions and `Must` for full-text relevance. - fmt.Printf("Found %d users\n", len(users)) - for _, user := range users { - fmt.Printf("User: %s (%s)\n", user.Name, user.Email) - } -} +### Term-level queries + +```go +orm.TermQuery("status", "active") // exact match (keyword/number/bool) +orm.TermsQuery("category", "a", "b", "c") // match any of the values (set membership) +orm.InQuery("id", []interface{}{"u1", "u2"}) // set membership, idiomatic for ID lists +orm.NotInQuery("status", []interface{}{"archived", "draft"}) +orm.ExistsQuery("deleted_at") // field is present +orm.PrefixQuery("name", "infini") // prefix match +orm.WildcardQuery("email", "*@infini.ltd") // * and ? wildcards +orm.RegexpQuery("code", "^INF-[0-9]+") // full regex (ES; SQLite approximates — see parity notes) +orm.FuzzyQuery("name", "infini", 2) // edit-distance match with explicit fuzziness ``` -### Complex Search with Text Queries +### Full-text queries ```go -func searchDocuments() { - ctx := orm.NewContext() +orm.MatchQuery("description", "fast search") // analyzed match +orm.MatchPhraseQuery("title", "log pattern", 1) // phrase with slop +orm.MultiMatchQuery([]string{"title", "body"}, "error") // OR across fields +orm.QueryStringQuery("message", "status:active AND (error OR warning)", "AND") // query-string syntax, explicit default operator +``` - builder := orm.NewQueryBuilderFromRequest(req, "title", "content") +On SQLite, `match`/`match_phrase`/`query_string` use FTS5 when a full-text plan exists for the field and fall back to equality/LIKE otherwise (see parity notes below). - // Enable body bytes for receiving additional Raw QueryDSL - builder.EnableBodyBytes() +### Range queries (fluent) - // Add date range filter - builder.Filter(orm.RangeQuery("created", util.MapStr{ - "gte": "2024-01-01", - })) +```go +orm.Range("price").Gte(100) // >= +orm.Range("price").Lt(1000) // < +orm.Range("created").Gte("2024-01-01").Lte("2024-12-31") // chained: both bounds + +// Build both bounds into ONE clause (recommended — a single range node): +qb.Filter(orm.MustQuery( + orm.Range("created").Gte(start), + orm.Range("created").Lte(end), +)) +``` - // Add pagination - builder.Size(20).From(0) +### Boolean composition - // Add aggregations - ctx.Set(orm.AggsTerms, "tags") +```go +qb := orm.NewQuery() +qb.Must(orm.MatchQuery("title", "error")) // scored, all must match +qb.Should(orm.MatchQuery("tags", "urgent")) // optional boost +qb.MinimumShouldMatch(1) // require at least one should +qb.Not(orm.TermQuery("status", "archived")) // must_not +qb.Filter(orm.TermQuery("tenant", "acme")) // non-scoring filter + +// Reusable sub-expression (grouping helpers produce one boolean clause): +cond := orm.FilterQuery( + orm.MustQuery(orm.TermQuery("env", "prod")), + orm.MustNotQuery(orm.ExistsQuery("deleted_at")), +) +qb.Filter(cond) +// orm.BoolQuery(orm.Filter|orm.Must|orm.Should|orm.MustNot, clauses...) is +// the explicit form of the same thing. +``` - var docs []Document - err, result := elastic.SearchV2WithResultItemMapper(ctx, &docs, builder, nil) - if err != nil { - log.Error("Search failed:", err) - return - } +### Sorting, pagination, source selection - // Process results - fmt.Printf("Found %d documents\n", len(docs)) -} +```go +qb.SortBy( + orm.Sort{Field: "priority", SortType: orm.DESC}, + orm.Sort{Field: "created", SortType: orm.ASC}, // tie-break +).From(0).Size(50) + +qb.Include("id", "name", "status") // _source_includes — reduces payload +qb.Exclude("large_blob") // _source_excludes +qb.Collapse("user_id") // field collapse (dedupe by field, ES) +``` + +### Fuzziness ladder + +`Fuzziness(n)` (0–5) applies progressive auto-fuzzy matching to match/multi_match text queries — useful for typo tolerance in user-typed filters: + +```go +qb := orm.NewQuery().Fuzziness(2).Must(orm.MatchQuery("name", userTyped)) +``` + +### Building from an HTTP request + +`NewQueryBuilderFromRequest` wires the URL parameter contract (`query`, `filter=field:value`, `sort=field:desc`, `from`, `size`, `_source_includes`, `fuzziness`, `default_fields`, and `agg[...]` aggregations — see *Query URL Parameters* / *Aggregation via URL Parameters*) into a builder, with optional default full-text fields: + +```go +builder, err := orm.NewQueryBuilderFromRequest(req, "title", "content") +if err != nil { /* 400 */ } +builder.EnableBodyBytes() // ALSO merge a raw ES DSL JSON body (ES backend only) ``` ### Delete by Query @@ -436,23 +479,195 @@ func searchDocuments() { func deleteOldUsers() { ctx := orm.NewContext() - // Create delete query - builder := orm.NewQueryBuilder() - builder.Filter(orm.RangeQuery("created", util.MapStr{ - "lt": "2022-01-01", // Delete users created before 2022 - })) + builder := orm.NewQuery(). + Filter(orm.Range("created").Lt("2022-01-01")) // created before 2022 - // Execute delete by query result, err := orm.DeleteByQuery(ctx, builder) if err != nil { log.Error("Delete by query failed:", err) return } - fmt.Printf("Deleted %d old users\n", result.Deleted) } ``` +### Backend parity notes (queries) + +| Capability | Elasticsearch | SQLite | +|---|---|---| +| term / terms / in / not_in / exists / prefix / wildcard / ranges | ✅ | ✅ (dates via epoch shadow columns) | +| match / phrase / multi_match / query_string | ✅ | ⚠️ FTS5 when available, else equality/LIKE; query-string operators not parsed | +| regexp / fuzzy | ✅ | ⚠️ approximated as substring LIKE | +| semantic / hybrid / nested | ✅ | ❌ compiled to a never-matching predicate (one-time warning) | +| Include / Exclude / Collapse | ✅ | ❌ currently ignored | +| Raw request-body DSL (`EnableBodyBytes`) | ✅ | ❌ ignored | + +--- + +## Aggregations + +Aggregations are a first-class operation: build an aggregation tree on a `QueryBuilder`, execute it with `orm.Aggregate`, and read back a **typed, recursive result model** — no ES-shaped JSON parsing. Bucket and metric aggregations are computed natively by each backend; pipeline aggregations are computed uniformly by the framework engine (`core/aggregate`) so every backend returns identical results. + +### Execution model and result types + +```go +// AggregationResult — root; keys match the names you gave in SetAggs. +// AggNode — one named agg; exactly one shape is populated: +// Value/ValueSet single metric (ValueSet distinguishes 0 from "no value") +// Values multi-value metric (e.g. percentiles) +// TopHit top document (top_hits) +// Buckets bucket list; each Bucket has Key (display), KeyRaw +// (epoch ms for time buckets), DocCount, and nested Aggs. +func Aggregate(ctx *Context, qb *QueryBuilder) (*AggregationResult, error) +``` + +Aggregations are attached with the fluent `SetAggs` (name/spec pairs, chainable) or the map form `SetAggregations`. The builder's WHERE clauses scope the aggregated document set, exactly like `SearchV2`. + +### Quick start — terms + nested metric + +```go +ctx := orm.NewContext() +orm.WithModel(ctx, &Order{}) + +avgPrice := orm.NewMetricAggregation(orm.MetricAvg, "price") +byStatus := &orm.TermsAggregation{Field: "status", Size: 10} +byStatus.AddNested("avg_price", avgPrice) // nest a metric under each bucket + +qb := orm.NewQuery().Filter(orm.Range("created").Gte("2024-01-01")) +qb.SetAggs("by_status", byStatus) + +res, err := orm.Aggregate(ctx, qb) +if err != nil { /* ... */ } + +for _, b := range res.Aggs["by_status"].Buckets { + fmt.Printf("status=%s orders=%d avg_price=%.2f\n", + b.Key, b.DocCount, b.Aggs["avg_price"].Value) +} +``` + +### Metric aggregations + +```go +orm.NewMetricAggregation(orm.MetricSum, "bytes") +orm.NewMetricAggregation(orm.MetricMin, "latency_ms") +orm.NewMetricAggregation(orm.MetricMax, "latency_ms") +orm.NewMetricAggregation(orm.MetricCount, "id") // value_count +orm.NewMetricAggregation(orm.MetricCardinality, "client_ip") // distinct count + +// Several metrics in one pass — SetAggs takes name/spec pairs: +qb.SetAggs( + "total_bytes", orm.NewMetricAggregation(orm.MetricSum, "bytes"), + "p95", &orm.PercentilesAggregation{Field: "latency_ms", Percents: []float64{50, 95, 99}}, + "worst", &orm.TopHitsAggregation{Size: 1, Sorts: []orm.Sort{{Field: "latency_ms", SortType: orm.DESC}}}, +) + +r, _ := orm.Aggregate(ctx, qb) +r.Aggs["total_bytes"].Value // float64 +r.Aggs["p95"].Values["95"] // multi-value metric +json.Unmarshal(*r.Aggs["worst"].TopHit, &order) // raw top document +``` + +### Bucket aggregations + +```go +// terms — group by (ordered by doc_count desc, key asc tie-break; Size truncates) +cats := &orm.TermsAggregation{Field: "category", Size: 20} + +// date_histogram — time buckets. Offset shifts bucket boundaries (e.g. align +// hourly buckets to "now" instead of the wall-clock hour). +now := time.Now().UTC() +hourly := &orm.DateHistogramAggregation{ + Field: "created", + Interval: "1h", + Offset: now.Sub(now.Truncate(time.Hour)), // e.g. 17m42s +} +hourly.AddNested("sum_bytes", orm.NewMetricAggregation(orm.MetricSum, "bytes")) + +// filter — aggregate a subset matching a simple query +errorsOnly := &orm.FilterAggregation{Query: map[string]interface{}{ + "term": map[string]interface{}{"level": "error"}, +}} + +// date_range — fixed time ranges +byQuarter := &orm.DateRangeAggregation{ + Field: "created", + Ranges: []interface{}{ + map[string]interface{}{"from": "2024-01-01", "to": "2024-04-01"}, + map[string]interface{}{"from": "2024-04-01", "to": "2024-07-01"}, + }, +} + +// auto_date_histogram — backend/framework picks an interval for ~N buckets +adaptive := &orm.AutoDateHistogramAggregation{Field: "created", Buckets: 30} + +qb.SetAggs("cats", cats, "hourly", hourly, "errors", errorsOnly) +``` + +### Multi-level nesting (one SQL per bucket node on SQLite) + +```go +// terms(stream) → date_histogram(1h) → sum(count): the pattern behind +// per-stream hourly trend dashboards. +streams := &orm.TermsAggregation{Field: "stream_id", Size: 1000} +trend := &orm.DateHistogramAggregation{Field: "bucket_start", Interval: "1h", + Offset: now.Sub(now.Truncate(time.Hour))} +trend.AddNested("value", orm.NewMetricAggregation(orm.MetricSum, "count")) +streams.AddNested("trend", trend) + +qb := orm.NewQuery().Filter(orm.Range("bucket_start").Gte(cutoff)) +qb.SetAggs("streams", streams) + +res, _ := orm.Aggregate(ctx, qb) +for _, s := range res.Aggs["streams"].Buckets { + for _, h := range s.Aggs["trend"].Buckets { + v := h.KeyRaw.(int64) // epoch milliseconds for time buckets + _ = v + } +} +``` + +### Pipeline aggregations + +Parent pipelines (`derivative`, `bucket_script`, `bucket_sort`) are declared **inside** the bucket aggregation they derive; sibling pipelines (`sum_bucket`, `max_bucket`) sit **next to** it at the same level: + +```go +dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} +dh.AddNested("sum_n", orm.NewMetricAggregation(orm.MetricSum, "n")) +dh.AddNested("derivative", &orm.DerivativeAggregation{BucketsPath: "sum_n"}) // Δ per bucket +dh.AddNested("ratio", &orm.BucketScriptAggregation{ // arithmetic + BucketsPath: map[string]string{"a": "sum_n", "b": "doc_count"}, + Script: "params.a / params.b", +}) +dh.AddNested("top3", &orm.BucketSortAggregation{ // sort + truncate buckets + Sort: []orm.BucketSortSpec{{Path: "sum_n", Desc: true}}, + Size: 3, +}) + +total := &orm.PipelineAggregation{Type: orm.MetricSumBucket, BucketsPath: "over_time>sum_n"} // total across buckets +peak := &orm.MaxBucketAggregation{BucketsPath: "over_time>sum_n"} // max bucket value + +qb.SetAggs("over_time", dh, "total", total, "peak", peak) + +res, _ := orm.Aggregate(ctx, qb) +res.Aggs["total"].Value // sum over all hourly buckets +res.Aggs["peak"].Value // max hourly sum +res.Aggs["over_time"].Buckets[0].Aggs["derivative"].ValueSet // false for the first bucket +``` + +`bucket_script` scripts support `+ - * / ( )` arithmetic over `params.*`; division by zero yields 0. `buckets_path` uses the two-segment `agg>metric` form. Pipelines are computed by the framework engine on **every** backend (ES native pipeline results are recomputed for parity), including zero-fill and deterministic ordering of time series. + +### Backend parity notes (aggregations) + +| Aggregation | Elasticsearch | SQLite | +|---|---|---| +| sum / avg / min / max / value_count / cardinality | ✅ | ✅ native SQL | +| percentiles | ✅ | ✅ exact nearest-rank | +| top_hits | ✅ | ⚠️ typed variant returns 1 document | +| median_absolute_deviation | ✅ | ❌ empty value | +| terms / date_histogram (offset) / date_range / filter / auto_date_histogram | ✅ | ✅ (interval whitelist 1m/1h/1d/1M; month ≈ 30d; terms `Include` ignored) | +| sampler | ✅ | ⚠️ full data set | +| pipelines (derivative / bucket_script / bucket_sort / sum_bucket / max_bucket) | ✅ | ✅ identical — computed by the framework engine | + ## Context Options The ORM provides various context options for controlling behavior: diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index 8e3c22ec3..1a519e01b 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -22,6 +22,9 @@ Information about release notes of INFINI Framework is provided here. - perf(sqlite): set per-connection PRAGMAs (WAL, busy_timeout, foreign_keys) via the DSN and enable 256 MiB mmap_size — applies to every pooled connection (not just one) and serves the metadata store via memory-mapped I/O instead of pread syscalls - perf(sqlite): auto-create expression indexes from elastic_mapping tags, now including nested object fields via dotted `$.parent.child` paths — the SQLite ORM indexes `json_extract(raw,'$.field')` for keyword/date/long/integer/boolean/double fields, turning full-table scans into B-tree lookups with zero query or model changes - feat: search provider injection + easysearch cluster CRUD module #398 +- feat(orm): cross-backend aggregation engine — metrics (min/max/sum/avg/value_count), bucket (terms/histogram/range) and pipeline aggregations run unchanged on the Elasticsearch and SQLite backends +- feat(crud): extend the generated CRUD with migration hooks driven by CocoAI's hand-written handlers — `ExtraOptions` per action (login, CORS, sensitive-field masking), custom `IDParam`, `CtxDecorate` orm-context markers, `UpdateMode` partial/full/`?replace=` with `ProtectedFields` + `PrepareUpdate`, best-effort `PostCreate/PostUpdate/PostDelete`, `PostGet` refinement, and `PrepareSearch`/`PostSearch` for injected filters and per-hit mapping; the MCP tool now registers on GET _search only (no duplicate tool names); `SkipActions` keeps hand-written endpoints where the generator cannot express the semantics (upsert/replace-keeping-system-fields, cache-first fetch); full-object update mode now merges through a map so `ProtectedFields` are restored from the loaded record, and `PrepareUpdate` receives the raw body as the delta +- feat(orm): fluent `SetAggs` query-builder API and a refactored SQLite SQL builder backing it ### 🐛 Bug fix - fix: expand configs.template when loading templated config files #391 diff --git a/modules/easysearch/cluster_api.go b/modules/easysearch/cluster_api.go index 877b056bc..e757610a1 100644 --- a/modules/easysearch/cluster_api.go +++ b/modules/easysearch/cluster_api.go @@ -3,18 +3,13 @@ package easysearch import ( - "crypto/tls" - "encoding/json" - "fmt" - "io" + "errors" "net/http" - "strings" - "time" "infini.sh/framework/core/api" + "infini.sh/framework/core/api/crud" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" - "infini.sh/framework/core/orm" "infini.sh/framework/core/security" "infini.sh/framework/core/util" ) @@ -26,10 +21,8 @@ import ( // or any non-elastic store). Routes use the /easysearch/ prefix (not // /elasticsearch/) so they don't collide with legacy /elasticsearch/ routes. // -// Handler conventions follow the standard module CRUD pattern (see -// coco/modules/integration): Write*JSON response envelopes, partial-field -// updates, and a _search endpoint driven by orm.NewQueryBuilderFromRequest -// (pagination/sort/filter/full-text via query string or request body). +// The five CRUD endpoints are generated by core/api/crud (this module was +// its pilot); only the pre-registration _test probe remains hand-written. // // Decoupled from modules/elastic: no live-client registration, no in-memory // metadata registry. Health status shown in responses is the value persisted @@ -57,151 +50,58 @@ func registerClusterAPI() { permClusterDelete = security.GetOrInitPermission("generic", "easysearch:cluster", security.Delete) permClusterSearch = security.GetOrInitPermission("generic", "easysearch:cluster", security.Search) ) - + permissionFor := func(action string) api.PermissionKey { + switch action { + case crud.ActionRead: + return permClusterRead + case crud.ActionCreate: + return permClusterCreate + case crud.ActionUpdate: + return permClusterUpdate + case crud.ActionDelete: + return permClusterDelete + case crud.ActionSearch: + return permClusterSearch + } + return "" + } + + // The five standard endpoints are generated (create / _search / get / + // update / delete — see core/api/crud for the conventions this module + // used to hand-copy). + crud.RegisterCRUD[elastic.ElasticsearchConfig](crud.Config[elastic.ElasticsearchConfig]{ + Prefix: "/easysearch", + Resource: "cluster", + Permission: permissionFor, + DefaultQueryFields: []string{"name"}, + MCP: true, + PrepareCreate: func(cfg *elastic.ElasticsearchConfig) error { + if cfg.Name == "" { + return errors.New("name is required") + } + if cfg.ID == "" { + cfg.ID = util.GetUUID() + } + if cfg.Distribution == "" { + cfg.Distribution = elastic.Elasticsearch + } + // Mark as dynamically-managed so the elasticsearch module's health + // loop persists status for it (the loop keys off this source value). + cfg.Source = elastic.ElasticsearchConfigSourceElasticsearch + cfg.Enabled = true + return nil + }, + GuardDelete: func(cfg *elastic.ElasticsearchConfig) error { + if cfg.Reserved { + return errors.New("reserved cluster cannot be deleted") + } + return nil + }, + }) + + // Pre-registration connectivity probe — deliberately bespoke. h := &ClusterAPI{} api.HandleUIMethod(api.POST, "/easysearch/_test", h.testConnection, api.RequirePermission(permClusterRead), api.AllowOPTIONSS(), api.Feature(api.FeatureCORS)) - api.HandleUIMethod(api.POST, "/easysearch/", h.createCluster, api.RequirePermission(permClusterCreate)) - api.HandleUIMethod(api.GET, "/easysearch/_search", h.searchClusters, api.RequirePermission(permClusterSearch)) - api.HandleUIMethod(api.POST, "/easysearch/_search", h.searchClusters, api.RequirePermission(permClusterSearch)) - api.HandleUIMethod(api.GET, "/easysearch/:id", h.getCluster, api.RequirePermission(permClusterRead)) - api.HandleUIMethod(api.PUT, "/easysearch/:id", h.updateCluster, api.RequirePermission(permClusterUpdate)) - api.HandleUIMethod(api.DELETE, "/easysearch/:id", h.deleteCluster, api.RequirePermission(permClusterDelete)) -} - -// createCluster — POST /easysearch/ -// Persists a cluster record. It becomes a live ES client when the -// elasticsearch module next loads clusters from the ORM (boot/reload). -func (h *ClusterAPI) createCluster(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { - var cfg elastic.ElasticsearchConfig - if err := h.DecodeJSON(req, &cfg); err != nil { - h.WriteError(w, err.Error(), http.StatusBadRequest) - return - } - if cfg.Name == "" { - h.WriteError(w, "name is required", http.StatusBadRequest) - return - } - if cfg.ID == "" { - cfg.ID = util.GetUUID() - } - if cfg.Distribution == "" { - cfg.Distribution = elastic.Elasticsearch - } - // Mark as dynamically-managed so the elasticsearch module's health loop - // persists status for it (the loop keys off this source value). - cfg.Source = elastic.ElasticsearchConfigSourceElasticsearch - cfg.Enabled = true - - ctx := orm.NewContextWithParent(req.Context()) - ctx.Refresh = orm.WaitForRefresh - if err := orm.Create(ctx, &cfg); err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteCreatedOKJSON(w, cfg.ID) -} - -// searchClusters — GET/POST /easysearch/_search -// Standard query-builder search (pagination, sorting, filtering, full-text on -// the name field via ?query=). Returns an ES-shaped SearchResponse; each hit's -// _source carries the cluster with its persisted health_status (in Labels) -// from the elasticsearch module's health loop. -func (h *ClusterAPI) searchClusters(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { - //handle url query args, convert to query builder - builder, err := orm.NewQueryBuilderFromRequest(req, "name") - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - builder.EnableBodyBytes() - if len(builder.Sorts()) == 0 { - builder.SortBy(orm.Sort{Field: "created", SortType: orm.DESC}) - } - - ctx := orm.NewContextWithParent(req.Context()) - orm.WithModel(ctx, &elastic.ElasticsearchConfig{}) - res, err := orm.SearchV2(ctx, builder) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - searchRes, err := parseSearchResponse(res) - if err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - - h.WriteJSON(w, searchRes, http.StatusOK) -} - -// getCluster — GET /easysearch/:id -func (h *ClusterAPI) getCluster(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.ByName("id") - cfg := elastic.ElasticsearchConfig{} - cfg.ID = id - ctx := orm.NewContextWithParent(req.Context()) - exists, err := orm.GetV2(ctx, &cfg) - if !exists || err != nil { - h.WriteGetMissingJSON(w, id) - return - } - h.WriteGetOKJSON(w, id, cfg) -} - -// updateCluster — PUT /easysearch/:id -// Partial update: only the fields present in the request body are changed -// (orm.UpdatePartialFields merges the delta onto the stored record). Secrets -// omitted from the body — e.g. the password, which GET responses return -// masked — keep their stored values, so a partial update never breaks the -// connection. -func (h *ClusterAPI) updateCluster(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.ByName("id") - obj := elastic.ElasticsearchConfig{} - obj.ID = id - - delta := util.MapStr{} - if err := h.DecodeJSON(req, &delta); err != nil { - h.WriteError(w, err.Error(), http.StatusBadRequest) - return - } - - ctx := orm.NewContextWithParent(req.Context()) - ctx.Refresh = orm.WaitForRefresh - if err := orm.UpdatePartialFields(ctx, &obj, delta); err != nil { - if strings.Contains(err.Error(), "not found") { - h.WriteOpRecordNotFoundJSON(w, id) - return - } - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteUpdatedOKJSON(w, obj.ID) -} - -// deleteCluster — DELETE /easysearch/:id -// Removes the cluster record. Reserved clusters (e.g. the system cluster) -// cannot be deleted. -func (h *ClusterAPI) deleteCluster(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - id := ps.ByName("id") - cfg := elastic.ElasticsearchConfig{} - cfg.ID = id - ctx := orm.NewContextWithParent(req.Context()) - exists, err := orm.GetV2(ctx, &cfg) - if !exists || err != nil { - h.WriteOpRecordNotFoundJSON(w, id) - return - } - if cfg.Reserved { - h.WriteError(w, "reserved cluster cannot be deleted", http.StatusForbidden) - return - } - ctx.Refresh = orm.WaitForRefresh - if err := orm.Delete(ctx, &cfg); err != nil { - h.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - h.WriteDeletedOKJSON(w, id) } // testConnection — POST /easysearch/_test @@ -226,7 +126,9 @@ func (h *ClusterAPI) testConnection(w http.ResponseWriter, req *http.Request, _ cfg.Distribution = elastic.Elasticsearch } - version, distribution, err := probeCluster(&cfg) + // Shared probe (core/elastic.ProbeCluster): connectivity + version, no + // adapter build, no persistence. + version, distribution, err := elastic.ProbeCluster(&cfg) if err != nil { h.WriteJSON(w, map[string]interface{}{ "connected": false, @@ -238,6 +140,18 @@ func (h *ClusterAPI) testConnection(w http.ResponseWriter, req *http.Request, _ if distribution == "" { distribution = cfg.Distribution } + + // Warm the shared config-keyed client cache with the probed version, so + // the eventual create/update builds a client without re-probing. + // Best-effort: the probe already validated connectivity. + if version != "" { + cfg.Version = version + if distribution != "" { + cfg.Distribution = distribution + } + _, _ = elastic.GetOrCreateClient(cfg) + } + h.WriteJSON(w, map[string]interface{}{ "connected": true, "version": version, @@ -248,84 +162,3 @@ func (h *ClusterAPI) testConnection(w http.ResponseWriter, req *http.Request, _ // ────────────────────────────────────────────────────────────────────────── // Helpers // ────────────────────────────────────────────────────────────────────────── - -// parseSearchResponse decodes an orm.SearchResult payload (ES-shaped JSON: -// {"hits":{"hits":[{"_source":{...}}]}}) into elastic.SearchResponse. Accepts -// both []byte and string payloads; nil/empty payloads yield a zero response. -func parseSearchResponse(res *orm.SearchResult) (elastic.SearchResponse, error) { - out := elastic.SearchResponse{} - if res == nil { - return out, nil - } - var raw []byte - switch payload := res.Payload.(type) { - case []byte: - raw = payload - case string: - raw = []byte(payload) - default: - return out, nil - } - if len(raw) == 0 { - return out, nil - } - if err := util.FromJSONBytes(raw, &out); err != nil { - return elastic.SearchResponse{}, err - } - return out, nil -} - -// probeTransport is the HTTP transport for connectivity probes: TLS verification -// disabled (ES clusters commonly use self-signed certs). -var probeTransport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, -} - -// probeCluster does a raw GET to the cluster's root endpoint to detect -// connectivity and the server version/distribution, without building a -// version-specific adapter. BasicAuth or X-API-TOKEN is applied when present. -func probeCluster(cfg *elastic.ElasticsearchConfig) (version, distribution string, err error) { - endpoint := cfg.Endpoint - if endpoint == "" && len(cfg.Endpoints) > 0 { - endpoint = cfg.Endpoints[0] - } - httpReq, err := http.NewRequest("GET", endpoint, nil) - if err != nil { - return "", "", err - } - if cfg.BasicAuth != nil && cfg.BasicAuth.Username != "" { - httpReq.SetBasicAuth(cfg.BasicAuth.Username, cfg.BasicAuth.Password.Get()) - } else if t := cfg.Token.Get(); t != "" { - httpReq.Header.Set("X-API-TOKEN", t) - } - - client := &http.Client{Timeout: 10 * time.Second, Transport: probeTransport} - resp, err := client.Do(httpReq) - if err != nil { - return "", "", err - } - defer resp.Body.Close() - body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - if resp.StatusCode >= 400 { - return "", "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(truncate(string(body), 200))) - } - var info struct { - Version struct { - Number string `json:"number"` - Distribution string `json:"distribution"` - } `json:"version"` - } - if err := json.Unmarshal(body, &info); err != nil { - return "", "", fmt.Errorf("parse version response: %w", err) - } - return info.Version.Number, info.Version.Distribution, nil -} - -// truncate caps s to n runes, appending "…" when truncated. -func truncate(s string, n int) string { - r := []rune(s) - if len(r) <= n { - return s - } - return string(r[:n]) + "…" -} diff --git a/modules/easysearch/cluster_api_test.go b/modules/easysearch/cluster_api_test.go index 99757745a..5da1852d2 100644 --- a/modules/easysearch/cluster_api_test.go +++ b/modules/easysearch/cluster_api_test.go @@ -2,83 +2,8 @@ package easysearch -import ( - "testing" - - "infini.sh/framework/core/orm" -) - -func TestParseSearchResponse(t *testing.T) { - payload := `{"hits":{"total":{"value":2},"hits":[{"_id":"a","_source":{"id":"a","name":"A","distribution":"easysearch"}},{"_id":"b","_source":{"id":"b","name":"B"}}]}}` - res, err := parseSearchResponse(&orm.SearchResult{Payload: []byte(payload)}) - if err != nil { - t.Fatalf("parse failed: %v", err) - } - if len(res.Hits.Hits) != 2 { - t.Fatalf("expected 2 hits, got %d (%+v)", len(res.Hits.Hits), res.Hits.Hits) - } - var nameA, nameB string - for _, hit := range res.Hits.Hits { - if hit.ID == "a" { - nameA, _ = hit.Source["name"].(string) - } - if hit.ID == "b" { - nameB, _ = hit.Source["name"].(string) - } - } - if nameA != "A" || nameB != "B" { - t.Fatalf("hit sources mismatch: %q, %q", nameA, nameB) - } -} - -func TestParseSearchResponse_EmptyAndMalformed(t *testing.T) { - checks := []struct { - name string - res *orm.SearchResult - wantErr bool - }{ - {"nil result", nil, false}, - {"nil payload", &orm.SearchResult{}, false}, - {"empty hits", &orm.SearchResult{Payload: []byte(`{"hits":{"hits":[]}}`)}, false}, - {"empty bytes", &orm.SearchResult{Payload: []byte(``)}, false}, - {"string payload", &orm.SearchResult{Payload: `{"hits":{"hits":[]}}`}, false}, - {"empty string", &orm.SearchResult{Payload: ``}, false}, - {"non-bytes payload", &orm.SearchResult{Payload: 12345}, false}, - {"malformed json", &orm.SearchResult{Payload: []byte(`not json`)}, true}, - } - for _, c := range checks { - t.Run(c.name, func(t *testing.T) { - res, err := parseSearchResponse(c.res) - if c.wantErr { - if err == nil { - t.Fatalf("expected an error, got %+v", res) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(res.Hits.Hits) != 0 { - t.Fatalf("expected no hits, got %d", len(res.Hits.Hits)) - } - }) - } -} - -func TestTruncate(t *testing.T) { - cases := []struct { - in string - n int - want string - }{ - {"abc", 5, "abc"}, // under limit → unchanged - {"abcdef", 3, "abc…"}, // over limit → cut + ellipsis - {"世界你好", 2, "世界…"}, // rune-aware, not byte-aware - {"", 3, ""}, // empty - } - for _, c := range cases { - if got := truncate(c.in, c.n); got != c.want { - t.Errorf("truncate(%q,%d) = %q, want %q", c.in, c.n, got, c.want) - } - } -} +// The search-response decode path is covered by core/elastic's +// TestDecodeSearchResult / TestDecodeHits — the module now uses +// elastic.DecodeSearchResult instead of a local helper. ProbeCluster and +// truncate likewise moved to core/elastic (client_provider.go) and are +// tested there. diff --git a/modules/elastic/adapter/elasticsearch/v0.go b/modules/elastic/adapter/elasticsearch/v0.go index be134b48f..eacf5ac91 100755 --- a/modules/elastic/adapter/elasticsearch/v0.go +++ b/modules/elastic/adapter/elasticsearch/v0.go @@ -631,6 +631,7 @@ func (c *ESAPIV0) QueryDSL(ctx context.Context, indexName string, queryArgs *[]u if resp.StatusCode >= 400 && resp.StatusCode != 404 { log.Error("invalid response: ", url, ",", string(queryDSL), ",", string(resp.Body)) + return nil, fmt.Errorf("search on [%s] failed: HTTP %d: %s", url, resp.StatusCode, string(resp.Body)) } if global.Env().IsDebug { diff --git a/modules/elastic/agg_conformance_integration_test.go b/modules/elastic/agg_conformance_integration_test.go new file mode 100644 index 000000000..f3ba349a2 --- /dev/null +++ b/modules/elastic/agg_conformance_integration_test.go @@ -0,0 +1,89 @@ +//go:build integration + +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "testing" + + "infini.sh/framework/core/aggregate/aggstest" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/modules/elastic/common" +) + +// elasticAggstestBackend adapts ElasticORM to the aggregation conformance +// suite. Requires a live cluster (the system cluster from elastic.* config): +// +// go test -tags integration ./modules/elastic/ -run TestAggConformance_Elastic +type elasticAggstestBackend struct { + handler *ElasticORM + index string +} + +func (e *elasticAggstestBackend) Setup(t *testing.T, docs []aggstest.Doc) (*orm.Context, func()) { + t.Helper() + client, err := common.GetElasticClient(elastic.GlobalSystemElasticsearchID) + if err != nil || client == nil { + t.Skipf("no live elasticsearch available: %v", err) + } + e.handler = &ElasticORM{Client: client} + e.index = "aggstest-conformance" + + // Create the index with explicit mappings: blind indexing lets dynamic + // mapping turn keyword-ish fields into text, and aggregations on text + // fields are rejected by ES/Easysearch by default. + _ = client.DeleteIndex(e.index) + if err := client.CreateIndex(e.index, map[string]interface{}{ + "mappings": map[string]interface{}{ + "properties": map[string]interface{}{ + "ts": map[string]interface{}{"type": "date"}, + "stream": map[string]interface{}{"type": "keyword"}, + "severity": map[string]interface{}{"type": "keyword"}, + "n": map[string]interface{}{"type": "integer"}, + }, + }, + }); err != nil { + t.Fatalf("create index: %v", err) + } + + for _, d := range docs { + id, _ := d["id"].(string) + if _, err := client.Index(e.index, "", id, d, ""); err != nil { + t.Fatalf("seed doc: %v", err) + } + } + if err := client.Refresh(e.index); err != nil { + t.Fatalf("refresh: %v", err) + } + t.Cleanup(func() { + _ = client.DeleteIndex(e.index) + }) + + ctx := orm.NewContext() + orm.WithModel(ctx, &aggstestDoc{}) + orm.WithIndices(ctx, e.index) + return ctx, func() {} +} + +// aggstestDoc maps the suite fixture fields for index resolution. +type aggstestDoc struct { + orm.ORMObjectBase + TS string `json:"ts,omitempty" elastic_mapping:"ts: { type: date }"` + Stream string `json:"stream,omitempty" elastic_mapping:"stream: { type: keyword }"` + Severity string `json:"severity,omitempty" elastic_mapping:"severity: { type: keyword }"` + N int `json:"n,omitempty" elastic_mapping:"n: { type: integer }"` +} + +func (e *elasticAggstestBackend) Aggregate(ctx *orm.Context, qb *orm.QueryBuilder) (*orm.AggregationResult, error) { + return e.handler.Aggregate(ctx, qb) +} + +func TestAggConformance_Elastic(t *testing.T) { + aggstest.RunConformance(t, &elasticAggstestBackend{}) +} + +// TestAggParity_SQLite_Elastic lives in the sqlite package +// (aggregate_parity_integration_test.go) — it needs both backends and the +// sqlite package is the cycle-free meeting point. diff --git a/modules/elastic/cluster_hook.go b/modules/elastic/cluster_hook.go new file mode 100644 index 000000000..45d6b0c33 --- /dev/null +++ b/modules/elastic/cluster_hook.go @@ -0,0 +1,95 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "sync" + + log "github.com/cihub/seelog" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/modules/elastic/common" +) + +// ────────────────────────────────────────────────────────────────────────── +// Cluster lifecycle hook. +// +// Clusters managed through the ORM (the /easysearch/ CRUD API served by +// modules/easysearch) used to become live clients only on the next boot, +// when ElasticModule.Start ran LoadClustersFromORM. This hook closes that +// gap: create/update/delete of an ElasticsearchConfig record is reflected +// in the live-client registry immediately. +// +// Loop guard: the health loop persists status back onto the same records +// (updateClusterHealthStatusViaORM → orm.Update → this hook). Those updates +// touch only non-connection fields, so re-initialization is skipped unless +// the connection identity (endpoints/credentials/version/distribution) +// actually changed — otherwise every health persist would re-probe the +// cluster. +// +// Note: logpilot-style consumers that resolve clusters straight from the +// ORM (ResolveClusterEndpoint) were already restart-free; this hook serves +// registry consumers (GetClient, the health loop itself, metadata). +// ────────────────────────────────────────────────────────────────────────── + +// registerClusterHookOnce guards the hook registration; RegisterDataOperationPostHook +// would otherwise accumulate duplicates across module lifecycles. +var registerClusterHookOnce sync.Once + +func registerClusterHook() { + registerClusterHookOnce.Do(func() { + orm.RegisterDataOperationPostHook(100, handleClusterChange, orm.OpCreate, orm.OpUpdate, orm.OpDelete) + }) +} + +// handleClusterChange keeps the live-client registry in sync with ORM +// cluster records. Non-cluster models pass through untouched. +func handleClusterChange(ctx *orm.Context, op orm.Operation, o interface{}) (*orm.Context, interface{}, error) { + cfg, ok := o.(*elastic.ElasticsearchConfig) + if !ok || cfg == nil { + return ctx, o, nil + } + + switch op { + case orm.OpCreate: + if _, err := common.InitElasticInstance(*cfg); err != nil { + log.Warnf("cluster %s (%s): live init after create failed: %v", cfg.ID, cfg.Name, err) + } else { + log.Debugf("cluster %s (%s): live client registered after create", cfg.ID, cfg.Name) + } + + case orm.OpUpdate: + prev := elastic.GetConfigNoPanic(cfg.ID) + switch { + case prev == nil: + // Not registered yet (e.g. record created while this hook was + // absent) — bring it live now. + if _, err := common.InitElasticInstance(*cfg); err != nil { + log.Warnf("cluster %s (%s): live init after update failed: %v", cfg.ID, cfg.Name, err) + } + case elastic.SameConnectionIdentity(*prev, *cfg): + // Labels/health-status-only change: keep the live client, but + // refresh the registered config so the registry is not stale. + if client := elastic.GetClientNoPanic(cfg.ID); client != nil { + elastic.RegisterInstance(*cfg, client) + } + default: + // Connection identity changed (endpoint/credentials/version): + // drop the cached self-contained client and re-register fresh. + elastic.InvalidateClient(*prev) + if _, err := common.InitElasticInstance(*cfg); err != nil { + log.Warnf("cluster %s (%s): live re-init after update failed: %v", cfg.ID, cfg.Name, err) + } else { + log.Debugf("cluster %s (%s): live client re-registered after connection change", cfg.ID, cfg.Name) + } + } + + case orm.OpDelete: + elastic.RemoveInstance(cfg.ID) + elastic.InvalidateClient(*cfg) + log.Debugf("cluster %s (%s): live client removed after delete", cfg.ID, cfg.Name) + } + + return ctx, o, nil +} diff --git a/modules/elastic/cluster_hook_test.go b/modules/elastic/cluster_hook_test.go new file mode 100644 index 000000000..aa5c86dee --- /dev/null +++ b/modules/elastic/cluster_hook_test.go @@ -0,0 +1,117 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/sqlite" +) + +// TestClusterLifecycleHook proves ORM cluster records and the live-client +// registry stay in sync without a restart: create registers a client, +// labels-only updates keep it (the health loop's persist path must not +// re-initialize), connection changes rebuild it, delete removes it. +func TestClusterLifecycleHook(t *testing.T) { + handler := &sqlite.SQLiteORM{Config: sqlite.SQLiteConfig{ + Enabled: true, + DBPath: filepath.Join(t.TempDir(), "hook.db"), + }} + require.NoError(t, handler.Open()) + t.Cleanup(func() { handler.Close() }) + require.NoError(t, handler.RegisterSchemaWithName(elastic.ElasticsearchConfig{}, "cluster")) + + orm.Register("sqlite", handler) + registerClusterHook() + + newCfg := func(id, endpoint string) elastic.ElasticsearchConfig { + cfg := elastic.ElasticsearchConfig{} + cfg.ID = id + cfg.Name = id + cfg.Endpoint = endpoint + cfg.Distribution = elastic.Elasticsearch + cfg.Version = "8.0.0" // preset → offline adapter selection, no probe + cfg.Enabled = true + return cfg + } + + t.Run("create registers live client", func(t *testing.T) { + cfg := newCfg("hook-1", "http://hook-1:9200") + require.NoError(t, orm.Create(orm.NewContext(), &cfg)) + assert.NotNil(t, elastic.GetClientNoPanic("hook-1")) + }) + + t.Run("labels-only update keeps the client", func(t *testing.T) { + before := elastic.GetClientNoPanic("hook-1") + + obj := elastic.ElasticsearchConfig{} + obj.ID = "hook-1" + require.NoError(t, orm.UpdatePartialFields(orm.NewContext(), &obj, util.MapStr{ + "labels": util.MapStr{"health_status": "green"}, // health-loop persist shape + })) + + after := elastic.GetClientNoPanic("hook-1") + assert.NotNil(t, after) + assert.Same(t, before, after, "labels-only change must not rebuild the client") + }) + + t.Run("connection change rebuilds the client", func(t *testing.T) { + before := elastic.GetClientNoPanic("hook-1") + + obj := elastic.ElasticsearchConfig{} + obj.ID = "hook-1" + require.NoError(t, orm.UpdatePartialFields(orm.NewContext(), &obj, util.MapStr{ + "endpoint": "http://hook-1-changed:9200", + })) + + after := elastic.GetClientNoPanic("hook-1") + assert.NotNil(t, after) + assert.NotSame(t, before, after, "endpoint change must rebuild the client") + reg := elastic.GetConfigNoPanic("hook-1") + require.NotNil(t, reg) + assert.Equal(t, "http://hook-1-changed:9200", reg.Endpoint) + }) + + t.Run("delete removes the client", func(t *testing.T) { + // Realistic path: load the full record first (the /easysearch/ API + // deletes a loaded config), then delete. + cfg := elastic.ElasticsearchConfig{} + cfg.ID = "hook-1" + exists, err := orm.GetV2(orm.NewContext(), &cfg) + require.NoError(t, err) + require.True(t, exists) + require.NoError(t, orm.Delete(orm.NewContext(), &cfg)) + assert.Nil(t, elastic.GetClientNoPanic("hook-1")) + assert.Nil(t, elastic.GetConfigNoPanic("hook-1")) + }) + + t.Run("delete with ID-only model must not panic", func(t *testing.T) { + cfg := elastic.ElasticsearchConfig{} + cfg.ID = "hook-2" + c2 := newCfg("hook-2", "http://hook-2:9200") + require.NoError(t, orm.Create(orm.NewContext(), &c2)) + bare := elastic.ElasticsearchConfig{} + bare.ID = "hook-2" + require.NoError(t, orm.Delete(orm.NewContext(), &bare)) + assert.Nil(t, elastic.GetClientNoPanic("hook-2")) + }) + + t.Run("non-cluster models pass through", func(t *testing.T) { + type bystander struct { + orm.ORMObjectBase + Name string `json:"name,omitempty"` + } + require.NoError(t, handler.RegisterSchemaWithName(bystander{}, "bystanders")) + b := bystander{Name: "x"} + b.ID = "b-1" + require.NoError(t, orm.Create(orm.NewContext(), &b)) // must not panic or register anything + assert.Nil(t, elastic.GetClientNoPanic("b-1")) + }) +} diff --git a/modules/elastic/cluster_loader.go b/modules/elastic/cluster_loader.go index 0a1a37c86..ab77236fc 100644 --- a/modules/elastic/cluster_loader.go +++ b/modules/elastic/cluster_loader.go @@ -3,9 +3,6 @@ package elastic import ( - "encoding/json" - "reflect" - log "github.com/cihub/seelog" "infini.sh/framework/core/elastic" @@ -19,52 +16,6 @@ import ( // depends on the elasticsearch client factory (modules/elastic/common). It will // move as part of the broader elasticsearch-module refactor. -// parseClusterHits extracts the _source array from an orm.SearchResult payload -// (ES-shaped JSON: {"hits":{"hits":[{"_source":{...}}]}}). Returns nil if the -// payload can't be parsed. -func parseClusterHits(res interface{}) json.RawMessage { - if res == nil { - return nil - } - type searchResult struct { - Hits struct { - Hits []struct { - Source json.RawMessage `json:"_source"` - } `json:"hits"` - } `json:"hits"` - } - rv := reflect.ValueOf(res) - if rv.Kind() == reflect.Ptr && !rv.IsNil() { - rv = rv.Elem() - } - payloadField := rv.FieldByName("Payload") - if !payloadField.IsValid() { - return nil - } - var raw []byte - switch v := payloadField.Interface().(type) { - case []byte: - raw = v - case string: - raw = []byte(v) - default: - return nil - } - var sr searchResult - if json.Unmarshal(raw, &sr) != nil { - return nil - } - if len(sr.Hits.Hits) == 0 { - return nil - } - out := make([]json.RawMessage, 0, len(sr.Hits.Hits)) - for _, hit := range sr.Hits.Hits { - out = append(out, hit.Source) - } - b, _ := json.Marshal(out) - return b -} - // LoadClustersFromORM loads dynamic clusters from the ORM backend (sqlite or // any non-elastic store) and registers a live client for each. Called from // ElasticModule.Start when RemoteConfigEnabled is false — i.e. when there's no @@ -79,9 +30,10 @@ func LoadClustersFromORM() { log.Warnf("load clusters from ORM: %v", err) return } - var clusters []elastic.ElasticsearchConfig - if hits := parseClusterHits(res); hits != nil { - _ = json.Unmarshal(hits, &clusters) + clusters, _, err := elastic.DecodeHits[elastic.ElasticsearchConfig](res) + if err != nil { + log.Warnf("load clusters from ORM: decode: %v", err) + return } for _, cfg := range clusters { if _, err := common.InitElasticInstance(cfg); err != nil { diff --git a/modules/elastic/contract_integration_test.go b/modules/elastic/contract_integration_test.go new file mode 100644 index 000000000..ef3df3bb8 --- /dev/null +++ b/modules/elastic/contract_integration_test.go @@ -0,0 +1,46 @@ +//go:build integration + +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "testing" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/orm/ormtest" + "infini.sh/framework/modules/elastic/common" +) + +// TestContract_Elastic runs the shared backend contract suite against a live +// cluster (the system cluster from the elastic.* config). Enable with: +// +// go test -tags integration ./modules/elastic/ -run TestContract_Elastic +func TestContract_Elastic(t *testing.T) { + client, err := common.GetElasticClient(elastic.GlobalSystemElasticsearchID) + if err != nil || client == nil { + t.Skipf("no live elasticsearch available for contract tests: %v", err) + } + // Fresh index with explicit mappings: drop leftovers from a previous run + // (fixed IDs would hit version conflicts) and prevent dynamic mapping from + // turning keyword fields into text (aggregations on text are rejected). + _ = client.DeleteIndex("contractmodel") + if err := client.CreateIndex("contractmodel", map[string]interface{}{ + "mappings": map[string]interface{}{ + "properties": map[string]interface{}{ + "name": map[string]interface{}{"type": "keyword"}, + "status": map[string]interface{}{"type": "keyword"}, + "body": map[string]interface{}{"type": "text"}, + "age": map[string]interface{}{"type": "integer"}, + "created": map[string]interface{}{"type": "date"}, + }, + }, + }); err != nil { + t.Fatalf("create contractmodel index: %v", err) + } + t.Cleanup(func() { _ = client.DeleteIndex("contractmodel") }) + ormtest.RunContractTests(t, func() orm.ORM { + return &ElasticORM{Client: client} + }) +} diff --git a/modules/elastic/integration_env_test.go b/modules/elastic/integration_env_test.go new file mode 100644 index 000000000..55aab73a6 --- /dev/null +++ b/modules/elastic/integration_env_test.go @@ -0,0 +1,51 @@ +//go:build integration + +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "os" + "path/filepath" + "testing" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/orm/ormtest" + "infini.sh/framework/modules/sqlite" +) + +// TestMain bootstraps the ORM handler + system-cluster record from CI's ES_* +// variables so the integration suites resolve a live cluster via +// common.GetElasticClient(GlobalSystemElasticsearchID). Without ES_ENDPOINT +// the suites skip gracefully (developer laptops). +func TestMain(m *testing.M) { + _ = seedORMBackend() + if true { + if err := ormtest.SeedSystemCluster(); err != nil && err != ormtest.ErrNoEndpoint { + println("integration bootstrap failed (tests will skip):", err.Error()) + } + } + os.Exit(m.Run()) +} + +// seedORMBackend registers a sqlite handler over a temp database (the +// cluster record must be persistable somewhere; sqlite keeps the fixture +// self-contained). +func seedORMBackend() error { + handler := &sqlite.SQLiteORM{ + Config: sqlite.SQLiteConfig{ + Enabled: true, + DBPath: filepath.Join(os.TempDir(), "orm-integration-test.db"), + }, + } + if err := handler.Open(); err != nil { + return err + } + if err := handler.RegisterSchemaWithName(elastic.ElasticsearchConfig{}, "cluster"); err != nil { + return err + } + defer func() { _ = recover() }() // duplicate registration is fine + orm.Register("sqlite-integration", handler) + return nil +} diff --git a/modules/elastic/module.go b/modules/elastic/module.go index 82b7415c0..7bad426fe 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -215,6 +215,11 @@ func (module *ElasticModule) Setup() { elastic.RegisterClientProvider(common.InitClientWithConfig) }) + // Keep the live-client registry in sync with ORM cluster records + // (create/update/delete via /easysearch/ takes effect immediately). + // Registered unconditionally for the same reason as the provider. + registerClusterHook() + moduleConfig = getDefaultConfig() exists, err := env.ParseConfig("elastic", &moduleConfig) diff --git a/modules/elastic/orm.go b/modules/elastic/orm.go index 7da6b0bf6..10c8c75e9 100755 --- a/modules/elastic/orm.go +++ b/modules/elastic/orm.go @@ -721,3 +721,16 @@ func (handler *ElasticORM) GroupBy(t interface{}, selectField, groupField string //return nil, finalResult return nil, nil } + +// Capabilities declares what the elastic backend honors: the full +// QueryBuilder surface (the DSL is native here). +func (handler *ElasticORM) Capabilities() api.Capabilities { + return api.Capabilities{ + FullText: true, + Aggregations: true, + Fuzzy: true, + Nested: true, + RequestBodyDSL: true, + Collapse: true, + } +} diff --git a/modules/elastic/orm/aggs.go b/modules/elastic/orm/aggs.go index 7306dfd7f..46afd43a3 100644 --- a/modules/elastic/orm/aggs.go +++ b/modules/elastic/orm/aggs.go @@ -47,6 +47,44 @@ type ESAggregation struct { TopHits map[string]interface{} `json:"top_hits,omitempty"` SumBucket *esPipelineAggregation `json:"sum_bucket,omitempty"` DateRange *esDateRangeAggregation `json:"date_range,omitempty"` + // Console-vocabulary completions (framework request model §5.1). ES + // computes these natively; the framework pipeline engine re-derives the + // pipeline ones afterwards so backends agree exactly. + MaxBucket *esPipelineAggregation `json:"max_bucket,omitempty"` + BucketScript *esBucketScriptAggregation `json:"bucket_script,omitempty"` + BucketSort *esBucketSortAggregation `json:"bucket_sort,omitempty"` + Sampler *esSamplerAggregation `json:"sampler,omitempty"` + AutoDateHist *esAutoDateHistogramAggregation `json:"auto_date_histogram,omitempty"` +} + +type esBucketScriptAggregation struct { + BucketsPath map[string]string `json:"buckets_path,omitempty"` + Script *esScript `json:"script,omitempty"` +} + +type esScript struct { + Source string `json:"source,omitempty"` +} + +type esBucketSortAggregation struct { + Sort []esBucketSortSpec `json:"sort,omitempty"` + From int `json:"from,omitempty"` + Size int `json:"size,omitempty"` +} + +type esBucketSortSpec struct { + Path string `json:"path,omitempty"` + Order string `json:"order,omitempty"` +} + +type esSamplerAggregation struct { + ShardSize int `json:"shard_size,omitempty"` +} + +type esAutoDateHistogramAggregation struct { + Field string `json:"field,omitempty"` + Buckets int `json:"buckets,omitempty"` + MinimumInterval string `json:"minimum_interval,omitempty"` } type esTermsAggregation struct { @@ -71,6 +109,7 @@ type esDateHistogramAggregation struct { Interval string `json:"interval,omitempty"` // Deprecated but still supported by ES Format string `json:"format,omitempty"` TimeZone string `json:"time_zone,omitempty"` + Offset string `json:"offset,omitempty"` // e.g. "+30m" } type esPipelineAggregation struct { @@ -157,6 +196,9 @@ func (c *AggreationBuilder) translateAggregation(agg orm.Aggregation) (*ESAggreg Format: v.Format, TimeZone: v.TimeZone, } + if v.Offset != 0 { + esAgg.DateHistogram.Offset = fmt.Sprintf("%+dm", int(v.Offset.Minutes())) + } switch v.IntervalField { case elastic.CalendarInterval: esAgg.DateHistogram.CalendarInterval = v.Interval @@ -182,6 +224,49 @@ func (c *AggreationBuilder) translateAggregation(agg orm.Aggregation) (*ESAggreg Ranges: v.Ranges, TimeZone: v.TimeZone, } + case *orm.MaxBucketAggregation: + esAgg.MaxBucket = &esPipelineAggregation{BucketsPath: v.BucketsPath} + case *orm.BucketScriptAggregation: + esAgg.BucketScript = &esBucketScriptAggregation{ + BucketsPath: v.BucketsPath, + Script: &esScript{Source: v.Script}, + } + case *orm.BucketSortAggregation: + sortSpecs := make([]esBucketSortSpec, 0, len(v.Sort)) + for _, s2 := range v.Sort { + order := "asc" + if s2.Desc { + order = "desc" + } + sortSpecs = append(sortSpecs, esBucketSortSpec{Path: s2.Path, Order: order}) + } + esAgg.BucketSort = &esBucketSortAggregation{Sort: sortSpecs, From: v.From, Size: v.Size} + case *orm.SamplerAggregation: + esAgg.Sampler = &esSamplerAggregation{ShardSize: v.ShardSize} + case *orm.AutoDateHistogramAggregation: + esAgg.AutoDateHist = &esAutoDateHistogramAggregation{ + Field: v.Field, + Buckets: v.Buckets, + MinimumInterval: v.MinimumInterval, + } + case *orm.TopHitsAggregation: + size := v.Size + if size <= 0 { + size = 1 // ES requires numHits > 0; the suite default is top-1 + } + params := map[string]interface{}{"size": size} + if len(v.Sorts) > 0 { + sorts := make([]interface{}, 0, len(v.Sorts)) + for _, s2 := range v.Sorts { + order := "desc" + if s2.SortType == orm.ASC { + order = "asc" + } + sorts = append(sorts, map[string]interface{}{s2.Field: map[string]string{"order": order}}) + } + params["sort"] = sorts + } + esAgg.TopHits = params default: return nil, fmt.Errorf("unsupported aggregation type: %T", v) } diff --git a/modules/elastic/orm_aggregate.go b/modules/elastic/orm_aggregate.go new file mode 100644 index 000000000..004389417 --- /dev/null +++ b/modules/elastic/orm_aggregate.go @@ -0,0 +1,195 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "encoding/json" + "strconv" + "time" + "fmt" + + "infini.sh/framework/core/aggregate" + api "infini.sh/framework/core/orm" +) + +// Aggregate implements orm.MetricsAPI for the elastic backend. +// +// Bucket and metric aggregations execute natively on the cluster through +// the existing SearchV2 + AggreationBuilder path; the ES-shaped response is +// parsed into the typed tree, and pipeline aggregations are then recomputed +// by the framework engine so elastic and sqlite agree exactly (design doc +// §6.1, option b). +func (handler *ElasticORM) Aggregate(ctx *api.Context, qb *api.QueryBuilder) (*api.AggregationResult, error) { + if qb == nil || len(qb.Aggs) == 0 { + return nil, fmt.Errorf("no aggregations set on the query builder") + } + res, err := handler.SearchV2(ctx, qb) + if err != nil { + return nil, err + } + var envelope struct { + Hits struct { + Total struct { + Value int64 `json:"value"` + } `json:"total"` + } `json:"hits"` + Aggregations map[string]interface{} `json:"aggregations"` + } + if payload, ok := res.Payload.([]byte); ok && len(payload) > 0 { + if err := json.Unmarshal(payload, &envelope); err != nil { + return nil, fmt.Errorf("parse aggregation response: %w", err) + } + } + if len(envelope.Aggregations) == 0 { + return &api.AggregationResult{Aggs: map[string]*api.AggNode{}}, nil + } + + nodes := parseESAggregations(envelope.Aggregations) + if envelope.Hits.Total.Value == 0 { + // Metric aggregations over zero matched docs report value:0.0 in ES - + // a placeholder, not a set value. Clear ValueSet so consumers (and + // the cross-backend conformance suite) read "no value", matching the + // sqlite backend and ES's own null-on-percentiles semantics. + for _, node := range nodes { + if node.ValueSet && len(node.Buckets) == 0 { + node.ValueSet = false + node.Value = 0 + } + } + } + result := &api.AggregationResult{Aggs: nodes} + if err := aggregate.ApplyPipelines(result, qb.Aggs); err != nil { + return nil, err + } + return result, nil +} + +// parseESAggregations converts ES-shaped aggregation maps into the typed +// tree (recursive over buckets and sub-aggregations). +func parseESAggregations(raw map[string]interface{}) map[string]*api.AggNode { + out := make(map[string]*api.AggNode, len(raw)) + for name, v := range raw { + m, ok := v.(map[string]interface{}) + if !ok { + continue + } + out[name] = parseESAggNode(m) + } + return out +} + +func parseESAggNode(m map[string]interface{}) *api.AggNode { + node := &api.AggNode{} + if rawBuckets, ok := m["buckets"].([]interface{}); ok { + buckets := make([]api.Bucket, 0, len(rawBuckets)) + for _, rb := range rawBuckets { + bm, ok := rb.(map[string]interface{}) + if !ok { + continue + } + bucket := api.Bucket{Aggs: map[string]*api.AggNode{}} + switch k := bm["key"].(type) { + case string: + bucket.Key = k + bucket.KeyRaw = k + case float64: + bucket.Key = fmt.Sprintf("%v", k) + // Numeric (epoch-millis) keys are normalized to int64 — the + // conformance suite asserts int64 KeyRaw and sqlite already + // yields int64 (UnixMilli). + if k == float64(int64(k)) { + bucket.KeyRaw = int64(k) + } else { + bucket.KeyRaw = k + } + case json.Number: + bucket.Key = k.String() + if i, err := k.Int64(); err == nil { + bucket.KeyRaw = i + } else { + bucket.KeyRaw = k + } + } + if kas, ok := bm["key_as_string"].(string); ok { + bucket.Key = normalizeESBucketKey(kas) + } + if dc, ok := bm["doc_count"].(float64); ok { + bucket.DocCount = int64(dc) + } + for k, v := range bm { + switch k { + case "key", "key_as_string", "doc_count", "doc_count_error_upper_bound": + continue + } + if sub, ok := v.(map[string]interface{}); ok { + bucket.Aggs[k] = parseESAggNode(sub) + } + } + buckets = append(buckets, bucket) + } + node.Buckets = buckets + return node + } + if v, ok := m["value"].(float64); ok { + node.Value = v + node.ValueSet = true + } else if _, ok := m["value"]; !ok { + // single-bucket aggregations (filter): inline scope, no buckets array + bucket := api.Bucket{Aggs: map[string]*api.AggNode{}} + if dc, ok := m["doc_count"].(float64); ok { + bucket.DocCount = int64(dc) + } + for k, v := range m { + if k == "doc_count" { + continue + } + if sub, ok := v.(map[string]interface{}); ok { + bucket.Aggs[k] = parseESAggNode(sub) + } + } + if len(bucket.Aggs) > 0 || bucket.DocCount > 0 { + node.Buckets = []api.Bucket{bucket} + } + } + if rawValues, ok := m["values"].(map[string]interface{}); ok { + values := make(map[string]float64, len(rawValues)) + for k, v := range rawValues { + if f, ok := v.(float64); ok { + // ES emits percentiles keys as "50.0"/"100.0"; normalize to + // the bare number ("50"/"100") so both backends share keys. + if fk, err := strconv.ParseFloat(k, 64); err == nil && fk == float64(int64(fk)) { + k = strconv.FormatInt(int64(fk), 10) + } + values[k] = f + } + } + node.Values = values + } + if hits, ok := m["hits"].(map[string]interface{}); ok { + if inner, ok := hits["hits"].([]interface{}); ok && len(inner) > 0 { + if hit, ok := inner[0].(map[string]interface{}); ok { + if src, ok := hit["_source"]; ok { + raw, _ := json.Marshal(src) + doc := json.RawMessage(raw) + node.TopHit = &doc + } + } + } + } + return node +} + + +// normalizeESBucketKey renders ES date_histogram key_as_string values in the +// canonical second-precision layout (2006-01-02T15:04:05, UTC, no millis) +// used by the cross-backend conformance suite — ES emits RFC3339 with +// milliseconds ("2026-08-13T00:00:00.000Z"), which broke parity with the +// sqlite backend's keys. +func normalizeESBucketKey(kas string) string { + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.000", "2006-01-02T15:04:05"} { + if ts, err := time.Parse(layout, kas); err == nil { + return ts.UTC().Format("2006-01-02T15:04:05") + } + } + return kas +} diff --git a/modules/elastic/orm_test.go b/modules/elastic/orm_test.go index 6f0950cb9..fec0f1376 100755 --- a/modules/elastic/orm_test.go +++ b/modules/elastic/orm_test.go @@ -44,7 +44,7 @@ type MyHost struct { Host string `json:"host,omitempty" elastic_meta:"_id" elastic_mapping:"host: { type: text, fields: { keyword: { type: keyword, ignore_above: 256 } } }"` Favicon string `json:"favicon,omitempty" elastic_mapping:"favicon: { type: keyword ,copy_to : [all_field_values]}"` Enabled bool `json:"enabled" elastic_mapping:"enabled: { type: boolean }"` - HostConfig *MyHostConfig `json:"host_configs,omitempty" elastic_mapping:"host_config:{type:object}"` + HostConfig *MyHostConfig `json:"host_config,omitempty" elastic_mapping:"host_config:{type:object}"` HostConfigs []MyHostConfig `json:"host_configs,omitempty" elastic_mapping:"host_configs:{type:object}"` } diff --git a/modules/sqlite/aggregate_conformance_test.go b/modules/sqlite/aggregate_conformance_test.go new file mode 100644 index 000000000..36311af08 --- /dev/null +++ b/modules/sqlite/aggregate_conformance_test.go @@ -0,0 +1,61 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package sqlite + +import ( + "path/filepath" + "testing" + + "infini.sh/framework/core/aggregate/aggstest" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +// conformModel covers the suite's fixture fields with proper mappings so +// generated columns and FTS apply during conformance runs. +type conformModel struct { + orm.ORMObjectBase + TS string `json:"ts,omitempty" elastic_mapping:"ts: { type: date }"` + Stream string `json:"stream,omitempty" elastic_mapping:"stream: { type: keyword }"` + Severity string `json:"severity,omitempty" elastic_mapping:"severity: { type: keyword }"` + N int `json:"n,omitempty" elastic_mapping:"n: { type: integer }"` +} + +// sqliteAggstestBackend adapts SQLiteORM to the conformance suite. +type sqliteAggstestBackend struct { + handler *SQLiteORM +} + +func (s *sqliteAggstestBackend) Setup(t *testing.T, docs []aggstest.Doc) (*orm.Context, func()) { + t.Helper() + s.handler = &SQLiteORM{Config: SQLiteConfig{ + Enabled: true, + DBPath: filepath.Join(t.TempDir(), "conformance.db"), + }} + if err := s.handler.Open(); err != nil { + t.Fatal(err) + } + if err := s.handler.RegisterSchemaWithName(conformModel{}, "conform_docs"); err != nil { + t.Fatal(err) + } + for _, d := range docs { + raw := util.MustToJSONBytes(d) + id, _ := d["id"].(string) + if _, err := s.handler.DB.Exec("INSERT INTO conform_docs (id, raw) VALUES (?, ?)", id, raw); err != nil { + t.Fatal(err) + } + } + ctx := orm.NewContext() + orm.WithModel(ctx, &conformModel{}) + return ctx, func() { s.handler.Close() } +} + +func (s *sqliteAggstestBackend) Aggregate(ctx *orm.Context, qb *orm.QueryBuilder) (*orm.AggregationResult, error) { + return s.handler.Aggregate(ctx, qb) +} + +// TestAggConformance_SQLite runs the shared aggregation conformance suite. +// Elastic wires the same suite behind the `integration` build tag. +func TestAggConformance_SQLite(t *testing.T) { + aggstest.RunConformance(t, &sqliteAggstestBackend{}) +} diff --git a/modules/sqlite/aggregate_offset_test.go b/modules/sqlite/aggregate_offset_test.go new file mode 100644 index 000000000..4ef1f0f53 --- /dev/null +++ b/modules/sqlite/aggregate_offset_test.go @@ -0,0 +1,75 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package sqlite + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "infini.sh/framework/core/orm" +) + +type offsetDoc struct { + orm.ORMObjectBase + TS time.Time `json:"ts,omitempty" elastic_mapping:"ts: { type: date }"` + Count int64 `json:"count,omitempty" elastic_mapping:"count: { type: long }"` +} + +// TestAggregate_DateHistogramOffset proves offset bucketing reproduces +// logpilot's now-anchored age slotting: slot(t) = floor((now-t)/1h) equals +// the histogram bucket of t with offset = now mod 1h, mapped via each +// bucket's right edge. Fixture instants avoid the exact boundary sliver +// (seconds truncation of the minute-granularity offset). +func TestAggregate_DateHistogramOffset(t *testing.T) { + handler := &SQLiteORM{Config: SQLiteConfig{Enabled: true, DBPath: filepath.Join(t.TempDir(), "off.db")}} + require.NoError(t, handler.Open()) + require.NoError(t, handler.RegisterSchemaWithName(offsetDoc{}, "offset_docs")) + defer handler.Close() + + now := time.Date(2026, 8, 13, 10, 37, 12, 0, time.UTC) + offset := now.Sub(now.Truncate(time.Hour)) + + seed := []struct { + minutesBack int + count int64 + wantSlot int // age slot, 0 = newest + }{ + {5, 3, 0}, // 10:32 + {61, 5, 1}, // 09:36 + {90, 7, 1}, // 09:07 + {182, 11, 3}, // 07:35 + } + for i, s := range seed { + d := offsetDoc{TS: now.Add(-time.Duration(s.minutesBack) * time.Minute), Count: s.count} + d.ID = fmt.Sprintf("d%d", i) + require.NoError(t, handler.Save(nil, &d)) + } + + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h", Offset: offset} + dh.AddNested("total", &orm.MetricAggregation{Type: orm.MetricSum, Field: "count"}) + ctx := orm.NewContext() + orm.WithModel(ctx, &offsetDoc{}) + qb := orm.NewQuery() + qb.SetAggregations(map[string]orm.Aggregation{"h": dh}) + res, err := handler.Aggregate(ctx, qb) + require.NoError(t, err) + + got := map[int]int64{} + for _, b := range res.Aggs["h"].Buckets { + require.NotNil(t, b.Aggs["total"], "zero-filled buckets must carry metric nodes") + // Map via the bucket's right edge: slot = floor((now - (start+1h))/h). + rightEdge := time.UnixMilli(b.KeyRaw.(int64)).Add(time.Hour) + slot := int(now.Sub(rightEdge).Hours()) + got[slot] += int64(b.Aggs["total"].Value) + } + want := map[int]int64{2: 0} // slot 2 zero-filled (gap hour, min_doc_count:0) + for _, s := range seed { + want[s.wantSlot] += s.count + } + assert.Equal(t, want, got) +} diff --git a/modules/sqlite/aggregate_parity_integration_test.go b/modules/sqlite/aggregate_parity_integration_test.go new file mode 100644 index 000000000..a3d5b9589 --- /dev/null +++ b/modules/sqlite/aggregate_parity_integration_test.go @@ -0,0 +1,97 @@ +package sqlite + +import ( + "os" + "path/filepath" + "testing" + + "infini.sh/framework/core/aggregate/aggstest" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/orm/ormtest" + elasticmod "infini.sh/framework/modules/elastic" + "infini.sh/framework/modules/elastic/common" +) + + +// elasticAggstestBackend adapts ElasticORM to the conformance suite for the +// parity run (the canonical elastic-side copy lives in modules/elastic; +// duplicated here because parity needs both backends and only this package +// can import elastic without a cycle). +type elasticAggstestBackend struct { + handler *elasticmod.ElasticORM + index string +} + +func (e *elasticAggstestBackend) Setup(t *testing.T, docs []aggstest.Doc) (*orm.Context, func()) { + t.Helper() + client, err := common.GetElasticClient(elastic.GlobalSystemElasticsearchID) + if err != nil || client == nil { + t.Skipf("no live elasticsearch available: %v", err) + } + e.handler = &elasticmod.ElasticORM{Client: client} + e.index = "aggstest-parity" + + _ = client.DeleteIndex(e.index) + if err := client.CreateIndex(e.index, map[string]interface{}{ + "mappings": map[string]interface{}{ + "properties": map[string]interface{}{ + "ts": map[string]interface{}{"type": "date"}, + "stream": map[string]interface{}{"type": "keyword"}, + "severity": map[string]interface{}{"type": "keyword"}, + "n": map[string]interface{}{"type": "integer"}, + }, + }, + }); err != nil { + t.Fatalf("create index: %v", err) + } + for _, d := range docs { + id, _ := d["id"].(string) + if _, err := client.Index(e.index, "", id, d, ""); err != nil { + t.Fatalf("seed doc: %v", err) + } + } + if err := client.Refresh(e.index); err != nil { + t.Fatalf("refresh: %v", err) + } + t.Cleanup(func() { _ = client.DeleteIndex(e.index) }) + + ctx := orm.NewContext() + orm.WithIndices(ctx, e.index) + return ctx, func() {} +} + +func (e *elasticAggstestBackend) Aggregate(ctx *orm.Context, qb *orm.QueryBuilder) (*orm.AggregationResult, error) { + return e.handler.Aggregate(ctx, qb) +} + +// TestAggParity_SQLite_Elastic deep-compares the two backends against the +// shared conformance suite. CI wires it via the integration workflow's +// ES_ENDPOINT fixture (the system cluster is seeded by the elastic +// package's integration TestMain); skips when no cluster is reachable. +func TestAggParity_SQLite_Elastic(t *testing.T) { + seedParityORM(t) + if err := ormtest.SeedSystemCluster(); err != nil { + t.Skipf("no live cluster fixture: %v", err) + } + aggstest.RunParity(t, &sqliteAggstestBackend{}, &elasticAggstestBackend{}) +} + + +// seedParityORM registers the sqlite ORM handler over a temp database for +// the parity run's cluster-record persistence. +func seedParityORM(t *testing.T) { + t.Helper() + handler := &SQLiteORM{Config: SQLiteConfig{ + Enabled: true, + DBPath: filepath.Join(os.TempDir(), "orm-parity-test.db"), + }} + if err := handler.Open(); err != nil { + t.Fatal(err) + } + if err := handler.RegisterSchemaWithName(elastic.ElasticsearchConfig{}, "cluster"); err != nil { + t.Fatal(err) + } + defer func() { _ = recover() }() + orm.Register("sqlite-parity", handler) +} diff --git a/modules/sqlite/contract_test.go b/modules/sqlite/contract_test.go new file mode 100644 index 000000000..d7f56c36c --- /dev/null +++ b/modules/sqlite/contract_test.go @@ -0,0 +1,24 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package sqlite + +import ( + "testing" + + "infini.sh/framework/core/orm" + "infini.sh/framework/core/orm/ormtest" +) + +// TestContract_SQLite runs the shared backend contract suite; elastic wires +// the same suite behind the `integration` build tag where a live cluster is +// available. +func TestContract_SQLite(t *testing.T) { + ormtest.RunContractTests(t, func() orm.ORM { + handler := &SQLiteORM{Config: SQLiteConfig{Enabled: true, DBPath: t.TempDir() + "/contract.db"}} + if err := handler.Open(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { handler.Close() }) + return handler + }) +} diff --git a/modules/sqlite/features_test.go b/modules/sqlite/features_test.go new file mode 100644 index 000000000..bbec23355 --- /dev/null +++ b/modules/sqlite/features_test.go @@ -0,0 +1,299 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package sqlite + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + sqliteOrm "infini.sh/framework/modules/sqlite/orm" +) + +// ftsDoc is a model exercising FTS (text) + flattened scalars together. +type ftsDoc struct { + orm.ORMObjectBase + Title string `json:"title,omitempty" elastic_mapping:"title: { type: text }"` + Body string `json:"body,omitempty" elastic_mapping:"body: { type: text }"` + Tags string `json:"tags,omitempty" elastic_mapping:"tags: { type: keyword }"` + N int `json:"n,omitempty" elastic_mapping:"n: { type: integer }"` +} + +func openFTSTestDB(t *testing.T) (*SQLiteORM, func()) { + t.Helper() + handler := &SQLiteORM{Config: SQLiteConfig{Enabled: true, DBPath: filepath.Join(t.TempDir(), "fts.db")}} + require.NoError(t, handler.Open()) + require.NoError(t, handler.RegisterSchemaWithName(ftsDoc{}, "fts_docs")) + return handler, func() { handler.Close() } +} + +func (h *SQLiteORM) saveFtsDoc(t *testing.T, id, title, body, tags string, n int) { + t.Helper() + doc := ftsDoc{Title: title, Body: body, Tags: tags, N: n} + doc.ID = id + require.NoError(t, h.Save(nil, &doc)) +} + +func decodeAggs(t *testing.T, res *orm.SearchResult) map[string]interface{} { + t.Helper() + var m map[string]interface{} + require.NoError(t, json.Unmarshal(res.Payload.([]byte), &m)) + return m +} + +func TestFTS_MatchQueries(t *testing.T) { + handler, cleanup := openFTSTestDB(t) + defer cleanup() + + handler.saveFtsDoc(t, "1", "hello world", "sqlite rocks", "a", 1) + handler.saveFtsDoc(t, "2", "goodbye world", "postgres rocks", "b", 2) + handler.saveFtsDoc(t, "3", "unrelated", "nothing here", "a", 3) + + ctx := orm.NewContext() + orm.WithModel(ctx, &ftsDoc{}) + + t.Run("match single word", func(t *testing.T) { + res, err := handler.SearchV2(ctx, orm.NewQuery().Filter(orm.MatchQuery("title", "hello"))) + require.NoError(t, err) + items, _, err := decodeLocal[ftsDoc](res) + require.NoError(t, err) + assert.Len(t, items, 1) + assert.Equal(t, "1", items[0].ID) + }) + + t.Run("match any word (OR semantics)", func(t *testing.T) { + res, err := handler.SearchV2(ctx, orm.NewQuery().Filter(orm.MatchQuery("title", "hello goodbye"))) + require.NoError(t, err) + items, _, err := decodeLocal[ftsDoc](res) + require.NoError(t, err) + assert.Len(t, items, 2) + }) + + t.Run("match phrase keeps order", func(t *testing.T) { + qb := orm.NewQuery().Filter(&orm.Clause{Field: "body", Operator: orm.QueryMatchPhrase, Value: "sqlite rocks"}) + res, err := handler.SearchV2(ctx, qb) + require.NoError(t, err) + items, _, err := decodeLocal[ftsDoc](res) + require.NoError(t, err) + assert.Len(t, items, 1) + assert.Equal(t, "1", items[0].ID) + }) + + t.Run("update resyncs index", func(t *testing.T) { + handler.saveFtsDoc(t, "1", "changed title", "sqlite rocks", "a", 1) + res, err := handler.SearchV2(ctx, orm.NewQuery().Filter(orm.MatchQuery("title", "hello"))) + require.NoError(t, err) + items, _, err := decodeLocal[ftsDoc](res) + require.NoError(t, err) + assert.Empty(t, items, "old term must be gone after update") + }) + + t.Run("delete resyncs index", func(t *testing.T) { + doc := ftsDoc{} + doc.ID = "2" + require.NoError(t, handler.Delete(nil, &doc)) + res, err := handler.SearchV2(ctx, orm.NewQuery().Filter(orm.MatchQuery("title", "goodbye"))) + require.NoError(t, err) + items, _, err := decodeLocal[ftsDoc](res) + require.NoError(t, err) + assert.Empty(t, items, "deleted doc must leave the index") + }) + + t.Run("term on keyword unaffected", func(t *testing.T) { + res, err := handler.SearchV2(ctx, orm.NewQuery().Filter(orm.TermQuery("tags", "a"))) + require.NoError(t, err) + items, _, err := decodeLocal[ftsDoc](res) + require.NoError(t, err) + assert.Len(t, items, 2) + }) +} + +func TestPagination_FromWithoutSize(t *testing.T) { + handler, cleanup := openFTSTestDB(t) + defer cleanup() + + for i := 0; i < 5; i++ { + handler.saveFtsDoc(t, fmt.Sprintf("p%d", i), "page", "body", "t", i) + } + + ctx := orm.NewContext() + orm.WithModel(ctx, &ftsDoc{}) + + // Regression: OFFSET without LIMIT used to emit invalid SQL. + res, err := handler.SearchV2(ctx, orm.NewQuery().From(2)) + require.NoError(t, err) + items, _, err := decodeLocal[ftsDoc](res) + require.NoError(t, err) + assert.Len(t, items, 3) +} + +func TestUnsupportedOperators_WarnAndDegrade(t *testing.T) { + handler, cleanup := openFTSTestDB(t) + defer cleanup() + handler.saveFtsDoc(t, "n1", "nested", "body", "t", 1) + + ctx := orm.NewContext() + orm.WithModel(ctx, &ftsDoc{}) + + for _, op := range []orm.QueryType{orm.QuerySemantic, orm.QueryHybrid, orm.QueryNested} { + qb := orm.NewQuery().Filter(&orm.Clause{Field: "title", Operator: op, Value: "x"}) + res, err := handler.SearchV2(ctx, qb) + require.NoError(t, err, "%s must not error (grace period)", op) + items, _, err := decodeLocal[ftsDoc](res) + require.NoError(t, err) + assert.Empty(t, items, "%s matches nothing", op) + } +} + +func TestCompositeIndex(t *testing.T) { + type compositeModel struct { + orm.ORMObjectBase + Status string `json:"status,omitempty" elastic_mapping:"status: { type: keyword }" sqlite_composite:"status,n"` + N int `json:"n,omitempty" elastic_mapping:"n: { type: integer }"` + } + handler := &SQLiteORM{Config: SQLiteConfig{Enabled: true, DBPath: filepath.Join(t.TempDir(), "comp.db")}} + require.NoError(t, handler.Open()) + defer handler.Close() + require.NoError(t, handler.RegisterSchemaWithName(compositeModel{}, "comp_docs")) + + idx := sqliteIndexDDL(t, handler.DB, "comp_docs") + found := false + for name, ddl := range idx { + if strings.HasPrefix(name, "ixcc_") && containsAll(ddl, `"status"`, `"n"`) { + found = true + } + } + assert.True(t, found, "composite index on (status, n) should exist; got %v", idx) + + // The composite actually serves two-field filters. + for i := 0; i < 100; i++ { + m := compositeModel{Status: fmt.Sprintf("s%d", i%4), N: i} + m.ID = fmt.Sprintf("c%d", i) + require.NoError(t, handler.Save(nil, &m)) + } + schema := lookupTableSchema("comp_docs") + require.NotNil(t, schema) + qb := orm.NewQuery(). + Filter(orm.TermQuery("status", "s1")). + Filter(orm.Range("n").Gte(50)) + qb.Build() + where, args := sqliteOrm.BuildWhereClause(qb, schema.resolver()) + require.NotEmpty(t, where) + detail := queryPlanDetail(t, handler.DB, "EXPLAIN QUERY PLAN SELECT id FROM comp_docs WHERE "+where, args...) + joined := fmt.Sprint(detail) + assert.Contains(t, joined, "ixcc_", "plan should use the composite index; got: %s", joined) + + var n int + require.NoError(t, handler.DB.QueryRow("SELECT COUNT(*) FROM comp_docs WHERE "+where, args...).Scan(&n)) + assert.Equal(t, 12, n) // s1: i%4==1, n>=50 → i ∈ {53,57,...,97} +} + +// containsAll reports whether s contains every substring. +func containsAll(s string, subs ...string) bool { + for _, sub := range subs { + if !strings.Contains(s, sub) { + return false + } + } + return true +} + +func TestAggregations(t *testing.T) { + handler, cleanup := openFTSTestDB(t) + defer cleanup() + + now := "2026-08-13T10:00:00Z" + for i := 0; i < 10; i++ { + doc := ftsDoc{ + Title: "t", + Body: "b", + Tags: fmt.Sprintf("tag%d", i%2), + N: i + 1, + } + doc.ID = fmt.Sprintf("a%d", i) + doc.Created = mustTime(now) + require.NoError(t, handler.Save(nil, &doc)) + } + + ctx := orm.NewContext() + orm.WithModel(ctx, &ftsDoc{}) + + t.Run("terms + nested metric", func(t *testing.T) { + terms := &orm.TermsAggregation{Field: "tags", Size: 10} + terms.AddNested("avg_n", &orm.MetricAggregation{Type: orm.MetricAvg, Field: "n"}) + qb := orm.NewQuery() + qb.SetAggregations(map[string]orm.Aggregation{"by_tag": terms}) + res, err := handler.SearchV2(ctx, qb) + require.NoError(t, err) + out := decodeAggs(t, res) + aggs := out["aggregations"].(map[string]interface{}) + byTag := aggs["by_tag"].(map[string]interface{}) + buckets := byTag["buckets"].([]interface{}) + require.Len(t, buckets, 2) + b0 := buckets[0].(map[string]interface{}) + assert.Equal(t, float64(5), b0["doc_count"]) + sub := b0["avg_n"].(map[string]interface{}) + assert.Equal(t, float64(5), sub["value"]) // tag0: (1+3+5+7+9)/5; ties on doc_count break by key asc + }) + + t.Run("metrics", func(t *testing.T) { + sum := &orm.MetricAggregation{Type: orm.MetricSum, Field: "n"} + qb := orm.NewQuery() + qb.SetAggregations(map[string]orm.Aggregation{"total": sum}) + res, err := handler.SearchV2(ctx, qb) + require.NoError(t, err) + out := decodeAggs(t, res) + total := out["aggregations"].(map[string]interface{})["total"].(map[string]interface{}) + assert.Equal(t, float64(55), total["value"]) + }) + + t.Run("date_histogram hourly", func(t *testing.T) { + dh := &orm.DateHistogramAggregation{Field: "created", Interval: "1h"} + qb := orm.NewQuery() + qb.SetAggregations(map[string]orm.Aggregation{"over_time": dh}) + res, err := handler.SearchV2(ctx, qb) + require.NoError(t, err) + out := decodeAggs(t, res) + buckets := out["aggregations"].(map[string]interface{})["over_time"].(map[string]interface{})["buckets"].([]interface{}) + require.Len(t, buckets, 1) + b := buckets[0].(map[string]interface{}) + assert.Equal(t, float64(10), b["doc_count"]) + assert.Equal(t, "2026-08-13T10:00:00", b["key_as_string"]) + }) + + t.Run("date_range", func(t *testing.T) { + dr := &orm.DateRangeAggregation{Field: "created", Ranges: []interface{}{ + map[string]interface{}{"from": "2026-08-13T09:00:00Z", "to": "2026-08-13T11:00:00Z"}, + map[string]interface{}{"from": "2026-08-14T00:00:00Z"}, + }} + qb := orm.NewQuery() + qb.SetAggregations(map[string]orm.Aggregation{"ranges": dr}) + res, err := handler.SearchV2(ctx, qb) + require.NoError(t, err) + out := decodeAggs(t, res) + buckets := out["aggregations"].(map[string]interface{})["ranges"].(map[string]interface{})["buckets"].([]interface{}) + require.Len(t, buckets, 2) + assert.Equal(t, float64(10), buckets[0].(map[string]interface{})["doc_count"]) + assert.Equal(t, float64(0), buckets[1].(map[string]interface{})["doc_count"]) + }) +} + +func mustTime(s string) *time.Time { + tm, err := time.Parse(time.RFC3339, s) + if err != nil { + panic(err) + } + return &tm +} + +func decodeLocal[T any](res *orm.SearchResult) ([]T, int64, error) { + return elastic.DecodeHits[T](res) +} diff --git a/modules/sqlite/flattened.go b/modules/sqlite/flattened.go new file mode 100644 index 000000000..943f63f85 --- /dev/null +++ b/modules/sqlite/flattened.go @@ -0,0 +1,584 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package sqlite + +import ( + "database/sql" + "fmt" + "reflect" + "strings" + "sync" + + log "github.com/cihub/seelog" + + "infini.sh/framework/core/global" + sqliteOrm "infini.sh/framework/modules/sqlite/orm" +) + +// ────────────────────────────────────────────────────────────────────────── +// Flattened storage model. +// +// Tables keep the document shape — (id, raw JSON) — as the source of truth, +// while mapped scalar leaves are promoted to VIRTUAL generated columns: +// +// "status" TEXT GENERATED ALWAYS AS (json_extract(raw,'$.status')) VIRTUAL +// +// Queries then hit real columns (plain B-tree indexes, composites possible, +// planner-friendly) instead of json_extract expression indexes. Generated +// columns are computed by SQLite itself, so the write path stays id-only + +// raw and can never drift. Text-mapped fields additionally get an FTS5 +// external-content table kept in sync by triggers (match queries use MATCH). +// +// Benchmark context (200k rows, see SEARCH_ORM_REFACTOR_PLAN appendix D): +// filter+sort+LIMIT via composite index ~59ms → ~12µs; two-field AND+range +// ~68ms → ~5.9ms; single-field lookups and GROUP BY are on par. +// ────────────────────────────────────────────────────────────────────────── + +// flattenedTypeAffinity maps elastic_mapping types to SQLite column +// affinities. json_extract returns TEXT for JSON strings, INTEGER for +// numbers/booleans, REAL for floats — matching these affinities keeps +// index comparisons type-correct. +var flattenedTypeAffinity = map[string]string{ + "keyword": "TEXT", + "date": "TEXT", + "long": "INTEGER", + "integer": "INTEGER", + "boolean": "INTEGER", + "double": "REAL", + "float": "REAL", +} + +// fieldInfo is one mapped leaf found on the model struct. +type fieldInfo struct { + Path string // dotted JSON path, e.g. "basic_auth.username" + ESType string // keyword/date/long/integer/boolean/double/float/text +} + +// columnInfo describes one promoted generated column. +type columnInfo struct { + Path string // dotted JSON path (also the quoted column name) + Affinity string // SQLite type affinity + Expr string // SQL expression referencing the column, e.g. ["status"] +} + +// ftsInfo describes one FTS5-synced text field. +type ftsInfo struct { + Path string // dotted JSON path + Column string // sanitized FTS column name (dots → underscores) + Expr string // SQL expression of the backing generated column +} + +// tableSchema is the flattened layout derived from a registered model. +type tableSchema struct { + Name string + Columns []columnInfo // scalar leaves promoted to generated columns + FTSFields []ftsInfo // text leaves synced into the FTS table + Composite [][]string // composite index column lists (sqlite_composite tag) + + // lookup maps JSON path → column expression / FTS info for the resolver. + columnByPath map[string]string + ftsByPath map[string]ftsInfo + // dateEpochByPath maps date-mapped paths to their integer-epoch shadow + // column expression — histogram bucketing does integer arithmetic on it + // instead of parsing the RFC3339 text per row. + dateEpochByPath map[string]string +} + +// resolver returns the query-side plan for a JSON path: the comparison +// expression (generated column when promoted, json_extract fallback +// otherwise) and the FTS target when the path is a text field. +func (s *tableSchema) resolver() sqliteOrm.FieldResolver { + return func(path string) (string, string, *sqliteOrm.FTSPlan) { + if s == nil { + return jsonExtractExpr(path), "", nil + } + if expr, ok := s.columnByPath[path]; ok { + var fts *sqliteOrm.FTSPlan + if f, ok := s.ftsByPath[path]; ok { + fts = &sqliteOrm.FTSPlan{Table: ftsTableName(s.Name), Column: f.Column} + } + return expr, s.dateEpochByPath[path], fts + } + if f, ok := s.ftsByPath[path]; ok { + return f.Expr, "", &sqliteOrm.FTSPlan{Table: ftsTableName(s.Name), Column: f.Column} + } + return jsonExtractExpr(path), "", nil + } +} + +func jsonExtractExpr(path string) string { + return fmt.Sprintf("json_extract(raw, '$.%s')", path) +} + +// registry of flattened schemas by table name. +var ( + schemaMu sync.RWMutex + schemaReg = map[string]*tableSchema{} +) + +func registerTableSchema(s *tableSchema) { + schemaMu.Lock() + defer schemaMu.Unlock() + schemaReg[s.Name] = s +} + +func lookupTableSchema(table string) *tableSchema { + schemaMu.RLock() + defer schemaMu.RUnlock() + return schemaReg[table] +} + +// collectFields walks the model struct — recursing into nested object +// structs and anonymous embeds — collecting every elastic_mapping leaf. +// Mirrors the promotion rules: scalar leaves become generated columns, +// text leaves also join the FTS table. Slices/maps are not recursed +// (json_extract cannot address array elements via dotted paths). +func collectFields(model interface{}) (scalars, texts []fieldInfo, composites [][]string) { + t := reflect.TypeOf(model) + for t != nil && t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t == nil || t.Kind() != reflect.Struct { + return nil, nil, nil + } + walkFields(t, "", &scalars, &texts, &composites) + return scalars, texts, composites +} + +func walkFields(t reflect.Type, prefix string, scalars, texts *[]fieldInfo, composites *[][]string) { + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + tag := strings.TrimSpace(field.Tag.Get("elastic_mapping")) + if tag == "-" { + continue + } + + // Composite index declarations: any field may carry a + // sqlite_composite:"a,b" tag listing column paths for one + // composite B-tree index (explicit opt-in, never inferred). + if comp := strings.TrimSpace(field.Tag.Get("sqlite_composite")); comp != "" { + var cols []string + for _, p := range strings.Split(comp, ",") { + if p = strings.TrimSpace(p); p != "" { + cols = append(cols, p) + } + } + if len(cols) > 1 { + *composites = append(*composites, cols) + } + } + + // Anonymous embedded struct without its own mapping tag: JSON-promoted + // fields (e.g. orm.ORMObjectBase) — recurse at the same path. + if field.Anonymous && tag == "" { + if ft := objectStructType(field.Type); ft != nil { + walkFields(ft, prefix, scalars, texts, composites) + } + continue + } + if tag == "" || mappingDisabled(tag) { + continue + } + + jsonField, esType, ok := parseMappingTag(tag) + if jsonField == "" { + continue + } + path := jsonField + if prefix != "" { + path = prefix + "." + jsonField + } + + if ok { + if flattenedTypeAffinity[esType] != "" { + *scalars = append(*scalars, fieldInfo{Path: path, ESType: esType}) + } else if esType == "text" { + *texts = append(*texts, fieldInfo{Path: path, ESType: esType}) + } + } + + if ft := objectStructType(field.Type); ft != nil { + walkFields(ft, path, scalars, texts, composites) + } + } +} + +// buildTableSchema derives the flattened layout for a model. +func buildTableSchema(tableName string, model interface{}) *tableSchema { + scalars, texts, composites := collectFields(model) + s := &tableSchema{ + Name: tableName, + Columns: make([]columnInfo, 0, len(scalars)), + FTSFields: make([]ftsInfo, 0, len(texts)), + Composite: composites, + columnByPath: map[string]string{}, + ftsByPath: map[string]ftsInfo{}, + dateEpochByPath: map[string]string{}, + } + for _, f := range scalars { + // "id" is the table's primary key and "raw" the document column — + // promoting either would collide with the real column. + if f.Path == "id" || f.Path == "raw" { + continue + } + col := columnInfo{ + Path: f.Path, + Affinity: flattenedTypeAffinity[f.ESType], + Expr: quoteIdent(f.Path), + } + s.Columns = append(s.Columns, col) + s.columnByPath[f.Path] = col.Expr + // Date fields get an integer-epoch shadow for bucketing math. + if f.ESType == "date" { + epochPath := dateEpochColumn(f.Path) + s.Columns = append(s.Columns, columnInfo{ + Path: epochPath, + Affinity: "INTEGER", + Expr: quoteIdent(epochPath), + }) + s.dateEpochByPath[f.Path] = quoteIdent(epochPath) + } + } + // The PK column serves id-path queries directly. + s.columnByPath["id"] = quoteIdent("id") + // Text fields are materialized as generated columns too — the FTS + // triggers read them (verified: triggers may reference VIRTUAL columns). + for _, f := range texts { + fts := ftsInfo{ + Path: f.Path, + Column: sanitizeForIndexName(f.Path), + Expr: quoteIdent(f.Path), + } + s.FTSFields = append(s.FTSFields, fts) + s.ftsByPath[f.Path] = fts + } + return s +} + +// ensureFlattenedTable creates or migrates the table to the flattened +// layout: generated columns inline, plain column indexes, FTS sync. +// Migration is a transactional rebuild when an existing table predates +// generated columns (they cannot be added via ALTER TABLE). +func ensureFlattenedTable(db *sql.DB, s *tableSchema) error { + rebuilt, err := createOrMigrateTable(db, s) + if err != nil { + return err + } + dropLegacyExpressionIndexes(db, s) + createColumnIndexes(db, s) + if rebuilt && len(s.FTSFields) > 0 { + // The rebuild reassigned rowids and dropped the old table's sync + // triggers — drop the stale FTS index so ensureFTS recreates and + // backfills it against the new rowids. + if _, err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS [%s]", ftsTableName(s.Name))); err != nil { + log.Warnf("sqlite: drop stale FTS table for %s: %v", s.Name, err) + } + } + ensureFTS(db, s) + return nil +} + +// createOrMigrateTable creates the table with generated columns, or +// rebuilds an older layout in place (same table name, data preserved). +// Returns rebuilt=true when a migration replaced the table (rowids change +// and the old table's triggers die, so the FTS index must be rebuilt). +func createOrMigrateTable(db *sql.DB, s *tableSchema) (bool, error) { + // Build the column DDL fragment once: reused by create and rebuild. + var colDefs []string + // STORED (not VIRTUAL): table scans and aggregate reads must not + // re-evaluate json_extract — a full-document JSON parse — per row per + // column. VIRTUAL is only free for index-only lookups (the index + // materializes the value); scans pay O(rows × columns) parses, which + // measured ~10s for a 3-level aggregation over 500k rows. The one-time + // write-time materialization is negligible for metadata-scale stores. + for _, c := range s.Columns { + if expr, ok := epochSourcePath(c.Path); ok { + // Integer-epoch shadow of a date field: histogram bucketing does + // arithmetic on it instead of parsing the RFC3339 text per row. + colDefs = append(colDefs, fmt.Sprintf("%s INTEGER GENERATED ALWAYS AS (CAST(strftime('%%s', json_extract(raw, '$.%s')) AS INTEGER)) STORED", + quoteIdent(c.Path), expr)) + continue + } + colDefs = append(colDefs, fmt.Sprintf("%s %s GENERATED ALWAYS AS (json_extract(raw, '$.%s')) STORED", + quoteIdent(c.Path), c.Affinity, c.Path)) + } + for _, f := range s.FTSFields { + colDefs = append(colDefs, fmt.Sprintf("%s TEXT GENERATED ALWAYS AS (json_extract(raw, '$.%s')) STORED", + quoteIdent(f.Path), f.Path)) + } + colsDDL := "" + if len(colDefs) > 0 { + colsDDL = ", " + strings.Join(colDefs, ", ") + } + + exists, err := tableExists(db, s.Name) + if err != nil { + return false, err + } + if !exists { + ddl := fmt.Sprintf("CREATE TABLE [%s] (id TEXT PRIMARY KEY, raw JSON NOT NULL%s)", s.Name, colsDDL) + if _, err := db.Exec(ddl); err != nil { + return false, fmt.Errorf("failed to create table %s: %w", s.Name, err) + } + if global.Env().IsDebug { + log.Debug("sqlite DDL: ", ddl) + } + return false, nil + } + + // Existing table: check whether every expected column is present. + have, err := tableColumns(db, s.Name) + if err != nil { + return false, err + } + missing := false + for _, c := range s.Columns { + if !have[c.Path] { + missing = true + break + } + } + if !missing { + for _, f := range s.FTSFields { + if !have[f.Path] { + missing = true + break + } + } + } + if !missing { + return false, nil + } + + // Generated columns cannot be ALTERed in — rebuild the table inside a + // transaction: create shadow with the new layout, copy id+raw, swap. + log.Infof("sqlite: migrating table %s to flattened layout (%d generated columns)", s.Name, len(s.Columns)+len(s.FTSFields)) + tx, err := db.Begin() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + shadow := s.Name + "__migrate" + if _, err := tx.Exec(fmt.Sprintf("DROP TABLE IF EXISTS [%s]", shadow)); err != nil { + return false, err + } + ddl := fmt.Sprintf("CREATE TABLE [%s] (id TEXT PRIMARY KEY, raw JSON NOT NULL%s)", shadow, colsDDL) + if _, err := tx.Exec(ddl); err != nil { + return false, fmt.Errorf("failed to create migration table for %s: %w", s.Name, err) + } + if _, err := tx.Exec(fmt.Sprintf("INSERT INTO [%s] (id, raw) SELECT id, raw FROM [%s]", shadow, s.Name)); err != nil { + return false, err + } + if _, err := tx.Exec(fmt.Sprintf("DROP TABLE [%s]", s.Name)); err != nil { + return false, err + } + if _, err := tx.Exec(fmt.Sprintf("ALTER TABLE [%s] RENAME TO [%s]", shadow, s.Name)); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, err + } + return true, nil +} + +// dropLegacyExpressionIndexes removes the pre-flattening json_extract +// expression indexes (name scheme ix__) so planner choices +// move to the new column indexes. +func dropLegacyExpressionIndexes(db *sql.DB, s *tableSchema) { + tableSafe := sanitizeForIndexName(s.Name) + for _, c := range s.Columns { + dropIndexIfKnown(db, indexNameFor(tableSafe, c.Path)) + } + for _, f := range s.FTSFields { + dropIndexIfKnown(db, indexNameFor(tableSafe, f.Path)) + } +} + +func dropIndexIfKnown(db *sql.DB, name string) { + if _, err := db.Exec(fmt.Sprintf("DROP INDEX IF EXISTS [%s]", name)); err != nil { + log.Warnf("sqlite: drop legacy index %s: %v", name, err) + } +} + +// createColumnIndexes builds plain B-tree indexes on promoted columns and +// the declared composites. All promoted scalars get a single-column index +// (parity with the old expression-index coverage). +func createColumnIndexes(db *sql.DB, s *tableSchema) { + tableSafe := sanitizeForIndexName(s.Name) + for _, c := range s.Columns { + if _, isEpoch := epochSourcePath(c.Path); isEpoch { + // Epoch shadows back the aggregation math only; the TEXT date + // index already covers equality/range queries. + continue + } + idxName := "ixc_" + tableSafe + "_" + sanitizeForIndexName(c.Path) + ddl := fmt.Sprintf("CREATE INDEX IF NOT EXISTS [%s] ON [%s](%s)", idxName, s.Name, c.Expr) + if _, err := db.Exec(ddl); err != nil { + log.Warnf("sqlite column index %s: %v", idxName, err) + } else if global.Env().IsDebug { + log.Debug("sqlite column index: ", ddl) + } + } + for i, cols := range s.Composite { + var exprs []string + var nameParts []string + for _, p := range cols { + // Column names are used verbatim: TEXT date columns keep their + // lexicographic ordering (walk/sort plans depend on it); covering + // aggregation composites may reference the integer epoch shadow + // explicitly, e.g. "bucket_start__epoch". + exprs = append(exprs, quoteIdent(p)) + nameParts = append(nameParts, sanitizeForIndexName(p)) + } + idxName := fmt.Sprintf("ixcc_%s_%d_%s", tableSafe, i, strings.Join(nameParts, "_")) + ddl := fmt.Sprintf("CREATE INDEX IF NOT EXISTS [%s] ON [%s](%s)", idxName, s.Name, strings.Join(exprs, ", ")) + if _, err := db.Exec(ddl); err != nil { + log.Warnf("sqlite composite index %s: %v", idxName, err) + } else if global.Env().IsDebug { + log.Debug("sqlite composite index: ", ddl) + } + } +} + +// ftsTableName is the FTS5 external-content table name for a table. +func ftsTableName(table string) string { + return "fts_" + sanitizeForIndexName(table) +} + +// ensureFTS creates the FTS5 external-content table plus AI/AU/AD triggers +// for text-mapped fields, and backfills existing rows. Column names are +// sanitized (dots → underscores); triggers reference the generated columns +// explicitly, so no name correspondence with the content table is needed. +// Failure degrades to LIKE-based search (resolver skips the FTS plan). +func ensureFTS(db *sql.DB, s *tableSchema) { + if len(s.FTSFields) == 0 { + return + } + fts := ftsTableName(s.Name) + + // Only backfill rows that predate the FTS table; probing for existence + // first avoids the full anti-join scan on every boot (the backfill + // SELECT runs a content-table scan otherwise). + existed := ftsAvailable(db, s.Name) + + var cols []string + for _, f := range s.FTSFields { + cols = append(cols, quoteIdent(f.Column)) + } + ddl := fmt.Sprintf("CREATE VIRTUAL TABLE IF NOT EXISTS [%s] USING fts5(%s, content='%s', content_rowid='rowid')", + fts, strings.Join(cols, ", "), s.Name) + if _, err := db.Exec(ddl); err != nil { + log.Warnf("sqlite FTS5 table %s: %v (falling back to LIKE search)", fts, err) + return + } + if global.Env().IsDebug { + log.Debug("sqlite FTS5: ", ddl) + } + if existed { + return // triggers are IF NOT EXISTS; no new rows to backfill + } + + var colList, newCols, oldCols, plainCols string + for i, f := range s.FTSFields { + if i > 0 { + colList += ", " + newCols += ", " + oldCols += ", " + plainCols += ", " + } + colList += quoteIdent(f.Column) + newCols += fmt.Sprintf("new.%s", quoteIdent(f.Path)) + oldCols += fmt.Sprintf("old.%s", quoteIdent(f.Path)) + plainCols += quoteIdent(f.Path) // the generated columns themselves + } + + statements := []string{ + // AI: index the new row's text columns. + fmt.Sprintf("CREATE TRIGGER IF NOT EXISTS [%[1]s_ai] AFTER INSERT ON [%[2]s] BEGIN INSERT INTO [%[1]s](rowid, %[3]s) VALUES (new.rowid, %[4]s); END", + fts, s.Name, colList, newCols), + // AU: FTS5 external-content delete of the old values, then insert + // of the new ones ('delete' is a command row, not a real insert). + fmt.Sprintf("CREATE TRIGGER IF NOT EXISTS [%[1]s_au] AFTER UPDATE ON [%[2]s] BEGIN INSERT INTO [%[1]s]([%[1]s], rowid, %[3]s) VALUES('delete', old.rowid, %[4]s); INSERT INTO [%[1]s](rowid, %[3]s) VALUES (new.rowid, %[5]s); END", + fts, s.Name, colList, oldCols, newCols), + // AD: remove the deleted row's entries. + fmt.Sprintf("CREATE TRIGGER IF NOT EXISTS [%[1]s_ad] AFTER DELETE ON [%[2]s] BEGIN INSERT INTO [%[1]s]([%[1]s], rowid, %[3]s) VALUES('delete', old.rowid, %[4]s); END", + fts, s.Name, colList, oldCols), + } + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + log.Warnf("sqlite FTS5 trigger on %s: %v", s.Name, err) + return + } + } + + // Backfill rows inserted before the FTS table existed. The automatic + // 'rebuild' command relies on column-name correspondence with the + // content table, which sanitized names break — populate explicitly + // from the generated columns themselves. + backfill := fmt.Sprintf("INSERT INTO [%s](rowid, %s) SELECT rowid, %s FROM [%s] WHERE rowid NOT IN (SELECT rowid FROM [%s])", + fts, colList, plainCols, s.Name, fts) + if _, err := db.Exec(backfill); err != nil { + log.Warnf("sqlite FTS5 backfill for %s: %v", s.Name, err) + } else if global.Env().IsDebug { + log.Debug("sqlite FTS5 backfill: ", backfill) + } +} + +// ftsAvailable reports whether the FTS table exists (LIKE fallback decides +// on this at query time; a missing table means ensureFTS degraded). +func ftsAvailable(db *sql.DB, table string) bool { + fts := ftsTableName(table) + var name string + err := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", fts).Scan(&name) + return err == nil +} + +func tableExists(db *sql.DB, table string) (bool, error) { + var name string + err := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// tableColumns returns the set of column names of an existing table, +// including VIRTUAL generated columns (hidden from pragma_table_info but +// visible in pragma_table_xinfo). +func tableColumns(db *sql.DB, table string) (map[string]bool, error) { + rows, err := db.Query("SELECT name FROM pragma_table_xinfo(?)", table) + if err != nil { + return nil, err + } + defer rows.Close() + cols := map[string]bool{} + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + cols[name] = true + } + return cols, rows.Err() +} + +// quoteIdent wraps an identifier in double quotes so dotted JSON paths +// work as column names ("basic_auth.username"). +func quoteIdent(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +// dateEpochColumn is the shadow-column name for a date path. +func dateEpochColumn(path string) string { return path + "__epoch" } + +// epochSourcePath reverses dateEpochColumn: returns the source date path +// when the column is an epoch shadow. +func epochSourcePath(col string) (string, bool) { + if strings.HasSuffix(col, "__epoch") { + return strings.TrimSuffix(col, "__epoch"), true + } + return "", false +} diff --git a/modules/sqlite/indexes.go b/modules/sqlite/indexes.go index 84635941f..355f049fb 100644 --- a/modules/sqlite/indexes.go +++ b/modules/sqlite/indexes.go @@ -3,19 +3,16 @@ package sqlite import ( - "database/sql" "fmt" "reflect" "strings" - - log "github.com/cihub/seelog" - - "infini.sh/framework/core/global" ) -// indexableElasticTypes lists the elastic_mapping types that map well to a -// SQLite B-tree expression index (equality / range / sort). text is excluded -// (needs FTS5), object/binary are excluded (no scalar column expression). +// indexableElasticTypes lists the elastic_mapping types promoted to SQLite +// generated columns (equality / range / sort). text is handled separately +// via FTS5; object/binary are excluded (no scalar column expression). +// Superseded as an index strategy by flattened.go, kept as the shared +// mapping-type vocabulary. var indexableElasticTypes = map[string]bool{ "keyword": true, "date": true, @@ -26,97 +23,6 @@ var indexableElasticTypes = map[string]bool{ "float": true, } -// createExpressionIndexes walks the model struct — recursing into nested -// object structs — and creates a SQLite expression index for every -// elastic_mapping field whose mapping type is B-tree-friendly. -// -// The index expression mirrors what the ORM's query builder emits -// (json_extract(raw,'$.field') for top-level fields and -// json_extract(raw,'$.parent.child') for nested ones), so existing -// TermQuery / Range / Sort clauses transparently use the index — including -// those on nested object fields — with no query changes required. SQLite -// supports indexes on expressions since 3.9 (2015); the modernc driver -// (pure-Go upstream SQLite) satisfies this. -// -// This is the key optimization that turns the JSON-blob storage model from -// full-table scans into index lookups: without it every WHERE/ORDER BY -// compiles to json_extract applied per-row. -func createExpressionIndexes(db *sql.DB, tableName string, model interface{}) { - t := reflect.TypeOf(model) - for t != nil && t.Kind() == reflect.Ptr { - t = t.Elem() - } - if t == nil || t.Kind() != reflect.Struct { - return - } - tableSafe := sanitizeForIndexName(tableName) - walkMappingIndexes(db, tableName, tableSafe, t, "") -} - -// walkMappingIndexes recurses through the struct's fields. jsonPathPrefix is -// the dotted JSON path of the enclosing object ("" at the top level); each -// indexed field's path is prefix+"."+jsonField, matching what the query -// builder emits and what json_extract expects. -func walkMappingIndexes(db *sql.DB, tableName, tableSafe string, t reflect.Type, jsonPathPrefix string) { - for i := 0; i < t.NumField(); i++ { - field := t.Field(i) - tag := strings.TrimSpace(field.Tag.Get("elastic_mapping")) - if tag == "-" { - continue - } - - // An anonymous embedded struct with no mapping tag of its own has its - // fields JSON-promoted to this level (e.g. orm.ORMObjectBase), so - // recurse at the same path rather than introducing a path segment. - if field.Anonymous && tag == "" { - if ft := objectStructType(field.Type); ft != nil { - walkMappingIndexes(db, tableName, tableSafe, ft, jsonPathPrefix) - } - continue - } - if tag == "" { - continue - } - if mappingDisabled(tag) { - continue - } - - jsonField, esType, ok := parseMappingTag(tag) - if jsonField == "" { - continue - } - path := jsonField - if jsonPathPrefix != "" { - path = jsonPathPrefix + "." + jsonField - } - - if ok && indexableElasticTypes[esType] { - createOneExpressionIndex(db, tableName, tableSafe, path) - } - - // Recurse into nested object structs so their scalar leaves get dotted - // indexes ($.parent.child). Slices/arrays/maps are intentionally NOT - // recursed: json_extract cannot address fields inside a JSON array via - // a dotted path, and map values have no declared struct tags to index. - if ft := objectStructType(field.Type); ft != nil { - walkMappingIndexes(db, tableName, tableSafe, ft, path) - } - } -} - -// createOneExpressionIndex issues CREATE INDEX for a single dotted JSON path. -func createOneExpressionIndex(db *sql.DB, tableName, tableSafe, jsonPath string) { - idxName := indexNameFor(tableSafe, jsonPath) - ddl := fmt.Sprintf(`CREATE INDEX IF NOT EXISTS [%s] ON [%s](json_extract(raw, '$.%s'))`, - idxName, tableName, jsonPath) - if _, err := db.Exec(ddl); err != nil { - // Non-fatal: a bad index shouldn't block schema registration. - log.Warnf("sqlite expression index %s on %s: %v", idxName, tableName, err) - } else if global.Env().IsDebug { - log.Debug("sqlite expression index: ", ddl) - } -} - // indexNameFor builds a SQLite-safe index name from the table and the dotted // JSON path, e.g. "ix_host_cpu_info_model". Dots and dashes become underscores. func indexNameFor(tableSafe, jsonPath string) string { @@ -209,7 +115,8 @@ func objectStructType(t reflect.Type) reflect.Type { } // sanitizeForIndexName replaces characters that are awkward in SQLite -// identifier names (table names like "logpilot-patterns" contain dashes). +// identifier names: dashes in table names ("logpilot-patterns") and dots in +// dotted JSON paths ("basic_auth.username") used for index/FTS column names. func sanitizeForIndexName(s string) string { - return strings.ReplaceAll(s, "-", "_") + return strings.NewReplacer("-", "_", ".", "_").Replace(s) } diff --git a/modules/sqlite/indexes_test.go b/modules/sqlite/indexes_test.go index 3cda9b8d7..2a2e1f0fd 100644 --- a/modules/sqlite/indexes_test.go +++ b/modules/sqlite/indexes_test.go @@ -12,28 +12,30 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "infini.sh/framework/core/elastic" "infini.sh/framework/core/orm" + sqliteOrm "infini.sh/framework/modules/sqlite/orm" ) // indexNestedSpec is a nested object struct whose scalar fields should get -// dotted-path indexes ($.cpu.model, $.cpu.physical_cpu). +// dotted-path generated columns ($.cpu.model, $.cpu.physical_cpu). type indexNestedSpec struct { Model string `json:"model,omitempty" elastic_mapping:"model: { type: keyword }"` PhysicalCPU int `json:"physical_cpu,omitempty" elastic_mapping:"physical_cpu: { type: integer }"` - // text is not B-tree-indexable. + // text goes to the FTS table, not a scalar column. Description string `json:"description,omitempty" elastic_mapping:"description: { type: text }"` } -// indexSliceItem is the element of a nested array; its fields must NOT get an -// index because json_extract cannot address fields inside a JSON array. +// indexSliceItem is the element of a nested array; its fields must NOT be +// promoted because json_extract cannot address fields inside a JSON array. type indexSliceItem struct { Key string `json:"key,omitempty" elastic_mapping:"key: { type: keyword }"` } -// indexRootModel exercises every branch of createExpressionIndexes: promoted -// ORMObjectBase fields, a top-level keyword, a nested struct object, a nested -// pointer-to-struct object, a slice of scalars, a slice of structs (nested), -// and an enabled:false object backed by a map. +// indexRootModel exercises every branch of the field walker: promoted +// ORMObjectBase fields, a top-level keyword, a nested struct object, a +// nested pointer-to-struct object, a slice of scalars, a slice of structs +// (nested), and an enabled:false object backed by a map. type indexRootModel struct { orm.ORMObjectBase Name string `json:"name,omitempty" elastic_mapping:"name: { type: keyword }"` @@ -69,46 +71,66 @@ func sqliteIndexDDL(t *testing.T, db *sql.DB, table string) map[string]string { return out } -func TestCreateExpressionIndexes_NestedObjects(t *testing.T) { +// columnIndexName is the flattened-scheme index name for a JSON path. +func columnIndexName(table, path string) string { + return "ixc_" + indexNameFor(table, path)[3:] +} + +func TestFlattened_NestedObjects(t *testing.T) { handler, cleanup := openIndexTestDB(t) defer cleanup() + + cols, err := tableColumns(handler.DB, "index_root") + require.NoError(t, err) idx := sqliteIndexDDL(t, handler.DB, "index_root") - // Each expected path must have an index whose DDL targets exactly that - // json_extract path (the expression must match the query builder verbatim - // for SQLite to use it). + // Every promoted scalar leaf (including dotted nested paths) becomes a + // generated column plus a plain column index. expectedPaths := []string{ - "id", "created", "updated", // promoted from orm.ORMObjectBase + "created", "updated", // promoted from orm.ORMObjectBase "name", "cpu.model", "cpu.physical_cpu", "disk.model", "disk.physical_cpu", "tags", } for _, p := range expectedPaths { - name := indexNameFor("index_root", p) - ddl, ok := idx[name] - require.Truef(t, ok, "expected index %s for path $.%s to exist", name, p) - wantExpr := fmt.Sprintf("json_extract(raw, '$.%s')", p) - assert.Containsf(t, ddl, wantExpr, "index %s DDL should target %s", name, wantExpr) + assert.Truef(t, cols[p], "expected generated column %q to exist", p) + idxName := columnIndexName("index_root", p) + ddl, ok := idx[idxName] + require.Truef(t, ok, "expected column index %s for path $.%s", idxName, p) + want := `"` + p + `"` + assert.Containsf(t, ddl, want, "index %s should target column %q", idxName, p) } - // Explicitly NOT indexed: text leaf, nested-array element, enabled:false - // subtree, the object/map roots themselves, and the _system map. + // Text leaves DO get generated columns (FTS trigger sources) but no + // B-tree index; array elements, enabled:false subtrees, object roots and + // the _system map get neither. + for _, p := range []string{ + "cpu.description", "disk.description", + } { + assert.Truef(t, cols[p], "text leaf %q should have a generated column (FTS source)", p) + _, has := idx[columnIndexName("index_root", p)] + assert.Falsef(t, has, "text leaf %q must not get a B-tree index", p) + } for _, p := range []string{ - "cpu.description", "items.key", "secret", "cpu", "disk", "items", "_system", } { - _, present := idx[indexNameFor("index_root", p)] - assert.Falsef(t, present, "no index should exist for $.%s", p) + assert.Falsef(t, cols[p], "no generated column should exist for $.%s", p) } + + // Text leaves sync into the FTS table under sanitized column names. + ftsCols, err := tableColumns(handler.DB, "fts_index_root") + require.NoError(t, err) + assert.True(t, ftsCols["cpu_description"], "text leaf should join the FTS table") + assert.True(t, ftsCols["disk_description"], "pointer text leaf should join the FTS table") } -// TestCreateExpressionIndexes_NestedQueryUsesIndex proves a query on a nested -// field actually picks up the dotted expression index (no full-table scan). -func TestCreateExpressionIndexes_NestedQueryUsesIndex(t *testing.T) { +// TestFlattened_NestedQueryUsesIndex proves a query translated through the +// resolver on a nested field hits the column index (no full-table scan). +func TestFlattened_NestedQueryUsesIndex(t *testing.T) { handler, cleanup := openIndexTestDB(t) defer cleanup() @@ -120,36 +142,98 @@ func TestCreateExpressionIndexes_NestedQueryUsesIndex(t *testing.T) { require.NoError(t, err) } + schema := lookupTableSchema("index_root") + require.NotNil(t, schema) + resolver := schema.resolver() + cases := []struct { - name string - path string // json_extract path used in WHERE - want string // index name expected in the plan + name string + path string + value string + want int }{ - {"nested keyword", "cpu.model", indexNameFor("index_root", "cpu.model")}, - {"nested pointer keyword", "disk.model", indexNameFor("index_root", "disk.model")}, - {"top-level keyword regression", "name", indexNameFor("index_root", "name")}, + {"nested keyword", "cpu.model", "m3", 10}, + {"nested pointer keyword", "disk.model", "d3", 10}, + {"top-level keyword regression", "name", "n3", 1}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - q := fmt.Sprintf("EXPLAIN QUERY PLAN SELECT id FROM index_root WHERE json_extract(raw, '$.%s') = 'm3'", tc.path) - detail := queryPlanDetail(t, handler.DB, q) + qb := orm.NewQuery().Filter(orm.TermQuery(tc.path, tc.value)) + qb.Build() + where, args := sqliteOrm.BuildWhereClause(qb, resolver) + require.NotEmpty(t, where) + + want := columnIndexName("index_root", tc.path) + q := "EXPLAIN QUERY PLAN SELECT id FROM index_root WHERE " + where + detail := queryPlanDetail(t, handler.DB, q, args...) joined := strings.Join(detail, "\n") - assert.Containsf(t, joined, tc.want, "plan should use index %s; got:\n%s", tc.want, joined) + assert.Containsf(t, joined, want, "plan should use index %s; got:\n%s", want, joined) + + // The clause must actually match rows via the generated column. + var n int + require.NoError(t, handler.DB.QueryRow("SELECT COUNT(*) FROM index_root WHERE "+where, args...).Scan(&n)) + assert.Equal(t, tc.want, n) }) } - // A non-indexed field (text) must NOT claim an index — it scans. - t.Run("text field scans", func(t *testing.T) { - q := "EXPLAIN QUERY PLAN SELECT id FROM index_root WHERE json_extract(raw, '$.cpu.description') = 'x'" - detail := queryPlanDetail(t, handler.DB, q) - joined := strings.Join(detail, "\n") - assert.NotContains(t, joined, "USING INDEX", "text field should not use an index; got:\n%s", joined) + // An unmapped path falls back to json_extract (correct, unindexed). + t.Run("unmapped path falls back", func(t *testing.T) { + expr, _, fts := resolver("items.key") + assert.Nil(t, fts) + assert.Equal(t, "json_extract(raw, '$.items.key')", expr) }) + + // The id path maps to the primary key column, not a generated column. + t.Run("id path maps to pk", func(t *testing.T) { + expr, _, fts := resolver("id") + assert.Nil(t, fts) + assert.Equal(t, `"id"`, expr) + }) +} + +// TestFlattened_MigratesLegacyTable proves an old-layout table (id+raw only, +// pre-generated-columns) is rebuilt in place with data preserved. +func TestFlattened_MigratesLegacyTable(t *testing.T) { + tmpDir := t.TempDir() + handler := &SQLiteORM{Config: SQLiteConfig{Enabled: true, DBPath: filepath.Join(tmpDir, "test.db")}} + require.NoError(t, handler.Open()) + defer handler.Close() + + // Create the legacy layout directly and seed rows. + _, err := handler.DB.Exec(`CREATE TABLE legacy_items (id TEXT PRIMARY KEY, raw JSON NOT NULL)`) + require.NoError(t, err) + for i := 0; i < 5; i++ { + raw := fmt.Sprintf(`{"id":"L%d","status":"st%d","name":"legacy-%d"}`, i, i%2, i) + _, err := handler.DB.Exec("INSERT INTO legacy_items (id, raw) VALUES (?, ?)", fmt.Sprintf("L%d", i), raw) + require.NoError(t, err) + } + + // Registering the model migrates the table. + require.NoError(t, handler.RegisterSchemaWithName(TestItem{}, "legacy_items")) + + cols, err := tableColumns(handler.DB, "legacy_items") + require.NoError(t, err) + assert.True(t, cols["status"], "status should be promoted after migration") + assert.True(t, cols["name"], "name should be promoted after migration") + + var count int + require.NoError(t, handler.DB.QueryRow("SELECT COUNT(*) FROM legacy_items").Scan(&count)) + assert.Equal(t, 5, count, "rows must survive the migration") + + // Filtered search works on the migrated column. + qb := orm.NewQuery().Filter(orm.TermQuery("status", "st1")) + ctx := orm.NewContext() + orm.WithModel(ctx, &TestItem{}) + res, err := handler.SearchV2(ctx, qb) + require.NoError(t, err) + items, _, err := elastic.DecodeHits[TestItem](res) + require.NoError(t, err) + assert.Len(t, items, 2) } -func queryPlanDetail(t *testing.T, db *sql.DB, query string) []string { +func queryPlanDetail(t *testing.T, db *sql.DB, query string, args ...interface{}) []string { t.Helper() - rows, err := db.Query(query) + rows, err := db.Query(query, args...) require.NoError(t, err) defer rows.Close() var out []string diff --git a/modules/sqlite/module.go b/modules/sqlite/module.go index fe5fe3c69..fa03b9434 100644 --- a/modules/sqlite/module.go +++ b/modules/sqlite/module.go @@ -25,6 +25,7 @@ package sqlite import ( "path/filepath" + "time" log "github.com/cihub/seelog" "infini.sh/framework/core/env" @@ -74,6 +75,25 @@ func (module *SQLiteModule) Start() error { if err := orm.InitSchema(); err != nil { return err } + + // Keep planner statistics fresh. Registration optimizes EMPTY + // tables; data arrives at runtime and without sqlite_stat1 the + // planner over/under-estimates and may skip covering composites + // (measured: 1.18s vs 0.76s per dashboard aggregation at 500k rows). + // PRAGMA optimize is near-free when nothing changed — the SQLite + // recommended cadence is "every few hours and after bulk changes". + task := global.BackgroundTask{} + task.Tag = "sqlite_optimize" + task.Func = func() { + if _, err := handler.DB.Exec("PRAGMA optimize"); err != nil { + log.Warnf("sqlite periodic PRAGMA optimize: %v", err) + } + } + task.Interval = 10 * time.Minute + task.InitialDelay = 10 * time.Minute + + // statsLoop refreshes planner statistics periodically. + global.RegisterBackgroundCallback(&task) } return nil diff --git a/modules/sqlite/orm.go b/modules/sqlite/orm.go index 7d1598c83..0926949a9 100644 --- a/modules/sqlite/orm.go +++ b/modules/sqlite/orm.go @@ -32,6 +32,7 @@ import ( "strings" log "github.com/cihub/seelog" + "infini.sh/framework/core/aggregate" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" api "infini.sh/framework/core/orm" @@ -48,6 +49,11 @@ type SQLiteORM struct { DB *sql.DB } +// defaultCacheSize is the per-connection page cache in KiB (negative = +// KiB units per SQLite convention). 64 MiB comfortably holds hot indexes +// for metadata-scale stores. +const defaultCacheSizeKB = -65536 + // defaultMmapSize caps how much of the database file SQLite serves via // memory-mapped I/O instead of pread syscalls. 256 MiB comfortably covers the // metadata store and leaves headroom; only virtual address space is reserved, @@ -67,8 +73,9 @@ func (handler *SQLiteORM) Open() error { // a single db.Exec only configures the one connection the pool happens to // use for that call, leaving the rest on defaults. dsn := fmt.Sprintf( - "file:%s?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)&_pragma=mmap_size(%d)", - handler.Config.DBPath, defaultMmapSize, + "file:%s?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)&_pragma=mmap_size(%d)"+ + "&_pragma=synchronous(NORMAL)&_pragma=temp_store(MEMORY)&_pragma=cache_size(%d)", + handler.Config.DBPath, defaultMmapSize, defaultCacheSizeKB, ) db, err := sql.Open("sqlite", dsn) if err != nil { @@ -95,24 +102,25 @@ func (handler *SQLiteORM) GetIndexName(o interface{}) string { return getTableName(o) } -// RegisterSchemaWithName creates the table for the given struct type if it does not exist. +// RegisterSchemaWithName creates the table for the given struct type if it +// does not exist, promoting mapped scalar leaves to VIRTUAL generated +// columns with plain B-tree indexes (plus FTS5 for text leaves). Existing +// tables from the pre-flattening layout are migrated in place. func (handler *SQLiteORM) RegisterSchemaWithName(t interface{}, indexName string) error { initTableName(t, indexName) tableName := handler.GetIndexName(t) - ddl := fmt.Sprintf("CREATE TABLE IF NOT EXISTS [%s] (id TEXT PRIMARY KEY, raw JSON NOT NULL)", tableName) - - if global.Env().IsDebug { - log.Debug("sqlite DDL: ", ddl) + schema := buildTableSchema(tableName, t) + if err := ensureFlattenedTable(handler.DB, schema); err != nil { + return err } - - _, err := handler.DB.Exec(ddl) - if err != nil { - return fmt.Errorf("failed to create table %s: %w", tableName, err) + registerTableSchema(schema) + // Refresh planner statistics after schema work; cheap, and without + // sqlite_stat1 the planner guesses row counts and may skip the very + // indexes we just created. + if _, err := handler.DB.Exec("PRAGMA optimize"); err != nil { + log.Warnf("sqlite PRAGMA optimize after registering %s: %v", tableName, err) } - // Build expression indexes from the model's elastic_mapping tags so - // json_extract-based filters/sorts use B-trees instead of full-table scans. - createExpressionIndexes(handler.DB, tableName, t) log.Debugf("sqlite schema registered: %s", tableName) return nil } @@ -492,9 +500,19 @@ func (handler *SQLiteORM) SearchV2(ctx *api.Context, qb *api.QueryBuilder) (*api if qb != nil { qb.Build() + if len(qb.RequestBodyBytesVal()) > 0 { + // Silent-ignore is worse than a signal: callers hand-merging ES DSL + // would get unfiltered results without noticing. + log.Warn("sqlite orm: request-body DSL is not supported and was ignored") + } } - where, args := sqliteOrm.BuildWhereClause(qb) + // Resolve mapped paths to generated columns; unmapped paths fall back + // to json_extract expressions. + schema := lookupTableSchema(indexName) + resolver := schema.resolver() + + where, args := sqliteOrm.BuildWhereClause(qb, resolver) // Count total countSQL := fmt.Sprintf("SELECT COUNT(*) FROM [%s]", indexName) @@ -518,15 +536,29 @@ func (handler *SQLiteORM) SearchV2(ctx *api.Context, qb *api.QueryBuilder) (*api if sorts := qb.Sorts(); len(sorts) > 0 { var sortParts []string for _, s := range sorts { - sortParts = append(sortParts, fmt.Sprintf("json_extract(raw, '$.%s') %s", s.Field, string(s.SortType))) + expr, epochExpr, _ := resolver(s.Field) + if epochExpr != "" { + // Integer epoch orders identically to the TEXT form and + // matches epoch-bearing composite indexes (walk plans). + expr = epochExpr + } + if s.Field == "_score" { + expr = "id" // no scoring in sqlite; stable tiebreaker + } + sortParts = append(sortParts, fmt.Sprintf("%s %s", expr, string(s.SortType))) } sqlStr += " ORDER BY " + strings.Join(sortParts, ", ") } + // Pagination: OFFSET without LIMIT is invalid SQL — treat an unset + // size as "no upper bound" (LIMIT -1). if qb.SizeVal() > 0 { sqlStr += fmt.Sprintf(" LIMIT %d", qb.SizeVal()) } if qb.FromVal() > 0 { + if qb.SizeVal() <= 0 { + sqlStr += " LIMIT -1" + } sqlStr += fmt.Sprintf(" OFFSET %d", qb.FromVal()) } } @@ -574,6 +606,21 @@ func (handler *SQLiteORM) SearchV2(ctx *api.Context, qb *api.QueryBuilder) (*api }, } + // Aggregations over the same filtered set (previously silently dropped). + // Runs through the typed Aggregate machinery; converted to the ES shape + // for this legacy response so existing consumers are unaffected. + if qb != nil && len(qb.Aggs) > 0 { + exec := &aggExecutor{handler: handler, index: indexName, resolve: resolver, where: where, args: args} + nodes, err := exec.execute(qb.Aggs) + if err != nil { + return nil, err + } + if err := aggregate.ApplyPipelines(&api.AggregationResult{Aggs: nodes}, qb.Aggs); err != nil { + return nil, err + } + response["aggregations"] = typedToESShape(nodes) + } + responseBytes := util.MustToJSONBytes(response) result.Payload = responseBytes result.Status = 200 @@ -603,7 +650,7 @@ func (handler *SQLiteORM) DeleteByQuery(ctx *api.Context, qb *api.QueryBuilder) } qb.Build() - where, args := sqliteOrm.BuildWhereClause(qb) + where, args := sqliteOrm.BuildWhereClause(qb, lookupTableSchema(indexName).resolver()) // Count before delete countSQL := fmt.Sprintf("SELECT COUNT(*) FROM [%s]", indexName) @@ -731,3 +778,17 @@ func buildLegacyWhere(conds []*api.Cond) ([]string, []interface{}) { } return clauses, args } + +// Capabilities declares what the sqlite backend honors. FullText and +// Aggregations are real (FTS5, GROUP BY); fuzzy is LIKE-approximated; the +// rest warn and degrade (see query_builder.go / aggs_exec.go). +func (handler *SQLiteORM) Capabilities() api.Capabilities { + return api.Capabilities{ + FullText: true, + Aggregations: true, + Fuzzy: false, + Nested: false, + RequestBodyDSL: false, + Collapse: false, + } +} diff --git a/modules/sqlite/orm/query_builder.go b/modules/sqlite/orm/query_builder.go index 207e6e511..1b5229295 100644 --- a/modules/sqlite/orm/query_builder.go +++ b/modules/sqlite/orm/query_builder.go @@ -9,14 +9,8 @@ // // Open Source licensed under AGPL V3: // This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. +// it under the terms of the License, or (at your option) any later version. +// See the GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . @@ -26,13 +20,33 @@ package orm import ( "fmt" "strings" + "sync" + "time" + + log "github.com/cihub/seelog" "infini.sh/framework/core/orm" ) -// BuildWhereClause translates an orm.QueryBuilder into a SQL WHERE clause string -// and corresponding parameter arguments. Returns empty string if no conditions exist. -func BuildWhereClause(qb *orm.QueryBuilder) (string, []interface{}) { +// FieldResolver maps a dotted JSON path to its query-side plan: the SQL +// comparison expression, the integer-epoch shadow expression for date +// fields (range predicates on it become integer comparisons — index- and +// covering-friendly, no per-row TEXT parse), and the FTS target when the +// path is a text field synced into an FTS5 table. A nil resolver (or nil +// plan parts) falls back to json_extract for everything. +type FieldResolver func(path string) (expr, epochExpr string, fts *FTSPlan) + +// FTSPlan identifies the FTS5 table/column backing a text field. +type FTSPlan struct { + Table string + Column string +} + +// BuildWhereClause translates an orm.QueryBuilder into a SQL WHERE clause +// string and corresponding parameter arguments. Returns empty string if no +// conditions exist. The resolver (may be nil) decides per path whether the +// comparison hits a promoted generated column or a json_extract fallback. +func BuildWhereClause(qb *orm.QueryBuilder, resolve FieldResolver) (string, []interface{}) { if qb == nil { return "", nil } @@ -42,19 +56,90 @@ func BuildWhereClause(qb *orm.QueryBuilder) (string, []interface{}) { return "", nil } - where, args := clauseToSQL(root) + where, args := clauseToSQL(root, resolve) return where, args } -// clauseToSQL recursively translates a Clause tree into SQL WHERE expression. -func clauseToSQL(clause *orm.Clause) (string, []interface{}) { +// ExprFor resolves a JSON path to its comparison expression via the +// resolver (generated column when promoted, json_extract otherwise). +func ExprFor(resolve FieldResolver, path string) string { + if resolve == nil { + return fmt.Sprintf("json_extract(raw, '$.%s')", path) + } + expr, _, _ := resolve(path) + return expr +} + +// RangeExprFor picks the comparison expression for a range predicate: the +// epoch shadow when one exists (the value is rewritten to integer epoch +// seconds by the caller), else the regular expression. +func RangeExprFor(resolve FieldResolver, path string) (expr string, epoch bool) { + if resolve == nil { + return fmt.Sprintf("json_extract(raw, '$.%s')", path), false + } + plain, epochExpr, _ := resolve(path) + if epochExpr != "" { + return epochExpr, true + } + return plain, false +} + +// ToEpochSeconds converts a range value to integer epoch seconds when +// possible (RFC3339 strings, time.Time, numeric epochs). +func ToEpochSeconds(v interface{}) (int64, bool) { + switch t := v.(type) { + case time.Time: + return t.Unix(), true + case string: + if ts, err := time.Parse(time.RFC3339, t); err == nil { + return ts.Unix(), true + } + case int64: + return t, true + case int: + return int64(t), true + case float64: + return int64(t), true + } + return 0, false +} + +func exprFor(resolve FieldResolver, path string) string { + return ExprFor(resolve, path) +} + +// rangeFor builds a range predicate, rewriting date fields with an epoch +// shadow to integer comparisons (the constant is parsed once, here). +func rangeFor(resolve FieldResolver, path string, op string, value interface{}) (string, []interface{}) { + expr, isEpoch := RangeExprFor(resolve, path) + if isEpoch { + if secs, ok := ToEpochSeconds(value); ok { + return fmt.Sprintf("%s %s ?", expr, op), []interface{}{secs} + } + } + return fmt.Sprintf("%s %s ?", expr, op), []interface{}{value} +} + +// unsupportedOnce deduplicates warnings for operators sqlite cannot honor; +// per-query spam would hide the first occurrence in logs. +var unsupportedOnce sync.Map + +func warnUnsupported(op orm.QueryType) { + if _, loaded := unsupportedOnce.LoadOrStore(op, true); !loaded { + log.Warnf("sqlite orm: %s queries are not supported on this backend; "+ + "the clause matches no documents (grace period: warning only, will become an error in a future release)", op) + } +} + +// clauseToSQL recursively translates a Clause tree into a SQL WHERE expression. +func clauseToSQL(clause *orm.Clause, resolve FieldResolver) (string, []interface{}) { if clause == nil { return "", nil } // Leaf node if clause.IsLeaf() { - return leafToSQL(clause) + return leafToSQL(clause, resolve) } var parts []string @@ -62,7 +147,7 @@ func clauseToSQL(clause *orm.Clause) (string, []interface{}) { // filter and must are combined with AND for _, sub := range clause.FilterClauses { - sql, args := clauseToSQL(sub) + sql, args := clauseToSQL(sub, resolve) if sql != "" { parts = append(parts, sql) allArgs = append(allArgs, args...) @@ -70,7 +155,7 @@ func clauseToSQL(clause *orm.Clause) (string, []interface{}) { } for _, sub := range clause.MustClauses { - sql, args := clauseToSQL(sub) + sql, args := clauseToSQL(sub, resolve) if sql != "" { parts = append(parts, sql) allArgs = append(allArgs, args...) @@ -79,7 +164,7 @@ func clauseToSQL(clause *orm.Clause) (string, []interface{}) { // must_not is combined with AND NOT for _, sub := range clause.MustNotClauses { - sql, args := clauseToSQL(sub) + sql, args := clauseToSQL(sub, resolve) if sql != "" { parts = append(parts, fmt.Sprintf("NOT (%s)", sql)) allArgs = append(allArgs, args...) @@ -111,7 +196,7 @@ func clauseToSQL(clause *orm.Clause) (string, []interface{}) { if shouldRequired { var shouldParts []string for _, sub := range clause.ShouldClauses { - sql, args := clauseToSQL(sub) + sql, args := clauseToSQL(sub, resolve) if sql != "" { shouldParts = append(shouldParts, sql) allArgs = append(allArgs, args...) @@ -140,80 +225,132 @@ func clauseToSQL(clause *orm.Clause) (string, []interface{}) { return "(" + strings.Join(parts, " AND ") + ")", allArgs } +// ftsMatchSQL builds a rowid subquery against the FTS5 table. Terms are +// quoted so user input can't inject FTS query syntax. A phrase query keeps +// its words in order inside one quoted string; a plain match joins its +// words with OR — matching ES's analyzed match semantics (any term). +func ftsMatchSQL(plan *FTSPlan, phrase bool, value interface{}) (string, []interface{}) { + raw := fmt.Sprintf("%v", value) + quote := func(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` } + term := quote(raw) + if !phrase { + if words := strings.Fields(raw); len(words) > 1 { + quoted := make([]string, len(words)) + for i, w := range words { + quoted[i] = quote(w) + } + term = strings.Join(quoted, " OR ") + } + } + return fmt.Sprintf("rowid IN (SELECT rowid FROM [%s] WHERE [%s] MATCH ?)", plan.Table, plan.Table), []interface{}{term} +} + // leafToSQL converts a single leaf Clause to a SQL fragment. -func leafToSQL(clause *orm.Clause) (string, []interface{}) { +func leafToSQL(clause *orm.Clause, resolve FieldResolver) (string, []interface{}) { field := clause.Field value := clause.Value - jsonPath := fmt.Sprintf("json_extract(raw, '$.%s')", field) + var fts *FTSPlan + if resolve != nil { + _, _, fts = resolve(field) + } switch clause.Operator { - case orm.QueryMatch, orm.QueryTerm: - return fmt.Sprintf("%s = ?", jsonPath), []interface{}{value} + case orm.QuerySemantic, orm.QueryHybrid, orm.QueryNested: + // These cannot be approximated on sqlite (vector search, nested docs). + // Previously they silently compiled to a never-matching equality; now + // the same empty result carries a one-time warning. + warnUnsupported(clause.Operator) + return "1 = 0", nil + + case orm.QueryMatch: + if fts != nil { + return ftsMatchSQL(fts, false, value) + } + return fmt.Sprintf("%s = ?", exprFor(resolve, field)), []interface{}{value} + + case orm.QueryTerm: + return fmt.Sprintf("%s = ?", exprFor(resolve, field)), []interface{}{value} case orm.QueryMultiMatch: - // field = "title,category" → split into multiple fields, match with OR + // field = "title,category" → split into multiple fields, match with OR. + // FTS-backed fields use MATCH; others fall back to LIKE containment. fields := strings.Split(field, ",") var parts []string var args []interface{} for _, f := range fields { f = strings.TrimSpace(f) - jp := fmt.Sprintf("json_extract(raw, '$.%s')", f) - parts = append(parts, fmt.Sprintf("%s LIKE ?", jp)) + var fPlan *FTSPlan + if resolve != nil { + _, _, fPlan = resolve(f) + } + if fPlan != nil { + sql, a := ftsMatchSQL(fPlan, false, value) + parts = append(parts, sql) + args = append(args, a...) + continue + } + parts = append(parts, fmt.Sprintf("%s LIKE ?", exprFor(resolve, f))) args = append(args, fmt.Sprintf("%%%v%%", value)) } return "(" + strings.Join(parts, " OR ") + ")", args case orm.QueryTerms, orm.QueryIn: - return termsToSQL(jsonPath, value) + return termsToSQL(exprFor(resolve, field), value) case orm.QueryNotIn: - sql, args := termsToSQL(jsonPath, value) + sql, args := termsToSQL(exprFor(resolve, field), value) if sql != "" { return "NOT " + sql, args } return "", nil case orm.QueryPrefix: - return fmt.Sprintf("%s LIKE ?", jsonPath), []interface{}{fmt.Sprintf("%v%%", value)} + return fmt.Sprintf("%s LIKE ?", exprFor(resolve, field)), []interface{}{fmt.Sprintf("%v%%", value)} case orm.QueryWildcard: val := strings.ReplaceAll(fmt.Sprintf("%v", value), "*", "%") val = strings.ReplaceAll(val, "?", "_") - return fmt.Sprintf("%s LIKE ?", jsonPath), []interface{}{val} + return fmt.Sprintf("%s LIKE ?", exprFor(resolve, field)), []interface{}{val} case orm.QueryRegexp: // SQLite doesn't have native regexp by default; fallback to LIKE - return fmt.Sprintf("%s LIKE ?", jsonPath), []interface{}{fmt.Sprintf("%%%v%%", value)} + return fmt.Sprintf("%s LIKE ?", exprFor(resolve, field)), []interface{}{fmt.Sprintf("%%%v%%", value)} case orm.QueryExists: - return fmt.Sprintf("%s IS NOT NULL", jsonPath), nil + return fmt.Sprintf("%s IS NOT NULL", exprFor(resolve, field)), nil case orm.QueryFuzzy: - // Fuzzy search approximated with LIKE - return fmt.Sprintf("%s LIKE ?", jsonPath), []interface{}{fmt.Sprintf("%%%v%%", value)} + // Fuzzy search approximated with LIKE (fuzziness distance not honored) + return fmt.Sprintf("%s LIKE ?", exprFor(resolve, field)), []interface{}{fmt.Sprintf("%%%v%%", value)} case orm.QueryMatchPhrase: - return fmt.Sprintf("%s LIKE ?", jsonPath), []interface{}{fmt.Sprintf("%%%v%%", value)} + if fts != nil { + return ftsMatchSQL(fts, true, value) + } + return fmt.Sprintf("%s LIKE ?", exprFor(resolve, field)), []interface{}{fmt.Sprintf("%%%v%%", value)} case orm.QueryQueryString: - return fmt.Sprintf("%s LIKE ?", jsonPath), []interface{}{fmt.Sprintf("%%%v%%", value)} + if fts != nil { + return ftsMatchSQL(fts, true, value) + } + return fmt.Sprintf("%s LIKE ?", exprFor(resolve, field)), []interface{}{fmt.Sprintf("%%%v%%", value)} case orm.QueryRangeGte: - return fmt.Sprintf("%s >= ?", jsonPath), []interface{}{value} + return rangeFor(resolve, field, ">=", value) case orm.QueryRangeLte: - return fmt.Sprintf("%s <= ?", jsonPath), []interface{}{value} + return rangeFor(resolve, field, "<=", value) case orm.QueryRangeGt: - return fmt.Sprintf("%s > ?", jsonPath), []interface{}{value} + return rangeFor(resolve, field, ">", value) case orm.QueryRangeLt: - return fmt.Sprintf("%s < ?", jsonPath), []interface{}{value} + return rangeFor(resolve, field, "<", value) default: // Fallback: treat as equals - return fmt.Sprintf("%s = ?", jsonPath), []interface{}{value} + return fmt.Sprintf("%s = ?", exprFor(resolve, field)), []interface{}{value} } } diff --git a/modules/sqlite/orm/query_builder_test.go b/modules/sqlite/orm/query_builder_test.go index 295fc2b6a..be84aaab2 100644 --- a/modules/sqlite/orm/query_builder_test.go +++ b/modules/sqlite/orm/query_builder_test.go @@ -31,7 +31,7 @@ import ( ) func TestBuildWhereClause_NilQueryBuilder(t *testing.T) { - where, args := BuildWhereClause(nil) + where, args := BuildWhereClause(nil, nil) assert.Equal(t, "", where) assert.Nil(t, args) } @@ -39,7 +39,7 @@ func TestBuildWhereClause_NilQueryBuilder(t *testing.T) { func TestBuildWhereClause_EmptyQueryBuilder(t *testing.T) { qb := orm.NewQuery() qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Equal(t, "", where) assert.Nil(t, args) } @@ -48,7 +48,7 @@ func TestBuildWhereClause_SingleTermFilter(t *testing.T) { qb := orm.NewQuery() qb.Filter(orm.TermQuery("status", "active")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Equal(t, "json_extract(raw, '$.status') = ?", where) assert.Equal(t, []interface{}{"active"}, args) } @@ -58,7 +58,7 @@ func TestBuildWhereClause_MultipleFilters(t *testing.T) { qb.Filter(orm.TermQuery("status", "active")) qb.Filter(orm.TermQuery("type", "node")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "json_extract(raw, '$.status') = ?") assert.Contains(t, where, "json_extract(raw, '$.type') = ?") assert.Equal(t, 2, len(args)) @@ -69,7 +69,7 @@ func TestBuildWhereClause_RangeQuery(t *testing.T) { qb.Filter(orm.Range("age").Gte(18)) qb.Filter(orm.Range("age").Lt(65)) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "json_extract(raw, '$.age') >= ?") assert.Contains(t, where, "json_extract(raw, '$.age') < ?") assert.Equal(t, 2, len(args)) @@ -79,7 +79,7 @@ func TestBuildWhereClause_PrefixQuery(t *testing.T) { qb := orm.NewQuery() qb.Filter(orm.PrefixQuery("name", "john")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Equal(t, "json_extract(raw, '$.name') LIKE ?", where) assert.Equal(t, []interface{}{"john%"}, args) } @@ -88,7 +88,7 @@ func TestBuildWhereClause_WildcardQuery(t *testing.T) { qb := orm.NewQuery() qb.Filter(orm.WildcardQuery("name", "j*n")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Equal(t, "json_extract(raw, '$.name') LIKE ?", where) assert.Equal(t, []interface{}{"j%n"}, args) } @@ -97,7 +97,7 @@ func TestBuildWhereClause_ExistsQuery(t *testing.T) { qb := orm.NewQuery() qb.Filter(orm.ExistsQuery("email")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Equal(t, "json_extract(raw, '$.email') IS NOT NULL", where) assert.Nil(t, args) } @@ -106,7 +106,7 @@ func TestBuildWhereClause_MustNotClause(t *testing.T) { qb := orm.NewQuery() qb.Not(orm.TermQuery("status", "deleted")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "NOT") assert.Contains(t, where, "json_extract(raw, '$.status') = ?") assert.Equal(t, []interface{}{"deleted"}, args) @@ -119,7 +119,7 @@ func TestBuildWhereClause_ShouldClauses(t *testing.T) { orm.TermQuery("status", "pending"), ) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "OR") assert.Equal(t, 2, len(args)) } @@ -128,7 +128,7 @@ func TestBuildWhereClause_TermsQuery(t *testing.T) { qb := orm.NewQuery() qb.Filter(orm.TermsQuery("status", []string{"active", "pending"})) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "IN") assert.Equal(t, 2, len(args)) } @@ -137,7 +137,7 @@ func TestBuildWhereClause_MatchQuery(t *testing.T) { qb := orm.NewQuery() qb.Must(orm.MatchQuery("title", "hello")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Equal(t, "json_extract(raw, '$.title') = ?", where) assert.Equal(t, []interface{}{"hello"}, args) } @@ -159,7 +159,7 @@ func TestBuildWhereClause_MustWrappingShouldQuery(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "OR") assert.Contains(t, where, "json_extract(raw, '$._system.owner_id') = ?") assert.Contains(t, where, "IN") @@ -176,7 +176,7 @@ func TestBuildWhereClause_OwnerOnlyNoSharing(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Equal(t, "json_extract(raw, '$._system.owner_id') = ?", where) assert.Equal(t, []interface{}{"user1"}, args) } @@ -193,7 +193,7 @@ func TestBuildWhereClause_ShouldWithSharedIDs(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "OR") assert.Contains(t, where, "IN") assert.Contains(t, where, "json_extract(raw, '$._system.owner_id') = ?") @@ -212,7 +212,7 @@ func TestBuildWhereClause_CategoryFilterWithOwner(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "OR") assert.Contains(t, where, "json_extract(raw, '$.datasource_id') = ?") assert.Contains(t, where, "json_extract(raw, '$._system.owner_id') = ?") @@ -239,7 +239,7 @@ func TestBuildWhereClause_NestedBoolInsideShould(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "LIKE ?") assert.Contains(t, where, "NOT") assert.Contains(t, where, "OR") @@ -271,7 +271,7 @@ func TestBuildWhereClause_ShouldWithMustNot_FolderDenyRules(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "OR") assert.Contains(t, where, "NOT") assert.NotEmpty(t, args) @@ -286,7 +286,7 @@ func TestBuildWhereClause_DottedFieldPaths(t *testing.T) { qb := orm.NewQuery() qb.Filter(orm.TermQuery("_system.owner_id", "user1")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Equal(t, "json_extract(raw, '$._system.owner_id') = ?", where) assert.Equal(t, []interface{}{"user1"}, args) } @@ -296,7 +296,7 @@ func TestBuildWhereClause_FilterWithMustQuery(t *testing.T) { qb := orm.NewQuery() qb.Filter(orm.MustQuery(orm.TermQuery("_system.owner_id", "user1"))) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "json_extract(raw, '$._system.owner_id') = ?") assert.Equal(t, []interface{}{"user1"}, args) } @@ -331,7 +331,7 @@ func TestBuildWhereClause_FullSearchHookSimulation(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.NotEmpty(t, where) assert.NotEmpty(t, args) @@ -389,7 +389,7 @@ func TestBuildWhereClause_MultipleFolderAllowAndDenyPaths(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.NotEmpty(t, where) assert.Contains(t, where, "OR") assert.Contains(t, where, "NOT") @@ -410,7 +410,7 @@ func TestBuildWhereClause_CategoryChildrenSharing(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "OR") assert.Contains(t, where, "IN") assert.Contains(t, where, "json_extract(raw, '$._system.owner_id') = ?") @@ -424,7 +424,7 @@ func TestBuildWhereClause_EmptyShouldStaysValid(t *testing.T) { qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) // Empty boolean should be simplified away assert.Equal(t, "", where) assert.Nil(t, args) @@ -444,7 +444,7 @@ func TestBuildWhereClause_SingleShouldMinShouldMatch1_IsMandatory(t *testing.T) qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) // Single should clause must produce a direct mandatory condition (no OR wrapping) assert.Equal(t, "json_extract(raw, '$._system.owner_id') = ?", where) assert.Equal(t, []interface{}{"user1"}, args) @@ -464,7 +464,7 @@ func TestBuildWhereClause_SingleShouldWithFilterAndMinShouldMatch1(t *testing.T) qb.Must(bq) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) assert.Contains(t, where, "json_extract(raw, '$.status') = ?") assert.Contains(t, where, "json_extract(raw, '$._system.owner_id') = ?") assert.Contains(t, where, "AND") @@ -480,7 +480,7 @@ func TestBuildWhereClause_OptionalShouldWithoutMinShouldMatch(t *testing.T) { qb.Should(orm.TermQuery("priority", "high")) qb.Build() - where, args := BuildWhereClause(qb) + where, args := BuildWhereClause(qb, nil) // Only the filter should appear; should is optional without min_should_match assert.Equal(t, "json_extract(raw, '$.status') = ?", where) assert.Equal(t, []interface{}{"active"}, args) diff --git a/modules/sqlite/orm_aggregate.go b/modules/sqlite/orm_aggregate.go new file mode 100644 index 000000000..65708b673 --- /dev/null +++ b/modules/sqlite/orm_aggregate.go @@ -0,0 +1,1165 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package sqlite + +import ( + "database/sql" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + "sync" + "time" + + log "github.com/cihub/seelog" + + "infini.sh/framework/core/aggregate" + api "infini.sh/framework/core/orm" + sqliteOrm "infini.sh/framework/modules/sqlite/orm" +) + +// ────────────────────────────────────────────────────────────────────────── +// Typed aggregation execution. +// +// Aggregate computes bucket/metric aggregations natively in SQL and hands +// pipeline aggregations to the framework engine (core/aggregate), so sqlite +// and elastic behave identically. +// +// Nesting strategy (design doc §6.2): ONE level query per bucket-aggregation +// node, grouped by (ancestor keys..., own key). Rows are partitioned by the +// ancestor tuple and linked back into the parent's buckets, so the query +// count scales with the spec-tree depth — not with the number of buckets +// (the P2 implementation issued one query per bucket value). +// percentiles and top_hits are per-bucket post passes (rare, documented). +// ────────────────────────────────────────────────────────────────────────── + +// Aggregate implements orm.MetricsAPI. +func (handler *SQLiteORM) Aggregate(ctx *api.Context, qb *api.QueryBuilder) (*api.AggregationResult, error) { + if qb == nil || len(qb.Aggs) == 0 { + return nil, fmt.Errorf("no aggregations set on the query builder") + } + indexName, err := handler.resolveAggregateIndex(ctx) + if err != nil { + return nil, err + } + qb.Build() + resolver := lookupTableSchema(indexName).resolver() + where, args := sqliteOrm.BuildWhereClause(qb, resolver) + + exec := &aggExecutor{handler: handler, index: indexName, resolve: resolver, schema: lookupTableSchema(indexName), where: where, args: args} + nodes, err := exec.execute(qb.Aggs) + if err != nil { + return nil, err + } + result := &api.AggregationResult{Aggs: nodes} + if err := aggregate.ApplyPipelines(result, qb.Aggs); err != nil { + return nil, err + } + return result, nil +} + +// epochExpr returns the integer-epoch shadow expression for a date field +// when the registered schema has one. +func (e *aggExecutor) epochExpr(field string) string { + if e.schema == nil { + return "" + } + return e.schema.dateEpochByPath[field] +} + +// resolveAggregateIndex resolves the target table like SearchV2. +func (handler *SQLiteORM) resolveAggregateIndex(ctx *api.Context) (string, error) { + var indexName string + if ctx != nil { + if indices := api.GetIndices(ctx); len(indices) > 0 { + indexName = indices[0] + } + if indexName == "" { + if pattern := api.GetIndexPattern(ctx); pattern != "" { + indexName = pattern + } + } + if indexName == "" { + if model := api.GetModel(ctx); model != nil { + indexName = handler.GetIndexName(model) + } + } + } + if indexName == "" { + return "", fmt.Errorf("cannot resolve table name from context") + } + return indexName, nil +} + +// ────────────────────────────────────────────────────────────────────────── +// Executor +// ────────────────────────────────────────────────────────────────────────── + +type aggExecutor struct { + handler *SQLiteORM + index string + resolve sqliteOrm.FieldResolver + schema *tableSchema // nil-safe: resolver falls back to json_extract + where string + args []interface{} +} + +// bucketRow is one row of a level query. +type bucketRow struct { + ancestors []interface{} // values of the ancestor group keys + key interface{} // this level's group key + docCount int64 + metrics []float64 // parallel to the level's metric specs +} + +// namedXxx pair an aggregation name with its spec for deterministic order. +type namedMetric struct { + name string + agg *api.MetricAggregation +} +type namedAgg struct { + name string + agg api.Aggregation +} + +type scopeChildren struct { + metrics []namedMetric // computed in the level query + buckets []namedAgg // recursive level queries + post []namedAgg // percentiles / top_hits: per-bucket post passes +} + +// splitChildren separates a bucket agg's nested scope by execution strategy. +func splitChildren(spec map[string]api.Aggregation) scopeChildren { + var c scopeChildren + for name, sub := range spec { + switch sub.(type) { + case *api.MetricAggregation: + c.metrics = append(c.metrics, namedMetric{name, sub.(*api.MetricAggregation)}) + case *api.TermsAggregation, *api.DateHistogramAggregation, *api.AutoDateHistogramAggregation: + c.buckets = append(c.buckets, namedAgg{name, sub}) + case *api.PercentilesAggregation, *api.TopHitsAggregation: + c.post = append(c.post, namedAgg{name, sub}) + } + } + sort.Slice(c.metrics, func(i, j int) bool { return c.metrics[i].name < c.metrics[j].name }) + sort.Slice(c.buckets, func(i, j int) bool { return c.buckets[i].name < c.buckets[j].name }) + sort.Slice(c.post, func(i, j int) bool { return c.post[i].name < c.post[j].name }) + return c +} + +// execute runs a top-level scope. +func (e *aggExecutor) execute(spec map[string]api.Aggregation) (map[string]*api.AggNode, error) { + out := map[string]*api.AggNode{} + for name, agg := range spec { + node, err := e.execOne(name, agg, nil, nil) + if err != nil { + return nil, err + } + if node == nil { + // Empty result sets leave the partition map unpopulated; callers + // expect a usable node (empty buckets), never nil. + node = &api.AggNode{Buckets: []api.Bucket{}} + } + out[name] = node + } + return out, nil +} + +// execOne executes one named aggregation. ancestors are the ancestor group +// expressions; scope pins ancestor VALUES (parent tuple) — either ancestors +// (level query) or scope (post pass), not both. +func (e *aggExecutor) execOne(name string, agg api.Aggregation, ancestors []string, scope *extraScope) (*api.AggNode, error) { + switch a := agg.(type) { + case *api.TermsAggregation: + parts, err := e.execBucketPartitioned(a, ancestors, termsKeyPlan{field: a.Field, size: a.Size}) + if err != nil { + return nil, err + } + return parts[""], nil // may be nil on an empty set; execute() nil-guards + + case *api.DateHistogramAggregation: + interval := a.Interval + if interval == "" && a.IntervalField != "" { + interval = a.IntervalField + } + format, ok := histogramFormatFor(interval) + if !ok { + warnAggUnsupported("date_histogram with interval " + interval) + return &api.AggNode{Buckets: []api.Bucket{}}, nil + } + parts, err := e.execBucketPartitioned(a, ancestors, dateKeyPlan{field: a.Field, format: format, interval: interval, offset: a.Offset, epochExpr: e.epochExpr(a.Field)}) + if err != nil { + return nil, err + } + return parts[""], nil + + case *api.AutoDateHistogramAggregation: + dh, err := e.autoIntervalDH(a) + if err != nil { + return nil, err + } + return e.execOne(name, dh, ancestors, scope) + + case *api.MetricAggregation: + v, err := e.topMetric(a) + if err != nil { + return nil, err + } + node := &api.AggNode{} + if v != nil { + node.Value = *v + node.ValueSet = true + } + return node, nil + + case *api.PercentilesAggregation: + vals, err := e.percentiles(a, scope) + if err != nil { + return nil, err + } + return &api.AggNode{Values: vals}, nil + + case *api.TopHitsAggregation: + return e.topHits(a, scope) + + case *api.DateRangeAggregation: + return e.dateRange(a) + + case *api.FilterAggregation: + return e.filterAgg(a) + + case *api.SamplerAggregation: + warnAggUnsupported("sampler (falling back to full data)") + return e.singleBucketScope(a.GetNested()) + + default: + // Pipelines get placeholder nodes; the framework engine fills them. + if isPipelineAgg(agg) { + return &api.AggNode{}, nil + } + warnAggUnsupported(fmt.Sprintf("%T", agg)) + return &api.AggNode{}, nil + } +} + +func isPipelineAgg(agg api.Aggregation) bool { + switch agg.(type) { + case *api.PipelineAggregation, *api.DerivativeAggregation, + *api.BucketScriptAggregation, *api.BucketSortAggregation, *api.MaxBucketAggregation: + return true + } + return false +} + +// ────────────────────────────────────────────────────────────────────────── +// Bucket level machinery (one query per bucket-agg node) +// ────────────────────────────────────────────────────────────────────────── + +// keyPlan produces the group-key expression and bucket identity for a level. +type keyPlan interface { + keyExpr(resolve sqliteOrm.FieldResolver) string + // bucketIdentity renders the row key into the typed Bucket fields. + bucketIdentity(key interface{}) (keyStr string, keyRaw interface{}) + // order/fill behavior + postProcess(buckets []api.Bucket) []api.Bucket +} + +type termsKeyPlan struct { + field string + size int +} + +func (p termsKeyPlan) keyExpr(r sqliteOrm.FieldResolver) string { return sqliteOrm.ExprFor(r, p.field) } +func (p termsKeyPlan) bucketIdentity(key interface{}) (string, interface{}) { + return fmt.Sprintf("%v", key), key +} + +// postProcess applies ES terms semantics: order by doc_count desc with +// key-ascending tie-break, then truncate to size. +func (p termsKeyPlan) postProcess(buckets []api.Bucket) []api.Bucket { + sort.SliceStable(buckets, func(i, j int) bool { + if buckets[i].DocCount != buckets[j].DocCount { + return buckets[i].DocCount > buckets[j].DocCount + } + return buckets[i].Key < buckets[j].Key + }) + if p.size > 0 && len(buckets) > p.size { + buckets = buckets[:p.size] + } + return buckets +} + +type dateKeyPlan struct { + field string + format string + interval string + offset time.Duration // ES date_histogram offset: bucket = floor((t-offset)/interval) + step time.Duration // resolved interval step (0 → derive from interval string) + epochExpr string // integer-epoch shadow column expression ("" → strftime fallback) +} + +// keyExpr buckets by integer division of the epoch seconds: +// +// (CAST(strftime('%%s', expr) AS INTEGER) - offsetSecs) / intervalSecs +// +// ONE strftime per row (the nested-modifier variant cost two string parses +// per row and only supported fixed formats); epoch division also handles +// arbitrary interval lengths (6h, 90m, ...). The returned key is the bucket +// INDEX; bucketIdentity renders it back to start-epoch/key-string. +func (p dateKeyPlan) keyExpr(r sqliteOrm.FieldResolver) string { + step := p.intervalStep() + if p.epochExpr != "" { + // Integer arithmetic on the materialized epoch shadow — no per-row + // RFC3339 parse. + return fmt.Sprintf("((%s - %d) / %d)", p.epochExpr, int(p.offset.Seconds()), int(step.Seconds())) + } + expr := sqliteOrm.ExprFor(r, p.field) + return fmt.Sprintf("((CAST(strftime('%%s', %s) AS INTEGER) - %d) / %d)", + expr, int(p.offset.Seconds()), int(step.Seconds())) +} + +func (p dateKeyPlan) intervalStep() time.Duration { + if p.step > 0 { + return p.step + } + switch strings.TrimSpace(p.interval) { + case "1m", "1minute", "minute": + return time.Minute + case "1h", "1hour", "hour": + return time.Hour + case "1d", "1day", "day": + return 24 * time.Hour + case "1w", "1week", "week": + return 7 * 24 * time.Hour + case "1M", "1month", "month": + return 30 * 24 * time.Hour // approximate; month buckets via 30d steps + } + return time.Hour +} + +// bucketIdentity renders a bucket index back to its start instant: the grid +// point plus the offset (ES offset semantics), formatted with the layout. +func (p dateKeyPlan) bucketIdentity(key interface{}) (string, interface{}) { + idx, ok := toInt64(key) + if !ok { + return fmt.Sprintf("%v", key), key + } + step := p.intervalStep() + start := time.Unix(idx*int64(step.Seconds())+int64(p.offset.Seconds()), 0).UTC() + layout := timeLayouts[p.format] + if layout == "" { + layout = "2006-01-02T15:04:05" + } + return start.Format(layout), start.UnixMilli() +} + +func toInt64(v interface{}) (int64, bool) { + switch n := v.(type) { + case int64: + return n, true + case int: + return int64(n), true + case float64: + return int64(n), true + } + return 0, false +} + +func (p dateKeyPlan) postProcess(buckets []api.Bucket) []api.Bucket { + sortBucketsByTime(buckets) + return zeroFill(buckets, p.interval, p.format) +} + +// execBucketPartitioned runs ONE level query for the bucket agg, grouped by +// (ancestors..., key), partitions rows by ancestor tuple, assembles each +// partition's buckets (metrics/sort/truncate/fill), recursively executes +// bucket children (their level queries group by this level too), and runs +// post-pass children per bucket. Returns nodes keyed by ancestor tuple. +func (e *aggExecutor) execBucketPartitioned(agg api.Aggregation, ancestors []string, plan keyPlan) (map[string]*api.AggNode, error) { + children := splitChildren(agg.GetNested()) + rows, err := e.levelQuery(plan, ancestors, children.metrics) + if err != nil { + return nil, err + } + + // Partition rows by ancestor tuple. + parts := map[string][]bucketRow{} + for _, r := range rows { + tk := tupleKey(r.ancestors) + parts[tk] = append(parts[tk], r) + } + + keyExpr := plan.keyExpr(e.resolve) + childAncestors := append(append([]string{}, ancestors...), keyExpr) + + out := map[string]*api.AggNode{} + for tk, prows := range parts { + node := &api.AggNode{Buckets: []api.Bucket{}} + // Tuple and source row per bucket key, so linkage and post-pass + // scoping survive postProcess reordering/truncation. + tupleByKey := make(map[string]string, len(prows)) + rowByKey := make(map[string]bucketRow, len(prows)) + for _, r := range prows { + keyStr, keyRaw := plan.bucketIdentity(r.key) + bucket := api.Bucket{Key: keyStr, KeyRaw: keyRaw, DocCount: r.docCount, Aggs: map[string]*api.AggNode{}} + attachMetrics(bucket.Aggs, children.metrics, r.metrics) + node.Buckets = append(node.Buckets, bucket) + // Child partition keys are tupleKey over the child's ancestors — + // this row's ancestors plus its own key. + tupleByKey[valueKey(keyRaw)] = tupleKey(append(append([]interface{}{}, r.ancestors...), r.key)) + rowByKey[valueKey(keyRaw)] = r + } + node.Buckets = plan.postProcess(node.Buckets) + fillEmptyBucketMetrics(node.Buckets, children.metrics) + + // Tuple/source-row lookup per surviving bucket; zero-filled buckets + // have no source row and get empty child nodes. + tupleOf := func(b api.Bucket) (string, bool) { + t, ok := tupleByKey[valueKey(b.KeyRaw)] + return t, ok + } + + // Bucket children: one level query each, partitioned by child ancestors. + for _, child := range children.buckets { + childParts, err := e.execOnePartitioned(child, childAncestors) + if err != nil { + return nil, err + } + for i := range node.Buckets { + tk2, ok := tupleOf(node.Buckets[i]) + if !ok { + node.Buckets[i].Aggs[child.name] = &api.AggNode{Buckets: []api.Bucket{}} + continue + } + if cn, ok := childParts[tk2]; ok { + node.Buckets[i].Aggs[child.name] = cn + } else { + node.Buckets[i].Aggs[child.name] = &api.AggNode{Buckets: []api.Bucket{}} + } + } + } + + // Post-pass children per bucket (scoped single-bucket queries). + for i := range node.Buckets { + row, ok := rowByKey[valueKey(node.Buckets[i].KeyRaw)] + if !ok { + continue + } + scoped := &extraScope{exprs: childAncestors, values: tupleValues(row)} + for _, post := range children.post { + switch pa := post.agg.(type) { + case *api.PercentilesAggregation: + vals, err := e.percentiles(pa, scoped) + if err != nil { + return nil, err + } + node.Buckets[i].Aggs[post.name] = &api.AggNode{Values: vals} + case *api.TopHitsAggregation: + n, err := e.topHits(pa, scoped) + if err != nil { + return nil, err + } + node.Buckets[i].Aggs[post.name] = n + } + } + } + + out[tk] = node + } + return out, nil +} + +// execOnePartitioned dispatches a bucket child to its partitioned execution. +func (e *aggExecutor) execOnePartitioned(child namedAgg, ancestors []string) (map[string]*api.AggNode, error) { + switch a := child.agg.(type) { + case *api.TermsAggregation: + return e.execBucketPartitioned(a, ancestors, termsKeyPlan{field: a.Field, size: a.Size}) + case *api.DateHistogramAggregation: + interval := a.Interval + if interval == "" && a.IntervalField != "" { + interval = a.IntervalField + } + format, ok := histogramFormatFor(interval) + if !ok { + warnAggUnsupported("date_histogram with interval " + interval) + return map[string]*api.AggNode{}, nil + } + return e.execBucketPartitioned(a, ancestors, dateKeyPlan{field: a.Field, format: format, interval: interval, offset: a.Offset, epochExpr: e.epochExpr(a.Field)}) + case *api.AutoDateHistogramAggregation: + dh, err := e.autoIntervalDH(a) + if err != nil { + return nil, err + } + return e.execOnePartitioned(namedAgg{child.name, dh}, ancestors) + } + return map[string]*api.AggNode{}, nil +} + +// levelQuery runs the per-node query: GROUP BY (ancestors..., key) with the +// level's metric aggregates. NULL keys are omitted (ES semantics: missing +// doc values produce no bucket). +func (e *aggExecutor) levelQuery(plan keyPlan, ancestors []string, metrics []namedMetric) ([]bucketRow, error) { + keyExpr := plan.keyExpr(e.resolve) + + var selects, groupBy []string + for _, a := range ancestors { + selects = append(selects, a) + groupBy = append(groupBy, a) + } + selects = append(selects, keyExpr+" AS agg_key", "COUNT(*) AS agg_dc") + groupBy = append(groupBy, "agg_key") + + metricExprs := make([]string, len(metrics)) + for i, m := range metrics { + if fn, ok := metricSQL(m.agg, e.resolve); ok { + metricExprs[i] = fn + } else { + metricExprs[i] = "NULL" + } + selects = append(selects, metricExprs[i]) + } + + cond := fmt.Sprintf("%s IS NOT NULL", keyExpr) + if e.where != "" { + cond = "(" + e.where + ") AND " + cond + } + + q := fmt.Sprintf("SELECT %s FROM [%s] WHERE %s GROUP BY %s", + strings.Join(selects, ", "), e.index, cond, strings.Join(groupBy, ", ")) + + rows, err := e.handler.DB.Query(q, e.args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []bucketRow + nAnc := len(ancestors) + for rows.Next() { + anc := make([]interface{}, nAnc) + scan := make([]interface{}, 0, nAnc+2+len(metrics)) + for i := range anc { + scan = append(scan, &anc[i]) + } + var key interface{} + var dc int64 + scan = append(scan, &key, &dc) + mvals := make([]sql.NullFloat64, len(metrics)) + for i := range mvals { + scan = append(scan, &mvals[i]) + } + if err := rows.Scan(scan...); err != nil { + return nil, err + } + row := bucketRow{ancestors: anc, key: key, docCount: dc} + for _, mv := range mvals { + row.metrics = append(row.metrics, mv.Float64) + } + out = append(out, row) + } + return out, rows.Err() +} + +// metricSQL translates a metric spec to its SQL aggregate expression. +func metricSQL(m *api.MetricAggregation, resolve sqliteOrm.FieldResolver) (string, bool) { + expr := sqliteOrm.ExprFor(resolve, m.Field) + switch m.Type { + case api.MetricCount: + return fmt.Sprintf("COUNT(%s)", expr), true + case api.MetricCardinality: + return fmt.Sprintf("COUNT(DISTINCT %s)", expr), true + case api.MetricSum: + return fmt.Sprintf("SUM(%s)", expr), true + case api.MetricAvg: + return fmt.Sprintf("AVG(%s)", expr), true + case api.MetricMin: + return fmt.Sprintf("MIN(%s)", expr), true + case api.MetricMax: + return fmt.Sprintf("MAX(%s)", expr), true + } + return "", false +} + +// attachMetrics writes the level query's metric values into a bucket scope. +func attachMetrics(scope map[string]*api.AggNode, metrics []namedMetric, vals []float64) { + for i, m := range metrics { + scope[m.name] = &api.AggNode{Value: vals[i], ValueSet: true} + } +} + +// fillEmptyBucketMetrics gives zero-filled buckets their sub-metric nodes, +// mirroring ES semantics on min_doc_count:0 buckets: count/sum/cardinality +// report 0; avg/min/max report no value. +func fillEmptyBucketMetrics(buckets []api.Bucket, metrics []namedMetric) { + if len(metrics) == 0 { + return + } + for i := range buckets { + if buckets[i].DocCount != 0 || len(buckets[i].Aggs) > 0 { + continue + } + if buckets[i].Aggs == nil { + buckets[i].Aggs = map[string]*api.AggNode{} + } + for _, m := range metrics { + switch m.agg.Type { + case api.MetricAvg, api.MetricMin, api.MetricMax: + buckets[i].Aggs[m.name] = &api.AggNode{} + default: // count, sum, cardinality + buckets[i].Aggs[m.name] = &api.AggNode{Value: 0, ValueSet: true} + } + } + } +} + +// ────────────────────────────────────────────────────────────────────────── +// Post-pass and single-value executions +// ────────────────────────────────────────────────────────────────────────── + +// extraScope pins ancestor values for per-bucket post passes. +type extraScope struct { + exprs []string + values []interface{} +} + +func (e *aggExecutor) scopedWhere(scope *extraScope) (string, []interface{}) { + cond := e.where + cargs := append([]interface{}{}, e.args...) + if scope != nil && len(scope.exprs) > 0 && len(scope.exprs) == len(scope.values) { + for i := range scope.exprs { + eq := fmt.Sprintf("%s IS ?", scope.exprs[i]) + if cond == "" { + cond = eq + } else { + cond = "(" + cond + ") AND " + eq + } + cargs = append(cargs, scope.values[i]) + } + } + if cond == "" { + return "", cargs + } + return " WHERE " + cond, cargs +} + +// tupleValues renders a row's full parent tuple (ancestors + own key). +func tupleValues(r bucketRow) []interface{} { + return append(append([]interface{}{}, r.ancestors...), r.key) +} + +func tupleKey(vals []interface{}) string { + parts := make([]string, len(vals)) + for i, v := range vals { + parts[i] = valueKey(v) + } + return strings.Join(parts, "\x00") +} + +func valueKey(v interface{}) string { return fmt.Sprintf("%v", v) } + +// topMetric computes a single-value metric over the whole scoped set. +func (e *aggExecutor) topMetric(a *api.MetricAggregation) (*float64, error) { + fn, ok := metricSQL(a, e.resolve) + if !ok { + warnAggUnsupported(a.Type + " metric") + return nil, nil + } + q := fmt.Sprintf("SELECT %s FROM [%s]", fn, e.index) + if e.where != "" { + q += " WHERE " + e.where + } + var v sql.NullFloat64 + if err := e.handler.DB.QueryRow(q, e.args...).Scan(&v); err != nil { + return nil, fmt.Errorf("%s aggregation on %s: %w", a.Type, a.Field, err) + } + if !v.Valid { + return nil, nil + } + f := v.Float64 + return &f, nil +} + +// percentiles computes exact nearest-rank percentiles (ES uses approximate +// TDigest; conformance compares with tolerance). +func (e *aggExecutor) percentiles(a *api.PercentilesAggregation, scope *extraScope) (map[string]float64, error) { + expr := sqliteOrm.ExprFor(e.resolve, a.Field) + cond, cargs := e.scopedWhere(scope) + + var total int64 + if err := e.handler.DB.QueryRow( + fmt.Sprintf("SELECT COUNT(%s) FROM [%s]%s", expr, e.index, cond), cargs...).Scan(&total); err != nil { + return nil, err + } + out := map[string]float64{} + if total == 0 { + return out, nil + } + percents := a.Percents + if len(percents) == 0 { + percents = []float64{1, 5, 25, 50, 75, 95, 99} + } + for _, p := range percents { + rank := int64(p/100.0*float64(total) + 0.5) + if rank < 1 { + rank = 1 + } + if rank > total { + rank = total + } + var v sql.NullFloat64 + q := fmt.Sprintf("SELECT %s FROM [%s]%s AND %s IS NOT NULL ORDER BY %s ASC LIMIT 1 OFFSET %d", + expr, e.index, whereOrTrue(cond), expr, expr, rank-1) + if err := e.handler.DB.QueryRow(q, cargs...).Scan(&v); err == nil && v.Valid { + out[strconv.FormatFloat(p, 'f', -1, 64)] = v.Float64 + } + } + return out, nil +} + +func whereOrTrue(cond string) string { + if cond == "" { + return " WHERE 1=1" + } + return cond +} + +// topHits fetches the top document of the scoped set. +func (e *aggExecutor) topHits(a *api.TopHitsAggregation, scope *extraScope) (*api.AggNode, error) { + orderBy, dir := "id", "DESC" + if len(a.Sorts) > 0 { + orderBy = sqliteOrm.ExprFor(e.resolve, a.Sorts[0].Field) + if a.Sorts[0].SortType == api.ASC { + dir = "ASC" + } + } + cond, cargs := e.scopedWhere(scope) + q := fmt.Sprintf("SELECT raw FROM [%s]%s ORDER BY %s %s LIMIT 1", e.index, cond, orderBy, dir) + var raw []byte + err := e.handler.DB.QueryRow(q, cargs...).Scan(&raw) + if err == sql.ErrNoRows { + return &api.AggNode{}, nil + } + if err != nil { + return nil, err + } + doc := json.RawMessage(raw) + return &api.AggNode{TopHit: &doc}, nil +} + +// dateRange counts documents per [from, to) range. +func (e *aggExecutor) dateRange(a *api.DateRangeAggregation) (*api.AggNode, error) { + expr := sqliteOrm.ExprFor(e.resolve, a.Field) + buckets := []api.Bucket{} + for _, r := range a.Ranges { + rangeMap, ok := r.(map[string]interface{}) + if !ok { + continue + } + var extra string + var extraArgs []interface{} + if from, ok := rangeMap["from"]; ok && from != nil { + extra = fmt.Sprintf("%s >= ?", expr) + extraArgs = append(extraArgs, from) + } + if to, ok := rangeMap["to"]; ok && to != nil { + if extra != "" { + extra += " AND " + } + extra += fmt.Sprintf("%s < ?", expr) + extraArgs = append(extraArgs, to) + } + if extra == "" { + continue + } + cond := extra + cargs := append([]interface{}{}, extraArgs...) + if e.where != "" { + cond = "(" + e.where + ") AND (" + extra + ")" + cargs = append(append([]interface{}{}, e.args...), extraArgs...) + } + var count int64 + if err := e.handler.DB.QueryRow( + fmt.Sprintf("SELECT COUNT(*) FROM [%s] WHERE %s", e.index, cond), cargs...).Scan(&count); err != nil { + return nil, fmt.Errorf("date_range aggregation on %s: %w", a.Field, err) + } + bucket := api.Bucket{DocCount: count, Aggs: map[string]*api.AggNode{}} + if from, ok := rangeMap["from"]; ok && from != nil { + bucket.KeyRaw = from + } + if key, ok := rangeMap["key"]; ok && key != nil { + bucket.Key = fmt.Sprintf("%v", key) + } else { + bucket.Key = fmt.Sprintf("%v-%v", rangeMap["from"], rangeMap["to"]) + } + buckets = append(buckets, bucket) + } + return &api.AggNode{Buckets: buckets}, nil +} + +// filterAgg runs a nested scope under an extra WHERE from the filter's +// query map (term/terms/range/bool.filter subset; anything else warns and +// yields an empty node — never a silent wrong result). +func (e *aggExecutor) filterAgg(a *api.FilterAggregation) (*api.AggNode, error) { + clause, err := filterQueryToClause(a.Query) + if err != nil || clause == nil { + if err != nil { + warnAggUnsupported("filter aggregation (" + err.Error() + ")") + } + return &api.AggNode{Buckets: []api.Bucket{{Aggs: map[string]*api.AggNode{}}}}, nil + } + qb := api.NewQuery().Filter(clause) + qb.Build() + fw, fa := sqliteOrm.BuildWhereClause(qb, e.resolve) + + combined, cargs := fw, append([]interface{}{}, fa...) + if e.where != "" && fw != "" { + combined = "(" + e.where + ") AND (" + fw + ")" + cargs = append(append([]interface{}{}, e.args...), fa...) + } + + sub := &aggExecutor{handler: e.handler, index: e.index, resolve: e.resolve, where: combined, args: cargs} + return sub.singleBucketScope(a.GetNested()) +} + +// singleBucketScope executes a nested scope as a single-bucket node with +// the total doc count (sampler fallback). +func (e *aggExecutor) singleBucketScope(nested map[string]api.Aggregation) (*api.AggNode, error) { + nodes, err := e.execute(nested) + if err != nil { + return nil, err + } + var dc int64 + q := fmt.Sprintf("SELECT COUNT(*) FROM [%s]", e.index) + if e.where != "" { + q += " WHERE " + e.where + } + _ = e.handler.DB.QueryRow(q, e.args...).Scan(&dc) + return &api.AggNode{Buckets: []api.Bucket{{DocCount: dc, Aggs: nodes}}}, nil +} + +// autoIntervalDH derives a fixed interval from the data range and returns +// an equivalent DateHistogramAggregation (design doc §4.2 fallback). +func (e *aggExecutor) autoIntervalDH(a *api.AutoDateHistogramAggregation) (*api.DateHistogramAggregation, error) { + target := a.Buckets + if target <= 0 { + target = 10 + } + expr := sqliteOrm.ExprFor(e.resolve, a.Field) + var minV, maxV sql.NullString + q := fmt.Sprintf("SELECT MIN(%s), MAX(%s) FROM [%s]", expr, expr, e.index) + if e.where != "" { + q += " WHERE " + e.where + } + if err := e.handler.DB.QueryRow(q, e.args...).Scan(&minV, &maxV); err != nil || !minV.Valid || !maxV.Valid { + return &api.DateHistogramAggregation{Field: a.Field, Interval: "1h"}, nil + } + interval := pickInterval(minV.String, maxV.String, target, a.MinimumInterval) + dh := &api.DateHistogramAggregation{Field: a.Field, Interval: interval} + for subName, sub := range a.GetNested() { + dh.AddNested(subName, sub) + } + return dh, nil +} + +// pickInterval chooses a fixed interval covering the range in ~target +// buckets, respecting minimum_interval as the floor. +func pickInterval(minV, maxV string, target int, minimumInterval string) string { + tMin, err1 := time.Parse(time.RFC3339, minV) + tMax, err2 := time.Parse(time.RFC3339, maxV) + if err1 != nil || err2 != nil || target <= 0 { + return "1h" + } + candidate := tMax.Sub(tMin) / time.Duration(target) + var floor time.Duration + switch minimumInterval { + case "minute": + floor = time.Minute + case "hour": + floor = time.Hour + case "day": + floor = 24 * time.Hour + case "week": + floor = 7 * 24 * time.Hour + case "month": + floor = 30 * 24 * time.Hour + } + if floor > 0 && candidate < floor { + candidate = floor + } + switch { + case candidate < time.Hour: + return "1m" + case candidate < 24*time.Hour: + return "1h" + case candidate < 30*24*time.Hour: + return "1d" + default: + return "1M" + } +} + +// filterQueryToClause translates the supported ES filter-query subset. +func filterQueryToClause(q map[string]interface{}) (*api.Clause, error) { + if q == nil { + return nil, nil + } + if term, ok := q["term"].(map[string]interface{}); ok && len(term) == 1 { + for f, v := range term { + return api.TermQuery(f, v), nil + } + } + if terms, ok := q["terms"].(map[string]interface{}); ok && len(terms) == 1 { + for f, v := range terms { + if list, ok := v.([]interface{}); ok { + return api.TermsQuery(f, list), nil + } + } + } + if rng, ok := q["range"].(map[string]interface{}); ok && len(rng) == 1 { + for f, v := range rng { + body, ok := v.(map[string]interface{}) + if !ok { + continue + } + var clauses []*api.Clause + if gte, ok := body["gte"]; ok { + clauses = append(clauses, &api.Clause{Field: f, Operator: api.QueryRangeGte, Value: gte}) + } + if gt, ok := body["gt"]; ok { + clauses = append(clauses, &api.Clause{Field: f, Operator: api.QueryRangeGt, Value: gt}) + } + if lte, ok := body["lte"]; ok { + clauses = append(clauses, &api.Clause{Field: f, Operator: api.QueryRangeLte, Value: lte}) + } + if lt, ok := body["lt"]; ok { + clauses = append(clauses, &api.Clause{Field: f, Operator: api.QueryRangeLt, Value: lt}) + } + if len(clauses) == 1 { + return clauses[0], nil + } + if len(clauses) > 1 { + return api.MustQuery(clauses...), nil + } + } + } + if b, ok := q["bool"].(map[string]interface{}); ok { + if filters, ok := b["filter"].([]interface{}); ok { + var clauses []*api.Clause + for _, f := range filters { + fm, ok := f.(map[string]interface{}) + if !ok { + continue + } + c, err := filterQueryToClause(fm) + if err != nil { + return nil, err + } + if c != nil { + clauses = append(clauses, c) + } + } + if len(clauses) == 1 { + return clauses[0], nil + } + if len(clauses) > 1 { + return api.MustQuery(clauses...), nil + } + return nil, nil + } + } + return nil, fmt.Errorf("unsupported filter query %v", keysOf(q)) +} + +func keysOf(m map[string]interface{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// ────────────────────────────────────────────────────────────────────────── +// Time-bucket helpers +// ────────────────────────────────────────────────────────────────────────── + +// histogramFormatFor maps interval strings to strftime bucket formats. +func histogramFormatFor(interval string) (string, bool) { + switch strings.TrimSpace(interval) { + case "1m", "1minute", "minute": + return "%Y-%m-%dT%H:%M:00", true + case "1h", "1hour", "hour": + return "%Y-%m-%dT%H:00:00", true + case "1d", "1day", "day": + return "%Y-%m-%dT00:00:00", true + case "1M", "1month", "month": + return "%Y-%m-01T00:00:00", true + } + return "", false +} + +var timeLayouts = map[string]string{ + "%Y-%m-%dT%H:%M:00": "2006-01-02T15:04:00", + "%Y-%m-%dT%H:00:00": "2006-01-02T15:00:00", + "%Y-%m-%dT00:00:00": "2006-01-02T00:00:00", + "%Y-%m-01T00:00:00": "2006-01-02T00:00:00", +} + +// epochOf converts a formatted bucket key to epoch milliseconds. +func epochOf(key, format string) int64 { + layout, ok := timeLayouts[format] + if !ok { + return 0 + } + t, err := time.Parse(layout, key) + if err != nil { + return 0 + } + return t.UnixMilli() +} + +func sortBucketsByTime(buckets []api.Bucket) { + sort.Slice(buckets, func(i, j int) bool { return buckets[i].Key < buckets[j].Key }) +} + +// stepOfInterval returns the zero-fill step (0 = no fill). +func stepOfInterval(interval string) time.Duration { + switch strings.TrimSpace(interval) { + case "1m", "1minute", "minute": + return time.Minute + case "1h", "1hour", "hour": + return time.Hour + case "1d", "1day", "day": + return 24 * time.Hour + } + return 0 +} + +// zeroFill inserts empty buckets between the first and last key (ES +// min_doc_count:0 + extended_bounds semantics; fixed intervals only). +// Bucket identity comes from KeyRaw (epoch ms) — string layouts stay +// presentation-only. +func zeroFill(buckets []api.Bucket, interval, format string) []api.Bucket { + step := stepOfInterval(interval) + if step == 0 || len(buckets) < 2 { + return buckets + } + layout := timeLayouts[format] + if layout == "" { + layout = "2006-01-02T15:04:05" + } + filled := make([]api.Bucket, 0, len(buckets)*2) + for i, b := range buckets { + filled = append(filled, b) + if i == len(buckets)-1 { + break + } + cur, ok1 := toInt64(b.KeyRaw) + next, ok2 := toInt64(buckets[i+1].KeyRaw) + if !ok1 || !ok2 { + continue + } + for t := cur + step.Milliseconds(); t < next; t += step.Milliseconds() { + ts := time.UnixMilli(t).UTC() + filled = append(filled, api.Bucket{ + Key: ts.Format(layout), KeyRaw: t, DocCount: 0, Aggs: map[string]*api.AggNode{}, + }) + } + } + return filled +} + +// ────────────────────────────────────────────────────────────────────────── +// ES-shape conversion (SearchV2 backward compatibility) +// ────────────────────────────────────────────────────────────────────────── + +// typedToESShape converts the typed aggregation tree to the ES-shaped maps +// SearchV2 has always returned, so existing consumers are unaffected while +// Aggregate callers get the typed model. +func typedToESShape(nodes map[string]*api.AggNode) map[string]interface{} { + out := map[string]interface{}{} + for name, node := range nodes { + out[name] = nodeToESShape(node) + } + return out +} + +func nodeToESShape(node *api.AggNode) map[string]interface{} { + if node == nil { + return map[string]interface{}{} + } + out := map[string]interface{}{} + if node.Values != nil { + out["values"] = node.Values + } + if node.TopHit != nil { + var doc interface{} + _ = json.Unmarshal(*node.TopHit, &doc) + out["hits"] = map[string]interface{}{ + "hits": map[string]interface{}{"hits": []interface{}{map[string]interface{}{"_source": doc}}}, + } + } + if node.ValueSet { + out["value"] = node.Value + } + if node.Buckets != nil { + buckets := make([]interface{}, 0, len(node.Buckets)) + for _, b := range node.Buckets { + bm := map[string]interface{}{} + if b.Key != "" { + bm["key"] = b.Key + } + if b.KeyRaw != nil { + bm["key_raw"] = b.KeyRaw + // Time buckets: numeric epoch-ms key + string form (ES). + if ms, ok := b.KeyRaw.(int64); ok && ms > 0 { + bm["key"] = ms + bm["key_as_string"] = b.Key + } + } + bm["doc_count"] = b.DocCount + for subName, subNode := range b.Aggs { + bm[subName] = nodeToESShape(subNode) + } + buckets = append(buckets, bm) + } + // Single-bucket aggs (filter/sampler) inline their scope ES-style. + if len(node.Buckets) == 1 && node.Buckets[0].Key == "" && node.Buckets[0].KeyRaw == nil { + single := map[string]interface{}{"doc_count": node.Buckets[0].DocCount} + for subName, subNode := range node.Buckets[0].Aggs { + single[subName] = nodeToESShape(subNode) + } + for k, v := range single { + out[k] = v + } + return out + } + out["buckets"] = buckets + } + return out +} + +// ────────────────────────────────────────────────────────────────────────── +// Unsupported-feature warnings (grace period: warn once, empty result) +// ────────────────────────────────────────────────────────────────────────── + +var aggUnsupportedOnce sync.Map + +func warnAggUnsupported(kind string) { + if _, loaded := aggUnsupportedOnce.LoadOrStore(kind, true); !loaded { + log.Warnf("sqlite orm: %s aggregation is not supported on this backend; it returns an empty result (grace period: warning only)", kind) + } +} + +func aggKind(agg api.Aggregation) string { + switch agg.(type) { + case *api.FilterAggregation: + return "filter" + case *api.PercentilesAggregation: + return "percentiles" + case *api.PipelineAggregation: + return "pipeline" + } + return fmt.Sprintf("%T", agg) +} diff --git a/modules/sqlite/perf_test.go b/modules/sqlite/perf_test.go new file mode 100644 index 000000000..16ce4e529 --- /dev/null +++ b/modules/sqlite/perf_test.go @@ -0,0 +1,234 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package sqlite + +// Performance benchmarks at logpilot-realistic scale (PatternStats-shaped): +// boot-time schema ensure, dashboard aggregations, and the remaining +// full-table fetches. Run with: +// +// go test ./modules/sqlite/ -run XXX -bench BenchmarkPerf -benchtime 1x -v +// +// (benchtime 1x: these benchmarks measure cold-ish single runs over a large +// fixture, not steady-state micro-ops.) + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "infini.sh/framework/core/orm" +) + +// perfStat mirrors logpilot's PatternStats shape. +type perfStat struct { + orm.ORMObjectBase + StreamID string `json:"stream_id" elastic_mapping:"stream_id:{type:keyword}" sqlite_composite:"stream_id,pattern_id,bucket_start__epoch,count"` + PatternID string `json:"pattern_id" elastic_mapping:"pattern_id:{type:keyword}" sqlite_composite:"pattern_id,bucket_start"` + BucketStart time.Time `json:"bucket_start" elastic_mapping:"bucket_start:{type:date}"` + Count int64 `json:"count" elastic_mapping:"count:{type:long}"` +} + +// perfStatSearch isolates the (stream,time) walk composite from the +// aggregation covering composite — both plans must coexist per query shape. +type perfStatSearch struct { + orm.ORMObjectBase + StreamID string `json:"stream_id" elastic_mapping:"stream_id:{type:keyword}" sqlite_composite:"stream_id,bucket_start__epoch"` + PatternID string `json:"pattern_id" elastic_mapping:"pattern_id:{type:keyword}"` + BucketStart time.Time `json:"bucket_start" elastic_mapping:"bucket_start:{type:date}"` + Count int64 `json:"count" elastic_mapping:"count:{type:long}"` +} + +// perfPattern mirrors Pattern (has a text field → FTS triggers + backfill). +type perfPattern struct { + orm.ORMObjectBase + StreamID string `json:"stream_id" elastic_mapping:"stream_id:{type:keyword}"` + Template string `json:"template" elastic_mapping:"template:{type:text}"` +} + +const perfRows = 500_000 + +func seedPerfStats(b *testing.B) *SQLiteORM { + b.Helper() + handler := &SQLiteORM{Config: SQLiteConfig{ + Enabled: true, + DBPath: filepath.Join(b.TempDir(), "perf.db"), + }} + if err := handler.Open(); err != nil { + b.Fatal(err) + } + if err := handler.RegisterSchemaWithName(perfStat{}, "perf_stats"); err != nil { + b.Fatal(err) + } + if err := handler.RegisterSchemaWithName(perfPattern{}, "perf_patterns"); err != nil { + b.Fatal(err) + } + + tx, err := handler.DB.Begin() + if err != nil { + b.Fatal(err) + } + stmt, err := tx.Prepare("INSERT INTO perf_stats (id, raw) VALUES (?, ?)") + if err != nil { + b.Fatal(err) + } + base := time.Date(2026, 8, 7, 0, 0, 0, 0, time.UTC) + patterns := 200 + for i := 0; i < perfRows; i++ { + day := i % 7 + minute := i % 1440 + ts := base.Add(time.Duration(day)*24*time.Hour + time.Duration(minute)*time.Minute) + raw := fmt.Sprintf(`{"id":"p%07d","stream_id":"s%d","pattern_id":"pat-%03d","bucket_start":%q,"count":%d}`, + i, i%4, i%patterns, ts.Format(time.RFC3339), 1+i%97) + if _, err := stmt.Exec(fmt.Sprintf("p%07d", i), raw); err != nil { + b.Fatal(err) + } + } + stmt.Close() + // A modest pattern table (FTS-bearing). + pstmt, err := tx.Prepare("INSERT INTO perf_patterns (id, raw) VALUES (?, ?)") + if err != nil { + b.Fatal(err) + } + for i := 0; i < patterns; i++ { + raw := fmt.Sprintf(`{"id":"pat-%03d","stream_id":"s%d","template":"error in module %d while processing request"}`, i, i%4, i) + if _, err := pstmt.Exec(fmt.Sprintf("pat-%03d", i), raw); err != nil { + b.Fatal(err) + } + } + pstmt.Close() + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + // Bulk-loaded after registration — refresh planner stats so index + // choices reflect real cardinalities (production: PRAGMA optimize runs + // on close / periodically). + if _, err := handler.DB.Exec("PRAGMA optimize"); err != nil { + b.Fatal(err) + } + return handler +} + +func perfCtx(handler *SQLiteORM) *orm.Context { + ctx := orm.NewContext() + orm.WithModel(ctx, &perfStat{}) + return ctx +} + +// BenchmarkPerfBootEnsure measures re-running ensureFlattenedTable on an +// existing populated database — the per-boot schema path every open pays. +func BenchmarkPerfBootEnsure(b *testing.B) { + handler := seedPerfStats(b) + defer handler.Close() + + // The pattern table's FTS backfill anti-join is the suspect. + schema := buildTableSchema("perf_patterns", perfPattern{}) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := ensureFlattenedTable(handler.DB, schema); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkPerfDashboardAggs measures the PatternOverview-shaped tree: +// terms(stream) → terms(pattern) → date_histogram(1h) → sum. +func BenchmarkPerfDashboardAggs(b *testing.B) { + handler := seedPerfStats(b) + defer handler.Close() + ctx := perfCtx(handler) + + streams := &orm.TermsAggregation{Field: "stream_id", Size: 1000} + patterns := &orm.TermsAggregation{Field: "pattern_id", Size: 10000} + dh := &orm.DateHistogramAggregation{Field: "bucket_start", Interval: "1h"} + dh.AddNested("total", &orm.MetricAggregation{Type: orm.MetricSum, Field: "count"}) + patterns.AddNested("trend", dh) + streams.AddNested("patterns", patterns) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Realistic window: PatternOverview aggregates the last 24h. + qb := orm.NewQuery().Filter(orm.Range("bucket_start").Gte("2026-08-12T00:00:00Z")) + qb.SetAggs("streams", streams) + if _, err := handler.Aggregate(ctx, qb); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkPerfHourlyTrend measures the Trends-shaped single series: +// offset 1h date_histogram + sum over the full table. +func BenchmarkPerfHourlyTrend(b *testing.B) { + handler := seedPerfStats(b) + defer handler.Close() + ctx := perfCtx(handler) + + now := time.Date(2026, 8, 14, 10, 37, 12, 0, time.UTC) + dh := &orm.DateHistogramAggregation{ + Field: "bucket_start", + Interval: "1h", + Offset: now.Sub(now.Truncate(time.Hour)), + } + dh.AddNested("total", &orm.MetricAggregation{Type: orm.MetricSum, Field: "count"}) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + qb := orm.NewQuery() + qb.SetAggs("trend", dh) + if _, err := handler.Aggregate(ctx, qb); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkPerfFullFetch measures the retired-but-still-present pattern: +// SearchV2 Size(10000) + its COUNT companion (two scans). +func BenchmarkPerfFullFetch(b *testing.B) { + handler := seedPerfStats(b) + defer handler.Close() + ctx := perfCtx(handler) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + qb := orm.NewQuery().Filter(orm.Range("bucket_start").Gte("2026-08-13T00:00:00Z")).Size(10000) + if _, err := handler.SearchV2(ctx, qb); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkPerfFilteredSearch measures a typical UI list query: term filter +// + sort by created DESC + size 50. +func BenchmarkPerfFilteredSearch(b *testing.B) { + handler := seedPerfStats(b) + defer handler.Close() + + // Same data registered with the (stream,time) composite only — the + // walk-ordered plan a filtered search wants. + if err := handler.RegisterSchemaWithName(perfStatSearch{}, "perf_search"); err != nil { + b.Fatal(err) + } + tx, err := handler.DB.Begin() + if err != nil { + b.Fatal(err) + } + if _, err := tx.Exec("INSERT INTO perf_search (id, raw) SELECT id, raw FROM perf_stats"); err != nil { + b.Fatal(err) + } + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + sctx := orm.NewContext() + orm.WithModel(sctx, &perfStatSearch{}) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + qb := orm.NewQuery(). + Filter(orm.TermQuery("stream_id", "s1")). + SortBy(orm.Sort{Field: "bucket_start", SortType: orm.DESC}). + Size(50) + if _, err := handler.SearchV2(sctx, qb); err != nil { + b.Fatal(err) + } + } +} diff --git a/modules/sqlite/scenario_test.go b/modules/sqlite/scenario_test.go new file mode 100644 index 000000000..0a3d219ee --- /dev/null +++ b/modules/sqlite/scenario_test.go @@ -0,0 +1,1014 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package sqlite + +// ────────────────────────────────────────────────────────────────────────── +// Comprehensive query & aggregation scenario (oracle-based verification). +// +// A realistic observability-events dataset exercises the full QueryBuilder +// operator matrix, boolean composition, promoted (generated-column) and +// unpromoted (json_extract fallback) paths, FTS text search, sorting and +// pagination, and the complete aggregation surface incl. pipelines. Expected +// results are computed by independent Go oracles over the in-memory fixture +// — the test validates the SQL machinery against straightforward reference +// implementations, not against hand-written constants. +// +// The suite drives the handler methods directly (no global orm.Register), +// so it coexists with the other sqlite tests in one binary. +// ────────────────────────────────────────────────────────────────────────── + +import ( + "encoding/json" + "fmt" + "math" + "math/rand" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" +) + +// ── scenario model ───────────────────────────────────────────────────────── + +type svcInfo struct { + Name string `json:"name,omitempty" elastic_mapping:"name:{type:keyword}"` + Version string `json:"version,omitempty" elastic_mapping:"version:{type:keyword}"` +} + +type scenarioEvent struct { + orm.ORMObjectBase + Stream string `json:"stream" elastic_mapping:"stream:{type:keyword}"` + Severity string `json:"severity" elastic_mapping:"severity:{type:keyword}"` + Region string `json:"region" elastic_mapping:"region:{type:keyword}"` + Host string `json:"host" elastic_mapping:"host:{type:keyword}"` + Message string `json:"message" elastic_mapping:"message:{type:text}"` + Status int `json:"status" elastic_mapping:"status:{type:integer}"` + Latency float64 `json:"latency" elastic_mapping:"latency:{type:double}"` + Verified bool `json:"verified" elastic_mapping:"verified:{type:boolean}"` + TS time.Time `json:"ts" elastic_mapping:"ts:{type:date}"` + Svc *svcInfo `json:"svc,omitempty" elastic_mapping:"svc:{type:object}"` + // Extra carries dynamic fields with NO mapping — queries on extra.* + // exercise the json_extract fallback path. + Extra map[string]interface{} `json:"extra,omitempty"` +} + +// ── fixture ──────────────────────────────────────────────────────────────── + +type scenario struct { + t *testing.T + handler *SQLiteORM + events []scenarioEvent // in-memory oracle copy, insertion order + byID map[string]*scenarioEvent +} + +var ( + scStreams = []string{"checkout", "search", "auth", "billing"} + scSeverities = []string{"info", "warning", "error", "fatal"} + scRegions = []string{"cn-east", "cn-north", "us-west"} + scHosts = []string{"node-1", "node-2", "node-3", "node-4", "node-5"} + scServices = []svcInfo{ + {Name: "api-gateway", Version: "v1"}, + {Name: "api-gateway", Version: "v2"}, + {Name: "order-svc", Version: "v1"}, + {Name: "user-svc", Version: "v3"}, + } + scMessages = []string{ + "request completed successfully", + "connection reset by peer", + "timeout while waiting for upstream", + "disk usage above threshold", + "authentication failed for user", + "database connection pool exhausted", + "rate limit exceeded for tenant", + "healthy heartbeat received", + } +) + +func newScenario(t *testing.T) *scenario { + t.Helper() + handler := &SQLiteORM{Config: SQLiteConfig{ + Enabled: true, + DBPath: filepath.Join(t.TempDir(), "scenario.db"), + }} + require.NoError(t, handler.Open()) + t.Cleanup(func() { handler.Close() }) + require.NoError(t, handler.RegisterSchemaWithName(scenarioEvent{}, "scenario_events")) + + s := &scenario{t: t, handler: handler, byID: map[string]*scenarioEvent{}} + rng := rand.New(rand.NewSource(20260813)) + + // Deterministic span: 2026-08-07..2026-08-13 (7 days). Timestamps are + // built absolutely (not additively) so minutes never overflow across + // hour/day boundaries into the engineered gaps. + for i := 0; i < 800; i++ { + // Skewed hour distribution with guaranteed gaps (zero-fill checks): + // skip hours 10..14 on day 2 and hour 3 on day 5. + day := rng.Intn(7) + hour := rng.Intn(24) + if day == 2 && hour >= 10 && hour <= 14 { + hour = 15 + } + if day == 5 && hour == 3 { + hour = 4 + } + minute := rng.Intn(60) + second := rng.Intn(60) + ts := time.Date(2026, 8, 7+day, hour, minute, second, 0, time.UTC) + sev := scSeverities[rng.Intn(len(scSeverities))] + if rng.Intn(10) == 0 { + sev = "fatal" // boost fatal to a deterministic minority + } + ev := scenarioEvent{ + Stream: scStreams[rng.Intn(len(scStreams))], + Severity: sev, + Region: scRegions[rng.Intn(len(scRegions))], + Host: scHosts[rng.Intn(len(scHosts))], + Message: scMessages[rng.Intn(len(scMessages))], + Status: 100 + rng.Intn(500), // [100,600) + Latency: float64(rng.Intn(90000)) / 100.0, // [0,900) ms, 2dp + Verified: rng.Intn(2) == 0, + TS: ts, + Svc: &scServices[rng.Intn(len(scServices))], + Extra: map[string]interface{}{ + "tenant": fmt.Sprintf("tenant-%d", rng.Intn(6)), + "attempt": float64(rng.Intn(3) + 1), + }, + } + ev.ID = fmt.Sprintf("ev-%04d", i) + require.NoError(t, handler.Save(nil, &ev)) + s.events = append(s.events, ev) + s.byID[ev.ID] = &s.events[len(s.events)-1] + } + return s +} + +func (s *scenario) ctx() *orm.Context { + ctx := orm.NewContext() + orm.WithModel(ctx, &scenarioEvent{}) + return ctx +} + +// ── query oracle ─────────────────────────────────────────────────────────── + +// query runs qb through SearchV2 and returns the decoded hits. +func (s *scenario) query(qb *orm.QueryBuilder) []scenarioEvent { + s.t.Helper() + qb.Build() // idempotent-safe: Build is invoked again by SearchV2 + res, err := s.handler.SearchV2(s.ctx(), qb) + require.NoError(s.t, err) + hits, _, err := elastic.DecodeHits[scenarioEvent](res) + require.NoError(s.t, err) + return hits +} + +func (s *scenario) queryIDs(qb *orm.QueryBuilder) []string { + s.t.Helper() + hits := s.query(qb) + ids := make([]string, 0, len(hits)) + for _, h := range hits { + ids = append(ids, h.ID) + } + sort.Strings(ids) + return ids +} + +// expect returns the sorted IDs of fixture events matching pred (the oracle). +func (s *scenario) expect(pred func(*scenarioEvent) bool) []string { + ids := []string{} + for i := range s.events { + if pred(&s.events[i]) { + ids = append(ids, s.events[i].ID) + } + } + sort.Strings(ids) + return ids +} + +func (s *scenario) expectIDs(pred func(*scenarioEvent) bool) map[string]struct{} { + out := map[string]struct{}{} + for _, id := range s.expect(pred) { + out[id] = struct{}{} + } + return out +} + +// tokenize mirrors FTS5 unicode61 tokenization for the fixture vocabulary +// (lowercase words split on spaces/punctuation). +func scTokenize(msg string) []string { + return strings.FieldsFunc(strings.ToLower(msg), func(r rune) bool { + return !('a' <= r && r <= 'z' || '0' <= r && r <= '9') + }) +} + +func scHasAnyWord(msg string, words []string) bool { + tokens := map[string]bool{} + for _, t := range scTokenize(msg) { + tokens[t] = true + } + for _, w := range words { + if tokens[strings.ToLower(w)] { + return true + } + } + return false +} + +func scHasPhrase(msg, phrase string) bool { + return strings.Contains(strings.ToLower(msg), strings.ToLower(phrase)) +} + +// ── query matrix ─────────────────────────────────────────────────────────── + +func TestScenario_Queries(t *testing.T) { + s := newScenario(t) + + t.Run("term keyword", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.TermQuery("stream", "checkout"))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return e.Stream == "checkout" }), got) + }) + + t.Run("terms multi-value", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.TermsQuery("severity", []interface{}{"error", "fatal"}))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return e.Severity == "error" || e.Severity == "fatal" + }), got) + }) + + t.Run("range int gte lt", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery(). + Filter(orm.Range("status").Gte(300)). + Filter(orm.Range("status").Lt(400))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return e.Status >= 300 && e.Status < 400 + }), got) + }) + + t.Run("range float gte", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.Range("latency").Gte(700.5))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return e.Latency >= 700.5 }), got) + }) + + t.Run("range date", func(t *testing.T) { + cutoff := time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC) + got := s.queryIDs(orm.NewQuery().Filter(orm.Range("ts").Gte(cutoff.Format(time.RFC3339)))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return !e.TS.Before(cutoff) }), got) + }) + + t.Run("bool term", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.TermQuery("verified", false))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return !e.Verified }), got) + }) + + t.Run("prefix", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(&orm.Clause{Field: "host", Operator: orm.QueryPrefix, Value: "node-1"})) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return strings.HasPrefix(e.Host, "node-1") }), got) + }) + + t.Run("wildcard", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(&orm.Clause{Field: "host", Operator: orm.QueryWildcard, Value: "node-?"})) + // node-? matches single-char suffix: node-1..node-5 all match via + // '?' → '_', i.e. hosts "node-X" — all five hosts. + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return len(e.Host) == 6 // "node-X" + }), got) + }) + + t.Run("exists nested object", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(&orm.Clause{Field: "svc", Operator: orm.QueryExists})) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return e.Svc != nil }), got) + }) + + t.Run("must_not", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.MustNotQuery(orm.TermQuery("region", "cn-east")))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return e.Region != "cn-east" }), got) + }) + + t.Run("should minimum_should_match 1", func(t *testing.T) { + qb := orm.NewQuery(). + Filter(orm.TermQuery("stream", "auth")). + Must(orm.ShouldQuery( + orm.TermQuery("region", "cn-east"), + orm.TermQuery("region", "us-west"), + ).Parameter("minimum_should_match", 1)) + got := s.queryIDs(qb) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return e.Stream == "auth" && (e.Region == "cn-east" || e.Region == "us-west") + }), got) + }) + + t.Run("nested dotted path term", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.TermQuery("svc.name", "order-svc"))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return e.Svc != nil && e.Svc.Name == "order-svc" }), got) + }) + + t.Run("unmapped dynamic path term (json_extract fallback)", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.TermQuery("extra.tenant", "tenant-3"))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { return e.Extra["tenant"] == "tenant-3" }), got) + }) + + t.Run("unmapped dynamic numeric range", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.Range("extra.attempt").Gte(3))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + v, _ := e.Extra["attempt"].(float64) + return v >= 3 + }), got) + }) + + t.Run("fulltext match single word", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.MatchQuery("message", "timeout"))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return scHasAnyWord(e.Message, []string{"timeout"}) + }), got) + }) + + t.Run("fulltext match multi word OR semantics", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(orm.MatchQuery("message", "timeout disk"))) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return scHasAnyWord(e.Message, []string{"timeout", "disk"}) + }), got) + }) + + t.Run("fulltext match phrase", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(&orm.Clause{ + Field: "message", Operator: orm.QueryMatchPhrase, Value: "connection reset", + })) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return scHasPhrase(e.Message, "connection reset") + }), got) + }) + + t.Run("fulltext query_string phrase", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(&orm.Clause{ + Field: "message", Operator: orm.QueryQueryString, Value: "pool exhausted", + })) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return scHasPhrase(e.Message, "pool exhausted") + }), got) + }) + + t.Run("multi_match text+keyword", func(t *testing.T) { + got := s.queryIDs(orm.NewQuery().Filter(&orm.Clause{ + Field: "message,host", Operator: orm.QueryMultiMatch, Value: "node-3", + })) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + return strings.Contains(e.Host, "node-3") || scHasAnyWord(e.Message, []string{"node-3"}) + }), got) + }) + + t.Run("composite: everything combined", func(t *testing.T) { + cutoff := time.Date(2026, 8, 9, 0, 0, 0, 0, time.UTC) + qb := orm.NewQuery(). + Filter(orm.TermsQuery("severity", []interface{}{"error", "fatal"})). + Filter(orm.Range("status").Gte(200)). + Filter(orm.Range("latency").Lt(500)). + Filter(orm.Range("ts").Gte(cutoff.Format(time.RFC3339))). + Filter(orm.MustNotQuery(orm.TermQuery("region", "us-west"))). + Must(orm.ShouldQuery( + orm.TermQuery("svc.name", "api-gateway"), + orm.TermQuery("svc.name", "user-svc"), + ).Parameter("minimum_should_match", 1)) + got := s.queryIDs(qb) + assert.ElementsMatch(t, s.expect(func(e *scenarioEvent) bool { + if e.Severity != "error" && e.Severity != "fatal" { + return false + } + if e.Status < 200 || e.Latency >= 500 || e.TS.Before(cutoff) || e.Region == "us-west" { + return false + } + return e.Svc != nil && (e.Svc.Name == "api-gateway" || e.Svc.Name == "user-svc") + }), got) + }) + + t.Run("sort multi-key with pagination", func(t *testing.T) { + qb := func(from, size int) *orm.QueryBuilder { + return orm.NewQuery(). + Filter(orm.TermQuery("stream", "search")). + SortBy(orm.Sort{Field: "status", SortType: orm.ASC}, + orm.Sort{Field: "ts", SortType: orm.DESC}). + From(from).Size(size) + } + // Oracle ordering: status asc, ts desc (stable insertion tiebreak). + type ev = *scenarioEvent + ordered := make([]ev, 0, len(s.events)) + for id := range s.expectIDs(func(e *scenarioEvent) bool { return e.Stream == "search" }) { + ordered = append(ordered, s.byID[id]) + } + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].Status != ordered[j].Status { + return ordered[i].Status < ordered[j].Status + } + return ordered[i].TS.After(ordered[j].TS) + }) + + page1 := s.query(qb(0, 10)) + require.Len(t, page1, 10) + for i := 0; i < 10; i++ { + assert.Equal(t, ordered[i].ID, page1[i].ID, "page1[%d]", i) + } + page2 := s.query(qb(10, 10)) + require.Len(t, page2, 10) + for i := 0; i < 10; i++ { + assert.Equal(t, ordered[10+i].ID, page2[i].ID, "page2[%d]", i) + } + }) + + t.Run("pagination from without size", func(t *testing.T) { + all := s.query(orm.NewQuery().Filter(orm.TermQuery("stream", "auth"))) + rest := s.query(orm.NewQuery().Filter(orm.TermQuery("stream", "auth")).From(5)) + require.Len(t, rest, len(all)-5) + }) + + t.Run("unsupported operators warn and match nothing", func(t *testing.T) { + for _, op := range []orm.QueryType{orm.QuerySemantic, orm.QueryHybrid, orm.QueryNested} { + res, err := s.handler.SearchV2(s.ctx(), orm.NewQuery().Filter(&orm.Clause{ + Field: "message", Operator: op, Value: "x", + })) + require.NoError(t, err, "%s must not error (grace period)", op) + hits, _, err := elastic.DecodeHits[scenarioEvent](res) + require.NoError(t, err) + assert.Empty(t, hits, "%s matches nothing", op) + } + }) +} + +// ── aggregation matrix ───────────────────────────────────────────────────── + +func (s *scenario) aggregate(qb *orm.QueryBuilder) *orm.AggregationResult { + s.t.Helper() + if qb == nil { + qb = orm.NewQuery() + } + res, err := s.handler.Aggregate(s.ctx(), qb) + require.NoError(s.t, err) + require.NotNil(s.t, res) + return res +} + +func feq(a, b float64) bool { return math.Abs(a-b) < 1e-6 } + +func TestScenario_Aggregations(t *testing.T) { + s := newScenario(t) + + t.Run("metrics full set", func(t *testing.T) { + var wantCount, wantSum float64 + wantMin, wantMax := math.MaxFloat64, -math.MaxFloat64 + distinct := map[float64]bool{} + for i := range s.events { + v := s.events[i].Latency + wantCount++ + wantSum += v + if v < wantMin { + wantMin = v + } + if v > wantMax { + wantMax = v + } + distinct[v] = true + } + m := func(typ, field string) *orm.MetricAggregation { + return &orm.MetricAggregation{Type: typ, Field: field} + } + qb := orm.NewQuery() + qb.SetAggregations(map[string]orm.Aggregation{ + "count": m(orm.MetricCount, "latency"), + "sum": m(orm.MetricSum, "latency"), + "avg": m(orm.MetricAvg, "latency"), + "min": m(orm.MetricMin, "latency"), + "max": m(orm.MetricMax, "latency"), + "card": m(orm.MetricCardinality, "latency"), + }) + res := s.aggregate(qb) + assert.True(t, feq(res.Aggs["count"].Value, wantCount)) + assert.True(t, feq(res.Aggs["sum"].Value, wantSum)) + assert.True(t, feq(res.Aggs["avg"].Value, wantSum/wantCount)) + assert.True(t, feq(res.Aggs["min"].Value, wantMin)) + assert.True(t, feq(res.Aggs["max"].Value, wantMax)) + assert.True(t, feq(res.Aggs["card"].Value, float64(len(distinct)))) + }) + + t.Run("terms with nested metric", func(t *testing.T) { + terms := &orm.TermsAggregation{Field: "severity", Size: 10} + terms.AddNested("sum_latency", &orm.MetricAggregation{Type: orm.MetricSum, Field: "latency"}) + terms.AddNested("avg_status", &orm.MetricAggregation{Type: orm.MetricAvg, Field: "status"}) + qb := orm.NewQuery() + qb.SetAggs("by_sev", terms) + res := s.aggregate(qb) + + want := map[string]struct { + count int + sumL float64 + sumS float64 + }{} + for i := range s.events { + w := want[s.events[i].Severity] + w.count++ + w.sumL += s.events[i].Latency + w.sumS += float64(s.events[i].Status) + want[s.events[i].Severity] = w + } + buckets := res.Aggs["by_sev"].Buckets + require.Len(t, buckets, len(want)) + for _, b := range buckets { + w, ok := want[b.Key] + require.True(t, ok, "unexpected bucket %q", b.Key) + assert.EqualValues(t, w.count, b.DocCount) + assert.True(t, feq(b.Aggs["sum_latency"].Value, w.sumL), "%s sum_latency", b.Key) + assert.True(t, feq(b.Aggs["avg_status"].Value, w.sumS/float64(w.count)), "%s avg_status", b.Key) + } + }) + + t.Run("terms size truncation keeps top by count", func(t *testing.T) { + terms := &orm.TermsAggregation{Field: "stream", Size: 2} + qb := orm.NewQuery() + qb.SetAggs("top_streams", terms) + res := s.aggregate(qb) + buckets := res.Aggs["top_streams"].Buckets + require.Len(t, buckets, 2) + assert.GreaterOrEqual(t, buckets[0].DocCount, buckets[1].DocCount, "count desc ordering") + }) + + t.Run("three-level nesting stream→severity→metric", func(t *testing.T) { + streams := &orm.TermsAggregation{Field: "stream", Size: 10} + sevs := &orm.TermsAggregation{Field: "severity", Size: 10} + sevs.AddNested("total", &orm.MetricAggregation{Type: orm.MetricSum, Field: "latency"}) + streams.AddNested("by_sev", sevs) + qb := orm.NewQuery() + qb.SetAggs("streams", streams) + res := s.aggregate(qb) + + type sevAgg struct { + count int + sumL float64 + } + want := map[string]map[string]sevAgg{} + for i := range s.events { + e := &s.events[i] + if want[e.Stream] == nil { + want[e.Stream] = map[string]sevAgg{} + } + w := want[e.Stream][e.Severity] + w.count++ + w.sumL += e.Latency + want[e.Stream][e.Severity] = w + } + require.Len(t, res.Aggs["streams"].Buckets, len(want)) + for _, sb := range res.Aggs["streams"].Buckets { + inner := want[sb.Key] + require.NotNil(t, inner, "stream %q", sb.Key) + sevNode := sb.Aggs["by_sev"] + require.NotNil(t, sevNode) + require.Len(t, sevNode.Buckets, len(inner)) + for _, pb := range sevNode.Buckets { + w := inner[pb.Key] + assert.EqualValues(t, w.count, pb.DocCount, "%s/%s", sb.Key, pb.Key) + assert.True(t, feq(pb.Aggs["total"].Value, w.sumL), "%s/%s sum", sb.Key, pb.Key) + } + } + }) + + t.Run("date_histogram daily with zero fill", func(t *testing.T) { + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1d"} + dh.AddNested("count", &orm.MetricAggregation{Type: orm.MetricCount, Field: "latency"}) + qb := orm.NewQuery() + qb.SetAggs("days", dh) + res := s.aggregate(qb) + + want := map[string]int{} + for i := range s.events { + day := s.events[i].TS.UTC().Format("2006-01-02") + want[day]++ + } + buckets := res.Aggs["days"].Buckets + require.Len(t, buckets, len(want)) + var total int64 + for _, b := range buckets { + day := strings.TrimSuffix(b.Key, "T00:00:00") + assert.EqualValues(t, want[day], b.DocCount, "day %s", day) + total += b.DocCount + } + assert.EqualValues(t, len(s.events), total) + }) + + t.Run("date_histogram hourly zero fill gaps", func(t *testing.T) { + // Scope to day 2 hours 8..18: fixture guarantees hours 10-14 empty. + from := time.Date(2026, 8, 9, 8, 0, 0, 0, time.UTC) + to := time.Date(2026, 8, 9, 19, 0, 0, 0, time.UTC) + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h"} + qb := orm.NewQuery(). + Filter(orm.Range("ts").Gte(from.Format(time.RFC3339))). + Filter(orm.Range("ts").Lte(to.Format(time.RFC3339))) + qb.SetAggs("hours", dh) + res := s.aggregate(qb) + + buckets := res.Aggs["hours"].Buckets + // 8..18 inclusive = 11 hours, all present (data or zero-filled). + require.Len(t, buckets, 11) + for h := 10; h <= 14; h++ { + key := fmt.Sprintf("2026-08-09T%02d:00:00", h) + var found bool + for _, b := range buckets { + if b.Key == key { + found = true + assert.EqualValues(t, 0, b.DocCount, "gap hour %s must be zero-filled", key) + } + } + assert.True(t, found, "gap hour %s present", key) + } + }) + + t.Run("date_histogram offset preserves totals", func(t *testing.T) { + // Offset shifts boundaries, not membership: total docs invariant. + offset := 37*time.Minute + 12*time.Second + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1h", Offset: offset} + qb := orm.NewQuery() + qb.SetAggs("h", dh) + res := s.aggregate(qb) + var total int64 + for _, b := range res.Aggs["h"].Buckets { + total += b.DocCount + } + assert.EqualValues(t, len(s.events), total) + }) + + t.Run("date_histogram monthly", func(t *testing.T) { + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1M"} + qb := orm.NewQuery() + qb.SetAggs("months", dh) + res := s.aggregate(qb) + buckets := res.Aggs["months"].Buckets + require.Len(t, buckets, 1) // all 7 days within 2026-08 + assert.EqualValues(t, len(s.events), buckets[0].DocCount) + }) + + t.Run("auto_date_histogram interval derivation", func(t *testing.T) { + // The fallback heuristic picks ceil-ish fixed intervals: a ~7d span + // with Buckets=6 → candidate ≈ 28h → 1d buckets; with Buckets=7 the + // candidate is 23.98h → 1h buckets (documented heuristic behavior). + adh := &orm.AutoDateHistogramAggregation{Field: "ts", Buckets: 6} + qb := orm.NewQuery() + qb.SetAggs("auto", adh) + res := s.aggregate(qb) + buckets := res.Aggs["auto"].Buckets + require.NotEmpty(t, buckets) + assert.InDelta(t, 8, len(buckets), 1.1, "daily buckets over the 7d span") + var total int64 + for _, b := range buckets { + total += b.DocCount + } + assert.EqualValues(t, len(s.events), total) + }) + + t.Run("date_range", func(t *testing.T) { + d1 := time.Date(2026, 8, 8, 0, 0, 0, 0, time.UTC) + dr := &orm.DateRangeAggregation{Field: "ts", Ranges: []interface{}{ + map[string]interface{}{"from": "2026-08-07T00:00:00Z", "to": d1.Format(time.RFC3339), "key": "day0"}, + map[string]interface{}{"from": d1.Format(time.RFC3339), "key": "rest"}, + }} + qb := orm.NewQuery() + qb.SetAggs("ranges", dr) + res := s.aggregate(qb) + buckets := res.Aggs["ranges"].Buckets + require.Len(t, buckets, 2) + wantDay0, wantRest := 0, 0 + for i := range s.events { + if s.events[i].TS.Before(d1) { + wantDay0++ + } else { + wantRest++ + } + } + assert.EqualValues(t, wantDay0, buckets[0].DocCount) + assert.EqualValues(t, wantRest, buckets[1].DocCount) + }) + + t.Run("filter bucket term + nested sum", func(t *testing.T) { + filter := &orm.FilterAggregation{Query: map[string]interface{}{ + "term": map[string]interface{}{"severity": "fatal"}, + }} + filter.AddNested("sum_latency", &orm.MetricAggregation{Type: orm.MetricSum, Field: "latency"}) + qb := orm.NewQuery() + qb.SetAggs("fatals", filter) + res := s.aggregate(qb) + buckets := res.Aggs["fatals"].Buckets + require.Len(t, buckets, 1) + var wantCount, wantSum float64 + for i := range s.events { + if s.events[i].Severity == "fatal" { + wantCount++ + wantSum += s.events[i].Latency + } + } + assert.EqualValues(t, wantCount, buckets[0].DocCount) + assert.True(t, feq(buckets[0].Aggs["sum_latency"].Value, wantSum)) + }) + + t.Run("filter bucket with range query", func(t *testing.T) { + filter := &orm.FilterAggregation{Query: map[string]interface{}{ + "range": map[string]interface{}{"status": map[string]interface{}{"gte": 500}}, + }} + qb := orm.NewQuery() + qb.SetAggs("high_status", filter) + res := s.aggregate(qb) + want := 0 + for i := range s.events { + if s.events[i].Status >= 500 { + want++ + } + } + assert.EqualValues(t, want, res.Aggs["high_status"].Buckets[0].DocCount) + }) + + t.Run("percentiles within rank tolerance", func(t *testing.T) { + p := &orm.PercentilesAggregation{Field: "latency", Percents: []float64{50, 95}} + qb := orm.NewQuery() + qb.SetAggs("p", p) + res := s.aggregate(qb) + vals := res.Aggs["p"].Values + require.NotEmpty(t, vals) + + all := make([]float64, 0, len(s.events)) + for i := range s.events { + all = append(all, s.events[i].Latency) + } + sort.Float64s(all) + rank := func(pct float64) float64 { + idx := int(pct / 100 * float64(len(all))) + if idx >= len(all) { + idx = len(all) - 1 + } + return all[idx] + } + // Exact nearest-rank; allow the neighboring ranks as tolerance. + for _, pct := range []float64{50, 95} { + want := rank(pct) + lo, hi := all[max(0, int(pct/100*float64(len(all)))-1)], all[min(len(all)-1, int(pct/100*float64(len(all)))+1)] + got := vals[fmt.Sprintf("%g", pct)] + assert.GreaterOrEqual(t, got, lo-1e-6, "p%v within rank tolerance", pct) + assert.LessOrEqual(t, got, hi+1e-6, "p%v within rank tolerance", pct) + _ = want + } + }) + + t.Run("top_hits per stream latest by ts", func(t *testing.T) { + streams := &orm.TermsAggregation{Field: "stream", Size: 10} + streams.AddNested("latest", &orm.TopHitsAggregation{ + Size: 1, + Sorts: []orm.Sort{{Field: "ts", SortType: orm.DESC}}, + }) + qb := orm.NewQuery() + qb.SetAggs("streams", streams) + res := s.aggregate(qb) + latest := map[string]*scenarioEvent{} + for i := range s.events { + e := &s.events[i] + if cur := latest[e.Stream]; cur == nil || e.TS.After(cur.TS) { + latest[e.Stream] = e + } + } + for _, sb := range res.Aggs["streams"].Buckets { + node := sb.Aggs["latest"] + require.NotNil(t, node, "stream %s", sb.Key) + require.NotNil(t, node.TopHit, "stream %s top hit", sb.Key) + var doc map[string]interface{} + require.NoError(t, jsonUnmarshal(*node.TopHit, &doc)) + // Tie-break by id: both the SQL (ORDER BY ts DESC) and the oracle + // pick a max-ts doc; with equal timestamps either is acceptable. + assert.Equal(t, latest[sb.Key].TS.Format(time.RFC3339), doc["ts"], "stream %s", sb.Key) + } + }) + + t.Run("pipelines on daily histogram", func(t *testing.T) { + dh := &orm.DateHistogramAggregation{Field: "ts", Interval: "1d"} + dh.AddNested("sum_latency", &orm.MetricAggregation{Type: orm.MetricSum, Field: "latency"}) + dh.AddNested("count", &orm.MetricAggregation{Type: orm.MetricCount, Field: "latency"}) + dh.AddNested("deriv", &orm.DerivativeAggregation{BucketsPath: "count"}) + dh.AddNested("avg_expr", &orm.BucketScriptAggregation{ + BucketsPath: map[string]string{"s": "sum_latency", "c": "count"}, + Script: "params.s / params.c", + }) + qb := orm.NewQuery() + qb.SetAggregations(map[string]orm.Aggregation{ + "days": dh, + "total": &orm.PipelineAggregation{Type: orm.MetricSumBucket, BucketsPath: "days>count"}, + "peak": &orm.MaxBucketAggregation{BucketsPath: "days>count"}, + }) + res := s.aggregate(qb) + + var counts []float64 + var sums []float64 + for _, b := range res.Aggs["days"].Buckets { + counts = append(counts, b.Aggs["count"].Value) + sums = append(sums, b.Aggs["sum_latency"].Value) + } + require.Len(t, counts, 7) + // derivative[0] unset (ES null semantics — the node may be absent); + // diffs match from bucket 1 on. + if d := res.Aggs["days"].Buckets[0].Aggs["deriv"]; d != nil { + assert.False(t, d.ValueSet) + } + for i := 1; i < len(counts); i++ { + assert.True(t, feq(res.Aggs["days"].Buckets[i].Aggs["deriv"].Value, counts[i]-counts[i-1]), "deriv[%d]", i) + assert.True(t, feq(res.Aggs["days"].Buckets[i].Aggs["avg_expr"].Value, sums[i]/counts[i]), "avg_expr[%d]", i) + } + var wantSum, wantPeak float64 + for _, c := range counts { + wantSum += c + if c > wantPeak { + wantPeak = c + } + } + assert.True(t, feq(res.Aggs["total"].Value, wantSum)) + assert.True(t, feq(res.Aggs["peak"].Value, wantPeak)) + }) + + t.Run("bucket_sort top-k by sub metric", func(t *testing.T) { + streams := &orm.TermsAggregation{Field: "stream", Size: 10} + streams.AddNested("total", &orm.MetricAggregation{Type: orm.MetricSum, Field: "latency"}) + streams.AddNested("sort", &orm.BucketSortAggregation{ + Sort: []orm.BucketSortSpec{{Path: "total", Desc: true}}, + Size: 2, + }) + qb := orm.NewQuery() + qb.SetAggs("streams", streams) + res := s.aggregate(qb) + + totals := map[string]float64{} + for i := range s.events { + totals[s.events[i].Stream] += s.events[i].Latency + } + type kv struct { + k string + v float64 + } + ranked := make([]kv, 0, len(totals)) + for k, v := range totals { + ranked = append(ranked, kv{k, v}) + } + sort.Slice(ranked, func(i, j int) bool { return ranked[i].v > ranked[j].v }) + + buckets := res.Aggs["streams"].Buckets + require.GreaterOrEqual(t, len(buckets), 2) + for i := 0; i < 2; i++ { + assert.Equal(t, ranked[i].k, buckets[i].Key, "rank %d", i) + } + }) + + t.Run("deep chain with sum_bucket per stream", func(t *testing.T) { + streams := &orm.TermsAggregation{Field: "stream", Size: 10} + days := &orm.DateHistogramAggregation{Field: "ts", Interval: "1d"} + days.AddNested("sum_latency", &orm.MetricAggregation{Type: orm.MetricSum, Field: "latency"}) + streams.AddNested("days", days) + streams.AddNested("grand", &orm.PipelineAggregation{Type: orm.MetricSumBucket, BucketsPath: "days>sum_latency"}) + qb := orm.NewQuery() + qb.SetAggs("streams", streams) + res := s.aggregate(qb) + + want := map[string]float64{} + for i := range s.events { + want[s.events[i].Stream] += s.events[i].Latency + } + for _, sb := range res.Aggs["streams"].Buckets { + assert.True(t, feq(sb.Aggs["grand"].Value, want[sb.Key]), "stream %s grand total", sb.Key) + } + }) + + t.Run("terms on nested dotted path", func(t *testing.T) { + terms := &orm.TermsAggregation{Field: "svc.name", Size: 10} + qb := orm.NewQuery() + qb.SetAggs("svcs", terms) + res := s.aggregate(qb) + want := map[string]int{} + for i := range s.events { + want[s.events[i].Svc.Name]++ + } + require.Len(t, res.Aggs["svcs"].Buckets, len(want)) + for _, b := range res.Aggs["svcs"].Buckets { + assert.EqualValues(t, want[b.Key], b.DocCount, "svc %s", b.Key) + } + }) + + t.Run("terms on unmapped dynamic path", func(t *testing.T) { + terms := &orm.TermsAggregation{Field: "extra.tenant", Size: 10} + qb := orm.NewQuery() + qb.SetAggs("tenants", terms) + res := s.aggregate(qb) + want := map[string]int{} + for i := range s.events { + t := s.events[i].Extra["tenant"].(string) + want[t]++ + } + require.Len(t, res.Aggs["tenants"].Buckets, len(want)) + for _, b := range res.Aggs["tenants"].Buckets { + assert.EqualValues(t, want[b.Key], b.DocCount, "tenant %s", b.Key) + } + }) + + t.Run("aggregation respects query filter", func(t *testing.T) { + terms := &orm.TermsAggregation{Field: "severity", Size: 10} + qb := orm.NewQuery().Filter(orm.TermQuery("stream", "auth")) + qb.SetAggs("sevs", terms) + res := s.aggregate(qb) + want := map[string]int{} + for i := range s.events { + if s.events[i].Stream == "auth" { + want[s.events[i].Severity]++ + } + } + require.Len(t, res.Aggs["sevs"].Buckets, len(want)) + for _, b := range res.Aggs["sevs"].Buckets { + assert.EqualValues(t, want[b.Key], b.DocCount) + } + }) + + t.Run("empty set behaviors", func(t *testing.T) { + sum := &orm.MetricAggregation{Type: orm.MetricSum, Field: "latency"} + terms := &orm.TermsAggregation{Field: "severity", Size: 10} + qb := orm.NewQuery().Filter(orm.TermQuery("stream", "no-such-stream")) + qb.SetAggs("total", sum, "sevs", terms) + res := s.aggregate(qb) + assert.False(t, res.Aggs["total"].ValueSet, "sum over empty set has no value") + assert.Empty(t, res.Aggs["sevs"].Buckets) + }) +} + +// ── cross-path consistency ───────────────────────────────────────────────── + +func TestScenario_CrossPathConsistency(t *testing.T) { + s := newScenario(t) + + t.Run("SearchV2 ES-shape == Aggregate typed", func(t *testing.T) { + terms := &orm.TermsAggregation{Field: "severity", Size: 10} + terms.AddNested("total", &orm.MetricAggregation{Type: orm.MetricSum, Field: "latency"}) + qb := orm.NewQuery().Filter(orm.Range("status").Gte(300)) + qb.SetAggs("by_sev", terms) + + // Typed path. + typed := s.aggregate(qb) + esMap := map[string]map[string]float64{} + for _, b := range typed.Aggs["by_sev"].Buckets { + esMap[b.Key] = map[string]float64{ + "doc_count": float64(b.DocCount), + "total": b.Aggs["total"].Value, + } + } + + // ES-shaped path (SearchV2 side channel). + res, err := s.handler.SearchV2(s.ctx(), qb) + require.NoError(t, err) + resp, err := elastic.DecodeSearchResult(res) + require.NoError(t, err) + require.NotNil(t, resp.Aggregations) + bySev, ok := resp.Aggregations["by_sev"] + require.True(t, ok) + require.Len(t, bySev.Buckets, len(esMap)) + for _, b := range bySev.Buckets { + key := fmt.Sprintf("%v", b["key"]) + want := esMap[key] + require.NotNil(t, want, "bucket %v", key) + assert.EqualValues(t, want["doc_count"], b["doc_count"], "%v doc_count", key) + if sub, ok := b["total"].(map[string]interface{}); ok { + assert.True(t, feq(want["total"], sub["value"].(float64)), "%v total", key) + } + } + }) + + t.Run("promoted and unmapped paths agree on identical data", func(t *testing.T) { + // region is promoted (generated column); extra.mirror holds the same + // values with no mapping. Both paths must select identically. + for i := range s.events { + s.events[i].Extra["mirror"] = s.events[i].Region + } + for i := range s.events { + raw := fmt.Sprintf(`{"extra":{"tenant":%q,"mirror":%q}}`, s.events[i].Extra["tenant"], s.events[i].Region) + _, err := s.handler.DB.Exec("UPDATE scenario_events SET raw = json_set(raw, '$.extra.mirror', ?) WHERE id = ?", + s.events[i].Region, s.events[i].ID) + require.NoError(t, err) + _ = raw + } + viaPromoted := s.queryIDs(orm.NewQuery().Filter(orm.TermQuery("region", "cn-north"))) + viaFallback := s.queryIDs(orm.NewQuery().Filter(orm.TermQuery("extra.mirror", "cn-north"))) + assert.ElementsMatch(t, viaPromoted, viaFallback) + }) +} + +// small helpers +func max(a, b int) int { + if a > b { + return a + } + return b +} +func min(a, b int) int { + if a < b { + return a + } + return b +} +func jsonUnmarshal(b []byte, v interface{}) error { return json.Unmarshal(b, v) }