fix: order strings byte-for-byte in the VM and search generators

generators/blevegen/blevegenerator_test.go

@@ -314,3 +314,110 @@ func (s schema) ColumnInfo(f string) (*gentypes.FieldType, bool) {
 		TypeName: c.String(),
 	}, true
 }
+
+// TestStringOrderingTermRange covers ordering operators against a string-typed
+// column. A NumericRangeQuery leaves both bounds nil for a string literal,
+// which silently drops the predicate and matches every document; the term
+// range has to compare byte-for-byte the way vm.operateStrings does.
+func TestStringOrderingTermRange(t *testing.T) {
+	// country must be untokenized; an analyzed field lowercases the terms and
+	// then compares them against a literal that was not lowercased.
+	countryMapping := bleve.NewKeywordFieldMapping()
+	countryMapping.Name = "country"
+	docMapping := bleve.NewDocumentMapping()
+	docMapping.AddFieldMappingsAt("country", countryMapping)
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.DefaultMapping = docMapping
+
+	idx, err := bleve.NewMemOnly(indexMapping)
+	require.NoError(t, err)
+	t.Cleanup(func() { idx.Close() })
+
+	for id, country := range map[string]string{
+		"d1": "Albania",
+		"d2": "Argentina",
+		"d3": "Zimbabwe",
+	} {
+		require.NoError(t, idx.Index(id, map[string]any{"country": country}))
+	}
+
+	countrySchema := schema{cols: map[string]value.ValueType{"country": value.StringType}}
+
+	tests := []struct {
+		filterQL string
+		want     int
+	}{
+		{`FILTER country < "Argentina"`, 1},  // Albania
+		{`FILTER country <= "Argentina"`, 2}, // Albania, Argentina
+		{`FILTER country > "Argentina"`, 1},  // Zimbabwe
+		{`FILTER country >= "Argentina"`, 2}, // Argentina, Zimbabwe
+		{`FILTER country < "Albania"`, 0},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.filterQL, func(t *testing.T) {
+			filter, err := rel.ParseFilterQL(tc.filterQL)
+			require.NoError(t, err)
+
+			payload, err := NewGenerator(time.Now(), nil, countrySchema).WalkExpr(filter.Filter)
+			require.NoError(t, err)
+
+			q, ok := payload.Filter.(query.Query)
+			require.True(t, ok)
+			res, err := idx.Search(bleve.NewSearchRequest(q))
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, int(res.Total))
+		})
+	}
+}
+
+// TestStringBetweenTermRange covers BETWEEN against a string-typed column,
+// which hit the same nil-bounds NumericRangeQuery defect as makeRange.
+func TestStringBetweenTermRange(t *testing.T) {
+	countryMapping := bleve.NewKeywordFieldMapping()
+	countryMapping.Name = "country"
+	docMapping := bleve.NewDocumentMapping()
+	docMapping.AddFieldMappingsAt("country", countryMapping)
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.DefaultMapping = docMapping
+
+	idx, err := bleve.NewMemOnly(indexMapping)
+	require.NoError(t, err)
+	t.Cleanup(func() { idx.Close() })
+
+	for id, country := range map[string]string{
+		"d1": "Albania",
+		"d2": "Argentina",
+		"d3": "Zimbabwe",
+	} {
+		require.NoError(t, idx.Index(id, map[string]any{"country": country}))
+	}
+
+	countrySchema := schema{cols: map[string]value.ValueType{"country": value.StringType}}
+
+	tests := []struct {
+		filterQL string
+		want     int
+	}{
+		{`FILTER country BETWEEN "AAA" AND "Argentina"`, 1},    // Albania; upper is exclusive
+		{`FILTER country BETWEEN "AAA" AND "Zimbabwe"`, 2},     // Albania, Argentina
+		{`FILTER country BETWEEN "Albania" AND "Zimbabwe"`, 1}, // lower is exclusive
+		{`FILTER country BETWEEN "AAA" AND "AAB"`, 0},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.filterQL, func(t *testing.T) {
+			filter, err := rel.ParseFilterQL(tc.filterQL)
+			require.NoError(t, err)
+
+			payload, err := NewGenerator(time.Now(), nil, countrySchema).WalkExpr(filter.Filter)
+			require.NoError(t, err)
+
+			q, ok := payload.Filter.(query.Query)
+			require.True(t, ok)
+			res, err := idx.Search(bleve.NewSearchRequest(q))
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, int(res.Total))
+		})
+	}
+}

generators/blevegen/bridgeutil.go

@@ -41,6 +41,9 @@ func makeRange(lhs *gentypes.FieldType, op lex.TokenType, rhs expr.Node) (query.
 			return nil, fmt.Errorf("Could not convert %T %v to float", rhsval, rhsval)
 		}
 		rhsval = fv
+	case value.StringType, value.StringsType, value.MapStringType:
+		// Untokenized string fields range byte-for-byte, so the literal has to
+		// survive as written; see makeTermRange below.
 	default:
 		if rhsstr, ok := rhsval.(string); ok {
 			if rhsf, err := strconv.ParseFloat(rhsstr, 64); err == nil {
@@ -52,6 +55,13 @@ func makeRange(lhs *gentypes.FieldType, op lex.TokenType, rhs expr.Node) (query.
 
 	fieldName := lhs.Field
 
+	switch lhs.Type {
+	case value.StringType, value.StringsType, value.MapStringType:
+		// A NumericRangeQuery leaves both bounds nil for a string literal, which
+		// silently drops the predicate instead of comparing anything.
+		return makeTermRange(fieldName, op, rhsval)
+	}
+
 	// Create a range query
 	rangeQuery := query.NewNumericRangeQuery(nil, nil)
 	rangeQuery.SetField(fieldName)
@@ -129,6 +139,38 @@ func makeRange(lhs *gentypes.FieldType, op lex.TokenType, rhs expr.Node) (query.
 	return rangeQuery, nil
 }
 
+// makeTermRange returns a byte-order term range for untokenized string fields,
+// matching vm.operateStrings.
+func makeTermRange(fieldName string, op lex.TokenType, rhsval any) (query.Query, error) {
+	term, ok := rhsval.(string)
+	if !ok {
+		return nil, fmt.Errorf("qlindex: string field range needs a string, got %T", rhsval)
+	}
+
+	if term == "" {
+		// bleve reads an empty bound as "unbounded", so it cannot express a
+		// range against the empty string.
+		return nil, fmt.Errorf("qlindex: cannot range %s against an empty string", op)
+	}
+
+	t, f := true, false
+	var rangeQuery *query.TermRangeQuery
+	switch op {
+	case lex.TokenGE:
+		rangeQuery = query.NewTermRangeInclusiveQuery(term, "", &t, &f)
+	case lex.TokenGT:
+		rangeQuery = query.NewTermRangeInclusiveQuery(term, "", &f, &f)
+	case lex.TokenLE:
+		rangeQuery = query.NewTermRangeInclusiveQuery("", term, &f, &t)
+	case lex.TokenLT:
+		rangeQuery = query.NewTermRangeInclusiveQuery("", term, &f, &f)
+	default:
+		return nil, fmt.Errorf("qlindex: unsupported range operator %s", op)
+	}
+	rangeQuery.SetField(fieldName)
+	return rangeQuery, nil
+}
+
 // makeDateRangeQuery creates a date range query for time-based fields
 func makeDateRangeQuery(fieldName string, op lex.TokenType, rhsval any) (query.Query, error) {
 	var timeVal time.Time
@@ -198,6 +240,13 @@ func makeBetween(lhs *gentypes.FieldType, lower, upper any) (query.Query, error)
 		return makeDateBetweenQuery(fieldName, lower, upper)
 	}
 
+	switch lhs.Type {
+	case value.StringType, value.StringsType, value.MapStringType:
+		// Same defect as makeRange had: string bounds match no numeric case
+		// below, leaving both bounds nil and dropping the predicate.
+		return makeTermBetween(fieldName, lower, upper)
+	}
+
 	// Create a numeric range query
 	rangeQuery := query.NewNumericRangeQuery(nil, nil)
 	rangeQuery.SetField(fieldName)
@@ -235,6 +284,26 @@ func makeBetween(lhs *gentypes.FieldType, lower, upper any) (query.Query, error)
 	return rangeQuery, nil
 }
 
+// makeTermBetween returns a byte-order term range for an untokenized string
+// field. Exclusive on both ends, matching vm.walkTernary and the gt/lt pair
+// esgen emits.
+func makeTermBetween(fieldName string, lower, upper any) (query.Query, error) {
+	lo, loOK := lower.(string)
+	hi, hiOK := upper.(string)
+	if !loOK || !hiOK {
+		return nil, fmt.Errorf("qlindex: string field BETWEEN needs string bounds, got %T and %T", lower, upper)
+	}
+	if lo == "" || hi == "" {
+		// bleve reads an empty bound as "unbounded".
+		return nil, fmt.Errorf("qlindex: cannot range BETWEEN against an empty string")
+	}
+
+	f := false
+	rangeQuery := query.NewTermRangeInclusiveQuery(lo, hi, &f, &f)
+	rangeQuery.SetField(fieldName)
+	return rangeQuery, nil
+}
+
 // makeDateBetweenQuery creates a date range query for BETWEEN operations on time fields
 func makeDateBetweenQuery(fieldName string, lower, upper any) (query.Query, error) {
 	// Create a date range query

generators/esgen/bridgeutil.go

@@ -45,6 +45,10 @@ func makeRange(lhs *gentypes.FieldType, op lex.TokenType, rhs expr.Node) (any, e
 			return nil, fmt.Errorf("Could not convert %T %v to float", rhsval, rhsval)
 		}
 		rhsval = fv
+	case value.StringType, value.StringsType, value.MapStringType:
+		// Keyword fields range byte-for-byte, so the literal has to reach ES as
+		// written. Coercing it would compare an epoch-millis number against a
+		// country name, and would not match vm.operateStrings either.
 	default:
 		if rhsstr, ok := rhsval.(string); ok {
 			if rhsf, err := strconv.ParseFloat(rhsstr, 64); err == nil {
@@ -299,11 +303,12 @@ func makeBetween(lhs *gentypes.FieldType, lower, upper any) (any, error) {
 // produce consistent queries.
 //
 // For IntType fields the value is converted to int64. For NumberType fields it
-// is converted to float64. For all other types (including TimeType) string
-// values are first tried as a float (epoch-millis strings like "1778310000000")
-// and then as an ISO date string (e.g. "2026-05-09"), which is converted to
-// epoch milliseconds. Values that cannot be coerced are returned unchanged so
-// that Elasticsearch can attempt its own parsing.
+// is converted to float64. String-typed fields keep the literal verbatim. For
+// all other types (including TimeType) string values are first tried as a float
+// (epoch-millis strings like "1778310000000") and then as an ISO date string
+// (e.g. "2026-05-09"), which is converted to epoch milliseconds. Values that
+// cannot be coerced are returned unchanged so that Elasticsearch can attempt
+// its own parsing.
 func coerceScalar(lhs *gentypes.FieldType, val any) any {
 	rhv := value.NewValue(val)
 	switch lhs.Type {
@@ -315,6 +320,10 @@ func coerceScalar(lhs *gentypes.FieldType, val any) any {
 		if fv, ok := value.ValueToFloat64(rhv); ok {
 			return fv
 		}
+	case value.StringType, value.StringsType, value.MapStringType:
+		// Same reason as makeRange: keyword fields range byte-for-byte, so the
+		// bound has to reach ES as written.
+		return val
 	default:
 		s, ok := val.(string)
 		if !ok {

generators/esgen/esgenerator_test.go

@@ -304,3 +304,97 @@ func assertJSONEqual(t *testing.T, want string, got any) {
 	require.NoError(t, json.Unmarshal([]byte(want), &wantNorm))
 	assert.Equal(t, wantNorm, gotNorm, "generated ES filter mismatch\nwant: %s\ngot:  %s", want, string(gotBytes))
 }
+
+// TestStringOrderingRange covers ordering operators against a string-typed
+// column. The literal must reach Elasticsearch as written: a keyword field
+// ranges byte-for-byte, so coercing a numeric- or date-looking literal to a
+// number would compare against a different value than vm.operateStrings does.
+func TestStringOrderingRange(t *testing.T) {
+	s := schema{cols: map[string]value.ValueType{
+		"country":  value.StringType,
+		"score":    value.StringType,
+		"signedup": value.StringType,
+		"visitct":  value.IntType,
+	}}
+	g := NewGenerator(time.Now(), nil, s)
+
+	tests := []struct {
+		name     string
+		filterQL string
+		field    string
+		want     RangeQry
+	}{
+		{"LtWord", `FILTER country < "Argentina"`, "country", RangeQry{LT: "Argentina"}},
+		{"GtWord", `FILTER country > "Argentina"`, "country", RangeQry{GT: "Argentina"}},
+		{"LeWord", `FILTER country <= "Argentina"`, "country", RangeQry{LTE: "Argentina"}},
+		{"GeWord", `FILTER country >= "Argentina"`, "country", RangeQry{GTE: "Argentina"}},
+		// "0.2" must stay a string; as a float it round-trips to a different
+		// term than the one the VM compares against.
+		{"NumericLooking", `FILTER score > "0.20"`, "score", RangeQry{GT: "0.20"}},
+		// Coerced to epoch millis this became a number whose leading digit made
+		// every ISO date in the index compare greater.
+		{"DateLooking", `FILTER signedup < "2026-05-09"`, "signedup", RangeQry{LT: "2026-05-09"}},
+		// Non-string columns keep their coercion.
+		{"IntColumnStillCoerces", `FILTER visitct < "10"`, "visitct", RangeQry{LT: int64(10)}},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			fs, err := rel.ParseFilterQL(tc.filterQL)
+			require.NoError(t, err)
+			p, err := g.WalkExpr(fs.Filter)
+			require.NoError(t, err)
+			r, ok := p.Filter.(*RangeFilter)
+			require.True(t, ok, "expected a range filter, got %T", p.Filter)
+			assert.Equal(t, tc.want, r.Range[tc.field])
+		})
+	}
+}
+
+// TestStringBetweenRange covers BETWEEN against a string-typed column.
+// coerceScalar coerced date- and numeric-looking bounds the same way makeRange
+// did, so an ISO bound reached ES as epoch millis and every keyword term
+// compared greater.
+func TestStringBetweenRange(t *testing.T) {
+	s := schema{cols: map[string]value.ValueType{
+		"country":  value.StringType,
+		"signedup": value.StringType,
+		"visitct":  value.IntType,
+	}}
+	g := NewGenerator(time.Now(), nil, s)
+
+	tests := []struct {
+		name      string
+		filterQL  string
+		field     string
+		wantLower any
+		wantUpper any
+	}{
+		{"Words", `FILTER country BETWEEN "AAA" AND "Argentina"`, "country", "AAA", "Argentina"},
+		{"DateLooking", `FILTER signedup BETWEEN "2026-01-01" AND "2026-12-31"`, "signedup", "2026-01-01", "2026-12-31"},
+		{"IntColumnStillCoerces", `FILTER visitct BETWEEN "1" AND "10"`, "visitct", int64(1), int64(10)},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			fs, err := rel.ParseFilterQL(tc.filterQL)
+			require.NoError(t, err)
+			p, err := g.WalkExpr(fs.Filter)
+			require.NoError(t, err)
+
+			b, ok := p.Filter.(*boolean)
+			require.True(t, ok, "expected a bool filter, got %T", p.Filter)
+			m, ok := b.Bool.(must)
+			require.True(t, ok, "expected a must clause, got %T", b.Bool)
+			require.Len(t, m.Filters, 2)
+
+			lower, ok := m.Filters[0].(*RangeFilter)
+			require.True(t, ok)
+			upper, ok := m.Filters[1].(*RangeFilter)
+			require.True(t, ok)
+
+			assert.Equal(t, tc.wantLower, lower.Range[tc.field].GT)
+			assert.Equal(t, tc.wantUpper, upper.Range[tc.field].LT)
+		})
+	}
+}