From 1f50d2d9063da8a61e310ca1b4a75d02afde2d3e Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 13 Aug 2026 21:33:16 +0800 Subject: [PATCH 1/4] feat(elastic): client provider injection + easysearch cluster CRUD module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - core/elastic: inject the client factory via RegisterClientProvider (modules/elastic registers InitClientWithConfig at setup, before the enabled check so disabled-module apps keep working); GetOrCreateClient and InvalidateClient now live in core with a config-keyed cache, resolving the cache.go compile deadlock without core importing modules - modules/easysearch: new cluster-management module following the standard CRUD conventions — GET/POST _search via the query builder, Write*JSON envelopes, partial-field updates, per-action permissions including Search, and a pre-registration _test probe - core/orm: UpdatePartialFields seeds the target from the stored record before merging the delta; previously fields absent from the delta were wiped on full-replace stores (sqlite) — regression tests cover field preservation and nested deep-merge - modules/elastic: decouple cluster loading from the REST API (cluster_loader.go), extract ModuleConfig, adapter/common updates - cleanup: drop duplicate SearchByTemplate from ScriptAPI, fix malformed elastic_mapping tags in View, rename core/elastic/orm.go -> mapper.go --- core/elastic/api.go | 3 +- core/elastic/client_provider.go | 156 +++++++++ core/elastic/client_provider_test.go | 160 +++++++++ core/elastic/{orm.go => mapper.go} | 0 core/elastic/view.go | 4 +- core/orm/orm.go | 10 + modules/easysearch/cluster_api.go | 331 ++++++++++++++++++ modules/easysearch/cluster_api_test.go | 84 +++++ modules/easysearch/module.go | 61 ++++ modules/elastic/adapter/elasticsearch/v0.go | 14 +- modules/elastic/adapter/elasticsearch/v2.go | 5 +- .../elastic/client_provider_wiring_test.go | 58 +++ modules/elastic/cluster_loader.go | 92 +++++ modules/elastic/common/config.go | 17 +- modules/elastic/config.go | 18 + modules/elastic/metadata.go | 42 +++ modules/elastic/module.go | 54 ++- modules/sqlite/orm_test.go | 70 ++++ 18 files changed, 1146 insertions(+), 33 deletions(-) create mode 100644 core/elastic/client_provider.go create mode 100644 core/elastic/client_provider_test.go rename core/elastic/{orm.go => mapper.go} (100%) create mode 100644 modules/easysearch/cluster_api.go create mode 100644 modules/easysearch/cluster_api_test.go create mode 100644 modules/easysearch/module.go create mode 100644 modules/elastic/client_provider_wiring_test.go create mode 100644 modules/elastic/cluster_loader.go create mode 100644 modules/elastic/config.go diff --git a/core/elastic/api.go b/core/elastic/api.go index 182fb9c17..f46b7b72d 100755 --- a/core/elastic/api.go +++ b/core/elastic/api.go @@ -170,7 +170,8 @@ type ScrollAPI interface { type ScriptAPI interface { ScriptExists(scriptName string) (bool, error) PutScript(scriptName string, script []byte) ([]byte, error) - SearchByTemplate(indexName, scriptName string, params map[string]interface{}) (*SearchResponse, error) + // SearchByTemplate is declared on TemplateAPI (same signature); adapters + // implement it once and satisfy both via embedding. //GetScript(scriptName string)([]byte,error) //DeleteScript(scriptName string)([]byte,error) } diff --git a/core/elastic/client_provider.go b/core/elastic/client_provider.go new file mode 100644 index 000000000..05f1058be --- /dev/null +++ b/core/elastic/client_provider.go @@ -0,0 +1,156 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "errors" + "fmt" + "strings" + "sync" +) + +// ────────────────────────────────────────────────────────────────────────── +// Client provider injection. +// +// The client factory (version probing + adapter selection, e.g. +// modules/elastic/common.InitClientWithConfig) lives in modules/elastic, +// which depends on this package — so it cannot be imported here without a +// dependency cycle. Instead, the module registers its factory once at setup +// time and this package exposes the config-keyed client cache on top of it. +// +// GetOrCreateClient therefore gives any caller (inline/stream config, a +// storage-backed cluster, or an ad-hoc probe config) a live, self-contained +// client without registering the cluster by ID first. +// ────────────────────────────────────────────────────────────────────────── + +// ClientProvider builds a live client from a connection config. Registered +// by modules/elastic at module setup (before its enabled check, so apps that +// disable the elastic module but still talk to ES keep working). +type ClientProvider func(cfg ElasticsearchConfig) (API, error) + +var ( + clientProvider ClientProvider + clientProviderMu sync.RWMutex +) + +// RegisterClientProvider installs the client factory. Panics on nil or on a +// duplicate registration — double wiring is a programming error. +func RegisterClientProvider(p ClientProvider) { + if p == nil { + panic("elastic: RegisterClientProvider called with nil provider") + } + clientProviderMu.Lock() + defer clientProviderMu.Unlock() + if clientProvider != nil { + panic("elastic: client provider already registered") + } + clientProvider = p +} + +// registeredProvider returns the installed factory, or an error explaining +// what is missing (clearer than a nil-pointer panic at the call site). +func registeredProvider() (ClientProvider, error) { + clientProviderMu.RLock() + defer clientProviderMu.RUnlock() + if clientProvider == nil { + return nil, errors.New("no client provider registered; modules/elastic registers one via elastic.RegisterClientProvider at setup") + } + return clientProvider, nil +} + +// clientCache reuses clients across calls that hand in identical configs. +// Keyed by connection identity (see configCacheKey), not by cluster ID. +// Bounded eviction (LRU/TTL) can be layered on later; for now a simple map +// since the number of distinct connection identities is small. +var ( + clientCache = map[string]API{} + clientCacheMu sync.RWMutex +) + +// GetOrCreateClient returns a live ES client for the given config, building +// one on the first request and reusing it for subsequent calls with the same +// connection identity. +// +// Unlike GetElasticClient(clusterID), the cluster does NOT need to be +// registered: the returned client is self-contained — its metadata is built +// from the config and attached via SetMetadata, so it never falls back to the +// ID-based registry (which would panic if the cluster isn't registered). +// +// Set cfg.Version (and cfg.Distribution) to skip the network version probe +// when the caller already knows them; otherwise the provider probes the +// cluster once (on cache miss) to select the right version-specific adapter. +func GetOrCreateClient(cfg ElasticsearchConfig) (API, error) { + provider, err := registeredProvider() + if err != nil { + return nil, err + } + + key := configCacheKey(cfg) + + clientCacheMu.RLock() + if c, ok := clientCache[key]; ok { + clientCacheMu.RUnlock() + return c, nil + } + clientCacheMu.RUnlock() + + client, err := provider(cfg) + if err != nil { + return nil, err + } + + // Make the client self-contained: attach a metadata built from the config + // so GetMetadata() returns it directly instead of looking the config up by + // ID (and panicking if the cluster isn't registered). + meta := &ElasticsearchMetadata{Config: &cfg} + if s, ok := client.(interface { + SetMetadata(*ElasticsearchMetadata) + }); ok { + s.SetMetadata(meta) + } + + clientCacheMu.Lock() + // A concurrent caller may have built one first; keep the winner. + if existing, ok := clientCache[key]; ok { + clientCacheMu.Unlock() + return existing, nil + } + clientCache[key] = client + clientCacheMu.Unlock() + return client, nil +} + +// InvalidateClient drops the cached client (if any) for the given config, so +// the next GetOrCreateClient rebuilds it — e.g. after a cluster's endpoint or +// credentials change. +func InvalidateClient(cfg ElasticsearchConfig) { + key := configCacheKey(cfg) + clientCacheMu.Lock() + delete(clientCache, key) + clientCacheMu.Unlock() +} + +// ResetClientCacheForTest clears the client cache and unregisters the client +// provider. Test-only. +func ResetClientCacheForTest() { + clientCacheMu.Lock() + clientCache = map[string]API{} + clientCacheMu.Unlock() + clientProviderMu.Lock() + clientProvider = nil + clientProviderMu.Unlock() +} + +// configCacheKey returns a canonical identity for a config's connection: two +// configs that reach the same cluster with the same auth/version share a key +// (and thus a client). Secrets are part of the key (held in memory only, never +// logged) so different credentials get different clients. +func configCacheKey(cfg ElasticsearchConfig) string { + var b strings.Builder + fmt.Fprintf(&b, "%s|%s|%s|", cfg.GetAnyEndpoint(), strings.Join(cfg.Endpoints, ","), cfg.Distribution) + if cfg.BasicAuth != nil { + fmt.Fprintf(&b, "%s:%s|", cfg.BasicAuth.Username, cfg.BasicAuth.Password.Get()) + } + fmt.Fprintf(&b, "%s|%s", cfg.Token.Get(), cfg.Version) + return b.String() +} diff --git a/core/elastic/client_provider_test.go b/core/elastic/client_provider_test.go new file mode 100644 index 000000000..d6cbdbcae --- /dev/null +++ b/core/elastic/client_provider_test.go @@ -0,0 +1,160 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "errors" + "testing" +) + +func mkCfg(endpoint, version string) ElasticsearchConfig { + c := ElasticsearchConfig{} + c.ID = "test-" + endpoint + c.Name = endpoint + c.Endpoint = endpoint + c.Distribution = Elasticsearch + c.Version = version // set → provider skips the network version probe + c.Enabled = true + return c +} + +// stubClient satisfies the fat API interface by embedding it (nil); tests +// only exercise pointer identity and metadata plumbing, never API calls. +type stubClient struct { + API + meta *ElasticsearchMetadata +} + +func (s *stubClient) SetMetadata(m *ElasticsearchMetadata) { s.meta = m } +func (s *stubClient) GetMetadata() *ElasticsearchMetadata { return s.meta } + +// stubProvider returns a fresh client per call so tests can tell builds apart. +func stubProvider(cfg ElasticsearchConfig) (API, error) { + return &stubClient{}, nil +} + +func TestConfigCacheKey(t *testing.T) { + a := mkCfg("http://a:9200", "8.0.0") + a2 := mkCfg("http://a:9200", "8.0.0") + b := mkCfg("http://b:9200", "8.0.0") + + if configCacheKey(a) != configCacheKey(a2) { + t.Fatal("identical configs must share a key") + } + if configCacheKey(a) == configCacheKey(b) { + t.Fatal("different endpoints must produce different keys") + } + + a3 := mkCfg("http://a:9200", "9.0.0") + if configCacheKey(a) == configCacheKey(a3) { + t.Fatal("different versions must produce different keys") + } +} + +func TestGetOrCreateClient_CachesByConfig(t *testing.T) { + ResetClientCacheForTest() + RegisterClientProvider(stubProvider) + cfg := mkCfg("http://x:9200", "8.0.0") + + c1, err := GetOrCreateClient(cfg) + if err != nil { + t.Fatalf("first GetOrCreateClient: %v", err) + } + c2, err := GetOrCreateClient(cfg) + if err != nil { + t.Fatalf("second GetOrCreateClient: %v", err) + } + if c1 != c2 { + t.Fatal("same config should return the same cached client pointer") + } + + other := mkCfg("http://y:9200", "8.0.0") + c3, err := GetOrCreateClient(other) + if err != nil { + t.Fatalf("GetOrCreateClient(other): %v", err) + } + if c1 == c3 { + t.Fatal("different config should return a different client") + } +} + +func TestGetOrCreateClient_SelfContained(t *testing.T) { + ResetClientCacheForTest() + RegisterClientProvider(stubProvider) + cfg := mkCfg("http://self:9200", "8.0.0") + + client, err := GetOrCreateClient(cfg) + if err != nil { + t.Fatalf("GetOrCreateClient: %v", err) + } + + // The client must carry its own metadata built from the config — it should + // NOT need the cluster to be registered by ID (no RegisterInstance ran). + mg, ok := client.(interface { + GetMetadata() *ElasticsearchMetadata + }) + if !ok { + t.Fatal("client does not expose GetMetadata") + } + md := mg.GetMetadata() + if md == nil || md.Config == nil || md.Config.Endpoint != "http://self:9200" { + t.Fatalf("self-contained metadata not set from config: %+v", md) + } +} + +func TestGetOrCreateClient_ProviderErrorsPropagate(t *testing.T) { + ResetClientCacheForTest() + RegisterClientProvider(func(cfg ElasticsearchConfig) (API, error) { + return nil, errors.New("boom") + }) + if _, err := GetOrCreateClient(mkCfg("http://err:9200", "8.0.0")); err == nil { + t.Fatal("provider error must propagate") + } +} + +func TestGetOrCreateClient_NoProviderRegistered(t *testing.T) { + ResetClientCacheForTest() + _, err := GetOrCreateClient(mkCfg("http://none:9200", "8.0.0")) + if err == nil || !contains(err.Error(), "no client provider registered") { + t.Fatalf("expected a clear missing-provider error, got: %v", err) + } +} + +func TestRegisterClientProvider_DoubleRegistrationPanics(t *testing.T) { + ResetClientCacheForTest() + RegisterClientProvider(stubProvider) + defer func() { + if recover() == nil { + t.Fatal("duplicate registration must panic") + } + ResetClientCacheForTest() + }() + RegisterClientProvider(stubProvider) +} + +func TestInvalidateClient(t *testing.T) { + ResetClientCacheForTest() + RegisterClientProvider(stubProvider) + cfg := mkCfg("http://inv:9200", "8.0.0") + + c1, _ := GetOrCreateClient(cfg) + InvalidateClient(cfg) + c2, _ := GetOrCreateClient(cfg) + + if c1 == c2 { + t.Fatal("InvalidateClient should force a rebuild on next GetOrCreateClient") + } +} + +func contains(s, sub string) bool { + return len(s) >= len(sub) && (s == sub || len(sub) == 0 || indexOf(s, sub) >= 0) +} + +func indexOf(s, sub string) int { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return i + } + } + return -1 +} diff --git a/core/elastic/orm.go b/core/elastic/mapper.go similarity index 100% rename from core/elastic/orm.go rename to core/elastic/mapper.go diff --git a/core/elastic/view.go b/core/elastic/view.go index 9bc7ee943..a2908f3c1 100644 --- a/core/elastic/view.go +++ b/core/elastic/view.go @@ -33,10 +33,10 @@ type View struct { ID string `json:"-" elastic_meta:"_id" elastic_mapping:"id: { type: keyword }"` ClusterID string `json:"cluster_id" elastic_mapping:"cluster_id:{type:keyword}"` Title string `json:"title" elastic_mapping:"title:{type:keyword}"` - ViewName string `json:"viewName" elastic_mapping:"view_name:{type:text}` + ViewName string `json:"viewName" elastic_mapping:"view_name:{type:text}"` TimeFieldName string `json:"timeFieldName" elastic_mapping:"timeFieldName:{type:keyword}"` Fields string `json:"fields" elastic_mapping:"fields:{type:text}"` - FieldFormatMap string `json:"fieldFormatMap" elastic_mapping:"fields:{type:text}` + FieldFormatMap string `json:"fieldFormatMap" elastic_mapping:"fields:{type:text}"` UpdatedAt time.Time `json:"updated_at,omitempty" elastic_mapping:"updated_at:{type:date}"` DefaultLayoutID string `json:"default_layout_id" elastic_mapping:"default_layout_id:{type:keyword}"` ComplexFields string `json:"complex_fields" elastic_mapping:"complex_fields:{type:text}"` diff --git a/core/orm/orm.go b/core/orm/orm.go index d1b6844ec..ca072494e 100755 --- a/core/orm/orm.go +++ b/core/orm/orm.go @@ -704,6 +704,16 @@ func saveOrUpdate(ctx *Context, o interface{}, delta util.MapStr, opType Operati if exists { if mergePartial && deltaNotEmpty { + // Seed the target with the stored state before overlaying the + // delta, so fields absent from the delta keep their stored + // values on full-replace stores (e.g. sqlite rewrites the whole + // row). Without this, a partial update would wipe every field + // not mentioned in the delta. + prevValue := reflect.ValueOf(prev) + if prevValue.Kind() == reflect.Ptr && !prevValue.IsNil() && + prevValue.Type().Elem() == rValue.Type().Elem() && rValue.Elem().CanSet() { + rValue.Elem().Set(prevValue.Elem()) + } if err := mergeMapToStruct(delta, rValue); err != nil { return err } diff --git a/modules/easysearch/cluster_api.go b/modules/easysearch/cluster_api.go new file mode 100644 index 000000000..877b056bc --- /dev/null +++ b/modules/easysearch/cluster_api.go @@ -0,0 +1,331 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package easysearch + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/security" + "infini.sh/framework/core/util" +) + +// ────────────────────────────────────────────────────────────────────────── +// Cluster management REST API — public, reusable, ORM-backed. +// +// CRUD over ElasticsearchConfig clusters using only the standard ORM (sqlite +// or any non-elastic store). Routes use the /easysearch/ prefix (not +// /elasticsearch/) so they don't collide with legacy /elasticsearch/ routes. +// +// Handler conventions follow the standard module CRUD pattern (see +// coco/modules/integration): Write*JSON response envelopes, partial-field +// updates, and a _search endpoint driven by orm.NewQueryBuilderFromRequest +// (pagination/sort/filter/full-text via query string or request body). +// +// Decoupled from modules/elastic: no live-client registration, no in-memory +// metadata registry. Health status shown in responses is the value persisted +// on each cluster's Labels by the elasticsearch module's health loop. +// ────────────────────────────────────────────────────────────────────────── + +// ClusterAPI provides REST CRUD for ElasticsearchConfig clusters. It embeds +// api.Handler for DecodeJSON/WriteJSON/WriteError. +type ClusterAPI struct { + api.Handler +} + +// registerClusterAPI registers the /easysearch/ CRUD routes. Called from +// Module.Setup so every app that registers the easysearch module gets cluster +// management. The /_test route is public (AllowPublicAccess) because it's a +// pre-registration connectivity probe (no cluster exists yet). +func registerClusterAPI() { + + // Generic permission keys for cluster management. Apps assign these to roles + // to control who can manage clusters. + var ( + permClusterRead = security.GetOrInitPermission("generic", "easysearch:cluster", security.Read) + permClusterCreate = security.GetOrInitPermission("generic", "easysearch:cluster", security.Create) + permClusterUpdate = security.GetOrInitPermission("generic", "easysearch:cluster", security.Update) + permClusterDelete = security.GetOrInitPermission("generic", "easysearch:cluster", security.Delete) + permClusterSearch = security.GetOrInitPermission("generic", "easysearch:cluster", security.Search) + ) + + h := &ClusterAPI{} + api.HandleUIMethod(api.POST, "/easysearch/_test", h.testConnection, api.RequirePermission(permClusterRead), api.AllowOPTIONSS(), api.Feature(api.FeatureCORS)) + api.HandleUIMethod(api.POST, "/easysearch/", h.createCluster, api.RequirePermission(permClusterCreate)) + api.HandleUIMethod(api.GET, "/easysearch/_search", h.searchClusters, api.RequirePermission(permClusterSearch)) + api.HandleUIMethod(api.POST, "/easysearch/_search", h.searchClusters, api.RequirePermission(permClusterSearch)) + api.HandleUIMethod(api.GET, "/easysearch/:id", h.getCluster, api.RequirePermission(permClusterRead)) + api.HandleUIMethod(api.PUT, "/easysearch/:id", h.updateCluster, api.RequirePermission(permClusterUpdate)) + api.HandleUIMethod(api.DELETE, "/easysearch/:id", h.deleteCluster, api.RequirePermission(permClusterDelete)) +} + +// createCluster — POST /easysearch/ +// Persists a cluster record. It becomes a live ES client when the +// elasticsearch module next loads clusters from the ORM (boot/reload). +func (h *ClusterAPI) createCluster(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + var cfg elastic.ElasticsearchConfig + if err := h.DecodeJSON(req, &cfg); err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if cfg.Name == "" { + h.WriteError(w, "name is required", http.StatusBadRequest) + return + } + if cfg.ID == "" { + cfg.ID = util.GetUUID() + } + if cfg.Distribution == "" { + cfg.Distribution = elastic.Elasticsearch + } + // Mark as dynamically-managed so the elasticsearch module's health loop + // persists status for it (the loop keys off this source value). + cfg.Source = elastic.ElasticsearchConfigSourceElasticsearch + cfg.Enabled = true + + ctx := orm.NewContextWithParent(req.Context()) + ctx.Refresh = orm.WaitForRefresh + if err := orm.Create(ctx, &cfg); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteCreatedOKJSON(w, cfg.ID) +} + +// searchClusters — GET/POST /easysearch/_search +// Standard query-builder search (pagination, sorting, filtering, full-text on +// the name field via ?query=). Returns an ES-shaped SearchResponse; each hit's +// _source carries the cluster with its persisted health_status (in Labels) +// from the elasticsearch module's health loop. +func (h *ClusterAPI) searchClusters(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + //handle url query args, convert to query builder + builder, err := orm.NewQueryBuilderFromRequest(req, "name") + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + builder.EnableBodyBytes() + if len(builder.Sorts()) == 0 { + builder.SortBy(orm.Sort{Field: "created", SortType: orm.DESC}) + } + + ctx := orm.NewContextWithParent(req.Context()) + orm.WithModel(ctx, &elastic.ElasticsearchConfig{}) + res, err := orm.SearchV2(ctx, builder) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + searchRes, err := parseSearchResponse(res) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteJSON(w, searchRes, http.StatusOK) +} + +// getCluster — GET /easysearch/:id +func (h *ClusterAPI) getCluster(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + cfg := elastic.ElasticsearchConfig{} + cfg.ID = id + ctx := orm.NewContextWithParent(req.Context()) + exists, err := orm.GetV2(ctx, &cfg) + if !exists || err != nil { + h.WriteGetMissingJSON(w, id) + return + } + h.WriteGetOKJSON(w, id, cfg) +} + +// updateCluster — PUT /easysearch/:id +// Partial update: only the fields present in the request body are changed +// (orm.UpdatePartialFields merges the delta onto the stored record). Secrets +// omitted from the body — e.g. the password, which GET responses return +// masked — keep their stored values, so a partial update never breaks the +// connection. +func (h *ClusterAPI) updateCluster(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + obj := elastic.ElasticsearchConfig{} + obj.ID = id + + delta := util.MapStr{} + if err := h.DecodeJSON(req, &delta); err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + + ctx := orm.NewContextWithParent(req.Context()) + ctx.Refresh = orm.WaitForRefresh + if err := orm.UpdatePartialFields(ctx, &obj, delta); err != nil { + if strings.Contains(err.Error(), "not found") { + h.WriteOpRecordNotFoundJSON(w, id) + return + } + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteUpdatedOKJSON(w, obj.ID) +} + +// deleteCluster — DELETE /easysearch/:id +// Removes the cluster record. Reserved clusters (e.g. the system cluster) +// cannot be deleted. +func (h *ClusterAPI) deleteCluster(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + cfg := elastic.ElasticsearchConfig{} + cfg.ID = id + ctx := orm.NewContextWithParent(req.Context()) + exists, err := orm.GetV2(ctx, &cfg) + if !exists || err != nil { + h.WriteOpRecordNotFoundJSON(w, id) + return + } + if cfg.Reserved { + h.WriteError(w, "reserved cluster cannot be deleted", http.StatusForbidden) + return + } + ctx.Refresh = orm.WaitForRefresh + if err := orm.Delete(ctx, &cfg); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteDeletedOKJSON(w, id) +} + +// testConnection — POST /easysearch/_test +// Pre-registration connectivity + version probe via a raw HTTP request. Does +// NOT persist or register the cluster. Public so it works before login is +// required (callers use it to validate a form before creating). +func (h *ClusterAPI) testConnection(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + var cfg elastic.ElasticsearchConfig + if err := h.DecodeJSON(req, &cfg); err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + endpoint := cfg.Endpoint + if endpoint == "" && len(cfg.Endpoints) > 0 { + endpoint = cfg.Endpoints[0] + } + if endpoint == "" { + h.WriteError(w, "endpoint is required", http.StatusBadRequest) + return + } + if cfg.Distribution == "" { + cfg.Distribution = elastic.Elasticsearch + } + + version, distribution, err := probeCluster(&cfg) + if err != nil { + h.WriteJSON(w, map[string]interface{}{ + "connected": false, + "error": err.Error(), + "distribution": cfg.Distribution, + }, http.StatusOK) + return + } + if distribution == "" { + distribution = cfg.Distribution + } + h.WriteJSON(w, map[string]interface{}{ + "connected": true, + "version": version, + "distribution": distribution, + }, http.StatusOK) +} + +// ────────────────────────────────────────────────────────────────────────── +// Helpers +// ────────────────────────────────────────────────────────────────────────── + +// parseSearchResponse decodes an orm.SearchResult payload (ES-shaped JSON: +// {"hits":{"hits":[{"_source":{...}}]}}) into elastic.SearchResponse. Accepts +// both []byte and string payloads; nil/empty payloads yield a zero response. +func parseSearchResponse(res *orm.SearchResult) (elastic.SearchResponse, error) { + out := elastic.SearchResponse{} + if res == nil { + return out, nil + } + var raw []byte + switch payload := res.Payload.(type) { + case []byte: + raw = payload + case string: + raw = []byte(payload) + default: + return out, nil + } + if len(raw) == 0 { + return out, nil + } + if err := util.FromJSONBytes(raw, &out); err != nil { + return elastic.SearchResponse{}, err + } + return out, nil +} + +// probeTransport is the HTTP transport for connectivity probes: TLS verification +// disabled (ES clusters commonly use self-signed certs). +var probeTransport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, +} + +// probeCluster does a raw GET to the cluster's root endpoint to detect +// connectivity and the server version/distribution, without building a +// version-specific adapter. BasicAuth or X-API-TOKEN is applied when present. +func probeCluster(cfg *elastic.ElasticsearchConfig) (version, distribution string, err error) { + endpoint := cfg.Endpoint + if endpoint == "" && len(cfg.Endpoints) > 0 { + endpoint = cfg.Endpoints[0] + } + httpReq, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return "", "", err + } + if cfg.BasicAuth != nil && cfg.BasicAuth.Username != "" { + httpReq.SetBasicAuth(cfg.BasicAuth.Username, cfg.BasicAuth.Password.Get()) + } else if t := cfg.Token.Get(); t != "" { + httpReq.Header.Set("X-API-TOKEN", t) + } + + client := &http.Client{Timeout: 10 * time.Second, Transport: probeTransport} + resp, err := client.Do(httpReq) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode >= 400 { + return "", "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(truncate(string(body), 200))) + } + var info struct { + Version struct { + Number string `json:"number"` + Distribution string `json:"distribution"` + } `json:"version"` + } + if err := json.Unmarshal(body, &info); err != nil { + return "", "", fmt.Errorf("parse version response: %w", err) + } + return info.Version.Number, info.Version.Distribution, nil +} + +// truncate caps s to n runes, appending "…" when truncated. +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/modules/easysearch/cluster_api_test.go b/modules/easysearch/cluster_api_test.go new file mode 100644 index 000000000..99757745a --- /dev/null +++ b/modules/easysearch/cluster_api_test.go @@ -0,0 +1,84 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package easysearch + +import ( + "testing" + + "infini.sh/framework/core/orm" +) + +func TestParseSearchResponse(t *testing.T) { + payload := `{"hits":{"total":{"value":2},"hits":[{"_id":"a","_source":{"id":"a","name":"A","distribution":"easysearch"}},{"_id":"b","_source":{"id":"b","name":"B"}}]}}` + res, err := parseSearchResponse(&orm.SearchResult{Payload: []byte(payload)}) + if err != nil { + t.Fatalf("parse failed: %v", err) + } + if len(res.Hits.Hits) != 2 { + t.Fatalf("expected 2 hits, got %d (%+v)", len(res.Hits.Hits), res.Hits.Hits) + } + var nameA, nameB string + for _, hit := range res.Hits.Hits { + if hit.ID == "a" { + nameA, _ = hit.Source["name"].(string) + } + if hit.ID == "b" { + nameB, _ = hit.Source["name"].(string) + } + } + if nameA != "A" || nameB != "B" { + t.Fatalf("hit sources mismatch: %q, %q", nameA, nameB) + } +} + +func TestParseSearchResponse_EmptyAndMalformed(t *testing.T) { + checks := []struct { + name string + res *orm.SearchResult + wantErr bool + }{ + {"nil result", nil, false}, + {"nil payload", &orm.SearchResult{}, false}, + {"empty hits", &orm.SearchResult{Payload: []byte(`{"hits":{"hits":[]}}`)}, false}, + {"empty bytes", &orm.SearchResult{Payload: []byte(``)}, false}, + {"string payload", &orm.SearchResult{Payload: `{"hits":{"hits":[]}}`}, false}, + {"empty string", &orm.SearchResult{Payload: ``}, false}, + {"non-bytes payload", &orm.SearchResult{Payload: 12345}, false}, + {"malformed json", &orm.SearchResult{Payload: []byte(`not json`)}, true}, + } + for _, c := range checks { + t.Run(c.name, func(t *testing.T) { + res, err := parseSearchResponse(c.res) + if c.wantErr { + if err == nil { + t.Fatalf("expected an error, got %+v", res) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(res.Hits.Hits) != 0 { + t.Fatalf("expected no hits, got %d", len(res.Hits.Hits)) + } + }) + } +} + +func TestTruncate(t *testing.T) { + cases := []struct { + in string + n int + want string + }{ + {"abc", 5, "abc"}, // under limit → unchanged + {"abcdef", 3, "abc…"}, // over limit → cut + ellipsis + {"世界你好", 2, "世界…"}, // rune-aware, not byte-aware + {"", 3, ""}, // empty + } + for _, c := range cases { + if got := truncate(c.in, c.n); got != c.want { + t.Errorf("truncate(%q,%d) = %q, want %q", c.in, c.n, got, c.want) + } + } +} diff --git a/modules/easysearch/module.go b/modules/easysearch/module.go new file mode 100644 index 000000000..cdfe1962f --- /dev/null +++ b/modules/easysearch/module.go @@ -0,0 +1,61 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package easysearch + +import ( + log "github.com/cihub/seelog" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" +) + +// Module owns cluster management (the /easysearch/ REST API) as a clean, +// ORM-backed data layer, decoupled from the elasticsearch module's live-client +// and health machinery. Apps opt in by registering it. +// +// It does NOT register live ES clients or read the in-memory metadata +// registry — cluster records become live clients when the elasticsearch module +// loads them (LoadClustersFromORM) on its own boot/reload cycle. Unifying live +// registration is a later refactor. +type Module struct { + moduleConfig moduleConfig +} + +type moduleConfig struct { + Enabled bool `json:"enabled" config:"enabled"` +} + +func (m *Module) Name() string { return "easysearch" } + +func (m *Module) Setup() { + + exists, err := env.ParseConfig("easysearch", &m.moduleConfig) + if exists && err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { + panic(err) + } + + if !m.moduleConfig.Enabled { + return + } + + // Register the cluster schema. This module owns it now (it was previously + // registered inside modules/elastic). The table is materialized later by + // the ORM backend's InitSchema() (e.g. sqlite), which runs after every + // module's Setup(). + if err := orm.RegisterSchemaWithIndexName(elastic.ElasticsearchConfig{}, "cluster"); err != nil { + log.Warnf("easysearch: register cluster schema: %v", err) + } + registerClusterAPI() +} + +func (m *Module) Start() error { + + if !m.moduleConfig.Enabled { + return nil + } + + return nil +} +func (m *Module) Stop() error { return nil } diff --git a/modules/elastic/adapter/elasticsearch/v0.go b/modules/elastic/adapter/elasticsearch/v0.go index 0a13b8bd8..be134b48f 100755 --- a/modules/elastic/adapter/elasticsearch/v0.go +++ b/modules/elastic/adapter/elasticsearch/v0.go @@ -77,6 +77,7 @@ func (c *ESAPIV0) GetActivePreferredEndpoint(host string) string { func (c *ESAPIV0) GetEndpoint() string { return c.GetMetadata().GetActiveEndpoint() } + func (c *ESAPIV0) GetMetadata() *elastic.ElasticsearchMetadata { c.metaLocker.Lock() defer c.metaLocker.Unlock() @@ -102,6 +103,17 @@ func (c *ESAPIV0) GetMetadata() *elastic.ElasticsearchMetadata { return c.metadata } +// SetMetadata attaches a prebuilt metadata so the client is self-contained: +// GetMetadata() returns it directly instead of looking the config up by ID in +// the global registry. This lets a client be built from a standalone config +// (see core/elastic.GetOrCreateClient) and used without first +// registering it by cluster ID. +func (c *ESAPIV0) SetMetadata(m *elastic.ElasticsearchMetadata) { + c.metaLocker.Lock() + c.metadata = m + c.metaLocker.Unlock() +} + func (c *ESAPIV0) GetVersion() elastic.Version { if c.Version.Number == "" && c.GetEndpoint() != "" { c.Version, _ = adapter.GetMajorVersion(c.GetMetadata()) @@ -1353,7 +1365,7 @@ func (s *ESAPIV0) NextScroll(ctx *elastic.APIContext, scrollTime string, scrollI url := fmt.Sprintf("%s/_search/scroll?scroll=%s&scroll_id=%s", s.GetEndpoint(), scrollTime, scrollId) - resp, err := adapter.RequestTimeout(ctx, util.Verb_GET, url, nil, s.metadata, time.Duration(s.metadata.Config.RequestTimeout)*time.Second) + resp, err := adapter.RequestTimeout(ctx, util.Verb_GET, url, nil, s.GetMetadata(), time.Duration(s.metadata.Config.RequestTimeout)*time.Second) if err != nil { return nil, err } diff --git a/modules/elastic/adapter/elasticsearch/v2.go b/modules/elastic/adapter/elasticsearch/v2.go index bf3a2a05c..ac3994580 100644 --- a/modules/elastic/adapter/elasticsearch/v2.go +++ b/modules/elastic/adapter/elasticsearch/v2.go @@ -27,12 +27,13 @@ import ( "context" "errors" "fmt" + "time" + log "github.com/cihub/seelog" "infini.sh/framework/core/elastic" "infini.sh/framework/core/global" "infini.sh/framework/core/util" "infini.sh/framework/modules/elastic/adapter" - "time" ) type ESAPIV2 struct { @@ -62,7 +63,7 @@ func (s *ESAPIV2) NextScroll(ctx *elastic.APIContext, scrollTime string, scrollI body["scroll"] = scrollTime bodyBytes := util.MustToJSONBytes(body) - resp, err := adapter.RequestTimeout(ctx, util.Verb_POST, url, bodyBytes, s.metadata, time.Duration(s.metadata.Config.RequestTimeout)*time.Second) + resp, err := adapter.RequestTimeout(ctx, util.Verb_POST, url, bodyBytes, s.GetMetadata(), time.Duration(s.metadata.Config.RequestTimeout)*time.Second) if err != nil { return nil, err } diff --git a/modules/elastic/client_provider_wiring_test.go b/modules/elastic/client_provider_wiring_test.go new file mode 100644 index 000000000..111a7cc46 --- /dev/null +++ b/modules/elastic/client_provider_wiring_test.go @@ -0,0 +1,58 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "testing" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/modules/elastic/common" +) + +// TestClientProviderWiring proves the factory hand-off end to end: the module +// registers common.InitClientWithConfig as the core client provider, and +// core.GetOrCreateClient builds a self-contained client from a bare config +// (version preset → no network probe, fully offline). +func TestClientProviderWiring(t *testing.T) { + elastic.ResetClientCacheForTest() + elastic.RegisterClientProvider(common.InitClientWithConfig) + + cfg := elastic.ElasticsearchConfig{} + cfg.ID = "wiring-test" + cfg.Name = "wiring-test" + cfg.Endpoint = "http://wiring-test:9200" + cfg.Distribution = elastic.Elasticsearch + cfg.Version = "8.0.0" // preset → adapter selection without probing + cfg.Enabled = true + + c1, err := elastic.GetOrCreateClient(cfg) + if err != nil { + t.Fatalf("GetOrCreateClient via real factory: %v", err) + } + if c1 == nil { + t.Fatal("factory returned a nil client") + } + + // Self-contained: metadata comes from the config, no RegisterInstance ran. + mg, ok := c1.(interface { + GetMetadata() *elastic.ElasticsearchMetadata + }) + if !ok { + t.Fatal("client does not expose GetMetadata") + } + md := mg.GetMetadata() + if md == nil || md.Config == nil || md.Config.Endpoint != cfg.Endpoint { + t.Fatalf("self-contained metadata not set from config: %+v", md) + } + + // Same config → same cached client; invalidate → rebuilt. + c2, err := elastic.GetOrCreateClient(cfg) + if err != nil || c2 != c1 { + t.Fatalf("same config must return the cached client (c2==c1: %v, err: %v)", c2 == c1, err) + } + elastic.InvalidateClient(cfg) + c3, err := elastic.GetOrCreateClient(cfg) + if err != nil || c3 == c1 { + t.Fatalf("after invalidate a new client must be built (c3==c1: %v, err: %v)", c3 == c1, err) + } +} diff --git a/modules/elastic/cluster_loader.go b/modules/elastic/cluster_loader.go new file mode 100644 index 000000000..0a1a37c86 --- /dev/null +++ b/modules/elastic/cluster_loader.go @@ -0,0 +1,92 @@ +/* Copyright © INFINI LTD. All rights reserved. */ + +package elastic + +import ( + "encoding/json" + "reflect" + + log "github.com/cihub/seelog" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/modules/elastic/common" +) + +// The cluster-management REST API (/easysearch/ CRUD) has moved to the +// dedicated, decoupled modules/easysearch package. This file keeps only the +// boot-time live-registration path, which still belongs here because it +// depends on the elasticsearch client factory (modules/elastic/common). It will +// move as part of the broader elasticsearch-module refactor. + +// parseClusterHits extracts the _source array from an orm.SearchResult payload +// (ES-shaped JSON: {"hits":{"hits":[{"_source":{...}}]}}). Returns nil if the +// payload can't be parsed. +func parseClusterHits(res interface{}) json.RawMessage { + if res == nil { + return nil + } + type searchResult struct { + Hits struct { + Hits []struct { + Source json.RawMessage `json:"_source"` + } `json:"hits"` + } `json:"hits"` + } + rv := reflect.ValueOf(res) + if rv.Kind() == reflect.Ptr && !rv.IsNil() { + rv = rv.Elem() + } + payloadField := rv.FieldByName("Payload") + if !payloadField.IsValid() { + return nil + } + var raw []byte + switch v := payloadField.Interface().(type) { + case []byte: + raw = v + case string: + raw = []byte(v) + default: + return nil + } + var sr searchResult + if json.Unmarshal(raw, &sr) != nil { + return nil + } + if len(sr.Hits.Hits) == 0 { + return nil + } + out := make([]json.RawMessage, 0, len(sr.Hits.Hits)) + for _, hit := range sr.Hits.Hits { + out = append(out, hit.Source) + } + b, _ := json.Marshal(out) + return b +} + +// LoadClustersFromORM loads dynamic clusters from the ORM backend (sqlite or +// any non-elastic store) and registers a live client for each. Called from +// ElasticModule.Start when RemoteConfigEnabled is false — i.e. when there's no +// "system ES" to read the cluster index from. This is what makes clusters +// created via the /easysearch/ API (served by modules/easysearch) usable as +// live clients after (re)start. +func LoadClustersFromORM() { + ctx := orm.NewContext().DirectAccess() + orm.WithModel(ctx, &elastic.ElasticsearchConfig{}) + res, err := orm.SearchV2(ctx, orm.NewQuery().Size(1000)) + if err != nil { + log.Warnf("load clusters from ORM: %v", err) + return + } + var clusters []elastic.ElasticsearchConfig + if hits := parseClusterHits(res); hits != nil { + _ = json.Unmarshal(hits, &clusters) + } + for _, cfg := range clusters { + if _, err := common.InitElasticInstance(cfg); err != nil { + log.Warnf("cluster %s (%s): init failed: %v", cfg.ID, cfg.Name, err) + } + } + log.Infof("loaded %d cluster(s) from ORM", len(clusters)) +} diff --git a/modules/elastic/common/config.go b/modules/elastic/common/config.go index ba51f55a0..45f81f447 100644 --- a/modules/elastic/common/config.go +++ b/modules/elastic/common/config.go @@ -25,9 +25,10 @@ package common import ( "fmt" - "infini.sh/framework/core/model" "strings" + "infini.sh/framework/core/model" + log "github.com/cihub/seelog" "infini.sh/framework/core/credential" elastic "infini.sh/framework/core/elastic" @@ -67,20 +68,6 @@ type CheckConfig struct { Interval string `config:"interval,omitempty"` } -type ModuleConfig struct { - Elasticsearch string `config:"elasticsearch"` - RemoteConfigEnabled bool `config:"remote_configs"` - ORMConfig ORMConfig `config:"orm"` - StoreConfig StoreConfig `config:"store"` - HealthCheckConfig CheckConfig `config:"health_check"` - NodeAvailabilityCheckConfig CheckConfig `config:"availability_check"` - MetadataRefresh CheckConfig `config:"metadata_refresh"` - ClusterSettingsCheckConfig CheckConfig `config:"cluster_settings_check"` - ClientTimeout string `config:"client_timeout"` - DeadNodeAvailabilityCheckInterval string `config:"dead_node_availability_check_interval,omitempty"` - SkipInitMetadataOnStart bool `config:"skip_init_metadata_on_start"` -} - func InitClientWithConfig(esConfig elastic.ElasticsearchConfig) (client elastic.API, err error) { var ( diff --git a/modules/elastic/config.go b/modules/elastic/config.go new file mode 100644 index 000000000..3eb00bf1c --- /dev/null +++ b/modules/elastic/config.go @@ -0,0 +1,18 @@ +package elastic + +import "infini.sh/framework/modules/elastic/common" + +type ModuleConfig struct { + Enabled bool `config:"enabled"` + Elasticsearch string `config:"elasticsearch"` + RemoteConfigEnabled bool `config:"remote_configs"` + ORMConfig common.ORMConfig `config:"orm"` + StoreConfig common.StoreConfig `config:"store"` + HealthCheckConfig common.CheckConfig `config:"health_check"` + NodeAvailabilityCheckConfig common.CheckConfig `config:"availability_check"` + MetadataRefresh common.CheckConfig `config:"metadata_refresh"` + ClusterSettingsCheckConfig common.CheckConfig `config:"cluster_settings_check"` + ClientTimeout string `config:"client_timeout"` + DeadNodeAvailabilityCheckInterval string `config:"dead_node_availability_check_interval,omitempty"` + SkipInitMetadataOnStart bool `config:"skip_init_metadata_on_start"` +} diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 60c07bdd9..de05e3ace 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -95,7 +95,49 @@ func (module *ElasticModule) clusterHealthCheck(clusterID string, force bool) { } } +// systemESConfigured reports whether a system Elasticsearch is registered. +func systemESConfigured() bool { + v := global.Lookup(elastic.GlobalSystemElasticsearchID) + s, ok := v.(string) + return ok && s != "" +} + +// updateClusterHealthStatusViaORM persists the health status to the cluster +// record through the ORM — used when clusters are managed without a system +// Elasticsearch (e.g. the /easysearch/ API backed by sqlite), where the +// cluster records live in the ORM rather than a system ES index. +func updateClusterHealthStatusViaORM(clusterID, healthStatus string) { + ctx := orm.NewContext().DirectAccess() + cfg := elastic.ElasticsearchConfig{} + cfg.ID = clusterID + exists, err := orm.GetV2(ctx, &cfg) + if err != nil { + log.Errorf("get cluster %s (orm) for health update: %v", clusterID, err) + return + } + if !exists { + return + } + if cfg.Labels == nil { + cfg.Labels = util.MapStr{} + } + if cur, ok := cfg.Labels["health_status"].(string); ok && cur == healthStatus { + return // unchanged + } + cfg.Labels["health_status"] = healthStatus + if err := orm.Save(ctx, &cfg); err != nil { + log.Errorf("save cluster %s health status (orm): %v", clusterID, err) + } +} + func updateClusterHealthStatus(clusterID string, healthStatus string) { + // Clusters managed via the ORM (no system Elasticsearch) have no system ES + // index to persist health to — fall back to the ORM instead of panicking + // on the system-ES lookup below. + if !systemESConfigured() { + updateClusterHealthStatusViaORM(clusterID, healthStatus) + return + } globalID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) diff --git a/modules/elastic/module.go b/modules/elastic/module.go index 2e26215ce..5f18272ce 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -57,18 +57,18 @@ func (module *ElasticModule) Name() string { } var ( - defaultConfig = common.ModuleConfig{ + defaultConfig = ModuleConfig{ RemoteConfigEnabled: false, HealthCheckConfig: common.CheckConfig{ - Enabled: true, + Enabled: false, Interval: "10s", }, NodeAvailabilityCheckConfig: common.CheckConfig{ - Enabled: true, + Enabled: false, Interval: "10s", }, MetadataRefresh: common.CheckConfig{ - Enabled: true, + Enabled: false, Interval: "30s", }, ORMConfig: common.ORMConfig{ @@ -90,7 +90,7 @@ var ( } ) -func getDefaultConfig() common.ModuleConfig { +func getDefaultConfig() ModuleConfig { return defaultConfig } @@ -197,16 +197,35 @@ func initElasticInstances(m []elastic.ElasticsearchConfig, source string) { } } -var moduleConfig = common.ModuleConfig{} +var moduleConfig = ModuleConfig{} + +// registerClientProviderOnce guards the factory hand-off to core/elastic — +// RegisterClientProvider panics on duplicates, and Setup must stay idempotent +// for tests and repeated module lifecycles. +var registerClientProviderOnce sync.Once func (module *ElasticModule) Setup() { + // Register the client factory with core/elastic BEFORE the enabled check: + // apps may disable this module (no live-cluster management, no ElasticORM) + // while still needing ES clients built from configs (e.g. logpilot log + // streams). core/elastic cannot import this package (dependency cycle), + // so injection is the only path. Pure registration, connects to nothing. + registerClientProviderOnce.Do(func() { + elastic.RegisterClientProvider(common.InitClientWithConfig) + }) + moduleConfig = getDefaultConfig() exists, err := env.ParseConfig("elastic", &moduleConfig) if exists && err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { panic(err) } + + if !moduleConfig.Enabled { + return + } + if exists { if moduleConfig.Elasticsearch != "" { global.Register(elastic.GlobalSystemElasticsearchID, moduleConfig.Elasticsearch) @@ -215,6 +234,9 @@ func (module *ElasticModule) Setup() { m := loadFileBasedElasticConfig() initElasticInstances(m, elastic.ElasticsearchConfigSourceFile) + + // The /easysearch/ cluster-management REST API now lives in the dedicated + // modules/easysearch module (registered by apps that want cluster CRUD). } func (module *ElasticModule) Stop() error { @@ -361,12 +383,9 @@ func InitSchema() { return } - //TODO move to dedicated module - err := orm.RegisterSchemaWithIndexName(elastic.ElasticsearchConfig{}, "cluster") - if err != nil { - panic(err) - } - err = orm.RegisterSchemaWithIndexName(elastic.NodeConfig{}, "node") + // The "cluster" schema (ElasticsearchConfig) is now owned and registered by + // the dedicated modules/easysearch module. + err := orm.RegisterSchemaWithIndexName(elastic.NodeConfig{}, "node") if err != nil { panic(err) } @@ -396,6 +415,10 @@ var ormInited bool func (module *ElasticModule) Start() error { + if !moduleConfig.Enabled { + return nil + } + if moduleConfig.ORMConfig.Enabled { client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) handler := ElasticORM{Client: client, Config: moduleConfig.ORMConfig} @@ -421,6 +444,12 @@ func (module *ElasticModule) Start() error { if moduleConfig.RemoteConfigEnabled { m := loadESBasedElasticConfig() initElasticInstances(m, elastic.ElasticsearchConfigSourceElasticsearch) + } else { + // No "system ES" to read clusters from — load dynamic clusters from + // the ORM backend instead (sqlite or any non-elastic store). This is + // the path used by apps that manage clusters via the /easysearch/ API + // without a dedicated system cluster. + LoadClustersFromORM() } if module.storeHandler != nil { @@ -463,6 +492,7 @@ func (module *ElasticModule) Start() error { return true }) } + if moduleConfig.HealthCheckConfig.Enabled { module.healthMap = sync.Map{} t := task.ScheduleTask{ diff --git a/modules/sqlite/orm_test.go b/modules/sqlite/orm_test.go index a3b83b476..f4bc1b061 100644 --- a/modules/sqlite/orm_test.go +++ b/modules/sqlite/orm_test.go @@ -732,3 +732,73 @@ func TestSQLiteORM_DeleteByQuery_NestedComplexQuery(t *testing.T) { require.NoError(t, err) assert.Equal(t, []string{"dq-3", "dq-4", "dq-5"}, requireSearchResultIDs(t, searchResult)) } + +// Regression tests: UpdatePartialFields must only change the fields present +// in the delta. SQLite Update rewrites the whole row, so the ORM has to seed +// the target object from the stored record before merging the delta — +// otherwise every field omitted from the delta is wiped. Both scenarios share +// one global orm.Register call (the registry rejects duplicates). +func TestSQLiteORM_UpdatePartialFields(t *testing.T) { + handler, cleanup := setupTestDB(t) + defer cleanup() + + orm.Register("sqlite", handler) + + t.Run("preserves fields omitted from the delta", func(t *testing.T) { + created := TestItem{Name: "orig-name", Status: "active", Age: 30} + created.ID = "item-1" + require.NoError(t, orm.Create(orm.NewContext(), &created)) + + obj := TestItem{} + obj.ID = "item-1" + err := orm.UpdatePartialFields(orm.NewContext(), &obj, map[string]interface{}{"name": "new-name"}) + require.NoError(t, err) + + got := TestItem{} + got.ID = "item-1" + exists, err := orm.GetV2(orm.NewContext(), &got) + require.NoError(t, err) + require.True(t, exists) + + assert.Equal(t, "new-name", got.Name) + assert.Equal(t, "active", got.Status, "status should survive a partial update that omits it") + assert.Equal(t, 30, got.Age, "age should survive a partial update that omits it") + }) + + // Partial updates must also deep-merge nested objects: a delta that + // touches one sub-field keeps the sibling sub-fields. + t.Run("deep-merges nested objects", func(t *testing.T) { + type Inner struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + } + type Outer struct { + orm.ORMObjectBase + Name string `json:"name,omitempty"` + Auth *Inner `json:"auth,omitempty"` + Count int `json:"count,omitempty"` + } + require.NoError(t, handler.RegisterSchemaWithName(Outer{}, "test_outers")) + + created := Outer{Name: "orig", Count: 7, Auth: &Inner{Username: "u1", Password: "secret"}} + created.ID = "outer-1" + require.NoError(t, orm.Create(orm.NewContext(), &created)) + + obj := Outer{} + obj.ID = "outer-1" + delta := map[string]interface{}{"name": "updated", "auth": map[string]interface{}{"username": "u2"}} + require.NoError(t, orm.UpdatePartialFields(orm.NewContext(), &obj, delta)) + + got := Outer{} + got.ID = "outer-1" + exists, err := orm.GetV2(orm.NewContext(), &got) + require.NoError(t, err) + require.True(t, exists) + + assert.Equal(t, "updated", got.Name) + assert.Equal(t, 7, got.Count) + require.NotNil(t, got.Auth) + assert.Equal(t, "u2", got.Auth.Username, "nested delta field applied") + assert.Equal(t, "secret", got.Auth.Password, "nested sibling field preserved") + }) +} From bbf6ed8675b4549270d470295705c8c4d383fa33 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 13 Aug 2026 21:41:40 +0800 Subject: [PATCH 2/4] chore: recovery previous configs --- modules/elastic/module.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/elastic/module.go b/modules/elastic/module.go index 5f18272ce..4ca888b0d 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -60,15 +60,15 @@ var ( defaultConfig = ModuleConfig{ RemoteConfigEnabled: false, HealthCheckConfig: common.CheckConfig{ - Enabled: false, + Enabled: true, Interval: "10s", }, NodeAvailabilityCheckConfig: common.CheckConfig{ - Enabled: false, + Enabled: true, Interval: "10s", }, MetadataRefresh: common.CheckConfig{ - Enabled: false, + Enabled: true, Interval: "30s", }, ORMConfig: common.ORMConfig{ @@ -385,6 +385,8 @@ func InitSchema() { // The "cluster" schema (ElasticsearchConfig) is now owned and registered by // the dedicated modules/easysearch module. + _ = orm.RegisterSchemaWithIndexName(elastic.ElasticsearchConfig{}, "cluster") + err := orm.RegisterSchemaWithIndexName(elastic.NodeConfig{}, "node") if err != nil { panic(err) From 1c066112026a8a4fa66d4b136d28e294a76a1902 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 13 Aug 2026 21:54:21 +0800 Subject: [PATCH 3/4] chore: add feature gateway to orm features --- core/config/system.go | 8 +++++++- core/env/env.go | 1 + modules/elastic/module.go | 20 ++++++++++++-------- modules/sqlite/indexes_test.go | 22 +++++++++++----------- modules/sqlite/orm.go | 2 +- 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/core/config/system.go b/core/config/system.go index ab95803ef..8824e028a 100755 --- a/core/config/system.go +++ b/core/config/system.go @@ -72,7 +72,7 @@ type NetworkConfig struct { SkipOccupiedPort bool `config:"skip_occupied_port" json:"skip_occupied_port,omitempty" elastic_mapping:"skip_occupied_port: { type: boolean }"` //ReusePort is nil when reuse_port is not configured, so applications can //tell "unset" apart from an explicit false and apply their own default. - ReusePort *bool `config:"reuse_port" json:"reuse_port,omitempty" elastic_mapping:"reuse_port: { type: boolean }"` + ReusePort *bool `config:"reuse_port" json:"reuse_port,omitempty" elastic_mapping:"reuse_port: { type: boolean }"` } // Helper function to report whether SO_REUSEPORT is explicitly enabled. @@ -164,6 +164,8 @@ type SystemConfig struct { PathConfig PathConfig `config:"path"` + ORMConfig ORMConfig `config:"orm"` + LoggingConfig LoggingConfig `config:"log"` AllowMultiInstance bool `config:"allow_multi_instance"` @@ -183,6 +185,10 @@ type SystemConfig struct { HTTPClientConfig map[string]HTTPClientConfig `config:"http_client"` } +type ORMConfig struct { + Enabled bool `config:"enabled"` +} + type CookieConfig struct { Store string `config:"store"` //cookie/filesystem StorePath string `config:"store_path"` //filesystem only diff --git a/core/env/env.go b/core/env/env.go index f81411044..f9015a8a0 100755 --- a/core/env/env.go +++ b/core/env/env.go @@ -335,6 +335,7 @@ func GetDefaultSystemConfig() config.SystemConfig { Log: "log", Config: "config", }, + ORMConfig: config.ORMConfig{Enabled: true}, AllowMultiInstance: false, MaxNumOfInstance: 5, diff --git a/modules/elastic/module.go b/modules/elastic/module.go index 4ca888b0d..82b7415c0 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -433,14 +433,18 @@ func (module *ElasticModule) Start() error { kv.Register("elastic", module.storeHandler) } - if moduleConfig.ORMConfig.Enabled { - if !ormInited { - //init template - InitTemplate(false) - //register schema - InitSchema() - ormInited = true + if global.Env().SystemConfig.ORMConfig.Enabled { + if moduleConfig.ORMConfig.Enabled { + if !ormInited { + //init template + InitTemplate(false) + //register schema + InitSchema() + ormInited = true + } } + + LoadClustersFromORM() } if moduleConfig.RemoteConfigEnabled { @@ -451,7 +455,7 @@ func (module *ElasticModule) Start() error { // the ORM backend instead (sqlite or any non-elastic store). This is // the path used by apps that manage clusters via the /easysearch/ API // without a dedicated system cluster. - LoadClustersFromORM() + } if module.storeHandler != nil { diff --git a/modules/sqlite/indexes_test.go b/modules/sqlite/indexes_test.go index 02aaa50b6..3cda9b8d7 100644 --- a/modules/sqlite/indexes_test.go +++ b/modules/sqlite/indexes_test.go @@ -36,11 +36,11 @@ type indexSliceItem struct { // and an enabled:false object backed by a map. type indexRootModel struct { orm.ORMObjectBase - Name string `json:"name,omitempty" elastic_mapping:"name: { type: keyword }"` - CPU indexNestedSpec `json:"cpu,omitempty" elastic_mapping:"cpu: { type: object }"` - Disk *indexNestedSpec `json:"disk,omitempty" elastic_mapping:"disk: { type: object }"` - Tags []string `json:"tags,omitempty" elastic_mapping:"tags: { type: keyword }"` - Items []indexSliceItem `json:"items,omitempty" elastic_mapping:"items: { type: nested }"` + Name string `json:"name,omitempty" elastic_mapping:"name: { type: keyword }"` + CPU indexNestedSpec `json:"cpu,omitempty" elastic_mapping:"cpu: { type: object }"` + Disk *indexNestedSpec `json:"disk,omitempty" elastic_mapping:"disk: { type: object }"` + Tags []string `json:"tags,omitempty" elastic_mapping:"tags: { type: keyword }"` + Items []indexSliceItem `json:"items,omitempty" elastic_mapping:"items: { type: nested }"` Secret map[string]interface{} `json:"secret,omitempty" elastic_mapping:"secret: { type: object, enabled: false }"` } @@ -166,17 +166,17 @@ func queryPlanDetail(t *testing.T, db *sql.DB, query string) []string { func TestParseMappingTag(t *testing.T) { cases := []struct { - tag string - field string - esType string - ok bool + tag string + field string + esType string + ok bool }{ {`stream_id:{type:keyword}`, "stream_id", "keyword", true}, {`created: { type: date }`, "created", "date", true}, {`cpu: { type: object }`, "cpu", "object", true}, {`stats: { properties: { x: { type: keyword } } }`, "stats", "", false}, // no top-level type: - {`s: { subtype: keyword }`, "s", "", false}, // "type:" inside "subtype:" must not match - {`enabled-only`, "", "", false}, // no colon + {`s: { subtype: keyword }`, "s", "", false}, // "type:" inside "subtype:" must not match + {`enabled-only`, "", "", false}, // no colon {"", "", "", false}, } for _, c := range cases { diff --git a/modules/sqlite/orm.go b/modules/sqlite/orm.go index 10cb5b97f..7d1598c83 100644 --- a/modules/sqlite/orm.go +++ b/modules/sqlite/orm.go @@ -32,12 +32,12 @@ import ( "strings" log "github.com/cihub/seelog" - _ "modernc.org/sqlite" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" api "infini.sh/framework/core/orm" "infini.sh/framework/core/util" sqliteOrm "infini.sh/framework/modules/sqlite/orm" + _ "modernc.org/sqlite" ) var ErrNotFound = errors.New("record not found") From 4f12830b1808e2908511561c246be57e72eb2a0f Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 13 Aug 2026 22:01:42 +0800 Subject: [PATCH 4/4] chore: update release notes --- docs/content.en/docs/release-notes/_index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index 085dbb9ba..8e3c22ec3 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -21,6 +21,7 @@ Information about release notes of INFINI Framework is provided here. - perf(sqlite): replace the WASM SQLite driver (ncruces/wazero) with modernc (pure-Go upstream SQLite) — removes the WASM interpreter + custom VFS overhead that dominated CPU profiles; no cgo, cross-compiles with `CGO_ENABLED=0`, no on-disk format change - perf(sqlite): set per-connection PRAGMAs (WAL, busy_timeout, foreign_keys) via the DSN and enable 256 MiB mmap_size — applies to every pooled connection (not just one) and serves the metadata store via memory-mapped I/O instead of pread syscalls - perf(sqlite): auto-create expression indexes from elastic_mapping tags, now including nested object fields via dotted `$.parent.child` paths — the SQLite ORM indexes `json_extract(raw,'$.field')` for keyword/date/long/integer/boolean/double fields, turning full-table scans into B-tree lookups with zero query or model changes +- feat: search provider injection + easysearch cluster CRUD module #398 ### 🐛 Bug fix - fix: expand configs.template when loading templated config files #391