Document the constraints around TSDB scrape interval and CEEMS API server's update interval

.golangci.yml

@@ -28,6 +28,9 @@ linters:
     - wrapcheck
     - wsl
   settings:
+    godot:
+      exclude:
+        - "(.*)@x-logo(.*)"
     wsl_v5:
       allow-first-in-block: true
       allow-whole-block: false

README.md

@@ -6,7 +6,7 @@
 | CI/CD   | [![ci](https://github.com/ceems-dev/ceems/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/ceems-dev/ceems/actions/workflows/ci.yml?query=branch%3Amain) [![CircleCI](https://dl.circleci.com/status-badge/img/circleci/UVxmfk5AT3EHZpsg3FdfaR/PKvLaAH1ahBZf8kBKHhCiA/tree/main.svg?style=svg&circle-token=CCIPRJ_67vq2cGkBpm9syySEp7tTW_7d4d6f3e8d72486acf477768f4f0a1d5235ab2a0)](https://dl.circleci.com/status-badge/redirect/circleci/UVxmfk5AT3EHZpsg3FdfaR/PKvLaAH1ahBZf8kBKHhCiA/tree/main)  [![Coverage](https://img.shields.io/badge/Coverage-78.2%25-brightgreen)](https://github.com/ceems-dev/ceems/actions/workflows/ci.yml?query=branch%3Amain)                                                                                          |
 | Docs    | [![docs](https://img.shields.io/badge/docs-passing-green?style=flat&link=https://ceems-dev.github.io/ceems/docs/)](https://ceems-dev.github.io/ceems/)  [![Go Doc](https://godoc.org/github.com/ceems-dev/ceems?status.svg)](http://godoc.org/github.com/ceems-dev/ceems) [![Discord](https://img.shields.io/discord/1466874652189003973?label=discord)](https://discord.gg/bedRvX5uVY)                                                                                                                                                                                                                            |
 | Package | [![Release](https://img.shields.io/github/v/release/ceems-dev/ceems.svg?include_prereleases)](https://github.com/ceems-dev/ceems/releases/latest) [![Releases downloads](https://img.shields.io/github/downloads/ceems-dev/ceems/total.svg)](https://github.com/ceems-dev/ceems/releases)                                                                                                                                                                    |
-| Meta    | [![GitHub License](https://img.shields.io/github/license/ceems-dev/ceems)](https://github.com/ceems-dev/ceems) [![Go Report Card](https://ceems-demo.myaddr.tools:2443/badge/github.com/ceems-dev/ceems)](https://ceems-demo.myaddr.tools:2443/report/github.com/ceems-dev/ceems) [![code style](https://img.shields.io/badge/code%20style-gofmt-blue.svg)](https://pkg.go.dev/cmd/gofmt) |
+| Meta    | [![GitHub License](https://img.shields.io/github/license/ceems-dev/ceems)](https://github.com/ceems-dev/ceems) [![Go Report Card](https://ceems-demo.myaddr.tools:2443/badge/github.com/ceems-dev/ceems)](https://ceems-demo.myaddr.tools:2443/report/github.com/ceems-dev/ceems) [![code style](https://img.shields.io/badge/code%20style-gofmt-blue.svg)](https://pkg.go.dev/cmd/gofmt) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/13395/badge)](https://www.bestpractices.dev/projects/13395) |
 
 <!-- markdown-link-check-enable -->
 

build/config/ceems_api_server/ceems_api_server.yml

@@ -710,13 +710,31 @@ updaters: []
   #     query_max_series: 50
 
   #     # Minimum number of samples that are guaranteed to available for executing the queries
-  #     # of the updater. It is expressed as proportion of `--query.max-samples` and takes a value
+  #     # of the updater. It is expressed as proportion of `query_max_series` and takes a value
   #     # between 0 to 1. A smaller value means smaller batch sizes.
   #     #
   #     # Default value is 0.5
   #     #
   #     query_min_samples: 0.5
 
+  #     # Scrape interval corresponding to the scrape targets that generate metrics provided in
+  #     # `queries` section.
+  #     #
+  #     # Default value `0s` means global scrape interval of the TSDB instance will be used.
+  #     #
+  #     # Units Supported: y, w, d, h, m, s, ms.
+  #     #
+  #     scrape_interval: 0s
+
+  #     # Evaluation interval corresponding to the recording rules that generate metrics provided in
+  #     # `queries` section.
+  #     #
+  #     # Default value `0s` means global evaluation interval of the TSDB instance will be used.
+  #     #
+  #     # Units Supported: y, w, d, h, m, s, ms.
+  #     #
+  #     evaluation_interval: 0s
+
   #     # Compute units that have total life time less than this value will be marked as ignored
   #     # in the CEEMS API DB. Compute units will only be marked as `ignored` and they are still
   #     # kept in the TSDB.

etc/prometheus/README.md

@@ -13,6 +13,9 @@ being scrapped by Prometheus successfully. We recommend to this tool to generate
 recording rules for your deployment. More details on how to generate rules can be
 found in [docs](https://ceems-dev.github.io/ceems/docs/usage/ceems-tool).
 
+> [!IMPORTANT]
+> ALWAYS USE `evaluation_interval` THAT IS SAME AS THE `scrape_interval` FOR BEST RESULTS.
+
 ## Rules
 
 The following recording rules files are provided for reference purposes and `ceems_tool`

pkg/api/cli/cli.go

@@ -132,11 +132,20 @@ func NewCEEMSServer() (*CEEMSServer, error) {
 //	@securityDefinitions.basic	BasicAuth
 //
 //	@servers.url				https://ceems-demo.myaddr.tools:6443/api/v1
-//	@servers.description		Test CEEMS API server URL.
+//	@servers.description		Demo CEEMS API server URL.
 //
-//	@externalDocs.url			https://ceems-dev.github.io/ceems/
+// @servers.url				{scheme}://{host}:{port}{basepath}
+// @servers.description		Current CEEMS API server URL.
+// @servers.variables.enum		scheme http
+// @servers.variables.enum		scheme https
+// @servers.variables.default	scheme http
+// @servers.variables.default	host localhost
+// @servers.variables.default	port 9020
+// @servers.variables.default	basepath /api/v1
 //
-//	@x-logo						{"url": "https://raw.githubusercontent.com/ceems-dev/ceems/refs/heads/main/website/static/img/logo.png", "altText": "CEEMS logo"}
+// @externalDocs.url			https://ceems-dev.github.io/ceems/
+//
+// @x-logo						{"url": "https://raw.githubusercontent.com/ceems-dev/ceems/refs/heads/main/website/static/img/logo.png", "altText": "CEEMS logo"}
 func (b *CEEMSServer) Main() error {
 	// CLI vars
 	var (

pkg/api/docs/swagger.json

@@ -1008,8 +1008,12 @@
                 ],
                 "servers": [
                     {
-                        "description": "Test CEEMS API server URL for health, swagger and debug endpoints only.",
+                        "description": "Demo CEEMS API server URL for health, swagger and debug endpoints only.",
                         "url": "https://ceems-demo.myaddr.tools:6443"
+                    },
+                    {
+                        "description": "Current CEEMS API server URL for health, swagger and debug endpoints only.",
+                        "url": "{scheme}://{host}:{port}{basepath}"
                     }
                 ],
                 "summary": "Health check endpoint for API server.",
@@ -2132,8 +2136,30 @@
     "openapi": "3.1.0",
     "servers": [
         {
-            "description": "Test CEEMS API server URL.",
+            "description": "Demo CEEMS API server URL.",
             "url": "https://ceems-demo.myaddr.tools:6443/api/v1"
+        },
+        {
+            "description": "Current CEEMS API server URL.",
+            "url": "{scheme}://{host}:{port}{basepath}",
+            "variables": {
+                "basepath": {
+                    "default": "/api/v1"
+                },
+                "host": {
+                    "default": "localhost"
+                },
+                "port": {
+                    "default": "9020"
+                },
+                "scheme": {
+                    "default": "http",
+                    "enum": [
+                        "http",
+                        "https"
+                    ]
+                }
+            }
         }
     ]
 }

pkg/api/docs/swagger.yaml

@@ -809,9 +809,12 @@ paths:
       security:
       - BasicAuth: []
       servers:
-      - description: Test CEEMS API server URL for health, swagger and debug endpoints
+      - description: Demo CEEMS API server URL for health, swagger and debug endpoints
           only.
         url: https://ceems-demo.myaddr.tools:6443
+      - description: Current CEEMS API server URL for health, swagger and debug endpoints
+          only.
+        url: '{scheme}://{host}:{port}{basepath}'
       summary: Health check endpoint for API server.
       tags:
       - health
@@ -1697,5 +1700,19 @@ paths:
       tags:
       - users
 servers:
-- description: Test CEEMS API server URL.
+- description: Demo CEEMS API server URL.
   url: https://ceems-demo.myaddr.tools:6443/api/v1
+- description: Current CEEMS API server URL.
+  url: '{scheme}://{host}:{port}{basepath}'
+  variables:
+    basepath:
+      default: /api/v1
+    host:
+      default: localhost
+    port:
+      default: "9020"
+    scheme:
+      default: http
+      enum:
+      - http
+      - https

pkg/api/http/server.go

@@ -338,7 +338,7 @@ func New(c *Config) (*CEEMSServer, error) {
 // Start launches CEEMS HTTP server godoc.
 func (s *CEEMSServer) Start(_ context.Context) error {
 	// Set swagger info
-	// docs.SwaggerInfo.BasePath = "/api/" + base.APIVersion
+	docs.SwaggerInfo.BasePath = "/api/" + base.APIVersion
 	docs.SwaggerInfo.Schemes = []string{"http", "https"}
 	docs.SwaggerInfo.Host = s.server.Addr
 
@@ -429,17 +429,25 @@ func (s *CEEMSServer) setWriteDeadline(deadline time.Duration, w http.ResponseWr
 //	@Description	This health endpoint can be used to check on the health of the API server
 //	@Description	in containerised and Kubernetes environments.
 //	@Description
-//	@Description			This endpoint does not need any user header. However, if basic auth
-//	@Description			is enabled, authentication header must be provided to get status.
-//	@servers.url			https://ceems-demo.myaddr.tools:6443
-//	@servers.description	Test CEEMS API server URL for health, swagger and debug endpoints only.
-//	@Security				BasicAuth
-//	@Tags					health
-//	@Produce				plain
-//	@Success				200	{string}	string	"OK"
-//	@Failure				401	{string}	string	"Unauthorized"
-//	@Failure				503	{string}	string	"KO"
-//	@Router					/health [get]
+//	@Description				This endpoint does not need any user header. However, if basic auth
+//	@Description				is enabled, authentication header must be provided to get status.
+//	@servers.url				https://ceems-demo.myaddr.tools:6443
+//	@servers.description		Demo CEEMS API server URL for health, swagger and debug endpoints only.
+//	@servers.url				{scheme}://{host}:{port}{basepath}
+//	@servers.description		Current CEEMS API server URL for health, swagger and debug endpoints only.
+//	@servers.variables.enum		scheme http
+//	@servers.variables.enum		scheme https
+//	@servers.variables.default	scheme http
+//	@servers.variables.default	host localhost
+//	@servers.variables.default	port 9020
+//	@servers.variables.default	basepath /api/v1
+//	@Security					BasicAuth
+//	@Tags						health
+//	@Produce					plain
+//	@Success					200	{string}	string	"OK"
+//	@Failure					401	{string}	string	"Unauthorized"
+//	@Failure					503	{string}	string	"KO"
+//	@Router						/health [get]
 //
 // GET /health
 // Get health status of the server.

pkg/api/updater/tsdb/tsdb.go

@@ -94,13 +94,15 @@ var (
 
 // config is the container for the configuration of a given TSDB instance.
 type tsdbConfig struct {
-	QueryMaxSeries  int64                        `yaml:"query_max_series"`
-	QueryMinSamples float64                      `yaml:"query_min_samples"`
-	CutoffDuration  model.Duration               `yaml:"cutoff_duration"`
-	QueryTimeout    model.Duration               `yaml:"query_timeout"`
-	DeleteIgnore    bool                         `yaml:"delete_ignored"`
-	Queries         map[string]map[string]string `yaml:"queries"`
-	LabelsToDrop    []string                     `yaml:"labels_to_drop"`
+	QueryMaxSeries     int64                        `yaml:"query_max_series"`
+	QueryMinSamples    float64                      `yaml:"query_min_samples"`
+	ScrapeInterval     model.Duration               `yaml:"scrape_interval"`
+	EvaluationInterval model.Duration               `yaml:"evaluation_interval"`
+	CutoffDuration     model.Duration               `yaml:"cutoff_duration"`
+	QueryTimeout       model.Duration               `yaml:"query_timeout"`
+	DeleteIgnore       bool                         `yaml:"delete_ignored"`
+	Queries            map[string]map[string]string `yaml:"queries"`
+	LabelsToDrop       []string                     `yaml:"labels_to_drop"`
 }
 
 // defaults set struct fields to default values.
@@ -155,6 +157,14 @@ func (c *tsdbConfig) validate() error {
 		return errors.New("query_timeout must be a valid time duration")
 	}
 
+	if c.ScrapeInterval < 0 {
+		return errors.New("scrape_interval must be a valid time duration")
+	}
+
+	if c.EvaluationInterval < 0 {
+		return errors.New("evaluation_interval must be a valid time duration")
+	}
+
 	return nil
 }
 
@@ -200,6 +210,11 @@ func New(instance updater.Instance, logger *slog.Logger) (updater.Updater, error
 		return nil, err
 	}
 
+	// Emit warning if scrape and evaluation intervals have not been configured
+	if config.ScrapeInterval <= 0 || config.EvaluationInterval <= 0 {
+		logger.Warn("No values configured for scrape_interval and/or evaluation_interval. It is highly recommended to configure updater with appropriate values for scrape_interval and evaluation_interval")
+	}
+
 	// Check HTTP client config
 	readPaths, err := common.CheckHTTPClientConfigFiles(&instance.Web.HTTPClientConfig)
 	if err != nil {
@@ -270,6 +285,8 @@ func (t *tsdbUpdater) fetchAggMetrics(
 
 	// If duration is less than rateInterval bail
 	if duration < settings.RateInterval {
+		t.Logger.Warn("Skipping updating units with TSDB updater as update_interval is too small. Use an update_interval that is > 10 * scrape interval of TSDB", "update_interval", duration, "scrape_interval", settings.ScrapeInterval)
+
 		return aggMetrics
 	}
 
@@ -359,6 +376,8 @@ func (t *tsdbUpdater) update(
 ) []models.Unit {
 	// Bail if TSDB is unavailable or there are no units to update
 	if !t.Available() || len(units) == 0 {
+		t.Logger.Warn("TSDB server unavailable or no units to update", "tsdb_server_available", t.Available(), "num_units", len(units))
+
 		return units
 	}
 
@@ -414,6 +433,21 @@ func (t *tsdbUpdater) update(
 	// Get rate and scrape intervals
 	settings := t.Settings(ctx)
 
+	// If scrape and evaluation intervals have been provided, use them instead of global value
+	if t.config.ScrapeInterval > 0 {
+		settings.ScrapeInterval = time.Duration(t.config.ScrapeInterval)
+		settings.RateInterval = 4 * time.Duration(t.config.ScrapeInterval)
+	}
+
+	if t.config.EvaluationInterval > 0 {
+		settings.EvaluationInterval = time.Duration(t.config.EvaluationInterval)
+	}
+
+	// Emit a warning if evaluation interval is larger than scrape interval
+	if t.config.EvaluationInterval > t.config.ScrapeInterval {
+		t.Logger.Warn("evaluation_interval is found to be greater than scrape_interval. It is recommended to use evalution_interval same as scrape_interval on TSDB server")
+	}
+
 	// Estimate a batch size based on scrape interval, duration, query max samples and total time series
 	samplesPerSeries := max(int64(duration.Seconds()/settings.ScrapeInterval.Seconds()), 1)
 	maxLabels := settings.QueryMaxSamples / (t.config.QueryMaxSeries * samplesPerSeries)

website/customMdGenerators.ts

@@ -20,6 +20,10 @@ ${servers
     if (!server.description) {
       server.description = 'Demo';
     }
+    // Ignore placeholder server
+    if (server.url.startsWith("{scheme}")) {
+      return
+    }
     return `| ${server.url} | ${server.description} | `.replace(/\n/g, "<br/>");
   })
   .join("\n")}

website/docs/00-introduction.md

@@ -12,7 +12,7 @@ slug: /
 | CI/CD   | [![ci](https://github.com/@ceemsOrg@/@ceemsRepo@/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/@ceemsOrg@/@ceemsRepo@/actions/workflows/ci.yml?query=branch%3Amain) [![CircleCI](https://dl.circleci.com/status-badge/img/circleci/UVxmfk5AT3EHZpsg3FdfaR/PKvLaAH1ahBZf8kBKHhCiA/tree/main.svg?style=svg&circle-token=CCIPRJ_67vq2cGkBpm9syySEp7tTW_7d4d6f3e8d72486acf477768f4f0a1d5235ab2a0)](https://dl.circleci.com/status-badge/redirect/circleci/UVxmfk5AT3EHZpsg3FdfaR/PKvLaAH1ahBZf8kBKHhCiA/tree/main)  [![Coverage](https://img.shields.io/badge/Coverage-78.2%25-brightgreen)](https://github.com/ceems-dev/ceems/actions/workflows/ci.yml?query=branch%3Amain)                                                                                          |
 | Docs    | [![docs](https://img.shields.io/badge/docs-passing-green?style=flat&link=https://github.com/@ceemsOrg@/@ceemsRepo@/blob/main/README.md)](https://github.com/@ceemsOrg@/@ceemsRepo@/blob/main/README.md)  [![Go Doc](https://godoc.org/github.com/@ceemsOrg@/@ceemsRepo@?status.svg)](http://godoc.org/github.com/@ceemsOrg@/@ceemsRepo@) [![Discord](https://img.shields.io/discord/1466874652189003973?label=discord)](https://discord.gg/bedRvX5uVY)                                                                                                                                                                                                                            |
 | Package | [![Release](https://img.shields.io/github/v/release/@ceemsOrg@/@ceemsRepo@.svg?include_prereleases)](https://github.com/@ceemsOrg@/@ceemsRepo@/releases/latest) [![Releases downloads](https://img.shields.io/github/downloads/@ceemsOrg@/@ceemsRepo@/total.svg)](https://github.com/@ceemsOrg@/@ceemsRepo@/releases)                                                                                                                                                                    |
-| Meta    | [![GitHub License](https://img.shields.io/github/license/@ceemsOrg@/@ceemsRepo@)](https://github.com/@ceemsOrg@/@ceemsRepo@) [![Go Report Card](https://ceems-demo.myaddr.tools:2443/badge/github.com/@ceemsOrg@/@ceemsRepo@)](https://ceems-demo.myaddr.tools:2443/report/github.com/@ceemsOrg@/@ceemsRepo@) [![code style](https://img.shields.io/badge/code%20style-gofmt-blue.svg)](https://pkg.go.dev/cmd/gofmt) |
+| Meta    | [![GitHub License](https://img.shields.io/github/license/@ceemsOrg@/@ceemsRepo@)](https://github.com/@ceemsOrg@/@ceemsRepo@) [![Go Report Card](https://ceems-demo.myaddr.tools:2443/badge/github.com/@ceemsOrg@/@ceemsRepo@)](https://ceems-demo.myaddr.tools:2443/report/github.com/@ceemsOrg@/@ceemsRepo@) [![code style](https://img.shields.io/badge/code%20style-gofmt-blue.svg)](https://pkg.go.dev/cmd/gofmt) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/13395/badge)](https://www.bestpractices.dev/projects/13395) |
 
 <!-- markdown-link-check-enable -->