Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,12 @@ logs/

# Dashboard UI
internal/dashboard/ui/node_modules/cp.log

# SQLite data. The provider's database normally lives in $CP_PATH, outside the
# repo — but CP_PATH can be pointed here during development, and the WAL and
# shared-memory sidecars appear beside it.
*.db
*.db-wal
*.db-shm
*.sqlite
*.sqlite3
10 changes: 10 additions & 0 deletions conf/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,16 @@ type ComputeNode struct {
HealthCheck HealthCheck `toml:"HealthCheck,omitempty"`
RequestLimits RequestLimits `toml:"RequestLimits,omitempty"`
Dashboard Dashboard `toml:"Dashboard,omitempty"`
RequestLog RequestLog `toml:"RequestLog,omitempty"`
}

// RequestLog controls how long the per-request history kept for the
// Transactions view is retained. It is one row per served request, so both
// limits exist: age for how far back it is useful, and a row cap so a burst
// cannot fill the disk before the age limit bites.
type RequestLog struct {
RetentionDays int `toml:"RetentionDays"` // Default: 7
MaxRows int64 `toml:"MaxRows"` // Default: 200000
}

// Dashboard configures the web UI served by `computing-provider dashboard`.
Expand Down
21 changes: 21 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,27 @@ overloaded GPU. Both can also be changed at runtime, globally or per model,
through the REST API — see the endpoint table in the
[main README](../README.md#rest-api).

### Request history

```toml
[RequestLog]
RetentionDays = 7 # how far back the Transactions view reaches (default 7)
MaxRows = 200000 # hard cap on stored rows (default 200000)
```

Every served request is stored, so the Transactions view — and its model and
source filters — survive a restart. Before this existed the list lived only in
a 1000-entry in-memory ring, so each restart emptied it, and a node that
restarts often kept almost no history at all.

Two limits rather than one: `RetentionDays` is how far back the data stays
useful, and `MaxRows` stops a traffic burst filling the disk before the age
limit ever applies. Whichever binds first wins, and pruning runs hourly.

Writes are batched and happen off the request path, so recording never delays
serving; under a burst large enough to fill the queue, records are dropped and
the count is logged rather than blocking inference.

### models.json field reference

Each key in `models.json` is the marketplace model ID (must match a value in `Models` above).
Expand Down
16 changes: 16 additions & 0 deletions internal/computing/inference_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ type InferenceMetrics struct {

// Request history (circular buffer)
requestHistory []RequestMetric
// requestSink persists each request, when a store is attached.
requestSink func(RequestMetric)
historyMu sync.RWMutex
maxHistorySize int
}
Expand Down Expand Up @@ -407,11 +409,25 @@ func (m *InferenceMetrics) GetPrometheusMetrics() string {
return sb.String()
}

// SetRequestSink installs a persistent store fed alongside the in-memory ring.
// The ring stays: it answers without touching the database, and it is what the
// model detail view reads.
func (m *InferenceMetrics) SetRequestSink(sink func(RequestMetric)) {
m.historyMu.Lock()
defer m.historyMu.Unlock()
m.requestSink = sink
}

// RecordRequest adds a request to the history circular buffer
func (m *InferenceMetrics) RecordRequest(req RequestMetric) {
m.historyMu.Lock()
defer m.historyMu.Unlock()

if m.requestSink != nil {
// Never blocks — the sink queues and returns.
m.requestSink(req)
}

if len(m.requestHistory) >= m.maxHistorySize {
// Remove oldest entry (circular buffer)
m.requestHistory = m.requestHistory[1:]
Expand Down
36 changes: 36 additions & 0 deletions internal/computing/inference_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ type InferenceService struct {
retryPolicy *RetryPolicy
gpuCollector *GPUMetricsCollector
metricsHistory *MetricsHistory
requestStore *RequestStore
alertMonitor *alertMonitor
selfCheck *selfCheckRunner
noticeLimiter *noticeLimiter
Expand Down Expand Up @@ -116,6 +117,17 @@ func NewInferenceService(nodeID, cpPath string) *InferenceService {
// Create metrics history for persistence
metricsHistory := NewMetricsHistory()

// Per-request history, so the Transactions view survives a restart.
// GetConfig is nil until InitConfig has run, which is not guaranteed for
// callers that construct the service directly; the store's own defaults
// cover that.
var retentionDays int
var maxRows int64
if cfg := conf.GetConfig(); cfg != nil {
retentionDays, maxRows = cfg.RequestLog.RetentionDays, cfg.RequestLog.MaxRows
}
requestStore := NewRequestStore(retentionDays, maxRows)

s := &InferenceService{
nodeID: nodeID,
cpPath: cpPath,
Expand All @@ -127,6 +139,7 @@ func NewInferenceService(nodeID, cpPath string) *InferenceService {
retryPolicy: retryPolicy,
gpuCollector: gpuCollector,
metricsHistory: metricsHistory,
requestStore: requestStore,
}

// Set up registry callbacks to update modelMappings for backward compatibility
Expand Down Expand Up @@ -345,6 +358,15 @@ func (s *InferenceService) Start() error {
s.alertMonitor.Start()
s.selfCheck.Start()

// Start persistent request history, and feed it from the metrics recorder.
if s.requestStore != nil {
if err := s.requestStore.Start(); err != nil {
logs.GetLogger().Warnf("Failed to start request history store: %v", err)
} else if s.client != nil && s.client.metrics != nil {
s.client.metrics.SetRequestSink(s.requestStore.Record)
}
}

// Start metrics history recorder
if s.metricsHistory != nil {
if err := s.metricsHistory.Start(func() *InferenceMetricsData {
Expand Down Expand Up @@ -384,6 +406,9 @@ func (s *InferenceService) Stop() {
if s.metricsHistory != nil {
s.metricsHistory.Stop()
}
if s.requestStore != nil {
s.requestStore.Stop()
}
if s.alertMonitor != nil {
s.alertMonitor.Stop()
}
Expand Down Expand Up @@ -1271,7 +1296,18 @@ func (s *InferenceService) GetRequestHistory(limit int, modelFilter string) []Re

// QueryRequestHistory returns one page of request history along with the total
// matching the filters.
//
// Served from the persistent store when there is one, so the list reaches back
// past the last restart. The in-memory ring is the fallback: it is all that
// exists before the store has started, or if the database is unavailable.
func (s *InferenceService) QueryRequestHistory(q RequestHistoryQuery) RequestHistoryPage {
if s.requestStore != nil {
if page, err := s.requestStore.Query(q); err == nil {
return page
} else {
logs.GetLogger().Warnf("Request history query failed, falling back to memory: %v", err)
}
}
if s.client == nil {
return RequestHistoryPage{Requests: []RequestMetric{}, Limit: q.Limit, Offset: q.Offset}
}
Expand Down
Loading
Loading